diff --git "a/checkpoint-26333/trainer_state.json" "b/checkpoint-26333/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-26333/trainer_state.json" @@ -0,0 +1,34237 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 26333, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027500001216928163, + "completions/max_length": 490.56666666666666, + "completions/max_terminated_length": 461.4, + "completions/mean_length": 188.9975112915039, + "completions/mean_terminated_length": 180.10747375488282, + "completions/min_length": 43.4, + "completions/min_terminated_length": 43.4, + "entropy": 0.2029941453287999, + "epoch": 0.0011392549272775604, + "eval/gt_acc_batch": 0.7313889125982921, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3513047993183136, + "kd/policy_loss": 0.013840306313553204, + "kd/rkl_advantage_mean": 0.24764675879268908, + "kd/rkl_advantage_p95": 3.8125994205474854, + "kd/rkl_advantage_std": 2.0226369639237722, + "kd/ssd_loss": 0.0, + "learning_rate": 9.98898720236965e-07, + "loss": 0.05536123116811117, + "num_tokens": 980727.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7313888847827912, + "rewards/gt_logging_reward/std": 0.43746271431446077, + "sampling/importance_sampling_ratio/max": 1.9902352770169576, + "sampling/importance_sampling_ratio/mean": 1.0039433201154073, + "sampling/importance_sampling_ratio/min": 0.4575092285871506, + "sampling/sampling_logp_difference/max": 0.27222853899002075, + "sampling/sampling_logp_difference/mean": 0.004479383436652521, + "step": 30, + "step_time": 7.885652181366458, + "student/entropy": 0.2029941453287999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722224300106365, + "completions/max_length": 503.46666666666664, + "completions/max_terminated_length": 465.2, + "completions/mean_length": 195.39556681315105, + "completions/mean_terminated_length": 185.8046076456706, + "completions/min_length": 45.5, + "completions/min_terminated_length": 45.5, + "entropy": 0.19655973594635726, + "epoch": 0.002278509854555121, + "eval/gt_acc_batch": 0.723888905843099, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.47802847623825073, + "kd/policy_loss": 0.013037882768549025, + "kd/rkl_advantage_mean": 0.3268238168520232, + "kd/rkl_advantage_p95": 3.9106563359498976, + "kd/rkl_advantage_std": 2.0042735169331234, + "kd/ssd_loss": 0.0, + "learning_rate": 9.977594653096876e-07, + "loss": 0.05215153694152832, + "num_tokens": 1983783.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.723888885974884, + "rewards/gt_logging_reward/std": 0.43908915122350056, + "sampling/importance_sampling_ratio/max": 1.9542729576428732, + "sampling/importance_sampling_ratio/mean": 0.99688547650973, + "sampling/importance_sampling_ratio/min": 0.4774436811606089, + "sampling/sampling_logp_difference/max": 0.2812481959660848, + "sampling/sampling_logp_difference/mean": 0.004415741958655417, + "step": 60, + "step_time": 7.903740958504689, + "student/entropy": 0.19655973594635726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030277779418975116, + "completions/max_length": 495.23333333333335, + "completions/max_terminated_length": 461.1666666666667, + "completions/mean_length": 195.65695444742838, + "completions/mean_terminated_length": 185.7668248494466, + "completions/min_length": 47.06666666666667, + "completions/min_terminated_length": 47.06666666666667, + "entropy": 0.18201517096410194, + "epoch": 0.0034177647818326813, + "eval/gt_acc_batch": 0.7461111307144165, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28869009017944336, + "kd/policy_loss": 0.012130236094041418, + "kd/rkl_advantage_mean": 0.27249475416998997, + "kd/rkl_advantage_p95": 3.8112845798333486, + "kd/rkl_advantage_std": 1.9830802311499913, + "kd/ssd_loss": 0.0, + "learning_rate": 9.9662021038241e-07, + "loss": 0.04852093855539958, + "num_tokens": 2993028.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7461111108462016, + "rewards/gt_logging_reward/std": 0.4324429521958033, + "sampling/importance_sampling_ratio/max": 1.985810387134552, + "sampling/importance_sampling_ratio/mean": 0.9974713385105133, + "sampling/importance_sampling_ratio/min": 0.4217121745149294, + "sampling/sampling_logp_difference/max": 0.3127308944861094, + "sampling/sampling_logp_difference/mean": 0.004246281879022718, + "step": 90, + "step_time": 8.030283141244825, + "student/entropy": 0.18201517096410194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334668229023, + "completions/max_length": 475.8666666666667, + "completions/max_terminated_length": 437.6666666666667, + "completions/mean_length": 184.11834309895832, + "completions/mean_terminated_length": 177.2191925048828, + "completions/min_length": 47.96666666666667, + "completions/min_terminated_length": 47.96666666666667, + "entropy": 0.1702964631219705, + "epoch": 0.004557019709110242, + "eval/gt_acc_batch": 0.7655555764834087, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2791326642036438, + "kd/policy_loss": 0.011101469253965964, + "kd/rkl_advantage_mean": 0.29705981047203145, + "kd/rkl_advantage_p95": 3.889968055486679, + "kd/rkl_advantage_std": 1.9908931851387024, + "kd/ssd_loss": 0.0, + "learning_rate": 9.954809554551323e-07, + "loss": 0.04440587361653646, + "num_tokens": 3966214.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7655555605888367, + "rewards/gt_logging_reward/std": 0.4159337828556697, + "sampling/importance_sampling_ratio/max": 1.946328834692637, + "sampling/importance_sampling_ratio/mean": 1.0006588041782378, + "sampling/importance_sampling_ratio/min": 0.47153339087963103, + "sampling/sampling_logp_difference/max": 0.2840041796366374, + "sampling/sampling_logp_difference/mean": 0.004205367582229277, + "step": 120, + "step_time": 7.771810823880757, + "student/entropy": 0.1702964631219705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016666667411724727, + "completions/max_length": 483.93333333333334, + "completions/max_terminated_length": 458.8666666666667, + "completions/mean_length": 187.3025115966797, + "completions/mean_terminated_length": 181.9511276245117, + "completions/min_length": 53.1, + "completions/min_terminated_length": 53.1, + "entropy": 0.17139346152544022, + "epoch": 0.005696274636387802, + "eval/gt_acc_batch": 0.7683333476384481, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28446802496910095, + "kd/policy_loss": 0.011136007619400819, + "kd/rkl_advantage_mean": 0.30794541554835936, + "kd/rkl_advantage_p95": 3.914899532000224, + "kd/rkl_advantage_std": 2.0140340367952985, + "kd/ssd_loss": 0.0, + "learning_rate": 9.943417005278546e-07, + "loss": 0.044544029235839847, + "num_tokens": 4941935.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7683333297570546, + "rewards/gt_logging_reward/std": 0.41198851565519967, + "sampling/importance_sampling_ratio/max": 1.8517481009165446, + "sampling/importance_sampling_ratio/mean": 0.9994672973950703, + "sampling/importance_sampling_ratio/min": 0.3872102633118629, + "sampling/sampling_logp_difference/max": 0.27680769165356955, + "sampling/sampling_logp_difference/mean": 0.004217145894654095, + "step": 150, + "step_time": 7.67959278972509, + "student/entropy": 0.17139346152544022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333448196451, + "completions/max_length": 483.3333333333333, + "completions/max_terminated_length": 439.8, + "completions/mean_length": 190.27195383707684, + "completions/mean_terminated_length": 183.5467737833659, + "completions/min_length": 49.56666666666667, + "completions/min_terminated_length": 49.56666666666667, + "entropy": 0.15777629570414622, + "epoch": 0.0068355295636653626, + "eval/gt_acc_batch": 0.7630555729071299, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3181924819946289, + "kd/policy_loss": 0.011039282781227181, + "kd/rkl_advantage_mean": 0.3438058839334796, + "kd/rkl_advantage_p95": 4.038741926352183, + "kd/rkl_advantage_std": 2.029553493857384, + "kd/ssd_loss": 0.0, + "learning_rate": 9.932024456005772e-07, + "loss": 0.044157135486602786, + "num_tokens": 5927162.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.763055553038915, + "rewards/gt_logging_reward/std": 0.41541487773259483, + "sampling/importance_sampling_ratio/max": 1.995250121752421, + "sampling/importance_sampling_ratio/mean": 1.0013793885707856, + "sampling/importance_sampling_ratio/min": 0.46634476880232495, + "sampling/sampling_logp_difference/max": 0.26239971319834393, + "sampling/sampling_logp_difference/mean": 0.003986104313905041, + "step": 180, + "step_time": 7.824581230175681, + "student/entropy": 0.15777629570414622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334382623433, + "completions/max_length": 483.43333333333334, + "completions/max_terminated_length": 433.03333333333336, + "completions/mean_length": 189.06250915527343, + "completions/mean_terminated_length": 183.01174875895182, + "completions/min_length": 54.06666666666667, + "completions/min_terminated_length": 54.06666666666667, + "entropy": 0.16184666175395251, + "epoch": 0.007974784490942924, + "eval/gt_acc_batch": 0.757500022649765, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31827056407928467, + "kd/policy_loss": 0.011123116644254576, + "kd/rkl_advantage_mean": 0.3897760251381745, + "kd/rkl_advantage_p95": 4.137957958380381, + "kd/rkl_advantage_std": 2.025299736857414, + "kd/ssd_loss": 0.0, + "learning_rate": 9.920631906732996e-07, + "loss": 0.04449246724446614, + "num_tokens": 6905203.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.75750000278155, + "rewards/gt_logging_reward/std": 0.4231238047281901, + "sampling/importance_sampling_ratio/max": 1.8898478945096333, + "sampling/importance_sampling_ratio/mean": 0.9948788781960806, + "sampling/importance_sampling_ratio/min": 0.4167682041724523, + "sampling/sampling_logp_difference/max": 0.2742826998233795, + "sampling/sampling_logp_difference/mean": 0.004086840362288058, + "step": 210, + "step_time": 7.628092233848292, + "student/entropy": 0.16184666175395251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890128582715, + "completions/max_length": 485.1333333333333, + "completions/max_terminated_length": 431.56666666666666, + "completions/mean_length": 187.1519536336263, + "completions/mean_terminated_length": 180.2242421468099, + "completions/min_length": 44.43333333333333, + "completions/min_terminated_length": 44.43333333333333, + "entropy": 0.15420423752317827, + "epoch": 0.009114039418220483, + "eval/gt_acc_batch": 0.7691666821638743, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29642724990844727, + "kd/policy_loss": 0.01012023645453155, + "kd/rkl_advantage_mean": 0.44419000793326024, + "kd/rkl_advantage_p95": 4.182680708169937, + "kd/rkl_advantage_std": 2.0236889561017355, + "kd/ssd_loss": 0.0, + "learning_rate": 9.909239357460222e-07, + "loss": 0.04048094352086385, + "num_tokens": 7871902.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7691666682561239, + "rewards/gt_logging_reward/std": 0.41583429972330727, + "sampling/importance_sampling_ratio/max": 1.9532995144526164, + "sampling/importance_sampling_ratio/mean": 0.9994506537914276, + "sampling/importance_sampling_ratio/min": 0.463630481561025, + "sampling/sampling_logp_difference/max": 0.28302475611368816, + "sampling/sampling_logp_difference/mean": 0.004033791902475059, + "step": 240, + "step_time": 7.651990339788608, + "student/entropy": 0.15420423752317827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055557422339915, + "completions/max_length": 499.3, + "completions/max_terminated_length": 466.2, + "completions/mean_length": 193.70028889973958, + "completions/mean_terminated_length": 184.71808115641275, + "completions/min_length": 50.4, + "completions/min_terminated_length": 50.4, + "entropy": 0.1553508721292019, + "epoch": 0.010253294345498045, + "eval/gt_acc_batch": 0.7497222383817037, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4439185857772827, + "kd/policy_loss": 0.010542551445541903, + "kd/rkl_advantage_mean": 0.3405587593772604, + "kd/rkl_advantage_p95": 4.016158455610276, + "kd/rkl_advantage_std": 2.0188363720973332, + "kd/ssd_loss": 0.0, + "learning_rate": 9.897846808187445e-07, + "loss": 0.04217021465301514, + "num_tokens": 8872327.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7497222205003102, + "rewards/gt_logging_reward/std": 0.42170828878879546, + "sampling/importance_sampling_ratio/max": 1.8455493092536925, + "sampling/importance_sampling_ratio/mean": 1.0015559593836467, + "sampling/importance_sampling_ratio/min": 0.45648245016733807, + "sampling/sampling_logp_difference/max": 0.32204410433769226, + "sampling/sampling_logp_difference/mean": 0.004114525183103979, + "step": 270, + "step_time": 7.861769156728405, + "student/entropy": 0.1553508721292019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556677281857, + "completions/max_length": 457.8666666666667, + "completions/max_terminated_length": 434.3, + "completions/mean_length": 177.24584350585937, + "completions/mean_terminated_length": 172.0651112874349, + "completions/min_length": 59.03333333333333, + "completions/min_terminated_length": 59.03333333333333, + "entropy": 0.15365676948179802, + "epoch": 0.011392549272775604, + "eval/gt_acc_batch": 0.779722253481547, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33860477805137634, + "kd/policy_loss": 0.010421268160765369, + "kd/rkl_advantage_mean": 0.40006949002854525, + "kd/rkl_advantage_p95": 4.114296293258667, + "kd/rkl_advantage_std": 2.0460882812738417, + "kd/ssd_loss": 0.0, + "learning_rate": 9.88645425891467e-07, + "loss": 0.04168507258097331, + "num_tokens": 9805596.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7797222276528676, + "rewards/gt_logging_reward/std": 0.4046100532015165, + "sampling/importance_sampling_ratio/max": 1.769742524623871, + "sampling/importance_sampling_ratio/mean": 0.99597261150678, + "sampling/importance_sampling_ratio/min": 0.4837028334538142, + "sampling/sampling_logp_difference/max": 0.29774684508641563, + "sampling/sampling_logp_difference/mean": 0.00405015309030811, + "step": 300, + "step_time": 7.470101887814235, + "student/entropy": 0.15365676948179802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055557484428086, + "completions/max_length": 489.3666666666667, + "completions/max_terminated_length": 461.6, + "completions/mean_length": 194.38917694091796, + "completions/mean_terminated_length": 185.24178924560547, + "completions/min_length": 44.06666666666667, + "completions/min_terminated_length": 44.06666666666667, + "entropy": 0.15853289403021337, + "epoch": 0.012531804200053166, + "eval/gt_acc_batch": 0.7630555768807729, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2567886710166931, + "kd/policy_loss": 0.011201375493934998, + "kd/rkl_advantage_mean": 0.4160057204309851, + "kd/rkl_advantage_p95": 4.156571886936823, + "kd/rkl_advantage_std": 2.04227860669295, + "kd/ssd_loss": 0.0, + "learning_rate": 9.875061709641895e-07, + "loss": 0.044805502891540526, + "num_tokens": 10812701.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.763055557012558, + "rewards/gt_logging_reward/std": 0.41593413253625233, + "sampling/importance_sampling_ratio/max": 1.88770885070165, + "sampling/importance_sampling_ratio/mean": 1.0014776666959126, + "sampling/importance_sampling_ratio/min": 0.43548253178596497, + "sampling/sampling_logp_difference/max": 0.33984741965929666, + "sampling/sampling_logp_difference/mean": 0.004208573054832717, + "step": 330, + "step_time": 7.850916166669534, + "student/entropy": 0.15853289403021337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017222223120431104, + "completions/max_length": 479.0, + "completions/max_terminated_length": 443.3333333333333, + "completions/mean_length": 189.57362009684246, + "completions/mean_terminated_length": 183.9298589070638, + "completions/min_length": 55.333333333333336, + "completions/min_terminated_length": 55.333333333333336, + "entropy": 0.13943199117978414, + "epoch": 0.013671059127330725, + "eval/gt_acc_batch": 0.7850000301996867, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2700779139995575, + "kd/policy_loss": 0.00939162780996412, + "kd/rkl_advantage_mean": 0.3628113689133897, + "kd/rkl_advantage_p95": 4.056593650579453, + "kd/rkl_advantage_std": 2.0081779569387437, + "kd/ssd_loss": 0.0, + "learning_rate": 9.863669160369118e-07, + "loss": 0.037566510836283366, + "num_tokens": 11807766.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7850000043710073, + "rewards/gt_logging_reward/std": 0.4038076023260752, + "sampling/importance_sampling_ratio/max": 1.8830128192901612, + "sampling/importance_sampling_ratio/mean": 1.000617935260137, + "sampling/importance_sampling_ratio/min": 0.4885498543580373, + "sampling/sampling_logp_difference/max": 0.32463822364807127, + "sampling/sampling_logp_difference/mean": 0.003892183490097523, + "step": 360, + "step_time": 7.804262704406089, + "student/entropy": 0.13943199117978414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.031111113106211025, + "completions/max_length": 489.3333333333333, + "completions/max_terminated_length": 447.93333333333334, + "completions/mean_length": 190.9025095621745, + "completions/mean_terminated_length": 180.78719685872395, + "completions/min_length": 53.93333333333333, + "completions/min_terminated_length": 53.93333333333333, + "entropy": 0.15333793237805365, + "epoch": 0.014810314054608286, + "eval/gt_acc_batch": 0.7547222435474396, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29113906621932983, + "kd/policy_loss": 0.010763291265660276, + "kd/rkl_advantage_mean": 0.41609783358095837, + "kd/rkl_advantage_p95": 4.173465975125631, + "kd/rkl_advantage_std": 2.028749202688535, + "kd/ssd_loss": 0.0, + "learning_rate": 9.852276611096342e-07, + "loss": 0.043053166071573896, + "num_tokens": 12794863.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7547222236792247, + "rewards/gt_logging_reward/std": 0.4254201134045919, + "sampling/importance_sampling_ratio/max": 1.8480279962221782, + "sampling/importance_sampling_ratio/mean": 0.9982978244622548, + "sampling/importance_sampling_ratio/min": 0.45347995261351265, + "sampling/sampling_logp_difference/max": 0.3424540042877197, + "sampling/sampling_logp_difference/mean": 0.00413392271536092, + "step": 390, + "step_time": 7.738586896366906, + "student/entropy": 0.15333793237805365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014166667498648167, + "completions/max_length": 473.6666666666667, + "completions/max_terminated_length": 445.93333333333334, + "completions/mean_length": 184.82306416829428, + "completions/mean_terminated_length": 180.31780141194662, + "completions/min_length": 55.56666666666667, + "completions/min_terminated_length": 55.56666666666667, + "entropy": 0.14132583755999803, + "epoch": 0.015949568981885848, + "eval/gt_acc_batch": 0.7955555737018585, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29947349429130554, + "kd/policy_loss": 0.009550300537375734, + "kd/rkl_advantage_mean": 0.3828863142931368, + "kd/rkl_advantage_p95": 4.193539774417877, + "kd/rkl_advantage_std": 2.0660841882228853, + "kd/ssd_loss": 0.0, + "learning_rate": 9.840884061823566e-07, + "loss": 0.03820120890935262, + "num_tokens": 13754402.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.795555563767751, + "rewards/gt_logging_reward/std": 0.3925007184346517, + "sampling/importance_sampling_ratio/max": 1.875376558303833, + "sampling/importance_sampling_ratio/mean": 1.0012089172999064, + "sampling/importance_sampling_ratio/min": 0.4797952324151993, + "sampling/sampling_logp_difference/max": 0.3346335828304291, + "sampling/sampling_logp_difference/mean": 0.003995176893658936, + "step": 420, + "step_time": 7.599777976481709, + "student/entropy": 0.14132583755999803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722224175930022, + "completions/max_length": 492.1333333333333, + "completions/max_terminated_length": 444.6, + "completions/mean_length": 191.62806599934896, + "completions/mean_terminated_length": 181.9193094889323, + "completions/min_length": 45.43333333333333, + "completions/min_terminated_length": 45.43333333333333, + "entropy": 0.14340751251826683, + "epoch": 0.017088823909163407, + "eval/gt_acc_batch": 0.7650000075499217, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2561100721359253, + "kd/policy_loss": 0.010143733047880233, + "kd/rkl_advantage_mean": 0.37719604197579126, + "kd/rkl_advantage_p95": 4.20889135201772, + "kd/rkl_advantage_std": 2.0824067334334058, + "kd/ssd_loss": 0.0, + "learning_rate": 9.829491512550791e-07, + "loss": 0.04057493209838867, + "num_tokens": 14752295.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7650000015894572, + "rewards/gt_logging_reward/std": 0.41605375409126283, + "sampling/importance_sampling_ratio/max": 2.015727432568868, + "sampling/importance_sampling_ratio/mean": 1.005115513006846, + "sampling/importance_sampling_ratio/min": 0.45442722340424857, + "sampling/sampling_logp_difference/max": 0.39062931934992473, + "sampling/sampling_logp_difference/mean": 0.004068104084581137, + "step": 450, + "step_time": 7.94331601227168, + "student/entropy": 0.14340751251826683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668020188808, + "completions/max_length": 493.26666666666665, + "completions/max_terminated_length": 450.8666666666667, + "completions/mean_length": 194.21556549072267, + "completions/mean_terminated_length": 186.7259063720703, + "completions/min_length": 56.46666666666667, + "completions/min_terminated_length": 56.46666666666667, + "entropy": 0.1411548318962256, + "epoch": 0.018228078836440967, + "eval/gt_acc_batch": 0.7563889106114705, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2622755169868469, + "kd/policy_loss": 0.009793725827087959, + "kd/rkl_advantage_mean": 0.37990102749705934, + "kd/rkl_advantage_p95": 4.182383513450622, + "kd/rkl_advantage_std": 2.075037932395935, + "kd/ssd_loss": 0.0, + "learning_rate": 9.818098963278015e-07, + "loss": 0.03917490641276042, + "num_tokens": 15752231.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7563888907432557, + "rewards/gt_logging_reward/std": 0.4220193445682526, + "sampling/importance_sampling_ratio/max": 1.9249449451764424, + "sampling/importance_sampling_ratio/mean": 0.9980144182840983, + "sampling/importance_sampling_ratio/min": 0.48571751912434896, + "sampling/sampling_logp_difference/max": 0.3790138145287832, + "sampling/sampling_logp_difference/mean": 0.004046919693549474, + "step": 480, + "step_time": 7.863551015628036, + "student/entropy": 0.1411548318962256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029444445949047805, + "completions/max_length": 488.8666666666667, + "completions/max_terminated_length": 456.26666666666665, + "completions/mean_length": 188.29223175048827, + "completions/mean_terminated_length": 178.7053227742513, + "completions/min_length": 50.36666666666667, + "completions/min_terminated_length": 50.36666666666667, + "entropy": 0.14622472779204448, + "epoch": 0.019367333763718526, + "eval/gt_acc_batch": 0.7755555848280589, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2487870752811432, + "kd/policy_loss": 0.009903502669961503, + "kd/rkl_advantage_mean": 0.4074699590482245, + "kd/rkl_advantage_p95": 4.28292181690534, + "kd/rkl_advantage_std": 2.1078390469153723, + "kd/ssd_loss": 0.0, + "learning_rate": 9.80670641400524e-07, + "loss": 0.03961400985717774, + "num_tokens": 16732571.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7755555510520935, + "rewards/gt_logging_reward/std": 0.4081752300262451, + "sampling/importance_sampling_ratio/max": 1.8574137608210246, + "sampling/importance_sampling_ratio/mean": 0.9997616767883301, + "sampling/importance_sampling_ratio/min": 0.3839781532684962, + "sampling/sampling_logp_difference/max": 0.3663038671016693, + "sampling/sampling_logp_difference/mean": 0.004202438510643939, + "step": 510, + "step_time": 7.860085966651483, + "student/entropy": 0.14622472779204448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556658655404, + "completions/max_length": 474.46666666666664, + "completions/max_terminated_length": 442.6666666666667, + "completions/mean_length": 181.78084309895834, + "completions/mean_terminated_length": 175.08450063069662, + "completions/min_length": 46.13333333333333, + "completions/min_terminated_length": 46.13333333333333, + "entropy": 0.1426143292337656, + "epoch": 0.02050658869099609, + "eval/gt_acc_batch": 0.7880555788675944, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3439692556858063, + "kd/policy_loss": 0.009800962308266511, + "kd/rkl_advantage_mean": 0.4476227140519768, + "kd/rkl_advantage_p95": 4.288417659203211, + "kd/rkl_advantage_std": 2.079409404595693, + "kd/ssd_loss": 0.0, + "learning_rate": 9.795313864732464e-07, + "loss": 0.03920385042826335, + "num_tokens": 17691302.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7880555470784505, + "rewards/gt_logging_reward/std": 0.3955184519290924, + "sampling/importance_sampling_ratio/max": 1.9108129064242045, + "sampling/importance_sampling_ratio/mean": 1.002501873175303, + "sampling/importance_sampling_ratio/min": 0.463145304719607, + "sampling/sampling_logp_difference/max": 0.3244284232457479, + "sampling/sampling_logp_difference/mean": 0.004124870981710652, + "step": 540, + "step_time": 7.847898686913928, + "student/entropy": 0.1426143292337656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01805555643513799, + "completions/max_length": 473.6333333333333, + "completions/max_terminated_length": 431.03333333333336, + "completions/mean_length": 183.23861999511718, + "completions/mean_terminated_length": 177.39219919840494, + "completions/min_length": 57.06666666666667, + "completions/min_terminated_length": 57.06666666666667, + "entropy": 0.13455340353151163, + "epoch": 0.02164584361827365, + "eval/gt_acc_batch": 0.7950000246365865, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27403488755226135, + "kd/policy_loss": 0.009416112274629995, + "kd/rkl_advantage_mean": 0.36873630223696335, + "kd/rkl_advantage_p95": 4.148748598496119, + "kd/rkl_advantage_std": 2.0706773926814397, + "kd/ssd_loss": 0.0, + "learning_rate": 9.783921315459688e-07, + "loss": 0.03766444524129232, + "num_tokens": 18651465.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7950000027815501, + "rewards/gt_logging_reward/std": 0.39625293711821236, + "sampling/importance_sampling_ratio/max": 1.9384885907173157, + "sampling/importance_sampling_ratio/mean": 1.0050499478975932, + "sampling/importance_sampling_ratio/min": 0.47230272988478345, + "sampling/sampling_logp_difference/max": 0.4102041006088257, + "sampling/sampling_logp_difference/mean": 0.004004332531864444, + "step": 570, + "step_time": 7.69368868387537, + "student/entropy": 0.13455340353151163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556969096264, + "completions/max_length": 485.53333333333336, + "completions/max_terminated_length": 442.03333333333336, + "completions/mean_length": 186.77945454915366, + "completions/mean_terminated_length": 180.0146275838216, + "completions/min_length": 58.766666666666666, + "completions/min_terminated_length": 58.766666666666666, + "entropy": 0.13927260829756657, + "epoch": 0.02278509854555121, + "eval/gt_acc_batch": 0.7733333388964335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24254149198532104, + "kd/policy_loss": 0.010074080315340931, + "kd/rkl_advantage_mean": 0.4601244098206128, + "kd/rkl_advantage_p95": 4.289574770132701, + "kd/rkl_advantage_std": 2.0753382553656894, + "kd/ssd_loss": 0.0, + "learning_rate": 9.772528766186914e-07, + "loss": 0.040296324094136554, + "num_tokens": 19627471.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7733333329359691, + "rewards/gt_logging_reward/std": 0.4112900694211324, + "sampling/importance_sampling_ratio/max": 1.96339408159256, + "sampling/importance_sampling_ratio/mean": 1.0031494975090027, + "sampling/importance_sampling_ratio/min": 0.46231493453184763, + "sampling/sampling_logp_difference/max": 0.36232064565022787, + "sampling/sampling_logp_difference/mean": 0.0041303595139955485, + "step": 600, + "step_time": 7.854809297442746, + "student/entropy": 0.13927260829756657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444571932157, + "completions/max_length": 484.56666666666666, + "completions/max_terminated_length": 463.46666666666664, + "completions/mean_length": 190.37056681315104, + "completions/mean_terminated_length": 182.43284352620444, + "completions/min_length": 57.833333333333336, + "completions/min_terminated_length": 57.833333333333336, + "entropy": 0.13945921019961435, + "epoch": 0.02392435347282877, + "eval/gt_acc_batch": 0.7700000087420146, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2700849175453186, + "kd/policy_loss": 0.009955698476793865, + "kd/rkl_advantage_mean": 0.44710651903102794, + "kd/rkl_advantage_p95": 4.316534155607224, + "kd/rkl_advantage_std": 2.066954290866852, + "kd/ssd_loss": 0.0, + "learning_rate": 9.761136216914138e-07, + "loss": 0.03982279300689697, + "num_tokens": 20611565.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7700000007947286, + "rewards/gt_logging_reward/std": 0.411551421880722, + "sampling/importance_sampling_ratio/max": 1.82971484263738, + "sampling/importance_sampling_ratio/mean": 0.9980163673559824, + "sampling/importance_sampling_ratio/min": 0.43132879535357155, + "sampling/sampling_logp_difference/max": 0.37539899349212646, + "sampling/sampling_logp_difference/mean": 0.0041238917348285515, + "step": 630, + "step_time": 7.880207194733278, + "student/entropy": 0.13945921019961435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001080334186, + "completions/max_length": 482.3333333333333, + "completions/max_terminated_length": 447.5, + "completions/mean_length": 192.34917653401692, + "completions/mean_terminated_length": 185.05050862630208, + "completions/min_length": 56.93333333333333, + "completions/min_terminated_length": 56.93333333333333, + "entropy": 0.13604440626998743, + "epoch": 0.02506360840010633, + "eval/gt_acc_batch": 0.7391666869322459, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26972514390945435, + "kd/policy_loss": 0.00971394965502744, + "kd/rkl_advantage_mean": 0.43573887475261774, + "kd/rkl_advantage_p95": 4.294135334094365, + "kd/rkl_advantage_std": 2.0947425991296766, + "kd/ssd_loss": 0.0, + "learning_rate": 9.749743667641363e-07, + "loss": 0.038855799039204914, + "num_tokens": 21611230.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7391666650772095, + "rewards/gt_logging_reward/std": 0.43218535085519155, + "sampling/importance_sampling_ratio/max": 1.986191991964976, + "sampling/importance_sampling_ratio/mean": 1.0037957549095153, + "sampling/importance_sampling_ratio/min": 0.44674121886491774, + "sampling/sampling_logp_difference/max": 0.35540513396263124, + "sampling/sampling_logp_difference/mean": 0.00408477804157883, + "step": 660, + "step_time": 7.846261426092436, + "student/entropy": 0.13604440626998743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777912716071, + "completions/max_length": 500.43333333333334, + "completions/max_terminated_length": 452.93333333333334, + "completions/mean_length": 194.68112131754557, + "completions/mean_terminated_length": 186.46819712320965, + "completions/min_length": 50.03333333333333, + "completions/min_terminated_length": 50.03333333333333, + "entropy": 0.1426631100475788, + "epoch": 0.02620286332738389, + "eval/gt_acc_batch": 0.7677777945995331, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3072415590286255, + "kd/policy_loss": 0.009705559837554271, + "kd/rkl_advantage_mean": 0.4660925720915354, + "kd/rkl_advantage_p95": 4.405044122536977, + "kd/rkl_advantage_std": 2.099466252326965, + "kd/ssd_loss": 0.0, + "learning_rate": 9.738351118368587e-07, + "loss": 0.03882224162419637, + "num_tokens": 22627594.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7677777787049611, + "rewards/gt_logging_reward/std": 0.41670821110407513, + "sampling/importance_sampling_ratio/max": 1.9280497749646506, + "sampling/importance_sampling_ratio/mean": 0.9964009046554565, + "sampling/importance_sampling_ratio/min": 0.43074339826901753, + "sampling/sampling_logp_difference/max": 0.37814860542615253, + "sampling/sampling_logp_difference/mean": 0.004066226266634961, + "step": 690, + "step_time": 8.091164884564932, + "student/entropy": 0.1426631100475788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01722222330669562, + "completions/max_length": 484.9, + "completions/max_terminated_length": 442.3, + "completions/mean_length": 182.7969533284505, + "completions/mean_terminated_length": 177.06156311035156, + "completions/min_length": 51.06666666666667, + "completions/min_terminated_length": 51.06666666666667, + "entropy": 0.1351904353747765, + "epoch": 0.02734211825466145, + "eval/gt_acc_batch": 0.7869444529215495, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2781446874141693, + "kd/policy_loss": 0.009253040524587657, + "kd/rkl_advantage_mean": 0.45914874530086913, + "kd/rkl_advantage_p95": 4.394913913806279, + "kd/rkl_advantage_std": 2.1079026718934375, + "kd/ssd_loss": 0.0, + "learning_rate": 9.72695856909581e-07, + "loss": 0.03701215982437134, + "num_tokens": 23577199.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.786944446961085, + "rewards/gt_logging_reward/std": 0.4002072314421336, + "sampling/importance_sampling_ratio/max": 1.7663983821868896, + "sampling/importance_sampling_ratio/mean": 0.995485124985377, + "sampling/importance_sampling_ratio/min": 0.4424729789296786, + "sampling/sampling_logp_difference/max": 0.3894407510757446, + "sampling/sampling_logp_difference/mean": 0.0040714646767204005, + "step": 720, + "step_time": 7.662151861124827, + "student/entropy": 0.1351904353747765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.011666667430351179, + "completions/max_length": 479.96666666666664, + "completions/max_terminated_length": 444.1666666666667, + "completions/mean_length": 185.07445526123047, + "completions/mean_terminated_length": 181.31953125, + "completions/min_length": 53.7, + "completions/min_terminated_length": 53.7, + "entropy": 0.1302966583520174, + "epoch": 0.028481373181939013, + "eval/gt_acc_batch": 0.8038889050483704, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22736547887325287, + "kd/policy_loss": 0.00871235299273394, + "kd/rkl_advantage_mean": 0.4638107375203011, + "kd/rkl_advantage_p95": 4.355591390530268, + "kd/rkl_advantage_std": 2.0973064690828322, + "kd/ssd_loss": 0.0, + "learning_rate": 9.715566019823034e-07, + "loss": 0.03484941323598226, + "num_tokens": 24545275.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8038888931274414, + "rewards/gt_logging_reward/std": 0.38952807982762655, + "sampling/importance_sampling_ratio/max": 1.9141941865285237, + "sampling/importance_sampling_ratio/mean": 1.0008125305175781, + "sampling/importance_sampling_ratio/min": 0.4044308945536613, + "sampling/sampling_logp_difference/max": 0.3755131463209788, + "sampling/sampling_logp_difference/mean": 0.003983167395927012, + "step": 750, + "step_time": 7.773384385940153, + "student/entropy": 0.1302966583520174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01555555642892917, + "completions/max_length": 496.8333333333333, + "completions/max_terminated_length": 452.3, + "completions/mean_length": 187.51250915527345, + "completions/mean_terminated_length": 182.51211496988932, + "completions/min_length": 58.06666666666667, + "completions/min_terminated_length": 58.06666666666667, + "entropy": 0.13024521954357623, + "epoch": 0.029620628109216573, + "eval/gt_acc_batch": 0.7900000194708506, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27903661131858826, + "kd/policy_loss": 0.008959814921642344, + "kd/rkl_advantage_mean": 0.47128258938901124, + "kd/rkl_advantage_p95": 4.425841889778773, + "kd/rkl_advantage_std": 2.125308553377787, + "kd/ssd_loss": 0.0, + "learning_rate": 9.70417347055026e-07, + "loss": 0.03583926757176717, + "num_tokens": 25526264.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7900000055631001, + "rewards/gt_logging_reward/std": 0.3976499944925308, + "sampling/importance_sampling_ratio/max": 1.8693394263585408, + "sampling/importance_sampling_ratio/mean": 0.9993054568767548, + "sampling/importance_sampling_ratio/min": 0.46665342648824054, + "sampling/sampling_logp_difference/max": 0.3991669982671738, + "sampling/sampling_logp_difference/mean": 0.004013178536357979, + "step": 780, + "step_time": 8.0328287954481, + "student/entropy": 0.13024521954357623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778965731462, + "completions/max_length": 479.46666666666664, + "completions/max_terminated_length": 447.3333333333333, + "completions/mean_length": 187.7672317504883, + "completions/mean_terminated_length": 180.35535685221353, + "completions/min_length": 56.86666666666667, + "completions/min_terminated_length": 56.86666666666667, + "entropy": 0.13549417797476054, + "epoch": 0.030759883036494132, + "eval/gt_acc_batch": 0.7911111315091451, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2940017282962799, + "kd/policy_loss": 0.009195210505276918, + "kd/rkl_advantage_mean": 0.4105372965617183, + "kd/rkl_advantage_p95": 4.2755749722321825, + "kd/rkl_advantage_std": 2.097934796412786, + "kd/ssd_loss": 0.0, + "learning_rate": 9.692780921277484e-07, + "loss": 0.036780846118927, + "num_tokens": 26507650.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7911111136277517, + "rewards/gt_logging_reward/std": 0.3993778293331464, + "sampling/importance_sampling_ratio/max": 1.8894100228945414, + "sampling/importance_sampling_ratio/mean": 1.0030367096265158, + "sampling/importance_sampling_ratio/min": 0.436696927746137, + "sampling/sampling_logp_difference/max": 0.37654449939727785, + "sampling/sampling_logp_difference/mean": 0.004130651662126184, + "step": 810, + "step_time": 7.920025414938573, + "student/entropy": 0.13549417797476054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555662761132, + "completions/max_length": 471.43333333333334, + "completions/max_terminated_length": 442.93333333333334, + "completions/mean_length": 188.99862213134764, + "completions/mean_terminated_length": 182.22716420491537, + "completions/min_length": 59.06666666666667, + "completions/min_terminated_length": 59.06666666666667, + "entropy": 0.13113776929676532, + "epoch": 0.031899137963771695, + "eval/gt_acc_batch": 0.7977777898311615, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3167026937007904, + "kd/policy_loss": 0.008760214527137577, + "kd/rkl_advantage_mean": 0.49579665608083207, + "kd/rkl_advantage_p95": 4.406716956694921, + "kd/rkl_advantage_std": 2.100488053758939, + "kd/ssd_loss": 0.0, + "learning_rate": 9.68138837200471e-07, + "loss": 0.03504086335500081, + "num_tokens": 27493437.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.797777775923411, + "rewards/gt_logging_reward/std": 0.3927893136938413, + "sampling/importance_sampling_ratio/max": 1.8255353331565858, + "sampling/importance_sampling_ratio/mean": 0.9953386028607686, + "sampling/importance_sampling_ratio/min": 0.4383034035563469, + "sampling/sampling_logp_difference/max": 0.41399598916371666, + "sampling/sampling_logp_difference/mean": 0.0040854938871537645, + "step": 840, + "step_time": 7.795423277063916, + "student/entropy": 0.13113776929676532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112466702857, + "completions/max_length": 471.8333333333333, + "completions/max_terminated_length": 431.1, + "completions/mean_length": 191.52001037597657, + "completions/mean_terminated_length": 183.71764577229817, + "completions/min_length": 63.36666666666667, + "completions/min_terminated_length": 63.36666666666667, + "entropy": 0.13077728723486265, + "epoch": 0.033038392891049255, + "eval/gt_acc_batch": 0.7825000246365865, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34009861946105957, + "kd/policy_loss": 0.009060778591083363, + "kd/rkl_advantage_mean": 0.3144032231997699, + "kd/rkl_advantage_p95": 4.078970756133398, + "kd/rkl_advantage_std": 2.047230197985967, + "kd/ssd_loss": 0.0, + "learning_rate": 9.669995822731933e-07, + "loss": 0.036243116855621337, + "num_tokens": 28489541.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7825000007947286, + "rewards/gt_logging_reward/std": 0.40601938863595327, + "sampling/importance_sampling_ratio/max": 1.9155193527539571, + "sampling/importance_sampling_ratio/mean": 1.0023401161034902, + "sampling/importance_sampling_ratio/min": 0.4083817412455877, + "sampling/sampling_logp_difference/max": 0.39253672361373904, + "sampling/sampling_logp_difference/mean": 0.004067242743136982, + "step": 870, + "step_time": 7.803381375456229, + "student/entropy": 0.13077728723486265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112628132104, + "completions/max_length": 491.76666666666665, + "completions/max_terminated_length": 456.3666666666667, + "completions/mean_length": 190.58778737386066, + "completions/mean_terminated_length": 182.11893056233723, + "completions/min_length": 56.333333333333336, + "completions/min_terminated_length": 56.333333333333336, + "entropy": 0.13490459906558197, + "epoch": 0.034177647818326815, + "eval/gt_acc_batch": 0.7894444664319357, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25146594643592834, + "kd/policy_loss": 0.00953551210113801, + "kd/rkl_advantage_mean": 0.4179676368056486, + "kd/rkl_advantage_p95": 4.329463563362757, + "kd/rkl_advantage_std": 2.101443925499916, + "kd/ssd_loss": 0.0, + "learning_rate": 9.658603273459157e-07, + "loss": 0.03814204533894857, + "num_tokens": 29474625.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7894444465637207, + "rewards/gt_logging_reward/std": 0.4031149516503016, + "sampling/importance_sampling_ratio/max": 1.977028783162435, + "sampling/importance_sampling_ratio/mean": 1.0031423171361287, + "sampling/importance_sampling_ratio/min": 0.4506333082914352, + "sampling/sampling_logp_difference/max": 0.3928524414698283, + "sampling/sampling_logp_difference/mean": 0.0041915961386015015, + "step": 900, + "step_time": 7.931726507640755, + "student/entropy": 0.13490459906558197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223468124866, + "completions/max_length": 478.2, + "completions/max_terminated_length": 431.8, + "completions/mean_length": 190.4394561767578, + "completions/mean_terminated_length": 184.04926554361978, + "completions/min_length": 57.96666666666667, + "completions/min_terminated_length": 57.96666666666667, + "entropy": 0.1333870379254222, + "epoch": 0.035316902745604374, + "eval/gt_acc_batch": 0.795000026623408, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2882496118545532, + "kd/policy_loss": 0.00902173495075355, + "kd/rkl_advantage_mean": 0.4238787419628352, + "kd/rkl_advantage_p95": 4.3318038562933605, + "kd/rkl_advantage_std": 2.094679540395737, + "kd/ssd_loss": 0.0, + "learning_rate": 9.647210724186382e-07, + "loss": 0.03608693679173788, + "num_tokens": 30476663.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7950000007947285, + "rewards/gt_logging_reward/std": 0.3967317114273707, + "sampling/importance_sampling_ratio/max": 1.8449629624684651, + "sampling/importance_sampling_ratio/mean": 0.9999599854151408, + "sampling/importance_sampling_ratio/min": 0.4286300371090571, + "sampling/sampling_logp_difference/max": 0.38531355063120526, + "sampling/sampling_logp_difference/mean": 0.004047162450539569, + "step": 930, + "step_time": 7.952911384574448, + "student/entropy": 0.1333870379254222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018055556590358415, + "completions/max_length": 493.73333333333335, + "completions/max_terminated_length": 451.9, + "completions/mean_length": 186.55139719645183, + "completions/mean_terminated_length": 180.5032943725586, + "completions/min_length": 45.43333333333333, + "completions/min_terminated_length": 45.43333333333333, + "entropy": 0.12980590853840113, + "epoch": 0.036456157672881934, + "eval/gt_acc_batch": 0.7861111402511597, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24491049349308014, + "kd/policy_loss": 0.008555913060748328, + "kd/rkl_advantage_mean": 0.4483588564898431, + "kd/rkl_advantage_p95": 4.356897507111231, + "kd/rkl_advantage_std": 2.1074685782194136, + "kd/ssd_loss": 0.0, + "learning_rate": 9.635818174913606e-07, + "loss": 0.03422364393870036, + "num_tokens": 31453608.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111124356588, + "rewards/gt_logging_reward/std": 0.4054222196340561, + "sampling/importance_sampling_ratio/max": 1.772775900363922, + "sampling/importance_sampling_ratio/mean": 0.9887248973051707, + "sampling/importance_sampling_ratio/min": 0.4422304277618726, + "sampling/sampling_logp_difference/max": 0.37954145272572837, + "sampling/sampling_logp_difference/mean": 0.004126150587884088, + "step": 960, + "step_time": 7.903867139919506, + "student/entropy": 0.12980590853840113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500000974784293, + "completions/max_length": 482.8333333333333, + "completions/max_terminated_length": 450.26666666666665, + "completions/mean_length": 181.49723103841146, + "completions/mean_terminated_length": 175.67654774983723, + "completions/min_length": 54.06666666666667, + "completions/min_terminated_length": 54.06666666666667, + "entropy": 0.13112400906781355, + "epoch": 0.03759541260015949, + "eval/gt_acc_batch": 0.7855555713176727, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24488502740859985, + "kd/policy_loss": 0.009096235865339015, + "kd/rkl_advantage_mean": 0.5158614825302114, + "kd/rkl_advantage_p95": 4.516036679347356, + "kd/rkl_advantage_std": 2.1424601902564366, + "kd/ssd_loss": 0.0, + "learning_rate": 9.62442562564083e-07, + "loss": 0.03638494809468587, + "num_tokens": 32405342.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555593967438, + "rewards/gt_logging_reward/std": 0.4048471430937449, + "sampling/importance_sampling_ratio/max": 1.9286951581637064, + "sampling/importance_sampling_ratio/mean": 1.0020315488179525, + "sampling/importance_sampling_ratio/min": 0.38562664290269216, + "sampling/sampling_logp_difference/max": 0.3808856993913651, + "sampling/sampling_logp_difference/mean": 0.00424522323689113, + "step": 990, + "step_time": 7.735459811434461, + "student/entropy": 0.13112400906781355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033888890563199915, + "completions/max_length": 494.3333333333333, + "completions/max_terminated_length": 455.43333333333334, + "completions/mean_length": 194.93556467692056, + "completions/mean_terminated_length": 183.98558756510417, + "completions/min_length": 56.766666666666666, + "completions/min_terminated_length": 56.766666666666666, + "entropy": 0.13406364638358353, + "epoch": 0.03873466752743705, + "eval/gt_acc_batch": 0.7783333520094554, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3306242823600769, + "kd/policy_loss": 0.009381091454997659, + "kd/rkl_advantage_mean": 0.5222811039498386, + "kd/rkl_advantage_p95": 4.4965941429138185, + "kd/rkl_advantage_std": 2.122713221112887, + "kd/ssd_loss": 0.0, + "learning_rate": 9.613033076368056e-07, + "loss": 0.03752436637878418, + "num_tokens": 33410774.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.778333330154419, + "rewards/gt_logging_reward/std": 0.40906290809313456, + "sampling/importance_sampling_ratio/max": 1.9052444219589233, + "sampling/importance_sampling_ratio/mean": 0.9960593402385711, + "sampling/importance_sampling_ratio/min": 0.40988250970840456, + "sampling/sampling_logp_difference/max": 0.39618947307268776, + "sampling/sampling_logp_difference/mean": 0.004212169861420989, + "step": 1020, + "step_time": 7.983822884177789, + "student/entropy": 0.13406364638358353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334928999346, + "completions/max_length": 500.96666666666664, + "completions/max_terminated_length": 465.8333333333333, + "completions/mean_length": 188.27556508382162, + "completions/mean_terminated_length": 179.7653615315755, + "completions/min_length": 53.2, + "completions/min_terminated_length": 53.2, + "entropy": 0.13963014855980874, + "epoch": 0.03987392245471462, + "eval/gt_acc_batch": 0.7869444688161215, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2319197952747345, + "kd/policy_loss": 0.009476867906050757, + "kd/rkl_advantage_mean": 0.5123576496417324, + "kd/rkl_advantage_p95": 4.489940039316813, + "kd/rkl_advantage_std": 2.1511889706055323, + "kd/ssd_loss": 0.0, + "learning_rate": 9.60164052709528e-07, + "loss": 0.03790746927261353, + "num_tokens": 34392310.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.786944446961085, + "rewards/gt_logging_reward/std": 0.4031680484612783, + "sampling/importance_sampling_ratio/max": 1.868604266643524, + "sampling/importance_sampling_ratio/mean": 1.0038973490397136, + "sampling/importance_sampling_ratio/min": 0.43526746531327565, + "sampling/sampling_logp_difference/max": 0.3888191262880961, + "sampling/sampling_logp_difference/mean": 0.004248286543103556, + "step": 1050, + "step_time": 8.008410968938067, + "student/entropy": 0.13963014855980874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.035833335326363645, + "completions/max_length": 499.0, + "completions/max_terminated_length": 454.8666666666667, + "completions/mean_length": 191.2777872721354, + "completions/mean_terminated_length": 179.53827718098958, + "completions/min_length": 59.1, + "completions/min_terminated_length": 59.1, + "entropy": 0.13994152272741, + "epoch": 0.04101317738199218, + "eval/gt_acc_batch": 0.7677777985731761, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2719656825065613, + "kd/policy_loss": 0.009729890661158909, + "kd/rkl_advantage_mean": 0.5908375907844553, + "kd/rkl_advantage_p95": 4.625113179286321, + "kd/rkl_advantage_std": 2.1743407130241392, + "kd/ssd_loss": 0.0, + "learning_rate": 9.590247977822503e-07, + "loss": 0.03891956806182861, + "num_tokens": 35380126.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7677777747313181, + "rewards/gt_logging_reward/std": 0.4150000015894572, + "sampling/importance_sampling_ratio/max": 1.9340392827987671, + "sampling/importance_sampling_ratio/mean": 0.9947075486183167, + "sampling/importance_sampling_ratio/min": 0.4014507070183754, + "sampling/sampling_logp_difference/max": 0.43332671423753105, + "sampling/sampling_logp_difference/mean": 0.004417717627560099, + "step": 1080, + "step_time": 7.928483367478475, + "student/entropy": 0.13994152272741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334680646657, + "completions/max_length": 495.1333333333333, + "completions/max_terminated_length": 442.73333333333335, + "completions/mean_length": 187.20528920491537, + "completions/mean_terminated_length": 178.68617553710936, + "completions/min_length": 57.4, + "completions/min_terminated_length": 57.4, + "entropy": 0.1327054822196563, + "epoch": 0.04215243230926974, + "eval/gt_acc_batch": 0.7869444708029429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3031732738018036, + "kd/policy_loss": 0.009455308564550553, + "kd/rkl_advantage_mean": 0.5121178429263333, + "kd/rkl_advantage_p95": 4.560929008324941, + "kd/rkl_advantage_std": 2.174414282043775, + "kd/ssd_loss": 0.0, + "learning_rate": 9.578855428549729e-07, + "loss": 0.037821229298909506, + "num_tokens": 36369729.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444410006206, + "rewards/gt_logging_reward/std": 0.4028739700714747, + "sampling/importance_sampling_ratio/max": 1.825038989384969, + "sampling/importance_sampling_ratio/mean": 0.9981901089350382, + "sampling/importance_sampling_ratio/min": 0.4426324799656868, + "sampling/sampling_logp_difference/max": 0.3975507736206055, + "sampling/sampling_logp_difference/mean": 0.004209040904728075, + "step": 1110, + "step_time": 8.02577666247574, + "student/entropy": 0.1327054822196563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02750000183780988, + "completions/max_length": 481.56666666666666, + "completions/max_terminated_length": 453.3333333333333, + "completions/mean_length": 186.12584279378254, + "completions/mean_terminated_length": 177.0022232055664, + "completions/min_length": 50.7, + "completions/min_terminated_length": 50.7, + "entropy": 0.13475909202049177, + "epoch": 0.0432916872365473, + "eval/gt_acc_batch": 0.782500026623408, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2897556722164154, + "kd/policy_loss": 0.009416640375275166, + "kd/rkl_advantage_mean": 0.4255038768130665, + "kd/rkl_advantage_p95": 4.313121604919433, + "kd/rkl_advantage_std": 2.1042641043663024, + "kd/ssd_loss": 0.0, + "learning_rate": 9.567462879276952e-07, + "loss": 0.03766656716664632, + "num_tokens": 37340934.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7825000007947286, + "rewards/gt_logging_reward/std": 0.408221623301506, + "sampling/importance_sampling_ratio/max": 1.8776241938273113, + "sampling/importance_sampling_ratio/mean": 0.9951621373494466, + "sampling/importance_sampling_ratio/min": 0.41814109484354656, + "sampling/sampling_logp_difference/max": 0.39207719365755717, + "sampling/sampling_logp_difference/mean": 0.004229770484380424, + "step": 1140, + "step_time": 7.784824705496431, + "student/entropy": 0.13475909202049177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014722222990045945, + "completions/max_length": 472.6666666666667, + "completions/max_terminated_length": 444.06666666666666, + "completions/mean_length": 182.723898824056, + "completions/mean_terminated_length": 177.81996307373046, + "completions/min_length": 49.36666666666667, + "completions/min_terminated_length": 49.36666666666667, + "entropy": 0.1366841400663058, + "epoch": 0.04443094216382486, + "eval/gt_acc_batch": 0.7875000178813935, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26732006669044495, + "kd/policy_loss": 0.009866254712687806, + "kd/rkl_advantage_mean": 0.38594752827387613, + "kd/rkl_advantage_p95": 4.242750970522563, + "kd/rkl_advantage_std": 2.09828284184138, + "kd/ssd_loss": 0.0, + "learning_rate": 9.556070330004176e-07, + "loss": 0.039465022087097165, + "num_tokens": 38306932.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7875, + "rewards/gt_logging_reward/std": 0.40186246037483214, + "sampling/importance_sampling_ratio/max": 2.042272484302521, + "sampling/importance_sampling_ratio/mean": 1.0045514742533366, + "sampling/importance_sampling_ratio/min": 0.40624864101409913, + "sampling/sampling_logp_difference/max": 0.4059317191441854, + "sampling/sampling_logp_difference/mean": 0.004361911839805543, + "step": 1170, + "step_time": 7.696019343108249, + "student/entropy": 0.1366841400663058 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444586833318, + "completions/max_length": 488.9, + "completions/max_terminated_length": 450.1333333333333, + "completions/mean_length": 190.08334452311198, + "completions/mean_terminated_length": 182.97108154296876, + "completions/min_length": 52.13333333333333, + "completions/min_terminated_length": 52.13333333333333, + "entropy": 0.13806713689118624, + "epoch": 0.04557019709110242, + "eval/gt_acc_batch": 0.7733333528041839, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2650133967399597, + "kd/policy_loss": 0.010004761141802494, + "kd/rkl_advantage_mean": 0.5138223921860724, + "kd/rkl_advantage_p95": 4.495609273513158, + "kd/rkl_advantage_std": 2.143854647874832, + "kd/ssd_loss": 0.0, + "learning_rate": 9.544677780731402e-07, + "loss": 0.04001904328664144, + "num_tokens": 39297368.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7733333309491476, + "rewards/gt_logging_reward/std": 0.4099621762832006, + "sampling/importance_sampling_ratio/max": 1.9180695295333863, + "sampling/importance_sampling_ratio/mean": 1.0017491082350414, + "sampling/importance_sampling_ratio/min": 0.4176797722776731, + "sampling/sampling_logp_difference/max": 0.4539886236190796, + "sampling/sampling_logp_difference/mean": 0.00429296368577828, + "step": 1200, + "step_time": 8.0293332656535, + "student/entropy": 0.13806713689118624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0213888899423182, + "completions/max_length": 494.8333333333333, + "completions/max_terminated_length": 455.7, + "completions/mean_length": 186.1877873738607, + "completions/mean_terminated_length": 179.12924346923828, + "completions/min_length": 60.43333333333333, + "completions/min_terminated_length": 60.43333333333333, + "entropy": 0.13894468024373055, + "epoch": 0.04670945201837998, + "eval/gt_acc_batch": 0.7758333484331766, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28859245777130127, + "kd/policy_loss": 0.009930307212440918, + "kd/rkl_advantage_mean": 0.5055101167488222, + "kd/rkl_advantage_p95": 4.523312179247538, + "kd/rkl_advantage_std": 2.176307301719983, + "kd/ssd_loss": 0.0, + "learning_rate": 9.533285231458625e-07, + "loss": 0.039721238613128665, + "num_tokens": 40271628.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7758333325386048, + "rewards/gt_logging_reward/std": 0.41134084860483805, + "sampling/importance_sampling_ratio/max": 1.9202594876289367, + "sampling/importance_sampling_ratio/mean": 0.9987711807092031, + "sampling/importance_sampling_ratio/min": 0.40990047256151835, + "sampling/sampling_logp_difference/max": 0.3990109403928121, + "sampling/sampling_logp_difference/mean": 0.004414699777650337, + "step": 1230, + "step_time": 7.970608511924123, + "student/entropy": 0.13894468024373055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667734583218, + "completions/max_length": 461.93333333333334, + "completions/max_terminated_length": 425.6333333333333, + "completions/mean_length": 185.57806599934895, + "completions/mean_terminated_length": 178.4705352783203, + "completions/min_length": 58.766666666666666, + "completions/min_terminated_length": 58.766666666666666, + "entropy": 0.1305646586542328, + "epoch": 0.04784870694565754, + "eval/gt_acc_batch": 0.799444462855657, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23523734509944916, + "kd/policy_loss": 0.00875447181169875, + "kd/rkl_advantage_mean": 0.4655029031060015, + "kd/rkl_advantage_p95": 4.329164098699888, + "kd/rkl_advantage_std": 2.086276273926099, + "kd/ssd_loss": 0.0, + "learning_rate": 9.52189268218585e-07, + "loss": 0.03501788377761841, + "num_tokens": 41244957.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7994444410006205, + "rewards/gt_logging_reward/std": 0.3884798968831698, + "sampling/importance_sampling_ratio/max": 1.881559463342031, + "sampling/importance_sampling_ratio/mean": 0.9924729486306508, + "sampling/importance_sampling_ratio/min": 0.42593758801619214, + "sampling/sampling_logp_difference/max": 0.39916374286015827, + "sampling/sampling_logp_difference/mean": 0.004212799986513953, + "step": 1260, + "step_time": 7.658384246181231, + "student/entropy": 0.1305646586542328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778704961143, + "completions/max_length": 481.4, + "completions/max_terminated_length": 448.9, + "completions/mean_length": 190.30445404052733, + "completions/mean_terminated_length": 184.59811197916667, + "completions/min_length": 53.8, + "completions/min_terminated_length": 53.8, + "entropy": 0.13205954780181248, + "epoch": 0.0489879618729351, + "eval/gt_acc_batch": 0.7675000230471293, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2250290960073471, + "kd/policy_loss": 0.009586874818584572, + "kd/rkl_advantage_mean": 0.3927052370427797, + "kd/rkl_advantage_p95": 4.232746241490046, + "kd/rkl_advantage_std": 2.0877994179725645, + "kd/ssd_loss": 0.0, + "learning_rate": 9.510500132913074e-07, + "loss": 0.038347506523132326, + "num_tokens": 42223237.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7674999992052715, + "rewards/gt_logging_reward/std": 0.41773091355959574, + "sampling/importance_sampling_ratio/max": 1.9180840929349263, + "sampling/importance_sampling_ratio/mean": 1.004743750890096, + "sampling/importance_sampling_ratio/min": 0.4335288678606351, + "sampling/sampling_logp_difference/max": 0.3657384693622589, + "sampling/sampling_logp_difference/mean": 0.0042173663464685285, + "step": 1290, + "step_time": 7.754713985145402, + "student/entropy": 0.13205954780181248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890345891316, + "completions/max_length": 480.5, + "completions/max_terminated_length": 443.3, + "completions/mean_length": 183.62750905354818, + "completions/mean_terminated_length": 176.38719533284504, + "completions/min_length": 53.5, + "completions/min_terminated_length": 53.5, + "entropy": 0.1295868429665764, + "epoch": 0.05012721680021266, + "eval/gt_acc_batch": 0.7997222403685252, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24079741537570953, + "kd/policy_loss": 0.009047037681254248, + "kd/rkl_advantage_mean": 0.4350222146138549, + "kd/rkl_advantage_p95": 4.302380998929341, + "kd/rkl_advantage_std": 2.096010450522105, + "kd/ssd_loss": 0.0, + "learning_rate": 9.499107583640299e-07, + "loss": 0.036188149452209474, + "num_tokens": 43188016.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7997222205003103, + "rewards/gt_logging_reward/std": 0.39699907501538595, + "sampling/importance_sampling_ratio/max": 1.9416816631952922, + "sampling/importance_sampling_ratio/mean": 1.001844576994578, + "sampling/importance_sampling_ratio/min": 0.43477134058872857, + "sampling/sampling_logp_difference/max": 0.40198151270548504, + "sampling/sampling_logp_difference/mean": 0.004196803597733378, + "step": 1320, + "step_time": 7.808941108019401, + "student/entropy": 0.1295868429665764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01861111226802071, + "completions/max_length": 483.8, + "completions/max_terminated_length": 446.73333333333335, + "completions/mean_length": 182.73889923095703, + "completions/mean_terminated_length": 176.54483337402343, + "completions/min_length": 46.86666666666667, + "completions/min_terminated_length": 46.86666666666667, + "entropy": 0.1320499664793412, + "epoch": 0.05126647172749022, + "eval/gt_acc_batch": 0.7905555685361226, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24322107434272766, + "kd/policy_loss": 0.0089542455971241, + "kd/rkl_advantage_mean": 0.4264907380488391, + "kd/rkl_advantage_p95": 4.335220678647359, + "kd/rkl_advantage_std": 2.097256257136663, + "kd/ssd_loss": 0.0, + "learning_rate": 9.487715034367523e-07, + "loss": 0.03581698338190715, + "num_tokens": 44151020.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7905555605888367, + "rewards/gt_logging_reward/std": 0.4011397918065389, + "sampling/importance_sampling_ratio/max": 1.8292698740959168, + "sampling/importance_sampling_ratio/mean": 1.0026962379614512, + "sampling/importance_sampling_ratio/min": 0.42409441620111465, + "sampling/sampling_logp_difference/max": 0.44165732661883034, + "sampling/sampling_logp_difference/mean": 0.004296521128465732, + "step": 1350, + "step_time": 7.874784729979001, + "student/entropy": 0.1320499664793412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334804823003, + "completions/max_length": 502.9, + "completions/max_terminated_length": 460.3333333333333, + "completions/mean_length": 192.06556447347006, + "completions/mean_terminated_length": 183.61007843017578, + "completions/min_length": 54.63333333333333, + "completions/min_terminated_length": 54.63333333333333, + "entropy": 0.13089757189154624, + "epoch": 0.05240572665476778, + "eval/gt_acc_batch": 0.7880555748939514, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29934316873550415, + "kd/policy_loss": 0.009186981463183959, + "kd/rkl_advantage_mean": 0.5305200241971761, + "kd/rkl_advantage_p95": 4.562812473376592, + "kd/rkl_advantage_std": 2.1609819849332172, + "kd/ssd_loss": 0.0, + "learning_rate": 9.476322485094748e-07, + "loss": 0.03674793243408203, + "num_tokens": 45160104.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7880555589993795, + "rewards/gt_logging_reward/std": 0.40707863569259645, + "sampling/importance_sampling_ratio/max": 1.970625642935435, + "sampling/importance_sampling_ratio/mean": 1.001666647195816, + "sampling/importance_sampling_ratio/min": 0.42452289164066315, + "sampling/sampling_logp_difference/max": 0.4395647903283437, + "sampling/sampling_logp_difference/mean": 0.004168329395664235, + "step": 1380, + "step_time": 8.31072727527159, + "student/entropy": 0.13089757189154624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779630074897, + "completions/max_length": 492.6333333333333, + "completions/max_terminated_length": 446.1, + "completions/mean_length": 191.01917673746746, + "completions/mean_terminated_length": 182.02425842285157, + "completions/min_length": 56.03333333333333, + "completions/min_terminated_length": 56.03333333333333, + "entropy": 0.13712634996821482, + "epoch": 0.05354498158204534, + "eval/gt_acc_batch": 0.7752777934074402, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2746981084346771, + "kd/policy_loss": 0.009554964965597415, + "kd/rkl_advantage_mean": 0.5726941334704558, + "kd/rkl_advantage_p95": 4.600284363826116, + "kd/rkl_advantage_std": 2.1604993959267933, + "kd/ssd_loss": 0.0, + "learning_rate": 9.464929935821972e-07, + "loss": 0.03821985721588135, + "num_tokens": 46145733.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7752777715524037, + "rewards/gt_logging_reward/std": 0.4115516314903895, + "sampling/importance_sampling_ratio/max": 1.9517006039619447, + "sampling/importance_sampling_ratio/mean": 0.9982248087724049, + "sampling/importance_sampling_ratio/min": 0.42693432917197544, + "sampling/sampling_logp_difference/max": 0.411659969886144, + "sampling/sampling_logp_difference/mean": 0.004319315419221917, + "step": 1410, + "step_time": 8.012428873990817, + "student/entropy": 0.13712634996821482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112435658773, + "completions/max_length": 474.73333333333335, + "completions/max_terminated_length": 446.56666666666666, + "completions/mean_length": 189.61028747558595, + "completions/mean_terminated_length": 182.14378458658854, + "completions/min_length": 54.266666666666666, + "completions/min_terminated_length": 54.266666666666666, + "entropy": 0.13169130471845467, + "epoch": 0.0546842365093229, + "eval/gt_acc_batch": 0.7775000075499217, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.331046998500824, + "kd/policy_loss": 0.0092339646575662, + "kd/rkl_advantage_mean": 0.41531487298004016, + "kd/rkl_advantage_p95": 4.238137894868851, + "kd/rkl_advantage_std": 2.0813306232293445, + "kd/ssd_loss": 0.0, + "learning_rate": 9.453537386549196e-07, + "loss": 0.036935861905415854, + "num_tokens": 47128386.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7774999976158142, + "rewards/gt_logging_reward/std": 0.4047900716463725, + "sampling/importance_sampling_ratio/max": 1.9343948403994242, + "sampling/importance_sampling_ratio/mean": 0.9925498286883037, + "sampling/importance_sampling_ratio/min": 0.41420688678820927, + "sampling/sampling_logp_difference/max": 0.4047835429509481, + "sampling/sampling_logp_difference/mean": 0.004293958866037428, + "step": 1440, + "step_time": 7.746310222602915, + "student/entropy": 0.13169130471845467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0269444459117949, + "completions/max_length": 493.1666666666667, + "completions/max_terminated_length": 446.7, + "completions/mean_length": 186.8672322591146, + "completions/mean_terminated_length": 177.87122751871746, + "completions/min_length": 54.2, + "completions/min_terminated_length": 54.2, + "entropy": 0.12774553907414277, + "epoch": 0.05582349143660046, + "eval/gt_acc_batch": 0.7861111362775167, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24788865447044373, + "kd/policy_loss": 0.008305798439929882, + "kd/rkl_advantage_mean": 0.4606620704755187, + "kd/rkl_advantage_p95": 4.401072019338608, + "kd/rkl_advantage_std": 2.1347546468178433, + "kd/ssd_loss": 0.0, + "learning_rate": 9.442144837276421e-07, + "loss": 0.03322318991025289, + "num_tokens": 48114532.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111104488373, + "rewards/gt_logging_reward/std": 0.4019089033206304, + "sampling/importance_sampling_ratio/max": 1.8723795930544536, + "sampling/importance_sampling_ratio/mean": 0.9937098880608877, + "sampling/importance_sampling_ratio/min": 0.39819308519363406, + "sampling/sampling_logp_difference/max": 0.3934435208638509, + "sampling/sampling_logp_difference/mean": 0.004123895149677992, + "step": 1470, + "step_time": 7.986871154994393, + "student/entropy": 0.12774553907414277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028611112323900064, + "completions/max_length": 485.6333333333333, + "completions/max_terminated_length": 461.8666666666667, + "completions/mean_length": 189.5636199951172, + "completions/mean_terminated_length": 180.39380950927733, + "completions/min_length": 55.766666666666666, + "completions/min_terminated_length": 55.766666666666666, + "entropy": 0.1362609348570307, + "epoch": 0.05696274636387803, + "eval/gt_acc_batch": 0.7936111211776733, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24917960166931152, + "kd/policy_loss": 0.0094911340798717, + "kd/rkl_advantage_mean": 0.4859275811783543, + "kd/rkl_advantage_p95": 4.420587690671285, + "kd/rkl_advantage_std": 2.1099185824394224, + "kd/ssd_loss": 0.0, + "learning_rate": 9.430752288003644e-07, + "loss": 0.03796453475952148, + "num_tokens": 49105249.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111152172088, + "rewards/gt_logging_reward/std": 0.4004330108563105, + "sampling/importance_sampling_ratio/max": 1.912660280863444, + "sampling/importance_sampling_ratio/mean": 1.0031083742777507, + "sampling/importance_sampling_ratio/min": 0.3929472347100576, + "sampling/sampling_logp_difference/max": 0.4075843930244446, + "sampling/sampling_logp_difference/mean": 0.004371285112574697, + "step": 1500, + "step_time": 7.849856868712232, + "student/entropy": 0.1362609348570307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001483907302, + "completions/max_length": 488.8, + "completions/max_terminated_length": 439.1, + "completions/mean_length": 182.81306533813478, + "completions/mean_terminated_length": 175.29583511352538, + "completions/min_length": 65.2, + "completions/min_terminated_length": 65.2, + "entropy": 0.12687780776371557, + "epoch": 0.058102001291155586, + "eval/gt_acc_batch": 0.7933333615461985, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29639458656311035, + "kd/policy_loss": 0.008688496709025155, + "kd/rkl_advantage_mean": 0.47396734592039136, + "kd/rkl_advantage_p95": 4.453593985239665, + "kd/rkl_advantage_std": 2.133880740404129, + "kd/ssd_loss": 0.0, + "learning_rate": 9.41935973873087e-07, + "loss": 0.03475399017333984, + "num_tokens": 50066408.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7933333277702331, + "rewards/gt_logging_reward/std": 0.39841359555721284, + "sampling/importance_sampling_ratio/max": 1.9006261746088664, + "sampling/importance_sampling_ratio/mean": 0.9991104125976562, + "sampling/importance_sampling_ratio/min": 0.4092116753260295, + "sampling/sampling_logp_difference/max": 0.3977258066336314, + "sampling/sampling_logp_difference/mean": 0.00413516325255235, + "step": 1530, + "step_time": 7.887949022060881, + "student/entropy": 0.12687780776371557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0269444459117949, + "completions/max_length": 484.93333333333334, + "completions/max_terminated_length": 452.53333333333336, + "completions/mean_length": 193.22639973958334, + "completions/mean_terminated_length": 184.7485336303711, + "completions/min_length": 55.233333333333334, + "completions/min_terminated_length": 55.233333333333334, + "entropy": 0.13641904791196188, + "epoch": 0.059241256218433146, + "eval/gt_acc_batch": 0.783611136674881, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30261698365211487, + "kd/policy_loss": 0.009951253417724123, + "kd/rkl_advantage_mean": 0.4300662286424389, + "kd/rkl_advantage_p95": 4.312958325942358, + "kd/rkl_advantage_std": 2.113129373391469, + "kd/ssd_loss": 0.0, + "learning_rate": 9.407967189458094e-07, + "loss": 0.039805010954538984, + "num_tokens": 51071663.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7836111068725586, + "rewards/gt_logging_reward/std": 0.4057087163130442, + "sampling/importance_sampling_ratio/max": 1.9647137880325318, + "sampling/importance_sampling_ratio/mean": 1.0016080379486083, + "sampling/importance_sampling_ratio/min": 0.4159156357248624, + "sampling/sampling_logp_difference/max": 0.428339276711146, + "sampling/sampling_logp_difference/mean": 0.004306183770919839, + "step": 1560, + "step_time": 7.867474790868195, + "student/entropy": 0.13641904791196188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01777777892226974, + "completions/max_length": 476.06666666666666, + "completions/max_terminated_length": 432.03333333333336, + "completions/mean_length": 185.55361989339193, + "completions/mean_terminated_length": 179.80084025065105, + "completions/min_length": 50.266666666666666, + "completions/min_terminated_length": 50.266666666666666, + "entropy": 0.12798440338422853, + "epoch": 0.060380511145710705, + "eval/gt_acc_batch": 0.8094444612661997, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23594437539577484, + "kd/policy_loss": 0.00869161959659929, + "kd/rkl_advantage_mean": 0.4818067514473417, + "kd/rkl_advantage_p95": 4.415740209817886, + "kd/rkl_advantage_std": 2.0991670499245325, + "kd/ssd_loss": 0.0, + "learning_rate": 9.396574640185319e-07, + "loss": 0.03476647535959879, + "num_tokens": 52043656.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8094444473584493, + "rewards/gt_logging_reward/std": 0.38414963483810427, + "sampling/importance_sampling_ratio/max": 1.8557328422864279, + "sampling/importance_sampling_ratio/mean": 0.9989226559797922, + "sampling/importance_sampling_ratio/min": 0.421547802289327, + "sampling/sampling_logp_difference/max": 0.4241507808367411, + "sampling/sampling_logp_difference/mean": 0.004141134621265034, + "step": 1590, + "step_time": 7.829480991147769, + "student/entropy": 0.12798440338422853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556857337555, + "completions/max_length": 494.96666666666664, + "completions/max_terminated_length": 456.26666666666665, + "completions/mean_length": 192.9372334798177, + "completions/mean_terminated_length": 184.76953887939453, + "completions/min_length": 56.13333333333333, + "completions/min_terminated_length": 56.13333333333333, + "entropy": 0.13910383880138397, + "epoch": 0.061519766072988265, + "eval/gt_acc_batch": 0.7761111319065094, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1981118768453598, + "kd/policy_loss": 0.009668007772415877, + "kd/rkl_advantage_mean": 0.502127900859341, + "kd/rkl_advantage_p95": 4.536231740315755, + "kd/rkl_advantage_std": 2.156936464707057, + "kd/ssd_loss": 0.0, + "learning_rate": 9.385182090912542e-07, + "loss": 0.038672037919362384, + "num_tokens": 53037478.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7761111080646514, + "rewards/gt_logging_reward/std": 0.4119443287452062, + "sampling/importance_sampling_ratio/max": 1.924413557847341, + "sampling/importance_sampling_ratio/mean": 1.0025873442490896, + "sampling/importance_sampling_ratio/min": 0.3829710433880488, + "sampling/sampling_logp_difference/max": 0.4407365103562673, + "sampling/sampling_logp_difference/mean": 0.004381851029271881, + "step": 1620, + "step_time": 7.88754960373432, + "student/entropy": 0.13910383880138397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556397885084, + "completions/max_length": 464.1666666666667, + "completions/max_terminated_length": 434.56666666666666, + "completions/mean_length": 189.06473185221355, + "completions/mean_terminated_length": 184.08736216227214, + "completions/min_length": 61.266666666666666, + "completions/min_terminated_length": 61.266666666666666, + "entropy": 0.12107625082135201, + "epoch": 0.06265902100026582, + "eval/gt_acc_batch": 0.8227778017520905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2539253234863281, + "kd/policy_loss": 0.007921620974472413, + "kd/rkl_advantage_mean": 0.39888008338360426, + "kd/rkl_advantage_p95": 4.2554416954517365, + "kd/rkl_advantage_std": 2.0922290345033008, + "kd/ssd_loss": 0.0, + "learning_rate": 9.373789541639767e-07, + "loss": 0.03168648680051168, + "num_tokens": 54024903.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8227777779102325, + "rewards/gt_logging_reward/std": 0.37089019318421684, + "sampling/importance_sampling_ratio/max": 1.8643449624379476, + "sampling/importance_sampling_ratio/mean": 0.9945861160755157, + "sampling/importance_sampling_ratio/min": 0.38786589801311494, + "sampling/sampling_logp_difference/max": 0.450235507885615, + "sampling/sampling_logp_difference/mean": 0.004055990069173276, + "step": 1650, + "step_time": 7.6910857483861035, + "student/entropy": 0.12107625082135201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030833335189769665, + "completions/max_length": 479.96666666666664, + "completions/max_terminated_length": 432.46666666666664, + "completions/mean_length": 189.30112050374348, + "completions/mean_terminated_length": 179.13788604736328, + "completions/min_length": 56.333333333333336, + "completions/min_terminated_length": 56.333333333333336, + "entropy": 0.13177461369584004, + "epoch": 0.06379827592754339, + "eval/gt_acc_batch": 0.7755555788675944, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21538788080215454, + "kd/policy_loss": 0.009356924623716623, + "kd/rkl_advantage_mean": 0.4908350238886972, + "kd/rkl_advantage_p95": 4.434452267487844, + "kd/rkl_advantage_std": 2.1075219531853993, + "kd/ssd_loss": 0.0, + "learning_rate": 9.362396992366992e-07, + "loss": 0.037427695592244466, + "num_tokens": 55017155.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7755555609862009, + "rewards/gt_logging_reward/std": 0.41068689624468485, + "sampling/importance_sampling_ratio/max": 2.0372203906377155, + "sampling/importance_sampling_ratio/mean": 1.0019435207049052, + "sampling/importance_sampling_ratio/min": 0.39773240784804026, + "sampling/sampling_logp_difference/max": 0.43994247118631996, + "sampling/sampling_logp_difference/mean": 0.004310186843698224, + "step": 1680, + "step_time": 7.9412532573643455, + "student/entropy": 0.13177461369584004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890575617552, + "completions/max_length": 500.26666666666665, + "completions/max_terminated_length": 457.43333333333334, + "completions/mean_length": 190.44639790852864, + "completions/mean_terminated_length": 181.8815887451172, + "completions/min_length": 52.733333333333334, + "completions/min_terminated_length": 52.733333333333334, + "entropy": 0.13354260058452685, + "epoch": 0.06493753085482094, + "eval/gt_acc_batch": 0.8072222391764323, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2548181712627411, + "kd/policy_loss": 0.008769306695709626, + "kd/rkl_advantage_mean": 0.5298034142547597, + "kd/rkl_advantage_p95": 4.522440363963445, + "kd/rkl_advantage_std": 2.1655157496531805, + "kd/ssd_loss": 0.0, + "learning_rate": 9.351004443094216e-07, + "loss": 0.035077230135599775, + "num_tokens": 56005258.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222272555033, + "rewards/gt_logging_reward/std": 0.38567766845226287, + "sampling/importance_sampling_ratio/max": 2.0129003802935284, + "sampling/importance_sampling_ratio/mean": 0.997453773021698, + "sampling/importance_sampling_ratio/min": 0.4268814002474149, + "sampling/sampling_logp_difference/max": 0.43880842526753744, + "sampling/sampling_logp_difference/mean": 0.004218905186280608, + "step": 1710, + "step_time": 7.921825190389063, + "student/entropy": 0.13354260058452685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777894089619, + "completions/max_length": 467.3666666666667, + "completions/max_terminated_length": 425.3333333333333, + "completions/mean_length": 183.33445383707684, + "completions/mean_terminated_length": 174.98929901123046, + "completions/min_length": 53.86666666666667, + "completions/min_terminated_length": 53.86666666666667, + "entropy": 0.13233609708646935, + "epoch": 0.06607678578209851, + "eval/gt_acc_batch": 0.8047222455342611, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3584451675415039, + "kd/policy_loss": 0.009240899370828023, + "kd/rkl_advantage_mean": 0.45176814281536887, + "kd/rkl_advantage_p95": 4.389229263861974, + "kd/rkl_advantage_std": 2.137334558367729, + "kd/ssd_loss": 0.0, + "learning_rate": 9.339611893821441e-07, + "loss": 0.03696360190709432, + "num_tokens": 56964294.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222197055817, + "rewards/gt_logging_reward/std": 0.38385874032974243, + "sampling/importance_sampling_ratio/max": 1.9180762767791748, + "sampling/importance_sampling_ratio/mean": 1.0078616797924043, + "sampling/importance_sampling_ratio/min": 0.40352218051751454, + "sampling/sampling_logp_difference/max": 0.3903858184814453, + "sampling/sampling_logp_difference/mean": 0.004334763833321631, + "step": 1740, + "step_time": 7.5688447172753515, + "student/entropy": 0.13233609708646935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779350678124, + "completions/max_length": 481.5, + "completions/max_terminated_length": 439.6333333333333, + "completions/mean_length": 190.05862121582032, + "completions/mean_terminated_length": 181.03280232747395, + "completions/min_length": 53.166666666666664, + "completions/min_terminated_length": 53.166666666666664, + "entropy": 0.13602967746555805, + "epoch": 0.06721604070937606, + "eval/gt_acc_batch": 0.7786111255486806, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2277403324842453, + "kd/policy_loss": 0.009074522543232887, + "kd/rkl_advantage_mean": 0.5186608690749078, + "kd/rkl_advantage_p95": 4.538462978601456, + "kd/rkl_advantage_std": 2.1583961407343546, + "kd/ssd_loss": 0.0, + "learning_rate": 9.328219344548665e-07, + "loss": 0.036298092206319174, + "num_tokens": 57949609.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7786111116409302, + "rewards/gt_logging_reward/std": 0.4105398029088974, + "sampling/importance_sampling_ratio/max": 1.9610918800036112, + "sampling/importance_sampling_ratio/mean": 0.9964329997698466, + "sampling/importance_sampling_ratio/min": 0.4393785903851191, + "sampling/sampling_logp_difference/max": 0.4592077414194743, + "sampling/sampling_logp_difference/mean": 0.00437441060009102, + "step": 1770, + "step_time": 7.899257109093014, + "student/entropy": 0.13602967746555805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334668229023, + "completions/max_length": 477.8333333333333, + "completions/max_terminated_length": 445.26666666666665, + "completions/mean_length": 187.36834309895832, + "completions/mean_terminated_length": 180.5821965535482, + "completions/min_length": 55.96666666666667, + "completions/min_terminated_length": 55.96666666666667, + "entropy": 0.13195363817115624, + "epoch": 0.06835529563665363, + "eval/gt_acc_batch": 0.7947222431500752, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24217137694358826, + "kd/policy_loss": 0.008946204841292153, + "kd/rkl_advantage_mean": 0.5176153003548583, + "kd/rkl_advantage_p95": 4.5280272424221035, + "kd/rkl_advantage_std": 2.135515178243319, + "kd/ssd_loss": 0.0, + "learning_rate": 9.316826795275889e-07, + "loss": 0.03578481674194336, + "num_tokens": 58922367.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222212950389, + "rewards/gt_logging_reward/std": 0.3924879292647044, + "sampling/importance_sampling_ratio/max": 1.8849701523780822, + "sampling/importance_sampling_ratio/mean": 0.9979813675085704, + "sampling/importance_sampling_ratio/min": 0.39969506512085595, + "sampling/sampling_logp_difference/max": 0.41199642022450766, + "sampling/sampling_logp_difference/mean": 0.004335533788738151, + "step": 1800, + "step_time": 7.9093199571361765, + "student/entropy": 0.13195363817115624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02222222356746594, + "completions/max_length": 483.53333333333336, + "completions/max_terminated_length": 454.23333333333335, + "completions/mean_length": 188.1958435058594, + "completions/mean_terminated_length": 180.89789326985678, + "completions/min_length": 53.43333333333333, + "completions/min_terminated_length": 53.43333333333333, + "entropy": 0.1322110606978337, + "epoch": 0.0694945505639312, + "eval/gt_acc_batch": 0.7866666813691457, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2800222635269165, + "kd/policy_loss": 0.00927802122120435, + "kd/rkl_advantage_mean": 0.5515845583130916, + "kd/rkl_advantage_p95": 4.65053463379542, + "kd/rkl_advantage_std": 2.1627071569363276, + "kd/ssd_loss": 0.0, + "learning_rate": 9.305434246003113e-07, + "loss": 0.03711208502451579, + "num_tokens": 59909456.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666714350382, + "rewards/gt_logging_reward/std": 0.40107011000315346, + "sampling/importance_sampling_ratio/max": 1.899915599822998, + "sampling/importance_sampling_ratio/mean": 1.0009732723236084, + "sampling/importance_sampling_ratio/min": 0.41238957742849985, + "sampling/sampling_logp_difference/max": 0.4790292024612427, + "sampling/sampling_logp_difference/mean": 0.004299720167182386, + "step": 1830, + "step_time": 7.9273036416581215, + "student/entropy": 0.1322110606978337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02916666828095913, + "completions/max_length": 497.43333333333334, + "completions/max_terminated_length": 461.03333333333336, + "completions/mean_length": 188.6047337849935, + "completions/mean_terminated_length": 179.02967020670573, + "completions/min_length": 49.733333333333334, + "completions/min_terminated_length": 49.733333333333334, + "entropy": 0.134600345355769, + "epoch": 0.07063380549120875, + "eval/gt_acc_batch": 0.7805555661519369, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31848758459091187, + "kd/policy_loss": 0.0094726511859335, + "kd/rkl_advantage_mean": 0.45058512369869275, + "kd/rkl_advantage_p95": 4.371852391958237, + "kd/rkl_advantage_std": 2.1258409549792607, + "kd/ssd_loss": 0.0, + "learning_rate": 9.294041696730338e-07, + "loss": 0.03789060910542806, + "num_tokens": 60898121.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7805555582046508, + "rewards/gt_logging_reward/std": 0.4102844427029292, + "sampling/importance_sampling_ratio/max": 2.022121028105418, + "sampling/importance_sampling_ratio/mean": 1.0026258409023285, + "sampling/importance_sampling_ratio/min": 0.4274846265713374, + "sampling/sampling_logp_difference/max": 0.44315466086069744, + "sampling/sampling_logp_difference/mean": 0.0043998717175175745, + "step": 1860, + "step_time": 7.966170869457225, + "student/entropy": 0.134600345355769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890476276476, + "completions/max_length": 494.26666666666665, + "completions/max_terminated_length": 461.5, + "completions/mean_length": 189.09417775472005, + "completions/mean_terminated_length": 181.22430674235025, + "completions/min_length": 55.666666666666664, + "completions/min_terminated_length": 55.666666666666664, + "entropy": 0.12791626217464605, + "epoch": 0.07177306041848631, + "eval/gt_acc_batch": 0.7988889118035635, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23677761852741241, + "kd/policy_loss": 0.008572325997132186, + "kd/rkl_advantage_mean": 0.5044254133788247, + "kd/rkl_advantage_p95": 4.585406603415807, + "kd/rkl_advantage_std": 2.1961194117863974, + "kd/ssd_loss": 0.0, + "learning_rate": 9.282649147457561e-07, + "loss": 0.03428930838902791, + "num_tokens": 61886188.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7988888919353485, + "rewards/gt_logging_reward/std": 0.3947013681133588, + "sampling/importance_sampling_ratio/max": 1.8421831369400024, + "sampling/importance_sampling_ratio/mean": 0.9961995999018352, + "sampling/importance_sampling_ratio/min": 0.4116328462958336, + "sampling/sampling_logp_difference/max": 0.41308249831199645, + "sampling/sampling_logp_difference/mean": 0.0042181566124781964, + "step": 1890, + "step_time": 7.958910997126562, + "student/entropy": 0.12791626217464605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001198301712, + "completions/max_length": 495.8666666666667, + "completions/max_terminated_length": 453.2, + "completions/mean_length": 190.811398824056, + "completions/mean_terminated_length": 184.44028065999348, + "completions/min_length": 59.5, + "completions/min_terminated_length": 59.5, + "entropy": 0.12937359996140002, + "epoch": 0.07291231534576387, + "eval/gt_acc_batch": 0.7880555868148804, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2507643401622772, + "kd/policy_loss": 0.008979624465185528, + "kd/rkl_advantage_mean": 0.48593370406500375, + "kd/rkl_advantage_p95": 4.4564838886260985, + "kd/rkl_advantage_std": 2.106365994612376, + "kd/ssd_loss": 0.0, + "learning_rate": 9.271256598184787e-07, + "loss": 0.03591849803924561, + "num_tokens": 62879117.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7880555550257365, + "rewards/gt_logging_reward/std": 0.40570049385229745, + "sampling/importance_sampling_ratio/max": 1.871559989452362, + "sampling/importance_sampling_ratio/mean": 0.9994694511095683, + "sampling/importance_sampling_ratio/min": 0.4205407828092575, + "sampling/sampling_logp_difference/max": 0.4400892317295074, + "sampling/sampling_logp_difference/mean": 0.00428825127116094, + "step": 1920, + "step_time": 7.8791185051668435, + "student/entropy": 0.12937359996140002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03250000181918343, + "completions/max_length": 501.56666666666666, + "completions/max_terminated_length": 450.8, + "completions/mean_length": 194.31778717041016, + "completions/mean_terminated_length": 183.59711151123048, + "completions/min_length": 57.166666666666664, + "completions/min_terminated_length": 57.166666666666664, + "entropy": 0.12474996813883384, + "epoch": 0.07405157027304143, + "eval/gt_acc_batch": 0.7783333619435628, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23270298540592194, + "kd/policy_loss": 0.008680731562587122, + "kd/rkl_advantage_mean": 0.5235075515462085, + "kd/rkl_advantage_p95": 4.505733462174733, + "kd/rkl_advantage_std": 2.106735266248385, + "kd/ssd_loss": 0.0, + "learning_rate": 9.259864048912012e-07, + "loss": 0.03472293217976888, + "num_tokens": 63888741.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7783333341280619, + "rewards/gt_logging_reward/std": 0.4085713585217794, + "sampling/importance_sampling_ratio/max": 1.9575131773948669, + "sampling/importance_sampling_ratio/mean": 1.0029576261838278, + "sampling/importance_sampling_ratio/min": 0.4283009946346283, + "sampling/sampling_logp_difference/max": 0.4523381094137828, + "sampling/sampling_logp_difference/mean": 0.004111053065086404, + "step": 1950, + "step_time": 8.10444961649288, + "student/entropy": 0.12474996813883384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02416666808227698, + "completions/max_length": 472.8333333333333, + "completions/max_terminated_length": 455.3666666666667, + "completions/mean_length": 192.26556549072265, + "completions/mean_terminated_length": 184.59081370035807, + "completions/min_length": 56.46666666666667, + "completions/min_terminated_length": 56.46666666666667, + "entropy": 0.13155975739161174, + "epoch": 0.07519082520031899, + "eval/gt_acc_batch": 0.7775000174840291, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28483787178993225, + "kd/policy_loss": 0.009256099399256831, + "kd/rkl_advantage_mean": 0.39859505960096914, + "kd/rkl_advantage_p95": 4.260186217228571, + "kd/rkl_advantage_std": 2.093841141462326, + "kd/ssd_loss": 0.0, + "learning_rate": 9.248471499639236e-07, + "loss": 0.037024402618408205, + "num_tokens": 64877249.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7775000015894572, + "rewards/gt_logging_reward/std": 0.40315716216961545, + "sampling/importance_sampling_ratio/max": 1.8879090468088786, + "sampling/importance_sampling_ratio/mean": 1.0004876752694447, + "sampling/importance_sampling_ratio/min": 0.3881937583287557, + "sampling/sampling_logp_difference/max": 0.4138429323832194, + "sampling/sampling_logp_difference/mean": 0.004333026086290677, + "step": 1980, + "step_time": 7.837078059573347, + "student/entropy": 0.13155975739161174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01805555655931433, + "completions/max_length": 486.4, + "completions/max_terminated_length": 447.43333333333334, + "completions/mean_length": 187.77889709472657, + "completions/mean_terminated_length": 181.87198944091796, + "completions/min_length": 60.56666666666667, + "completions/min_terminated_length": 60.56666666666667, + "entropy": 0.13058917224407196, + "epoch": 0.07633008012759655, + "eval/gt_acc_batch": 0.7944444497426351, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2517739236354828, + "kd/policy_loss": 0.009205438285910835, + "kd/rkl_advantage_mean": 0.465671230328735, + "kd/rkl_advantage_p95": 4.359647141893705, + "kd/rkl_advantage_std": 2.0911360065142315, + "kd/ssd_loss": 0.0, + "learning_rate": 9.23707895036646e-07, + "loss": 0.036821754773457845, + "num_tokens": 65861029.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7944444417953491, + "rewards/gt_logging_reward/std": 0.4000958055257797, + "sampling/importance_sampling_ratio/max": 1.9076964298884074, + "sampling/importance_sampling_ratio/mean": 0.9961376329263051, + "sampling/importance_sampling_ratio/min": 0.4364798714717229, + "sampling/sampling_logp_difference/max": 0.443558531999588, + "sampling/sampling_logp_difference/mean": 0.004243813408538699, + "step": 2010, + "step_time": 7.965713440747155, + "student/entropy": 0.13058917224407196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0277777794127663, + "completions/max_length": 468.0, + "completions/max_terminated_length": 437.7, + "completions/mean_length": 191.1175099690755, + "completions/mean_terminated_length": 181.93260192871094, + "completions/min_length": 57.96666666666667, + "completions/min_terminated_length": 57.96666666666667, + "entropy": 0.13233683096865814, + "epoch": 0.0774693350548741, + "eval/gt_acc_batch": 0.782500022649765, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2849023938179016, + "kd/policy_loss": 0.008967649209080265, + "kd/rkl_advantage_mean": 0.5760604279736677, + "kd/rkl_advantage_p95": 4.565486985445022, + "kd/rkl_advantage_std": 2.1440737187862395, + "kd/ssd_loss": 0.0, + "learning_rate": 9.225686401093684e-07, + "loss": 0.03587059577306111, + "num_tokens": 66850452.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7825000007947286, + "rewards/gt_logging_reward/std": 0.4072064975897471, + "sampling/importance_sampling_ratio/max": 1.9765862186749776, + "sampling/importance_sampling_ratio/mean": 1.000109883149465, + "sampling/importance_sampling_ratio/min": 0.43475004931290945, + "sampling/sampling_logp_difference/max": 0.4214184045791626, + "sampling/sampling_logp_difference/mean": 0.004355892795138061, + "step": 2040, + "step_time": 7.868182047569038, + "student/entropy": 0.13233683096865814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667790462572, + "completions/max_length": 482.06666666666666, + "completions/max_terminated_length": 451.23333333333335, + "completions/mean_length": 185.57001088460285, + "completions/mean_terminated_length": 179.32699432373047, + "completions/min_length": 56.86666666666667, + "completions/min_terminated_length": 56.86666666666667, + "entropy": 0.1306519312163194, + "epoch": 0.07860858998215167, + "eval/gt_acc_batch": 0.7930555721124013, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22426144778728485, + "kd/policy_loss": 0.008937614793345953, + "kd/rkl_advantage_mean": 0.4725372999285658, + "kd/rkl_advantage_p95": 4.467708683013916, + "kd/rkl_advantage_std": 2.1575969139734905, + "kd/ssd_loss": 0.0, + "learning_rate": 9.214293851820909e-07, + "loss": 0.03575046459833781, + "num_tokens": 67821464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555562178294, + "rewards/gt_logging_reward/std": 0.3979779620965322, + "sampling/importance_sampling_ratio/max": 1.9353002866109212, + "sampling/importance_sampling_ratio/mean": 0.9949861109256745, + "sampling/importance_sampling_ratio/min": 0.37038032660881676, + "sampling/sampling_logp_difference/max": 0.43952191472053526, + "sampling/sampling_logp_difference/mean": 0.004331361634346346, + "step": 2070, + "step_time": 7.758045650382216, + "student/entropy": 0.1306519312163194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668175409238, + "completions/max_length": 490.76666666666665, + "completions/max_terminated_length": 454.7, + "completions/mean_length": 192.8047317504883, + "completions/mean_terminated_length": 184.96310577392578, + "completions/min_length": 57.36666666666667, + "completions/min_terminated_length": 57.36666666666667, + "entropy": 0.13108663304398457, + "epoch": 0.07974784490942924, + "eval/gt_acc_batch": 0.7961111307144165, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31515657901763916, + "kd/policy_loss": 0.00930598470925664, + "kd/rkl_advantage_mean": 0.557286356560265, + "kd/rkl_advantage_p95": 4.571281848351161, + "kd/rkl_advantage_std": 2.1340308596690494, + "kd/ssd_loss": 0.0, + "learning_rate": 9.202901302548133e-07, + "loss": 0.03722394307454427, + "num_tokens": 68823425.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111088593801, + "rewards/gt_logging_reward/std": 0.39907184143861135, + "sampling/importance_sampling_ratio/max": 1.985925034681956, + "sampling/importance_sampling_ratio/mean": 1.000217455625534, + "sampling/importance_sampling_ratio/min": 0.4095621605714162, + "sampling/sampling_logp_difference/max": 0.4749327818552653, + "sampling/sampling_logp_difference/mean": 0.004257549671456218, + "step": 2100, + "step_time": 7.872133801978392, + "student/entropy": 0.13108663304398457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333448196451, + "completions/max_length": 478.43333333333334, + "completions/max_terminated_length": 444.6, + "completions/mean_length": 186.3211196899414, + "completions/mean_terminated_length": 179.52101186116536, + "completions/min_length": 60.06666666666667, + "completions/min_terminated_length": 60.06666666666667, + "entropy": 0.13381838761270046, + "epoch": 0.08088709983670679, + "eval/gt_acc_batch": 0.7872222443421681, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2583805322647095, + "kd/policy_loss": 0.009222612485367184, + "kd/rkl_advantage_mean": 0.5216386028099805, + "kd/rkl_advantage_p95": 4.520686920483907, + "kd/rkl_advantage_std": 2.1466214428345363, + "kd/ssd_loss": 0.0, + "learning_rate": 9.191508753275358e-07, + "loss": 0.03689045906066894, + "num_tokens": 69794789.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7872222224871318, + "rewards/gt_logging_reward/std": 0.4005415221055349, + "sampling/importance_sampling_ratio/max": 1.853285543123881, + "sampling/importance_sampling_ratio/mean": 0.9982494692007701, + "sampling/importance_sampling_ratio/min": 0.41366642514864604, + "sampling/sampling_logp_difference/max": 0.43381335337956745, + "sampling/sampling_logp_difference/mean": 0.0044001693604514, + "step": 2130, + "step_time": 7.756042513608311, + "student/entropy": 0.13381838761270046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01805555643513799, + "completions/max_length": 472.8, + "completions/max_terminated_length": 442.6666666666667, + "completions/mean_length": 179.66750996907552, + "completions/mean_terminated_length": 173.6427241007487, + "completions/min_length": 51.46666666666667, + "completions/min_terminated_length": 51.46666666666667, + "entropy": 0.12885212413966657, + "epoch": 0.08202635476398436, + "eval/gt_acc_batch": 0.810833356777827, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3199542760848999, + "kd/policy_loss": 0.008512443996733055, + "kd/rkl_advantage_mean": 0.5504339669598266, + "kd/rkl_advantage_p95": 4.5083488365014395, + "kd/rkl_advantage_std": 2.1296819110711414, + "kd/ssd_loss": 0.0, + "learning_rate": 9.180116204002582e-07, + "loss": 0.03404978116353353, + "num_tokens": 70740232.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8108333349227905, + "rewards/gt_logging_reward/std": 0.3798377806941668, + "sampling/importance_sampling_ratio/max": 1.8914802432060243, + "sampling/importance_sampling_ratio/mean": 1.002708731094996, + "sampling/importance_sampling_ratio/min": 0.44367477695147195, + "sampling/sampling_logp_difference/max": 0.4106687476237615, + "sampling/sampling_logp_difference/mean": 0.004249967130211492, + "step": 2160, + "step_time": 7.752001272196261, + "student/entropy": 0.12885212413966657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444446060806514, + "completions/max_length": 485.46666666666664, + "completions/max_terminated_length": 459.1666666666667, + "completions/mean_length": 188.53945566813152, + "completions/mean_terminated_length": 180.44600779215494, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.12847886104136705, + "epoch": 0.08316560969126191, + "eval/gt_acc_batch": 0.791111143430074, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.38358309864997864, + "kd/policy_loss": 0.008954476113043104, + "kd/rkl_advantage_mean": 0.5600560375644515, + "kd/rkl_advantage_p95": 4.63530224164327, + "kd/rkl_advantage_std": 2.1912126739819846, + "kd/ssd_loss": 0.0, + "learning_rate": 9.168723654729806e-07, + "loss": 0.03581790924072266, + "num_tokens": 71730190.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7911111096541087, + "rewards/gt_logging_reward/std": 0.3988018115361532, + "sampling/importance_sampling_ratio/max": 1.9492147564888, + "sampling/importance_sampling_ratio/mean": 1.0023767451445262, + "sampling/importance_sampling_ratio/min": 0.4121244788169861, + "sampling/sampling_logp_difference/max": 0.40340741475423175, + "sampling/sampling_logp_difference/mean": 0.0042830877316494785, + "step": 2190, + "step_time": 8.082188894727732, + "student/entropy": 0.12847886104136705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223381201427, + "completions/max_length": 475.56666666666666, + "completions/max_terminated_length": 441.3, + "completions/mean_length": 183.92556508382162, + "completions/mean_terminated_length": 176.56497243245443, + "completions/min_length": 57.9, + "completions/min_terminated_length": 57.9, + "entropy": 0.1243789363031586, + "epoch": 0.08430486461853948, + "eval/gt_acc_batch": 0.8094444751739502, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2651103138923645, + "kd/policy_loss": 0.008534347357150788, + "kd/rkl_advantage_mean": 0.5753608885531624, + "kd/rkl_advantage_p95": 4.659590444962183, + "kd/rkl_advantage_std": 2.164811690648397, + "kd/ssd_loss": 0.0, + "learning_rate": 9.15733110545703e-07, + "loss": 0.0341373880704244, + "num_tokens": 72697154.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8094444433848064, + "rewards/gt_logging_reward/std": 0.3843872144818306, + "sampling/importance_sampling_ratio/max": 1.9825315276781719, + "sampling/importance_sampling_ratio/mean": 1.0088355084260305, + "sampling/importance_sampling_ratio/min": 0.43042065054178236, + "sampling/sampling_logp_difference/max": 0.4561076323191325, + "sampling/sampling_logp_difference/mean": 0.004204935526164869, + "step": 2220, + "step_time": 7.821717067854479, + "student/entropy": 0.1243789363031586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445936630168, + "completions/max_length": 487.7, + "completions/max_terminated_length": 451.93333333333334, + "completions/mean_length": 185.12139739990235, + "completions/mean_terminated_length": 176.955441792806, + "completions/min_length": 52.8, + "completions/min_terminated_length": 52.8, + "entropy": 0.129484648630023, + "epoch": 0.08544411954581703, + "eval/gt_acc_batch": 0.8050000270207723, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3025919198989868, + "kd/policy_loss": 0.008739575621439144, + "kd/rkl_advantage_mean": 0.6011017846642062, + "kd/rkl_advantage_p95": 4.68336132367452, + "kd/rkl_advantage_std": 2.198494995633761, + "kd/ssd_loss": 0.0, + "learning_rate": 9.145938556184255e-07, + "loss": 0.03495830694834391, + "num_tokens": 73667079.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8050000031789144, + "rewards/gt_logging_reward/std": 0.38644814242919284, + "sampling/importance_sampling_ratio/max": 1.9399807453155518, + "sampling/importance_sampling_ratio/mean": 1.0035587271054587, + "sampling/importance_sampling_ratio/min": 0.4109939277172089, + "sampling/sampling_logp_difference/max": 0.4741142471631368, + "sampling/sampling_logp_difference/mean": 0.004305462857397894, + "step": 2250, + "step_time": 7.9964593221782705, + "student/entropy": 0.129484648630023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223753730456, + "completions/max_length": 488.6333333333333, + "completions/max_terminated_length": 459.06666666666666, + "completions/mean_length": 188.71112111409505, + "completions/mean_terminated_length": 181.4258071899414, + "completions/min_length": 59.333333333333336, + "completions/min_terminated_length": 59.333333333333336, + "entropy": 0.13618510228892167, + "epoch": 0.0865833744730946, + "eval/gt_acc_batch": 0.7858333607514699, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26028093695640564, + "kd/policy_loss": 0.008713893995930751, + "kd/rkl_advantage_mean": 0.5672770070649373, + "kd/rkl_advantage_p95": 4.659412030378977, + "kd/rkl_advantage_std": 2.1844313472509382, + "kd/ssd_loss": 0.0, + "learning_rate": 9.134546006911479e-07, + "loss": 0.03485557238260905, + "num_tokens": 74641191.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.785833336909612, + "rewards/gt_logging_reward/std": 0.4077138970295588, + "sampling/importance_sampling_ratio/max": 1.8542234619458517, + "sampling/importance_sampling_ratio/mean": 0.9892552455266317, + "sampling/importance_sampling_ratio/min": 0.39006411482890446, + "sampling/sampling_logp_difference/max": 0.4547816276550293, + "sampling/sampling_logp_difference/mean": 0.004381965977760653, + "step": 2280, + "step_time": 7.927389533216289, + "student/entropy": 0.13618510228892167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01500000087544322, + "completions/max_length": 477.7, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 184.48667755126954, + "completions/mean_terminated_length": 179.61041819254558, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.12824787913511196, + "epoch": 0.08772262940037216, + "eval/gt_acc_batch": 0.8016666909058888, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2790432572364807, + "kd/policy_loss": 0.00864245427850013, + "kd/rkl_advantage_mean": 0.5562232778407633, + "kd/rkl_advantage_p95": 4.596868836879731, + "kd/rkl_advantage_std": 2.1463759650786716, + "kd/ssd_loss": 0.0, + "learning_rate": 9.123153457638704e-07, + "loss": 0.03456981976826986, + "num_tokens": 75601087.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.801666667064031, + "rewards/gt_logging_reward/std": 0.3928664356470108, + "sampling/importance_sampling_ratio/max": 1.9825396259625754, + "sampling/importance_sampling_ratio/mean": 1.0014914254347482, + "sampling/importance_sampling_ratio/min": 0.40905782481034597, + "sampling/sampling_logp_difference/max": 0.4472511967023214, + "sampling/sampling_logp_difference/mean": 0.004358638504830499, + "step": 2310, + "step_time": 7.795646638136047, + "student/entropy": 0.12824787913511196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0247222236978511, + "completions/max_length": 483.03333333333336, + "completions/max_terminated_length": 448.6666666666667, + "completions/mean_length": 187.61445337931315, + "completions/mean_terminated_length": 179.74809036254882, + "completions/min_length": 60.266666666666666, + "completions/min_terminated_length": 60.266666666666666, + "entropy": 0.1265005715812246, + "epoch": 0.08886188432764972, + "eval/gt_acc_batch": 0.7919444700082143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2810114324092865, + "kd/policy_loss": 0.008615959022426978, + "kd/rkl_advantage_mean": 0.4957778957167951, + "kd/rkl_advantage_p95": 4.550545448064804, + "kd/rkl_advantage_std": 2.162119206786156, + "kd/ssd_loss": 0.0, + "learning_rate": 9.111760908365929e-07, + "loss": 0.03446383476257324, + "num_tokens": 76594507.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7919444421927134, + "rewards/gt_logging_reward/std": 0.400081005692482, + "sampling/importance_sampling_ratio/max": 1.8127265691757202, + "sampling/importance_sampling_ratio/mean": 0.9961942970752716, + "sampling/importance_sampling_ratio/min": 0.4229888717333476, + "sampling/sampling_logp_difference/max": 0.37418672839800515, + "sampling/sampling_logp_difference/mean": 0.00419707950980713, + "step": 2340, + "step_time": 8.22678322234812, + "student/entropy": 0.1265005715812246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112652967375, + "completions/max_length": 476.26666666666665, + "completions/max_terminated_length": 430.73333333333335, + "completions/mean_length": 182.51112060546876, + "completions/mean_terminated_length": 174.89916585286457, + "completions/min_length": 52.96666666666667, + "completions/min_terminated_length": 52.96666666666667, + "entropy": 0.1289219186330835, + "epoch": 0.09000113925492728, + "eval/gt_acc_batch": 0.8111111303170522, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28333938121795654, + "kd/policy_loss": 0.00867459579797772, + "kd/rkl_advantage_mean": 0.5293283195312445, + "kd/rkl_advantage_p95": 4.567687068382899, + "kd/rkl_advantage_std": 2.1652725537618003, + "kd/ssd_loss": 0.0, + "learning_rate": 9.100368359093152e-07, + "loss": 0.03469838301340739, + "num_tokens": 77552115.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8111111144224803, + "rewards/gt_logging_reward/std": 0.3824382558465004, + "sampling/importance_sampling_ratio/max": 1.895455006758372, + "sampling/importance_sampling_ratio/mean": 1.003621083498001, + "sampling/importance_sampling_ratio/min": 0.39692553877830505, + "sampling/sampling_logp_difference/max": 0.4748138646284739, + "sampling/sampling_logp_difference/mean": 0.00436567230305324, + "step": 2370, + "step_time": 7.9887411564898985, + "student/entropy": 0.1289219186330835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02722222398345669, + "completions/max_length": 481.9, + "completions/max_terminated_length": 442.6333333333333, + "completions/mean_length": 186.71334330240884, + "completions/mean_terminated_length": 177.7746617635091, + "completions/min_length": 55.5, + "completions/min_terminated_length": 55.5, + "entropy": 0.13743510966499647, + "epoch": 0.09114039418220483, + "eval/gt_acc_batch": 0.7658333500226339, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27445101737976074, + "kd/policy_loss": 0.009643352860196804, + "kd/rkl_advantage_mean": 0.5966234654188156, + "kd/rkl_advantage_p95": 4.704431922237078, + "kd/rkl_advantage_std": 2.176137657960256, + "kd/ssd_loss": 0.0, + "learning_rate": 9.088975809820377e-07, + "loss": 0.038573408126831056, + "num_tokens": 78512619.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7658333301544189, + "rewards/gt_logging_reward/std": 0.41176370630661646, + "sampling/importance_sampling_ratio/max": 1.8626338322957356, + "sampling/importance_sampling_ratio/mean": 0.9989446997642517, + "sampling/importance_sampling_ratio/min": 0.41120772461096444, + "sampling/sampling_logp_difference/max": 0.4143080830574036, + "sampling/sampling_logp_difference/mean": 0.004561686042385797, + "step": 2400, + "step_time": 7.828006503769817, + "student/entropy": 0.13743510966499647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.034166668790082136, + "completions/max_length": 487.26666666666665, + "completions/max_terminated_length": 460.2, + "completions/mean_length": 195.2527877807617, + "completions/mean_terminated_length": 184.14296112060546, + "completions/min_length": 55.9, + "completions/min_terminated_length": 55.9, + "entropy": 0.13822718095034361, + "epoch": 0.0922796491094824, + "eval/gt_acc_batch": 0.7561111251513163, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3636430501937866, + "kd/policy_loss": 0.00965895431387859, + "kd/rkl_advantage_mean": 0.638688262825599, + "kd/rkl_advantage_p95": 4.699793750047684, + "kd/rkl_advantage_std": 2.1665239999691646, + "kd/ssd_loss": 0.0, + "learning_rate": 9.077583260547601e-07, + "loss": 0.03863582213719686, + "num_tokens": 79515505.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7561111132303874, + "rewards/gt_logging_reward/std": 0.42501756449540457, + "sampling/importance_sampling_ratio/max": 1.9506949583689372, + "sampling/importance_sampling_ratio/mean": 0.9988558570543925, + "sampling/importance_sampling_ratio/min": 0.37841168691714605, + "sampling/sampling_logp_difference/max": 0.4947968681653341, + "sampling/sampling_logp_difference/mean": 0.004528520093299448, + "step": 2430, + "step_time": 8.054076588549652, + "student/entropy": 0.13822718095034361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028888891140619915, + "completions/max_length": 496.8666666666667, + "completions/max_terminated_length": 459.3333333333333, + "completions/mean_length": 197.1725107828776, + "completions/mean_terminated_length": 187.96822814941407, + "completions/min_length": 55.666666666666664, + "completions/min_terminated_length": 55.666666666666664, + "entropy": 0.1277294119199117, + "epoch": 0.09341890403675995, + "eval/gt_acc_batch": 0.7830555677413941, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25965452194213867, + "kd/policy_loss": 0.009027881823324908, + "kd/rkl_advantage_mean": 0.5254877035661291, + "kd/rkl_advantage_p95": 4.502410932381948, + "kd/rkl_advantage_std": 2.106247592965762, + "kd/ssd_loss": 0.0, + "learning_rate": 9.066190711274826e-07, + "loss": 0.03611152569452922, + "num_tokens": 80530478.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7830555538336436, + "rewards/gt_logging_reward/std": 0.40618504385153453, + "sampling/importance_sampling_ratio/max": 1.971566092967987, + "sampling/importance_sampling_ratio/mean": 0.9997199316819508, + "sampling/importance_sampling_ratio/min": 0.41501439809799195, + "sampling/sampling_logp_difference/max": 0.4329114278157552, + "sampling/sampling_logp_difference/mean": 0.0042702968465164306, + "step": 2460, + "step_time": 8.195509743981528, + "student/entropy": 0.1277294119199117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02222222356746594, + "completions/max_length": 488.23333333333335, + "completions/max_terminated_length": 445.1333333333333, + "completions/mean_length": 184.1086201985677, + "completions/mean_terminated_length": 176.73757985432943, + "completions/min_length": 55.7, + "completions/min_terminated_length": 55.7, + "entropy": 0.13043890030433733, + "epoch": 0.09455815896403752, + "eval/gt_acc_batch": 0.8186111390590668, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2930452823638916, + "kd/policy_loss": 0.00854595378623344, + "kd/rkl_advantage_mean": 0.5432878953715166, + "kd/rkl_advantage_p95": 4.610193755229314, + "kd/rkl_advantage_std": 2.1820219000180563, + "kd/ssd_loss": 0.0, + "learning_rate": 9.054798162002049e-07, + "loss": 0.03418382008870443, + "num_tokens": 81491389.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8186111092567444, + "rewards/gt_logging_reward/std": 0.38035005231698354, + "sampling/importance_sampling_ratio/max": 1.9551400581995646, + "sampling/importance_sampling_ratio/mean": 0.9915592789649963, + "sampling/importance_sampling_ratio/min": 0.3871398086349169, + "sampling/sampling_logp_difference/max": 0.5042222360769908, + "sampling/sampling_logp_difference/mean": 0.00438570948317647, + "step": 2490, + "step_time": 7.838864750756572, + "student/entropy": 0.13043890030433733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02833333474894365, + "completions/max_length": 506.7, + "completions/max_terminated_length": 462.7, + "completions/mean_length": 196.07445576985677, + "completions/mean_terminated_length": 186.93275299072266, + "completions/min_length": 56.56666666666667, + "completions/min_terminated_length": 56.56666666666667, + "entropy": 0.12967719994485377, + "epoch": 0.09569741389131509, + "eval/gt_acc_batch": 0.7811111430327098, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26822954416275024, + "kd/policy_loss": 0.009079055338709925, + "kd/rkl_advantage_mean": 0.6136037112524112, + "kd/rkl_advantage_p95": 4.723098788658778, + "kd/rkl_advantage_std": 2.187207872668902, + "kd/ssd_loss": 0.0, + "learning_rate": 9.043405612729275e-07, + "loss": 0.03631621996561686, + "num_tokens": 82498705.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7811111092567444, + "rewards/gt_logging_reward/std": 0.40334372719128925, + "sampling/importance_sampling_ratio/max": 1.9551493604977925, + "sampling/importance_sampling_ratio/mean": 0.9996428032716115, + "sampling/importance_sampling_ratio/min": 0.4240936776002248, + "sampling/sampling_logp_difference/max": 0.44321104685465496, + "sampling/sampling_logp_difference/mean": 0.004373557625027994, + "step": 2520, + "step_time": 8.132418876424587, + "student/entropy": 0.12967719994485377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018055556683490674, + "completions/max_length": 470.6666666666667, + "completions/max_terminated_length": 418.1333333333333, + "completions/mean_length": 182.11056416829427, + "completions/mean_terminated_length": 176.12134958902996, + "completions/min_length": 53.233333333333334, + "completions/min_terminated_length": 53.233333333333334, + "entropy": 0.1342774836346507, + "epoch": 0.09683666881859264, + "eval/gt_acc_batch": 0.7858333587646484, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24099889397621155, + "kd/policy_loss": 0.009323345495310302, + "kd/rkl_advantage_mean": 0.4866545496857725, + "kd/rkl_advantage_p95": 4.470307143529256, + "kd/rkl_advantage_std": 2.136853490273158, + "kd/ssd_loss": 0.0, + "learning_rate": 9.0320130634565e-07, + "loss": 0.0372933824857076, + "num_tokens": 83457847.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7858333249886831, + "rewards/gt_logging_reward/std": 0.39770562052726743, + "sampling/importance_sampling_ratio/max": 1.9260886589686075, + "sampling/importance_sampling_ratio/mean": 0.9984665671984355, + "sampling/importance_sampling_ratio/min": 0.40174026240905125, + "sampling/sampling_logp_difference/max": 0.5191780765851338, + "sampling/sampling_logp_difference/mean": 0.004489704562971989, + "step": 2550, + "step_time": 7.829716186807491, + "student/entropy": 0.1342774836346507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018055556745578846, + "completions/max_length": 477.53333333333336, + "completions/max_terminated_length": 444.03333333333336, + "completions/mean_length": 185.80945434570313, + "completions/mean_terminated_length": 180.01238962809245, + "completions/min_length": 54.63333333333333, + "completions/min_terminated_length": 54.63333333333333, + "entropy": 0.1341927339633306, + "epoch": 0.0979759237458702, + "eval/gt_acc_batch": 0.7916666865348816, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2536690831184387, + "kd/policy_loss": 0.009400840136610593, + "kd/rkl_advantage_mean": 0.5568274569697678, + "kd/rkl_advantage_p95": 4.577443089087804, + "kd/rkl_advantage_std": 2.138621539870898, + "kd/ssd_loss": 0.0, + "learning_rate": 9.020620514183723e-07, + "loss": 0.037603362401326494, + "num_tokens": 84434537.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7916666746139527, + "rewards/gt_logging_reward/std": 0.39548064519961673, + "sampling/importance_sampling_ratio/max": 1.9541515866915384, + "sampling/importance_sampling_ratio/mean": 1.0015944302082063, + "sampling/importance_sampling_ratio/min": 0.3928253730138143, + "sampling/sampling_logp_difference/max": 0.4888993302981059, + "sampling/sampling_logp_difference/mean": 0.004441790600928168, + "step": 2580, + "step_time": 7.883022433325338, + "student/entropy": 0.1341927339633306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112566043933, + "completions/max_length": 483.46666666666664, + "completions/max_terminated_length": 446.43333333333334, + "completions/mean_length": 190.1341776529948, + "completions/mean_terminated_length": 181.83374837239583, + "completions/min_length": 55.36666666666667, + "completions/min_terminated_length": 55.36666666666667, + "entropy": 0.1284243743866682, + "epoch": 0.09911517867314776, + "eval/gt_acc_batch": 0.773333348830541, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2707442045211792, + "kd/policy_loss": 0.008845166879473254, + "kd/rkl_advantage_mean": 0.46266511380672454, + "kd/rkl_advantage_p95": 4.441837922732035, + "kd/rkl_advantage_std": 2.1361097087462744, + "kd/ssd_loss": 0.0, + "learning_rate": 9.009227964910948e-07, + "loss": 0.03538066546122233, + "num_tokens": 85431156.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7733333329359691, + "rewards/gt_logging_reward/std": 0.4092830717563629, + "sampling/importance_sampling_ratio/max": 1.9331445773442586, + "sampling/importance_sampling_ratio/mean": 0.9974872887134552, + "sampling/importance_sampling_ratio/min": 0.40437979201475777, + "sampling/sampling_logp_difference/max": 0.4334670344988505, + "sampling/sampling_logp_difference/mean": 0.004338604576575259, + "step": 2610, + "step_time": 8.039176417258568, + "student/entropy": 0.1284243743866682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016666667660077412, + "completions/max_length": 480.96666666666664, + "completions/max_terminated_length": 437.6, + "completions/mean_length": 185.89695383707684, + "completions/mean_terminated_length": 180.45635935465495, + "completions/min_length": 57.5, + "completions/min_terminated_length": 57.5, + "entropy": 0.13133317635705075, + "epoch": 0.10025443360042532, + "eval/gt_acc_batch": 0.7988889018694559, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33449363708496094, + "kd/policy_loss": 0.008953722327714787, + "kd/rkl_advantage_mean": 0.5451364499206345, + "kd/rkl_advantage_p95": 4.603030594189962, + "kd/rkl_advantage_std": 2.145874415834745, + "kd/ssd_loss": 0.0, + "learning_rate": 8.997835415638172e-07, + "loss": 0.03581489721934001, + "num_tokens": 86410369.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7988888919353485, + "rewards/gt_logging_reward/std": 0.3921315347154935, + "sampling/importance_sampling_ratio/max": 1.918578024705251, + "sampling/importance_sampling_ratio/mean": 1.007682885726293, + "sampling/importance_sampling_ratio/min": 0.4189666564265887, + "sampling/sampling_logp_difference/max": 0.40926923155784606, + "sampling/sampling_logp_difference/mean": 0.00427223058262219, + "step": 2640, + "step_time": 7.882418575227105, + "student/entropy": 0.13133317635705075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018611111957579852, + "completions/max_length": 493.6333333333333, + "completions/max_terminated_length": 468.43333333333334, + "completions/mean_length": 190.97223205566405, + "completions/mean_terminated_length": 184.96208699544272, + "completions/min_length": 63.36666666666667, + "completions/min_terminated_length": 63.36666666666667, + "entropy": 0.13132069098452728, + "epoch": 0.10139368852770288, + "eval/gt_acc_batch": 0.7741667012373606, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25463974475860596, + "kd/policy_loss": 0.009242772068440293, + "kd/rkl_advantage_mean": 0.5823033522814512, + "kd/rkl_advantage_p95": 4.650602527459463, + "kd/rkl_advantage_std": 2.153051367402077, + "kd/ssd_loss": 0.0, + "learning_rate": 8.986442866365396e-07, + "loss": 0.036971092224121094, + "num_tokens": 87391933.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7741666694482168, + "rewards/gt_logging_reward/std": 0.4141622046629588, + "sampling/importance_sampling_ratio/max": 1.9754610776901245, + "sampling/importance_sampling_ratio/mean": 1.0021492660045623, + "sampling/importance_sampling_ratio/min": 0.3904266655445099, + "sampling/sampling_logp_difference/max": 0.4780644436677297, + "sampling/sampling_logp_difference/mean": 0.004398190408634642, + "step": 2670, + "step_time": 7.954217938353152, + "student/entropy": 0.13132069098452728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223604718842, + "completions/max_length": 486.26666666666665, + "completions/max_terminated_length": 453.8666666666667, + "completions/mean_length": 188.47417653401692, + "completions/mean_terminated_length": 180.43152720133463, + "completions/min_length": 55.333333333333336, + "completions/min_terminated_length": 55.333333333333336, + "entropy": 0.12492078511665265, + "epoch": 0.10253294345498044, + "eval/gt_acc_batch": 0.8030555705229442, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.244247704744339, + "kd/policy_loss": 0.00832951344588461, + "kd/rkl_advantage_mean": 0.5099521154072135, + "kd/rkl_advantage_p95": 4.483334050575892, + "kd/rkl_advantage_std": 2.1368101994196573, + "kd/ssd_loss": 0.0, + "learning_rate": 8.975050317092621e-07, + "loss": 0.033318054676055905, + "num_tokens": 88359576.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555526415507, + "rewards/gt_logging_reward/std": 0.3893969794114431, + "sampling/importance_sampling_ratio/max": 2.0530823787053425, + "sampling/importance_sampling_ratio/mean": 1.009898457924525, + "sampling/importance_sampling_ratio/min": 0.4220739091436068, + "sampling/sampling_logp_difference/max": 0.43813617825508117, + "sampling/sampling_logp_difference/mean": 0.004195696270714203, + "step": 2700, + "step_time": 7.917494105997806, + "student/entropy": 0.12492078511665265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019444445458551247, + "completions/max_length": 490.46666666666664, + "completions/max_terminated_length": 443.7, + "completions/mean_length": 186.33612111409505, + "completions/mean_terminated_length": 180.01663157145182, + "completions/min_length": 59.63333333333333, + "completions/min_terminated_length": 59.63333333333333, + "entropy": 0.12733064573258163, + "epoch": 0.10367219838225801, + "eval/gt_acc_batch": 0.793888908624649, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26950204372406006, + "kd/policy_loss": 0.008816314496410389, + "kd/rkl_advantage_mean": 0.6282241777982562, + "kd/rkl_advantage_p95": 4.679370059569677, + "kd/rkl_advantage_std": 2.134985715150833, + "kd/ssd_loss": 0.0, + "learning_rate": 8.963657767819846e-07, + "loss": 0.03526525497436524, + "num_tokens": 89326578.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7938888887564342, + "rewards/gt_logging_reward/std": 0.39312157879273096, + "sampling/importance_sampling_ratio/max": 1.9670658230781555, + "sampling/importance_sampling_ratio/mean": 1.0060882647832234, + "sampling/importance_sampling_ratio/min": 0.4259149859348933, + "sampling/sampling_logp_difference/max": 0.43084608713785805, + "sampling/sampling_logp_difference/mean": 0.004235842498019338, + "step": 2730, + "step_time": 7.977963447582442, + "student/entropy": 0.12733064573258163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016666667660077412, + "completions/max_length": 487.3333333333333, + "completions/max_terminated_length": 447.1333333333333, + "completions/mean_length": 188.76028849283855, + "completions/mean_terminated_length": 183.2480962117513, + "completions/min_length": 61.2, + "completions/min_terminated_length": 61.2, + "entropy": 0.1279523404315114, + "epoch": 0.10481145330953556, + "eval/gt_acc_batch": 0.8111111402511597, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2640644609928131, + "kd/policy_loss": 0.00848029184077556, + "kd/rkl_advantage_mean": 0.5692271318286657, + "kd/rkl_advantage_p95": 4.649699354171753, + "kd/rkl_advantage_std": 2.159418644507726, + "kd/ssd_loss": 0.0, + "learning_rate": 8.952265218547069e-07, + "loss": 0.03392117023468018, + "num_tokens": 90299411.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8111111124356588, + "rewards/gt_logging_reward/std": 0.38412694732348124, + "sampling/importance_sampling_ratio/max": 1.8974849939346314, + "sampling/importance_sampling_ratio/mean": 0.9985783139864604, + "sampling/importance_sampling_ratio/min": 0.4559721330801646, + "sampling/sampling_logp_difference/max": 0.4708087642987569, + "sampling/sampling_logp_difference/mean": 0.004288840813872715, + "step": 2760, + "step_time": 7.826985008506259, + "student/entropy": 0.1279523404315114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334513008596, + "completions/max_length": 482.06666666666666, + "completions/max_terminated_length": 455.46666666666664, + "completions/mean_length": 183.5077870686849, + "completions/mean_terminated_length": 176.67318369547527, + "completions/min_length": 54.63333333333333, + "completions/min_terminated_length": 54.63333333333333, + "entropy": 0.1293431148553888, + "epoch": 0.10595070823681313, + "eval/gt_acc_batch": 0.7947222451368968, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.334206223487854, + "kd/policy_loss": 0.009016885379484545, + "kd/rkl_advantage_mean": 0.5772790737605343, + "kd/rkl_advantage_p95": 4.689591989914576, + "kd/rkl_advantage_std": 2.191928951938947, + "kd/ssd_loss": 0.0, + "learning_rate": 8.940872669274294e-07, + "loss": 0.03606754938761393, + "num_tokens": 91257703.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222193082174, + "rewards/gt_logging_reward/std": 0.3933519000808398, + "sampling/importance_sampling_ratio/max": 1.8908846735954286, + "sampling/importance_sampling_ratio/mean": 0.9974142014980316, + "sampling/importance_sampling_ratio/min": 0.3986753324667613, + "sampling/sampling_logp_difference/max": 0.46421138842900594, + "sampling/sampling_logp_difference/mean": 0.0043472337691734236, + "step": 2790, + "step_time": 7.86610577335426, + "student/entropy": 0.1293431148553888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001390775046, + "completions/max_length": 487.26666666666665, + "completions/max_terminated_length": 461.3666666666667, + "completions/mean_length": 185.4466756184896, + "completions/mean_terminated_length": 178.0888427734375, + "completions/min_length": 47.2, + "completions/min_terminated_length": 47.2, + "entropy": 0.1341731216137608, + "epoch": 0.10708996316409068, + "eval/gt_acc_batch": 0.7841666758060455, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.288561075925827, + "kd/policy_loss": 0.009394078220551213, + "kd/rkl_advantage_mean": 0.5921128570723037, + "kd/rkl_advantage_p95": 4.710111363728841, + "kd/rkl_advantage_std": 2.186418343583743, + "kd/ssd_loss": 0.0, + "learning_rate": 8.929480120001519e-07, + "loss": 0.03757631778717041, + "num_tokens": 92217671.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.784166673819224, + "rewards/gt_logging_reward/std": 0.4062733858823776, + "sampling/importance_sampling_ratio/max": 1.91035106976827, + "sampling/importance_sampling_ratio/mean": 0.9994420568148296, + "sampling/importance_sampling_ratio/min": 0.37972700546185173, + "sampling/sampling_logp_difference/max": 0.4596379160881042, + "sampling/sampling_logp_difference/mean": 0.004473197739571333, + "step": 2820, + "step_time": 7.825856989622116, + "student/entropy": 0.1341731216137608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444578140974, + "completions/max_length": 484.3, + "completions/max_terminated_length": 453.23333333333335, + "completions/mean_length": 192.96056569417317, + "completions/mean_terminated_length": 185.20463053385416, + "completions/min_length": 55.7, + "completions/min_terminated_length": 55.7, + "entropy": 0.1357055296500524, + "epoch": 0.10822921809136825, + "eval/gt_acc_batch": 0.7722222407658895, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29710280895233154, + "kd/policy_loss": 0.009339032826634746, + "kd/rkl_advantage_mean": 0.6207599075511098, + "kd/rkl_advantage_p95": 4.786616762479146, + "kd/rkl_advantage_std": 2.1960125962893167, + "kd/ssd_loss": 0.0, + "learning_rate": 8.918087570728742e-07, + "loss": 0.03735613028208415, + "num_tokens": 93229705.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.772222222884496, + "rewards/gt_logging_reward/std": 0.41294146378835045, + "sampling/importance_sampling_ratio/max": 1.9031977852185566, + "sampling/importance_sampling_ratio/mean": 1.0009443660577138, + "sampling/importance_sampling_ratio/min": 0.40976707090934117, + "sampling/sampling_logp_difference/max": 0.44607590039571127, + "sampling/sampling_logp_difference/mean": 0.004281426329786579, + "step": 2850, + "step_time": 8.19971856539681, + "student/entropy": 0.1357055296500524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02583333496004343, + "completions/max_length": 487.76666666666665, + "completions/max_terminated_length": 454.1666666666667, + "completions/mean_length": 185.3658426920573, + "completions/mean_terminated_length": 176.65553232828776, + "completions/min_length": 57.43333333333333, + "completions/min_terminated_length": 57.43333333333333, + "entropy": 0.12650156350185474, + "epoch": 0.1093684730186458, + "eval/gt_acc_batch": 0.7933333516120911, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3176569640636444, + "kd/policy_loss": 0.008896940671062718, + "kd/rkl_advantage_mean": 0.5651173517418404, + "kd/rkl_advantage_p95": 4.694011131922404, + "kd/rkl_advantage_std": 2.2001018931468326, + "kd/ssd_loss": 0.0, + "learning_rate": 8.906695021455968e-07, + "loss": 0.03558776378631592, + "num_tokens": 94189790.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7933333337306976, + "rewards/gt_logging_reward/std": 0.39701737960179645, + "sampling/importance_sampling_ratio/max": 1.968362327416738, + "sampling/importance_sampling_ratio/mean": 1.0021819472312927, + "sampling/importance_sampling_ratio/min": 0.44424338738123575, + "sampling/sampling_logp_difference/max": 0.4665427347024282, + "sampling/sampling_logp_difference/mean": 0.004246408158602814, + "step": 2880, + "step_time": 7.841231532418169, + "student/entropy": 0.12650156350185474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01666666759798924, + "completions/max_length": 462.8666666666667, + "completions/max_terminated_length": 441.1, + "completions/mean_length": 188.96973215738933, + "completions/mean_terminated_length": 183.48166198730468, + "completions/min_length": 49.3, + "completions/min_terminated_length": 49.3, + "entropy": 0.1263409563029806, + "epoch": 0.11050772794592337, + "eval/gt_acc_batch": 0.8027778049310048, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3114893138408661, + "kd/policy_loss": 0.008268149144714698, + "kd/rkl_advantage_mean": 0.46100068429659585, + "kd/rkl_advantage_p95": 4.421410803000132, + "kd/rkl_advantage_std": 2.1155777404705685, + "kd/ssd_loss": 0.0, + "learning_rate": 8.895302472183192e-07, + "loss": 0.03307259678840637, + "num_tokens": 95174321.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8027777771155039, + "rewards/gt_logging_reward/std": 0.38886961390574776, + "sampling/importance_sampling_ratio/max": 1.8528289556503297, + "sampling/importance_sampling_ratio/mean": 0.9955129702885945, + "sampling/importance_sampling_ratio/min": 0.4217707778016726, + "sampling/sampling_logp_difference/max": 0.4762453258037567, + "sampling/sampling_logp_difference/mean": 0.004242754727602005, + "step": 2910, + "step_time": 7.914536713626391, + "student/entropy": 0.1263409563029806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777779151995976, + "completions/max_length": 480.7, + "completions/max_terminated_length": 444.2, + "completions/mean_length": 188.05973103841146, + "completions/mean_terminated_length": 180.63280639648437, + "completions/min_length": 60.03333333333333, + "completions/min_terminated_length": 60.03333333333333, + "entropy": 0.12963624962915976, + "epoch": 0.11164698287320092, + "eval/gt_acc_batch": 0.7877777953942616, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25547686219215393, + "kd/policy_loss": 0.0091092527921622, + "kd/rkl_advantage_mean": 0.5409802442067303, + "kd/rkl_advantage_p95": 4.570285373926163, + "kd/rkl_advantage_std": 2.1494710832834243, + "kd/ssd_loss": 0.0, + "learning_rate": 8.883909922910417e-07, + "loss": 0.03643701871236165, + "num_tokens": 96146608.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7877777775128683, + "rewards/gt_logging_reward/std": 0.4004997327923775, + "sampling/importance_sampling_ratio/max": 1.9795308709144592, + "sampling/importance_sampling_ratio/mean": 1.003355618317922, + "sampling/importance_sampling_ratio/min": 0.4325621897975604, + "sampling/sampling_logp_difference/max": 0.4222909927368164, + "sampling/sampling_logp_difference/mean": 0.004409500785792868, + "step": 2940, + "step_time": 7.845258369828419, + "student/entropy": 0.12963624962915976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029444446011135977, + "completions/max_length": 497.6, + "completions/max_terminated_length": 452.73333333333335, + "completions/mean_length": 186.3055653889974, + "completions/mean_terminated_length": 176.55448506673176, + "completions/min_length": 47.7, + "completions/min_terminated_length": 47.7, + "entropy": 0.12982990108430387, + "epoch": 0.11278623780047849, + "eval/gt_acc_batch": 0.7850000242392222, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30163827538490295, + "kd/policy_loss": 0.009107812658961241, + "kd/rkl_advantage_mean": 0.5615044108572571, + "kd/rkl_advantage_p95": 4.723702734708786, + "kd/rkl_advantage_std": 2.21654703716437, + "kd/ssd_loss": 0.0, + "learning_rate": 8.87251737363764e-07, + "loss": 0.0364312489827474, + "num_tokens": 97119532.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7850000023841858, + "rewards/gt_logging_reward/std": 0.405627104640007, + "sampling/importance_sampling_ratio/max": 1.9664215246836345, + "sampling/importance_sampling_ratio/mean": 1.0001489380995432, + "sampling/importance_sampling_ratio/min": 0.38202548027038574, + "sampling/sampling_logp_difference/max": 0.46672728657722473, + "sampling/sampling_logp_difference/mean": 0.004370495770126581, + "step": 2970, + "step_time": 8.081611769630884, + "student/entropy": 0.12982990108430387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779381722212, + "completions/max_length": 481.8333333333333, + "completions/max_terminated_length": 457.8333333333333, + "completions/mean_length": 192.47112019856772, + "completions/mean_terminated_length": 183.46061197916666, + "completions/min_length": 56.233333333333334, + "completions/min_terminated_length": 56.233333333333334, + "entropy": 0.13391398737827936, + "epoch": 0.11392549272775605, + "eval/gt_acc_batch": 0.7908333520094554, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31095436215400696, + "kd/policy_loss": 0.009304822529278075, + "kd/rkl_advantage_mean": 0.553014601006483, + "kd/rkl_advantage_p95": 4.591026719411214, + "kd/rkl_advantage_std": 2.149878619114558, + "kd/ssd_loss": 0.0, + "learning_rate": 8.861124824364865e-07, + "loss": 0.03721928596496582, + "num_tokens": 98114596.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7908333361148834, + "rewards/gt_logging_reward/std": 0.39829020301500956, + "sampling/importance_sampling_ratio/max": 1.9340099612871806, + "sampling/importance_sampling_ratio/mean": 0.9997600456078847, + "sampling/importance_sampling_ratio/min": 0.3979719529549281, + "sampling/sampling_logp_difference/max": 0.47996949354807533, + "sampling/sampling_logp_difference/mean": 0.0044729281449690465, + "step": 3000, + "step_time": 7.91545682354675, + "student/entropy": 0.13391398737827936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667852550744, + "completions/max_length": 485.1, + "completions/max_terminated_length": 454.96666666666664, + "completions/mean_length": 189.883620707194, + "completions/mean_terminated_length": 183.66282297770184, + "completions/min_length": 57.13333333333333, + "completions/min_terminated_length": 57.13333333333333, + "entropy": 0.12644131345053514, + "epoch": 0.1150647476550336, + "eval/gt_acc_batch": 0.7922222554683686, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2448025792837143, + "kd/policy_loss": 0.008910949272103608, + "kd/rkl_advantage_mean": 0.5127500921060952, + "kd/rkl_advantage_p95": 4.604007281859716, + "kd/rkl_advantage_std": 2.1944486021995546, + "kd/ssd_loss": 0.0, + "learning_rate": 8.849732275092089e-07, + "loss": 0.035643792152404784, + "num_tokens": 99099281.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222236792247, + "rewards/gt_logging_reward/std": 0.3964786320924759, + "sampling/importance_sampling_ratio/max": 1.9348843892415364, + "sampling/importance_sampling_ratio/mean": 1.0020498156547546, + "sampling/importance_sampling_ratio/min": 0.4202634369333585, + "sampling/sampling_logp_difference/max": 0.41174968481063845, + "sampling/sampling_logp_difference/mean": 0.0042539237843205535, + "step": 3030, + "step_time": 7.904931049289492, + "student/entropy": 0.12644131345053514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01833333435157935, + "completions/max_length": 476.03333333333336, + "completions/max_terminated_length": 444.53333333333336, + "completions/mean_length": 187.8555669148763, + "completions/mean_terminated_length": 181.8478001912435, + "completions/min_length": 60.06666666666667, + "completions/min_terminated_length": 60.06666666666667, + "entropy": 0.12160934892793497, + "epoch": 0.11620400258231117, + "eval/gt_acc_batch": 0.8072222431500753, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.265241801738739, + "kd/policy_loss": 0.008438106981338933, + "kd/rkl_advantage_mean": 0.5222618892245615, + "kd/rkl_advantage_p95": 4.517312947909037, + "kd/rkl_advantage_std": 2.1241929054260256, + "kd/ssd_loss": 0.0, + "learning_rate": 8.838339725819313e-07, + "loss": 0.03375242551167806, + "num_tokens": 100085817.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222272555033, + "rewards/gt_logging_reward/std": 0.38527297377586367, + "sampling/importance_sampling_ratio/max": 1.937837302684784, + "sampling/importance_sampling_ratio/mean": 1.0022961934407553, + "sampling/importance_sampling_ratio/min": 0.4192844410737356, + "sampling/sampling_logp_difference/max": 0.4281455318133036, + "sampling/sampling_logp_difference/mean": 0.004147628229111433, + "step": 3060, + "step_time": 7.935295636769539, + "student/entropy": 0.12160934892793497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01861111211280028, + "completions/max_length": 486.06666666666666, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 189.7658432006836, + "completions/mean_terminated_length": 183.73353932698566, + "completions/min_length": 64.63333333333334, + "completions/min_terminated_length": 64.63333333333334, + "entropy": 0.12537016390512387, + "epoch": 0.11734325750958872, + "eval/gt_acc_batch": 0.7930555840333303, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.388371080160141, + "kd/policy_loss": 0.0087150711818443, + "kd/rkl_advantage_mean": 0.47350176858211246, + "kd/rkl_advantage_p95": 4.398908207813899, + "kd/rkl_advantage_std": 2.095533405741056, + "kd/ssd_loss": 0.0, + "learning_rate": 8.826947176546539e-07, + "loss": 0.03486028512318929, + "num_tokens": 101077710.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555522441864, + "rewards/gt_logging_reward/std": 0.4004454096158346, + "sampling/importance_sampling_ratio/max": 1.9231550137201945, + "sampling/importance_sampling_ratio/mean": 0.9995372414588928, + "sampling/importance_sampling_ratio/min": 0.41753091613451637, + "sampling/sampling_logp_difference/max": 0.4601218303044637, + "sampling/sampling_logp_difference/mean": 0.004274066219416757, + "step": 3090, + "step_time": 8.022318035088635, + "student/entropy": 0.12537016390512387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666668076068162, + "completions/max_length": 497.6666666666667, + "completions/max_terminated_length": 435.1666666666667, + "completions/mean_length": 185.15723164876303, + "completions/mean_terminated_length": 177.98362681070964, + "completions/min_length": 56.86666666666667, + "completions/min_terminated_length": 56.86666666666667, + "entropy": 0.12795597265164058, + "epoch": 0.11848251243686629, + "eval/gt_acc_batch": 0.7991666793823242, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28930187225341797, + "kd/policy_loss": 0.008806529788610835, + "kd/rkl_advantage_mean": 0.55240562291195, + "kd/rkl_advantage_p95": 4.598730285962422, + "kd/rkl_advantage_std": 2.1570199698209764, + "kd/ssd_loss": 0.0, + "learning_rate": 8.815554627273763e-07, + "loss": 0.03522611856460571, + "num_tokens": 102047124.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7991666634877522, + "rewards/gt_logging_reward/std": 0.3915647322932879, + "sampling/importance_sampling_ratio/max": 2.00159508784612, + "sampling/importance_sampling_ratio/mean": 0.9988702615102132, + "sampling/importance_sampling_ratio/min": 0.34816933423280716, + "sampling/sampling_logp_difference/max": 0.4654865086078644, + "sampling/sampling_logp_difference/mean": 0.004354878274413446, + "step": 3120, + "step_time": 7.986426722312657, + "student/entropy": 0.12795597265164058 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445998718343, + "completions/max_length": 467.73333333333335, + "completions/max_terminated_length": 438.43333333333334, + "completions/mean_length": 180.44500986735025, + "completions/mean_terminated_length": 172.2989242553711, + "completions/min_length": 52.9, + "completions/min_terminated_length": 52.9, + "entropy": 0.1321094830830892, + "epoch": 0.11962176736414384, + "eval/gt_acc_batch": 0.8136111279328664, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2480178028345108, + "kd/policy_loss": 0.008777907448044667, + "kd/rkl_advantage_mean": 0.5383117169917871, + "kd/rkl_advantage_p95": 4.536651798089346, + "kd/rkl_advantage_std": 2.1649298737446467, + "kd/ssd_loss": 0.0, + "learning_rate": 8.804162078000987e-07, + "loss": 0.03511163393656413, + "num_tokens": 102990126.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813611110051473, + "rewards/gt_logging_reward/std": 0.3783667981624603, + "sampling/importance_sampling_ratio/max": 1.908983329931895, + "sampling/importance_sampling_ratio/mean": 0.9979601184527079, + "sampling/importance_sampling_ratio/min": 0.410832779109478, + "sampling/sampling_logp_difference/max": 0.4460064470767975, + "sampling/sampling_logp_difference/mean": 0.004480631245921056, + "step": 3150, + "step_time": 7.703203705182144, + "student/entropy": 0.1321094830830892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556720743575, + "completions/max_length": 481.8666666666667, + "completions/max_terminated_length": 447.4, + "completions/mean_length": 190.03112182617187, + "completions/mean_terminated_length": 183.36178588867188, + "completions/min_length": 51.53333333333333, + "completions/min_terminated_length": 51.53333333333333, + "entropy": 0.13199725324908892, + "epoch": 0.12076102229142141, + "eval/gt_acc_batch": 0.7861111342906952, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26870352029800415, + "kd/policy_loss": 0.009176970007441316, + "kd/rkl_advantage_mean": 0.5988997920105855, + "kd/rkl_advantage_p95": 4.655586703618368, + "kd/rkl_advantage_std": 2.149715925256411, + "kd/ssd_loss": 0.0, + "learning_rate": 8.792769528728211e-07, + "loss": 0.03670788606007894, + "num_tokens": 103973710.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111124356588, + "rewards/gt_logging_reward/std": 0.4016463900605837, + "sampling/importance_sampling_ratio/max": 2.050091246763865, + "sampling/importance_sampling_ratio/mean": 1.0013919691244761, + "sampling/importance_sampling_ratio/min": 0.3806546022494634, + "sampling/sampling_logp_difference/max": 0.4668731908003489, + "sampling/sampling_logp_difference/mean": 0.004434997580635051, + "step": 3180, + "step_time": 7.865223933236363, + "student/entropy": 0.13199725324908892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778835346302, + "completions/max_length": 485.3333333333333, + "completions/max_terminated_length": 448.3333333333333, + "completions/mean_length": 188.7697321573893, + "completions/mean_terminated_length": 182.18024648030598, + "completions/min_length": 57.8, + "completions/min_terminated_length": 57.8, + "entropy": 0.13337789283444484, + "epoch": 0.12190027721869898, + "eval/gt_acc_batch": 0.7766666869322459, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30028414726257324, + "kd/policy_loss": 0.009340588449655721, + "kd/rkl_advantage_mean": 0.5577432734891772, + "kd/rkl_advantage_p95": 4.56463151772817, + "kd/rkl_advantage_std": 2.1621671726306277, + "kd/ssd_loss": 0.0, + "learning_rate": 8.781376979455436e-07, + "loss": 0.03736236095428467, + "num_tokens": 104950185.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7766666650772095, + "rewards/gt_logging_reward/std": 0.4093223611513774, + "sampling/importance_sampling_ratio/max": 2.022914628187815, + "sampling/importance_sampling_ratio/mean": 1.0096119443575542, + "sampling/importance_sampling_ratio/min": 0.4428839564323425, + "sampling/sampling_logp_difference/max": 0.42456708550453187, + "sampling/sampling_logp_difference/mean": 0.004422038040744761, + "step": 3210, + "step_time": 7.90896871748846, + "student/entropy": 0.13337789283444484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890041659276, + "completions/max_length": 485.53333333333336, + "completions/max_terminated_length": 453.3666666666667, + "completions/mean_length": 193.2819559733073, + "completions/mean_terminated_length": 185.54942982991537, + "completions/min_length": 48.4, + "completions/min_terminated_length": 48.4, + "entropy": 0.1277780519798398, + "epoch": 0.12303953214597653, + "eval/gt_acc_batch": 0.7797222395737966, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3451780080795288, + "kd/policy_loss": 0.008912410268870493, + "kd/rkl_advantage_mean": 0.5381518165425708, + "kd/rkl_advantage_p95": 4.678116275866826, + "kd/rkl_advantage_std": 2.1909276922543843, + "kd/ssd_loss": 0.0, + "learning_rate": 8.769984430182659e-07, + "loss": 0.03564964532852173, + "num_tokens": 105968432.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7797222276528676, + "rewards/gt_logging_reward/std": 0.4090959757566452, + "sampling/importance_sampling_ratio/max": 1.9940569917360942, + "sampling/importance_sampling_ratio/mean": 1.0004310230414073, + "sampling/importance_sampling_ratio/min": 0.41443876127401985, + "sampling/sampling_logp_difference/max": 0.4682272712389628, + "sampling/sampling_logp_difference/mean": 0.004273979055384795, + "step": 3240, + "step_time": 8.241763534043761, + "student/entropy": 0.1277780519798398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890389353038, + "completions/max_length": 488.3, + "completions/max_terminated_length": 453.73333333333335, + "completions/mean_length": 190.14417673746746, + "completions/mean_terminated_length": 181.58126220703124, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.13247901070863008, + "epoch": 0.1241787870732541, + "eval/gt_acc_batch": 0.787500015894572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30201953649520874, + "kd/policy_loss": 0.009239215487226223, + "kd/rkl_advantage_mean": 0.6205948137290155, + "kd/rkl_advantage_p95": 4.720363096396128, + "kd/rkl_advantage_std": 2.166735730568568, + "kd/ssd_loss": 0.0, + "learning_rate": 8.758591880909884e-07, + "loss": 0.0369568665822347, + "num_tokens": 106968887.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7875, + "rewards/gt_logging_reward/std": 0.39916381339232127, + "sampling/importance_sampling_ratio/max": 1.9217944264411926, + "sampling/importance_sampling_ratio/mean": 0.9941265602906545, + "sampling/importance_sampling_ratio/min": 0.4073998168110847, + "sampling/sampling_logp_difference/max": 0.43743375738461815, + "sampling/sampling_logp_difference/mean": 0.004402770381420851, + "step": 3270, + "step_time": 8.298590180262302, + "student/entropy": 0.13247901070863008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030555557428548732, + "completions/max_length": 497.76666666666665, + "completions/max_terminated_length": 471.6333333333333, + "completions/mean_length": 194.0466766357422, + "completions/mean_terminated_length": 184.24701131184895, + "completions/min_length": 49.4, + "completions/min_terminated_length": 49.4, + "entropy": 0.1351915016149481, + "epoch": 0.12531804200053165, + "eval/gt_acc_batch": 0.7686111191908519, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2448456883430481, + "kd/policy_loss": 0.00939669026993215, + "kd/rkl_advantage_mean": 0.5985618131545682, + "kd/rkl_advantage_p95": 4.730362848440806, + "kd/rkl_advantage_std": 2.1924953987201055, + "kd/ssd_loss": 0.0, + "learning_rate": 8.747199331637109e-07, + "loss": 0.037586760520935056, + "num_tokens": 107972199.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7686111172040303, + "rewards/gt_logging_reward/std": 0.41108149737119676, + "sampling/importance_sampling_ratio/max": 1.9616144100824993, + "sampling/importance_sampling_ratio/mean": 1.0011666397253671, + "sampling/importance_sampling_ratio/min": 0.42680716862281165, + "sampling/sampling_logp_difference/max": 0.4030303080876668, + "sampling/sampling_logp_difference/mean": 0.004450731181229154, + "step": 3300, + "step_time": 8.085356685208778, + "student/entropy": 0.1351915016149481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277778909852108, + "completions/max_length": 489.26666666666665, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 186.99139963785808, + "completions/mean_terminated_length": 178.57846323649088, + "completions/min_length": 55.233333333333334, + "completions/min_terminated_length": 55.233333333333334, + "entropy": 0.13238962894926468, + "epoch": 0.12645729692780922, + "eval/gt_acc_batch": 0.7877777814865112, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2529484033584595, + "kd/policy_loss": 0.008619312464725226, + "kd/rkl_advantage_mean": 0.6833821083574245, + "kd/rkl_advantage_p95": 4.863298573096594, + "kd/rkl_advantage_std": 2.2282499402761458, + "kd/ssd_loss": 0.0, + "learning_rate": 8.735806782364334e-07, + "loss": 0.03447724978129069, + "num_tokens": 108948720.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7877777874469757, + "rewards/gt_logging_reward/std": 0.40146307051181795, + "sampling/importance_sampling_ratio/max": 1.9197857896486918, + "sampling/importance_sampling_ratio/mean": 0.9870763500531514, + "sampling/importance_sampling_ratio/min": 0.43445735772450766, + "sampling/sampling_logp_difference/max": 0.4601202170054118, + "sampling/sampling_logp_difference/mean": 0.00447044720252355, + "step": 3330, + "step_time": 8.037400667757417, + "student/entropy": 0.13238962894926468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.036944446184982854, + "completions/max_length": 482.1, + "completions/max_terminated_length": 452.06666666666666, + "completions/mean_length": 188.72723286946615, + "completions/mean_terminated_length": 176.61573028564453, + "completions/min_length": 46.733333333333334, + "completions/min_terminated_length": 46.733333333333334, + "entropy": 0.13653970242788394, + "epoch": 0.12759655185508678, + "eval/gt_acc_batch": 0.7863889078299204, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23837272822856903, + "kd/policy_loss": 0.009488712132830793, + "kd/rkl_advantage_mean": 0.5568264831788838, + "kd/rkl_advantage_p95": 4.644654782613118, + "kd/rkl_advantage_std": 2.1731474498907724, + "kd/ssd_loss": 0.0, + "learning_rate": 8.724414233091558e-07, + "loss": 0.03795484701792399, + "num_tokens": 109936050.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.786388882001241, + "rewards/gt_logging_reward/std": 0.40324352582295736, + "sampling/importance_sampling_ratio/max": 2.0410133918126423, + "sampling/importance_sampling_ratio/mean": 1.0044897417227427, + "sampling/importance_sampling_ratio/min": 0.40215475906928383, + "sampling/sampling_logp_difference/max": 0.45568832953770955, + "sampling/sampling_logp_difference/mean": 0.004498862063822647, + "step": 3360, + "step_time": 7.999358086194843, + "student/entropy": 0.13653970242788394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668020188808, + "completions/max_length": 490.6666666666667, + "completions/max_terminated_length": 445.96666666666664, + "completions/mean_length": 183.08889821370443, + "completions/mean_terminated_length": 175.2589314778646, + "completions/min_length": 54.46666666666667, + "completions/min_terminated_length": 54.46666666666667, + "entropy": 0.12626711533715326, + "epoch": 0.12873580678236432, + "eval/gt_acc_batch": 0.8111111283302307, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26094377040863037, + "kd/policy_loss": 0.008689720319428792, + "kd/rkl_advantage_mean": 0.5621983289747732, + "kd/rkl_advantage_p95": 4.551067258914312, + "kd/rkl_advantage_std": 2.139119583368301, + "kd/ssd_loss": 0.0, + "learning_rate": 8.713021683818782e-07, + "loss": 0.0347588857014974, + "num_tokens": 110895850.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8111111164093018, + "rewards/gt_logging_reward/std": 0.37525113026301066, + "sampling/importance_sampling_ratio/max": 1.8438990791638692, + "sampling/importance_sampling_ratio/mean": 1.0001079320907593, + "sampling/importance_sampling_ratio/min": 0.4195111900568008, + "sampling/sampling_logp_difference/max": 0.4220865488052368, + "sampling/sampling_logp_difference/mean": 0.0042311886325478556, + "step": 3390, + "step_time": 7.98834578403815, + "student/entropy": 0.12626711533715326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030555557211240134, + "completions/max_length": 482.6666666666667, + "completions/max_terminated_length": 452.26666666666665, + "completions/mean_length": 194.74445343017578, + "completions/mean_terminated_length": 185.01844329833983, + "completions/min_length": 52.86666666666667, + "completions/min_terminated_length": 52.86666666666667, + "entropy": 0.1302453612908721, + "epoch": 0.1298750617096419, + "eval/gt_acc_batch": 0.768888904651006, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24697674810886383, + "kd/policy_loss": 0.008794553405217205, + "kd/rkl_advantage_mean": 0.5280657878456017, + "kd/rkl_advantage_p95": 4.533943629264831, + "kd/rkl_advantage_std": 2.146011910835902, + "kd/ssd_loss": 0.0, + "learning_rate": 8.701629134546007e-07, + "loss": 0.035178216298421223, + "num_tokens": 111907426.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7688888907432556, + "rewards/gt_logging_reward/std": 0.4114924043416977, + "sampling/importance_sampling_ratio/max": 1.933761723836263, + "sampling/importance_sampling_ratio/mean": 0.9991441210110982, + "sampling/importance_sampling_ratio/min": 0.3814587106307348, + "sampling/sampling_logp_difference/max": 0.44338545004526775, + "sampling/sampling_logp_difference/mean": 0.004357165734594067, + "step": 3420, + "step_time": 8.202975747094024, + "student/entropy": 0.1302453612908721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334513008596, + "completions/max_length": 481.2, + "completions/max_terminated_length": 457.6, + "completions/mean_length": 189.1997309366862, + "completions/mean_terminated_length": 182.47071024576823, + "completions/min_length": 58.5, + "completions/min_terminated_length": 58.5, + "entropy": 0.1301593027388056, + "epoch": 0.13101431663691945, + "eval/gt_acc_batch": 0.7936111370722453, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2319832295179367, + "kd/policy_loss": 0.008673139702295884, + "kd/rkl_advantage_mean": 0.5553556760462622, + "kd/rkl_advantage_p95": 4.539742328723272, + "kd/rkl_advantage_std": 2.1206376055876413, + "kd/ssd_loss": 0.0, + "learning_rate": 8.69023658527323e-07, + "loss": 0.03469256560007731, + "num_tokens": 112898833.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111072699229, + "rewards/gt_logging_reward/std": 0.39466269115606945, + "sampling/importance_sampling_ratio/max": 1.8678322672843932, + "sampling/importance_sampling_ratio/mean": 0.9977221747239431, + "sampling/importance_sampling_ratio/min": 0.4058547422289848, + "sampling/sampling_logp_difference/max": 0.37442789872487386, + "sampling/sampling_logp_difference/mean": 0.004325135479060312, + "step": 3450, + "step_time": 8.108620758506003, + "student/entropy": 0.1301593027388056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001396983863, + "completions/max_length": 481.6, + "completions/max_terminated_length": 451.23333333333335, + "completions/mean_length": 194.38084309895834, + "completions/mean_terminated_length": 186.57857767740884, + "completions/min_length": 52.9, + "completions/min_terminated_length": 52.9, + "entropy": 0.13400991975019375, + "epoch": 0.13215357156419702, + "eval/gt_acc_batch": 0.7869444708029429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24049454927444458, + "kd/policy_loss": 0.008904942965212588, + "kd/rkl_advantage_mean": 0.5046597116005918, + "kd/rkl_advantage_p95": 4.539095809062322, + "kd/rkl_advantage_std": 2.115022851030032, + "kd/ssd_loss": 0.0, + "learning_rate": 8.678844036000456e-07, + "loss": 0.035619775454203285, + "num_tokens": 113904988.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444489479065, + "rewards/gt_logging_reward/std": 0.3916092832883199, + "sampling/importance_sampling_ratio/max": 1.9139215151468914, + "sampling/importance_sampling_ratio/mean": 0.998860776424408, + "sampling/importance_sampling_ratio/min": 0.4372386733690898, + "sampling/sampling_logp_difference/max": 0.4241244912147522, + "sampling/sampling_logp_difference/mean": 0.004352133977226913, + "step": 3480, + "step_time": 8.119304089977716, + "student/entropy": 0.13400991975019375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779065072538, + "completions/max_length": 485.96666666666664, + "completions/max_terminated_length": 445.6, + "completions/mean_length": 188.73556518554688, + "completions/mean_terminated_length": 180.48362833658854, + "completions/min_length": 53.06666666666667, + "completions/min_terminated_length": 53.06666666666667, + "entropy": 0.13260286673903465, + "epoch": 0.1332928264914746, + "eval/gt_acc_batch": 0.7891666869322459, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2622111439704895, + "kd/policy_loss": 0.008634296084831778, + "kd/rkl_advantage_mean": 0.537520574987866, + "kd/rkl_advantage_p95": 4.569740730524063, + "kd/rkl_advantage_std": 2.146898920337359, + "kd/ssd_loss": 0.0, + "learning_rate": 8.66745148672768e-07, + "loss": 0.03453719218571981, + "num_tokens": 114883076.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7891666670640309, + "rewards/gt_logging_reward/std": 0.4042519986629486, + "sampling/importance_sampling_ratio/max": 1.9484352310498556, + "sampling/importance_sampling_ratio/mean": 0.9981636106967926, + "sampling/importance_sampling_ratio/min": 0.41927977204322814, + "sampling/sampling_logp_difference/max": 0.45506683985392254, + "sampling/sampling_logp_difference/mean": 0.004316694235118727, + "step": 3510, + "step_time": 8.016562550724483, + "student/entropy": 0.13260286673903465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01944444583108028, + "completions/max_length": 484.0, + "completions/max_terminated_length": 437.6666666666667, + "completions/mean_length": 184.0636205037435, + "completions/mean_terminated_length": 177.8388478597005, + "completions/min_length": 51.1, + "completions/min_terminated_length": 51.1, + "entropy": 0.13540159004429977, + "epoch": 0.13443208141875213, + "eval/gt_acc_batch": 0.7802778085072836, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25266408920288086, + "kd/policy_loss": 0.00931189118224817, + "kd/rkl_advantage_mean": 0.5511935097941508, + "kd/rkl_advantage_p95": 4.618194834391276, + "kd/rkl_advantage_std": 2.173896497488022, + "kd/ssd_loss": 0.0, + "learning_rate": 8.656058937454904e-07, + "loss": 0.037247570355733235, + "num_tokens": 115839081.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7802777787049612, + "rewards/gt_logging_reward/std": 0.4015271455049515, + "sampling/importance_sampling_ratio/max": 1.9464171489079793, + "sampling/importance_sampling_ratio/mean": 0.9974754492441813, + "sampling/importance_sampling_ratio/min": 0.4063983072837194, + "sampling/sampling_logp_difference/max": 0.4629483828941981, + "sampling/sampling_logp_difference/mean": 0.0045270572726925215, + "step": 3540, + "step_time": 7.8362607803506155, + "student/entropy": 0.13540159004429977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334513008596, + "completions/max_length": 485.1333333333333, + "completions/max_terminated_length": 450.8666666666667, + "completions/mean_length": 189.6013977050781, + "completions/mean_terminated_length": 182.6419642130534, + "completions/min_length": 57.266666666666666, + "completions/min_terminated_length": 57.266666666666666, + "entropy": 0.13055024159451326, + "epoch": 0.1355713363460297, + "eval/gt_acc_batch": 0.7788889149824778, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23441430926322937, + "kd/policy_loss": 0.009325501035588483, + "kd/rkl_advantage_mean": 0.5786990081658587, + "kd/rkl_advantage_p95": 4.636624004443487, + "kd/rkl_advantage_std": 2.1888610114653906, + "kd/ssd_loss": 0.0, + "learning_rate": 8.644666388182128e-07, + "loss": 0.037301997343699135, + "num_tokens": 116823806.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7788888871669769, + "rewards/gt_logging_reward/std": 0.4113510290781657, + "sampling/importance_sampling_ratio/max": 1.9666248321533204, + "sampling/importance_sampling_ratio/mean": 0.9968909382820129, + "sampling/importance_sampling_ratio/min": 0.3710144822796186, + "sampling/sampling_logp_difference/max": 0.4799356559912364, + "sampling/sampling_logp_difference/mean": 0.004412760701961815, + "step": 3570, + "step_time": 7.977315610387207, + "student/entropy": 0.13055024159451326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777918924888, + "completions/max_length": 499.2, + "completions/max_terminated_length": 458.96666666666664, + "completions/mean_length": 186.19112141927084, + "completions/mean_terminated_length": 177.73036295572916, + "completions/min_length": 53.96666666666667, + "completions/min_terminated_length": 53.96666666666667, + "entropy": 0.12850381328413885, + "epoch": 0.13671059127330726, + "eval/gt_acc_batch": 0.7952777922153473, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32929104566574097, + "kd/policy_loss": 0.008902148574513073, + "kd/rkl_advantage_mean": 0.6130951015899578, + "kd/rkl_advantage_p95": 4.75553472439448, + "kd/rkl_advantage_std": 2.1760454813639325, + "kd/ssd_loss": 0.0, + "learning_rate": 8.633273838909353e-07, + "loss": 0.035608593622843424, + "num_tokens": 117793478.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777763207753, + "rewards/gt_logging_reward/std": 0.397961688041687, + "sampling/importance_sampling_ratio/max": 1.8282899936040242, + "sampling/importance_sampling_ratio/mean": 0.9934767206509908, + "sampling/importance_sampling_ratio/min": 0.39634723713000614, + "sampling/sampling_logp_difference/max": 0.4875128984451294, + "sampling/sampling_logp_difference/mean": 0.004293988971039653, + "step": 3600, + "step_time": 8.183716369051641, + "student/entropy": 0.12850381328413885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033055557621022066, + "completions/max_length": 501.03333333333336, + "completions/max_terminated_length": 464.3333333333333, + "completions/mean_length": 192.74139862060548, + "completions/mean_terminated_length": 182.01934763590495, + "completions/min_length": 54.96666666666667, + "completions/min_terminated_length": 54.96666666666667, + "entropy": 0.13772923015058042, + "epoch": 0.13784984620058482, + "eval/gt_acc_batch": 0.7919444680213928, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4877491593360901, + "kd/policy_loss": 0.009148767363512889, + "kd/rkl_advantage_mean": 0.6430709361952419, + "kd/rkl_advantage_p95": 4.856299110253652, + "kd/rkl_advantage_std": 2.2608909249305724, + "kd/ssd_loss": 0.0, + "learning_rate": 8.621881289636577e-07, + "loss": 0.03659507433573405, + "num_tokens": 118792947.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7919444461663564, + "rewards/gt_logging_reward/std": 0.3969426900148392, + "sampling/importance_sampling_ratio/max": 1.9295590281486512, + "sampling/importance_sampling_ratio/mean": 0.9980403006076812, + "sampling/importance_sampling_ratio/min": 0.4249534060557683, + "sampling/sampling_logp_difference/max": 0.4415054082870483, + "sampling/sampling_logp_difference/mean": 0.004379628342576325, + "step": 3630, + "step_time": 8.211108769231942, + "student/entropy": 0.13772923015058042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02861111310000221, + "completions/max_length": 492.3, + "completions/max_terminated_length": 462.53333333333336, + "completions/mean_length": 193.63917643229166, + "completions/mean_terminated_length": 184.320046488444, + "completions/min_length": 61.766666666666666, + "completions/min_terminated_length": 61.766666666666666, + "entropy": 0.1250526058798035, + "epoch": 0.1389891011278624, + "eval/gt_acc_batch": 0.7861111283302307, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2157025784254074, + "kd/policy_loss": 0.008474511333042755, + "kd/rkl_advantage_mean": 0.5564098631807913, + "kd/rkl_advantage_p95": 4.690655622879664, + "kd/rkl_advantage_std": 2.177954180041949, + "kd/ssd_loss": 0.0, + "learning_rate": 8.610488740363801e-07, + "loss": 0.03389804760615031, + "num_tokens": 119793912.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111144224803, + "rewards/gt_logging_reward/std": 0.4022726277510325, + "sampling/importance_sampling_ratio/max": 1.9469571709632874, + "sampling/importance_sampling_ratio/mean": 0.9901086886723837, + "sampling/importance_sampling_ratio/min": 0.4241572543978691, + "sampling/sampling_logp_difference/max": 0.4286548952261607, + "sampling/sampling_logp_difference/mean": 0.004182238814731439, + "step": 3660, + "step_time": 8.055642103306793, + "student/entropy": 0.1250526058798035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445905586084, + "completions/max_length": 474.56666666666666, + "completions/max_terminated_length": 448.43333333333334, + "completions/mean_length": 194.18306477864584, + "completions/mean_terminated_length": 186.5422332763672, + "completions/min_length": 58.166666666666664, + "completions/min_terminated_length": 58.166666666666664, + "entropy": 0.12626226376742125, + "epoch": 0.14012835605513993, + "eval/gt_acc_batch": 0.7863889118035634, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23186156153678894, + "kd/policy_loss": 0.008528966208299, + "kd/rkl_advantage_mean": 0.541322452078263, + "kd/rkl_advantage_p95": 4.5639243940512335, + "kd/rkl_advantage_std": 2.13153993388017, + "kd/ssd_loss": 0.0, + "learning_rate": 8.599096191091027e-07, + "loss": 0.03411586284637451, + "num_tokens": 120812619.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7863888919353486, + "rewards/gt_logging_reward/std": 0.4029636343320211, + "sampling/importance_sampling_ratio/max": 1.8730605880419413, + "sampling/importance_sampling_ratio/mean": 0.9944485267003377, + "sampling/importance_sampling_ratio/min": 0.43048641632000606, + "sampling/sampling_logp_difference/max": 0.4312378684679667, + "sampling/sampling_logp_difference/mean": 0.004237973092434307, + "step": 3690, + "step_time": 8.109928004796773, + "student/entropy": 0.12626226376742125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445998718343, + "completions/max_length": 499.7, + "completions/max_terminated_length": 461.9, + "completions/mean_length": 192.49445444742838, + "completions/mean_terminated_length": 184.62982177734375, + "completions/min_length": 55.666666666666664, + "completions/min_terminated_length": 55.666666666666664, + "entropy": 0.12696559578180314, + "epoch": 0.1412676109824175, + "eval/gt_acc_batch": 0.8119444588820139, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3019994795322418, + "kd/policy_loss": 0.008440619639198605, + "kd/rkl_advantage_mean": 0.6472775445940594, + "kd/rkl_advantage_p95": 4.821461766958237, + "kd/rkl_advantage_std": 2.227101605137189, + "kd/ssd_loss": 0.0, + "learning_rate": 8.58770364181825e-07, + "loss": 0.03376247882843018, + "num_tokens": 121803511.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.811944454908371, + "rewards/gt_logging_reward/std": 0.3849032700061798, + "sampling/importance_sampling_ratio/max": 1.9297804196675619, + "sampling/importance_sampling_ratio/mean": 0.9985670487085978, + "sampling/importance_sampling_ratio/min": 0.439083065589269, + "sampling/sampling_logp_difference/max": 0.45107114712397256, + "sampling/sampling_logp_difference/mean": 0.004259158484637737, + "step": 3720, + "step_time": 8.133531683667874, + "student/entropy": 0.12696559578180314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777779120951892, + "completions/max_length": 494.43333333333334, + "completions/max_terminated_length": 465.76666666666665, + "completions/mean_length": 195.31028798421224, + "completions/mean_terminated_length": 188.24877014160157, + "completions/min_length": 53.13333333333333, + "completions/min_terminated_length": 53.13333333333333, + "entropy": 0.12976726250102122, + "epoch": 0.14240686590969506, + "eval/gt_acc_batch": 0.7769444713989894, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.19312074780464172, + "kd/policy_loss": 0.0087610934007292, + "kd/rkl_advantage_mean": 0.5740872770547867, + "kd/rkl_advantage_p95": 4.656116048494975, + "kd/rkl_advantage_std": 2.151873853802681, + "kd/ssd_loss": 0.0, + "learning_rate": 8.576311092545475e-07, + "loss": 0.03504437208175659, + "num_tokens": 122818020.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7769444445768993, + "rewards/gt_logging_reward/std": 0.40281115571657816, + "sampling/importance_sampling_ratio/max": 1.8301790634791055, + "sampling/importance_sampling_ratio/mean": 0.9951868911584218, + "sampling/importance_sampling_ratio/min": 0.4119135975837708, + "sampling/sampling_logp_difference/max": 0.45305673281351727, + "sampling/sampling_logp_difference/mean": 0.004307411580036084, + "step": 3750, + "step_time": 8.086361345136538, + "student/entropy": 0.12976726250102122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028888890861223142, + "completions/max_length": 484.8666666666667, + "completions/max_terminated_length": 464.8666666666667, + "completions/mean_length": 190.31167653401693, + "completions/mean_terminated_length": 180.82097574869792, + "completions/min_length": 52.96666666666667, + "completions/min_terminated_length": 52.96666666666667, + "entropy": 0.13808030740668376, + "epoch": 0.14354612083697263, + "eval/gt_acc_batch": 0.7783333559830984, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2946498394012451, + "kd/policy_loss": 0.00938511304363298, + "kd/rkl_advantage_mean": 0.5895082773718362, + "kd/rkl_advantage_p95": 4.561515768369039, + "kd/rkl_advantage_std": 2.1249737332264584, + "kd/ssd_loss": 0.0, + "learning_rate": 8.564918543272699e-07, + "loss": 0.03754044771194458, + "num_tokens": 123804726.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7783333381017049, + "rewards/gt_logging_reward/std": 0.4050659328699112, + "sampling/importance_sampling_ratio/max": 1.971240222454071, + "sampling/importance_sampling_ratio/mean": 0.9934919357299805, + "sampling/importance_sampling_ratio/min": 0.3956456502278646, + "sampling/sampling_logp_difference/max": 0.4436298648516337, + "sampling/sampling_logp_difference/mean": 0.00435984677169472, + "step": 3780, + "step_time": 7.9574565231023975, + "student/entropy": 0.13808030740668376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001012037198, + "completions/max_length": 469.46666666666664, + "completions/max_terminated_length": 439.1666666666667, + "completions/mean_length": 187.12167714436848, + "completions/mean_terminated_length": 180.75076700846355, + "completions/min_length": 57.166666666666664, + "completions/min_terminated_length": 57.166666666666664, + "entropy": 0.13251610522468885, + "epoch": 0.14468537576425017, + "eval/gt_acc_batch": 0.7975000282128651, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.41638410091400146, + "kd/policy_loss": 0.00902582513129649, + "kd/rkl_advantage_mean": 0.5862570943310856, + "kd/rkl_advantage_p95": 4.606290302673975, + "kd/rkl_advantage_std": 2.1336263755957288, + "kd/ssd_loss": 0.0, + "learning_rate": 8.553525993999924e-07, + "loss": 0.0361033042271932, + "num_tokens": 124779356.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7975000003973644, + "rewards/gt_logging_reward/std": 0.392922568321228, + "sampling/importance_sampling_ratio/max": 1.9322321573893229, + "sampling/importance_sampling_ratio/mean": 0.9996098061402638, + "sampling/importance_sampling_ratio/min": 0.4147029717763265, + "sampling/sampling_logp_difference/max": 0.42061358292897544, + "sampling/sampling_logp_difference/mean": 0.004438786565636595, + "step": 3810, + "step_time": 7.876054451447756, + "student/entropy": 0.13251610522468885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01694444545234243, + "completions/max_length": 479.8333333333333, + "completions/max_terminated_length": 436.53333333333336, + "completions/mean_length": 183.59612121582032, + "completions/mean_terminated_length": 177.94090220133464, + "completions/min_length": 57.766666666666666, + "completions/min_terminated_length": 57.766666666666666, + "entropy": 0.12859336622059345, + "epoch": 0.14582463069152773, + "eval/gt_acc_batch": 0.8013889054457347, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3059675395488739, + "kd/policy_loss": 0.008941694678893934, + "kd/rkl_advantage_mean": 0.5909876501653344, + "kd/rkl_advantage_p95": 4.609255439043045, + "kd/rkl_advantage_std": 2.1475826650857925, + "kd/ssd_loss": 0.0, + "learning_rate": 8.542133444727147e-07, + "loss": 0.035766780376434326, + "num_tokens": 125736798.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888895511627, + "rewards/gt_logging_reward/std": 0.3891641512513161, + "sampling/importance_sampling_ratio/max": 1.900791331132253, + "sampling/importance_sampling_ratio/mean": 1.0055002550284067, + "sampling/importance_sampling_ratio/min": 0.44749155392249423, + "sampling/sampling_logp_difference/max": 0.40686410466829936, + "sampling/sampling_logp_difference/mean": 0.0042761899453277385, + "step": 3840, + "step_time": 7.885675118297028, + "student/entropy": 0.12859336622059345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334444711604, + "completions/max_length": 476.26666666666665, + "completions/max_terminated_length": 441.6333333333333, + "completions/mean_length": 187.06084289550782, + "completions/mean_terminated_length": 181.1723856608073, + "completions/min_length": 64.6, + "completions/min_terminated_length": 64.6, + "entropy": 0.12036727238446474, + "epoch": 0.1469638856188053, + "eval/gt_acc_batch": 0.8288889050483703, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21585385501384735, + "kd/policy_loss": 0.007807196591359874, + "kd/rkl_advantage_mean": 0.5831190113152843, + "kd/rkl_advantage_p95": 4.754276239871979, + "kd/rkl_advantage_std": 2.2196255713701247, + "kd/ssd_loss": 0.0, + "learning_rate": 8.530740895454373e-07, + "loss": 0.031228786706924437, + "num_tokens": 126713065.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8288888931274414, + "rewards/gt_logging_reward/std": 0.36962173581123353, + "sampling/importance_sampling_ratio/max": 1.8702951073646545, + "sampling/importance_sampling_ratio/mean": 0.9955123623212179, + "sampling/importance_sampling_ratio/min": 0.424912183980147, + "sampling/sampling_logp_difference/max": 0.4764505346616109, + "sampling/sampling_logp_difference/mean": 0.004113429140609999, + "step": 3870, + "step_time": 7.873756031416511, + "student/entropy": 0.12036727238446474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03000000175088644, + "completions/max_length": 482.56666666666666, + "completions/max_terminated_length": 448.03333333333336, + "completions/mean_length": 192.46139882405598, + "completions/mean_terminated_length": 182.69854532877605, + "completions/min_length": 51.36666666666667, + "completions/min_terminated_length": 51.36666666666667, + "entropy": 0.12716003296275932, + "epoch": 0.14810314054608287, + "eval/gt_acc_batch": 0.7825000147024791, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2505664527416229, + "kd/policy_loss": 0.008980624920999011, + "kd/rkl_advantage_mean": 0.6183612528412292, + "kd/rkl_advantage_p95": 4.813882062832515, + "kd/rkl_advantage_std": 2.2132724155982335, + "kd/ssd_loss": 0.0, + "learning_rate": 8.519348346181598e-07, + "loss": 0.035922507445017494, + "num_tokens": 127717358.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7825000047683716, + "rewards/gt_logging_reward/std": 0.4025372336308161, + "sampling/importance_sampling_ratio/max": 1.9249574780464171, + "sampling/importance_sampling_ratio/mean": 1.0055257042249044, + "sampling/importance_sampling_ratio/min": 0.40461665342251457, + "sampling/sampling_logp_difference/max": 0.4361312707265218, + "sampling/sampling_logp_difference/mean": 0.004229902957255642, + "step": 3900, + "step_time": 8.010640232280517, + "student/entropy": 0.12716003296275932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01777777907749017, + "completions/max_length": 490.1333333333333, + "completions/max_terminated_length": 456.4, + "completions/mean_length": 183.76695454915364, + "completions/mean_terminated_length": 177.86695861816406, + "completions/min_length": 46.9, + "completions/min_terminated_length": 46.9, + "entropy": 0.13071482709298532, + "epoch": 0.14924239547336043, + "eval/gt_acc_batch": 0.8030555685361226, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21536394953727722, + "kd/policy_loss": 0.0085913473352169, + "kd/rkl_advantage_mean": 0.6167706399302308, + "kd/rkl_advantage_p95": 4.794722936550776, + "kd/rkl_advantage_std": 2.2393792817989984, + "kd/ssd_loss": 0.0, + "learning_rate": 8.507955796908821e-07, + "loss": 0.034365391731262206, + "num_tokens": 128679967.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555546283722, + "rewards/gt_logging_reward/std": 0.3914008448521296, + "sampling/importance_sampling_ratio/max": 1.887408939997355, + "sampling/importance_sampling_ratio/mean": 1.0004679143428803, + "sampling/importance_sampling_ratio/min": 0.44575444211562476, + "sampling/sampling_logp_difference/max": 0.4034588098526001, + "sampling/sampling_logp_difference/mean": 0.004369302784713606, + "step": 3930, + "step_time": 7.921837930567563, + "student/entropy": 0.13071482709298532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015000000906487307, + "completions/max_length": 482.9, + "completions/max_terminated_length": 441.53333333333336, + "completions/mean_length": 183.66584243774415, + "completions/mean_terminated_length": 178.80771814982097, + "completions/min_length": 51.3, + "completions/min_terminated_length": 51.3, + "entropy": 0.1309664336964488, + "epoch": 0.15038165040063797, + "eval/gt_acc_batch": 0.7900000214576721, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2550804018974304, + "kd/policy_loss": 0.008917909204804649, + "kd/rkl_advantage_mean": 0.5318890278460457, + "kd/rkl_advantage_p95": 4.5937477350234985, + "kd/rkl_advantage_std": 2.1663916617631913, + "kd/ssd_loss": 0.0, + "learning_rate": 8.496563247636046e-07, + "loss": 0.035671643416086835, + "num_tokens": 129644308.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7900000015894572, + "rewards/gt_logging_reward/std": 0.3956454336643219, + "sampling/importance_sampling_ratio/max": 1.888447646299998, + "sampling/importance_sampling_ratio/mean": 1.0059825201829276, + "sampling/importance_sampling_ratio/min": 0.41258290559053423, + "sampling/sampling_logp_difference/max": 0.4695527990659078, + "sampling/sampling_logp_difference/mean": 0.004362455933975677, + "step": 3960, + "step_time": 7.947355321375653, + "student/entropy": 0.1309664336964488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333482965827, + "completions/max_length": 493.0, + "completions/max_terminated_length": 447.76666666666665, + "completions/mean_length": 187.94500885009765, + "completions/mean_terminated_length": 180.33023732503256, + "completions/min_length": 56.733333333333334, + "completions/min_terminated_length": 56.733333333333334, + "entropy": 0.1282873999948303, + "epoch": 0.15152090532791554, + "eval/gt_acc_batch": 0.7961111307144165, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2680627107620239, + "kd/policy_loss": 0.008808373221351454, + "kd/rkl_advantage_mean": 0.6283982713085909, + "kd/rkl_advantage_p95": 4.722524823745092, + "kd/rkl_advantage_std": 2.1692258606354398, + "kd/ssd_loss": 0.0, + "learning_rate": 8.48517069836327e-07, + "loss": 0.035233497619628906, + "num_tokens": 130626662.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111088593801, + "rewards/gt_logging_reward/std": 0.39642598827679953, + "sampling/importance_sampling_ratio/max": 1.9171944936116536, + "sampling/importance_sampling_ratio/mean": 0.9993879159291585, + "sampling/importance_sampling_ratio/min": 0.4389844780166944, + "sampling/sampling_logp_difference/max": 0.4655621826648712, + "sampling/sampling_logp_difference/mean": 0.004273882888567945, + "step": 3990, + "step_time": 8.047604063828476, + "student/entropy": 0.1282873999948303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778866390386, + "completions/max_length": 492.4, + "completions/max_terminated_length": 459.5, + "completions/mean_length": 189.8327880859375, + "completions/mean_terminated_length": 183.2701665242513, + "completions/min_length": 53.166666666666664, + "completions/min_terminated_length": 53.166666666666664, + "entropy": 0.1299325656145811, + "epoch": 0.1526601602551931, + "eval/gt_acc_batch": 0.7847222487131754, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24568936228752136, + "kd/policy_loss": 0.009243122106029962, + "kd/rkl_advantage_mean": 0.5238471609850724, + "kd/rkl_advantage_p95": 4.631309457619985, + "kd/rkl_advantage_std": 2.19391452173392, + "kd/ssd_loss": 0.0, + "learning_rate": 8.473778149090494e-07, + "loss": 0.03697249094645182, + "num_tokens": 131620908.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7847222208976745, + "rewards/gt_logging_reward/std": 0.40395907759666444, + "sampling/importance_sampling_ratio/max": 1.9699184934298197, + "sampling/importance_sampling_ratio/mean": 1.0056901971499126, + "sampling/importance_sampling_ratio/min": 0.4527740334471067, + "sampling/sampling_logp_difference/max": 0.4292052914698919, + "sampling/sampling_logp_difference/mean": 0.004380018962547183, + "step": 4020, + "step_time": 8.108341849094723, + "student/entropy": 0.1299325656145811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223660598197, + "completions/max_length": 493.0, + "completions/max_terminated_length": 453.3666666666667, + "completions/mean_length": 190.1541763305664, + "completions/mean_terminated_length": 183.0502151489258, + "completions/min_length": 60.666666666666664, + "completions/min_terminated_length": 60.666666666666664, + "entropy": 0.13141200070579848, + "epoch": 0.15379941518247067, + "eval/gt_acc_batch": 0.8022222379843394, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2678760290145874, + "kd/policy_loss": 0.008843089612976958, + "kd/rkl_advantage_mean": 0.6553081595959763, + "kd/rkl_advantage_p95": 4.819316520293554, + "kd/rkl_advantage_std": 2.190024987856547, + "kd/ssd_loss": 0.0, + "learning_rate": 8.462385599817718e-07, + "loss": 0.035372364521026614, + "num_tokens": 132607495.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8022222260634104, + "rewards/gt_logging_reward/std": 0.39186418056488037, + "sampling/importance_sampling_ratio/max": 1.9844186027844748, + "sampling/importance_sampling_ratio/mean": 1.0023646175861358, + "sampling/importance_sampling_ratio/min": 0.4091393311818441, + "sampling/sampling_logp_difference/max": 0.4160718103249868, + "sampling/sampling_logp_difference/mean": 0.004390641332914432, + "step": 4050, + "step_time": 8.02225600798071, + "student/entropy": 0.13141200070579848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334680646657, + "completions/max_length": 481.76666666666665, + "completions/max_terminated_length": 452.46666666666664, + "completions/mean_length": 185.7327870686849, + "completions/mean_terminated_length": 177.23232676188152, + "completions/min_length": 47.9, + "completions/min_terminated_length": 47.9, + "entropy": 0.12927385376145442, + "epoch": 0.1549386701097482, + "eval/gt_acc_batch": 0.7952777922153473, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3615736961364746, + "kd/policy_loss": 0.008323279265702392, + "kd/rkl_advantage_mean": 0.4607571441990634, + "kd/rkl_advantage_p95": 4.408920383453369, + "kd/rkl_advantage_std": 2.1355039536952973, + "kd/ssd_loss": 0.0, + "learning_rate": 8.450993050544944e-07, + "loss": 0.03329311807950338, + "num_tokens": 133578677.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777802944183, + "rewards/gt_logging_reward/std": 0.3934675469994545, + "sampling/importance_sampling_ratio/max": 1.9203877607981363, + "sampling/importance_sampling_ratio/mean": 0.9961386839548747, + "sampling/importance_sampling_ratio/min": 0.40807383954524995, + "sampling/sampling_logp_difference/max": 0.431478151679039, + "sampling/sampling_logp_difference/mean": 0.004261350080681344, + "step": 4080, + "step_time": 7.902688853178794, + "student/entropy": 0.12927385376145442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334643393756, + "completions/max_length": 495.0, + "completions/max_terminated_length": 464.6, + "completions/mean_length": 192.14778747558594, + "completions/mean_terminated_length": 184.61016031901042, + "completions/min_length": 51.5, + "completions/min_terminated_length": 51.5, + "entropy": 0.13545831721276044, + "epoch": 0.15607792503702578, + "eval/gt_acc_batch": 0.7855555752913157, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2718941569328308, + "kd/policy_loss": 0.009231335087679326, + "kd/rkl_advantage_mean": 0.6478290696628392, + "kd/rkl_advantage_p95": 4.811893941958745, + "kd/rkl_advantage_std": 2.183758536974589, + "kd/ssd_loss": 0.0, + "learning_rate": 8.439600501272167e-07, + "loss": 0.03692533572514852, + "num_tokens": 134578281.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555574099223, + "rewards/gt_logging_reward/std": 0.4042228748401006, + "sampling/importance_sampling_ratio/max": 1.9205192049344382, + "sampling/importance_sampling_ratio/mean": 0.9998274505138397, + "sampling/importance_sampling_ratio/min": 0.40738620758056643, + "sampling/sampling_logp_difference/max": 0.43125993212064107, + "sampling/sampling_logp_difference/mean": 0.0044761765748262405, + "step": 4110, + "step_time": 8.082483251288068, + "student/entropy": 0.13545831721276044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223530213037, + "completions/max_length": 485.43333333333334, + "completions/max_terminated_length": 459.3333333333333, + "completions/mean_length": 190.36112111409506, + "completions/mean_terminated_length": 183.98695017496746, + "completions/min_length": 47.7, + "completions/min_terminated_length": 47.7, + "entropy": 0.12813850914438565, + "epoch": 0.15721717996430334, + "eval/gt_acc_batch": 0.7922222375869751, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28710395097732544, + "kd/policy_loss": 0.008951588733665024, + "kd/rkl_advantage_mean": 0.5302395096087518, + "kd/rkl_advantage_p95": 4.6482244769732155, + "kd/rkl_advantage_std": 2.1745707402626673, + "kd/ssd_loss": 0.0, + "learning_rate": 8.428207951999392e-07, + "loss": 0.03580636183420817, + "num_tokens": 135567621.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222216924032, + "rewards/gt_logging_reward/std": 0.4008854548136393, + "sampling/importance_sampling_ratio/max": 1.9571528633435566, + "sampling/importance_sampling_ratio/mean": 0.9992006381352743, + "sampling/importance_sampling_ratio/min": 0.41946633607149125, + "sampling/sampling_logp_difference/max": 0.42390459378560386, + "sampling/sampling_logp_difference/mean": 0.004311849487324556, + "step": 4140, + "step_time": 8.069770083529875, + "student/entropy": 0.12813850914438565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223474333683, + "completions/max_length": 487.4, + "completions/max_terminated_length": 442.9, + "completions/mean_length": 187.0519546508789, + "completions/mean_terminated_length": 179.7940892537435, + "completions/min_length": 51.13333333333333, + "completions/min_terminated_length": 51.13333333333333, + "entropy": 0.13038908429443835, + "epoch": 0.1583564348915809, + "eval/gt_acc_batch": 0.7930555740992228, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3027611970901489, + "kd/policy_loss": 0.009152335289400071, + "kd/rkl_advantage_mean": 0.6023480845615268, + "kd/rkl_advantage_p95": 4.736371819178263, + "kd/rkl_advantage_std": 2.195637721816699, + "kd/ssd_loss": 0.0, + "learning_rate": 8.416815402726617e-07, + "loss": 0.036609347661336264, + "num_tokens": 136534576.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555562178294, + "rewards/gt_logging_reward/std": 0.39814168711503345, + "sampling/importance_sampling_ratio/max": 1.9007872621218362, + "sampling/importance_sampling_ratio/mean": 1.004242604970932, + "sampling/importance_sampling_ratio/min": 0.3928728853662809, + "sampling/sampling_logp_difference/max": 0.4371136208375295, + "sampling/sampling_logp_difference/mean": 0.0043490045160676045, + "step": 4170, + "step_time": 8.01301650107683, + "student/entropy": 0.13038908429443835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02777777922650178, + "completions/max_length": 498.73333333333335, + "completions/max_terminated_length": 461.1333333333333, + "completions/mean_length": 194.81389872233072, + "completions/mean_terminated_length": 185.9985778808594, + "completions/min_length": 49.43333333333333, + "completions/min_terminated_length": 49.43333333333333, + "entropy": 0.13223536722362042, + "epoch": 0.15949568981885848, + "eval/gt_acc_batch": 0.786388901869456, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2800457179546356, + "kd/policy_loss": 0.009084603279673805, + "kd/rkl_advantage_mean": 0.6758867027238011, + "kd/rkl_advantage_p95": 4.853655767440796, + "kd/rkl_advantage_std": 2.2197127838929496, + "kd/ssd_loss": 0.0, + "learning_rate": 8.40542285345384e-07, + "loss": 0.03633841673533122, + "num_tokens": 137537498.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7863888879617055, + "rewards/gt_logging_reward/std": 0.4031791776418686, + "sampling/importance_sampling_ratio/max": 1.9349221388498943, + "sampling/importance_sampling_ratio/mean": 0.9986820340156555, + "sampling/importance_sampling_ratio/min": 0.40646717250347136, + "sampling/sampling_logp_difference/max": 0.4193293869495392, + "sampling/sampling_logp_difference/mean": 0.004404727757597963, + "step": 4200, + "step_time": 8.124026103814442, + "student/entropy": 0.13223536722362042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01694444545234243, + "completions/max_length": 484.43333333333334, + "completions/max_terminated_length": 447.6666666666667, + "completions/mean_length": 184.49473368326824, + "completions/mean_terminated_length": 178.87979125976562, + "completions/min_length": 50.8, + "completions/min_terminated_length": 50.8, + "entropy": 0.12642300768444936, + "epoch": 0.16063494474613602, + "eval/gt_acc_batch": 0.7991666853427887, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29206791520118713, + "kd/policy_loss": 0.00826465521628658, + "kd/rkl_advantage_mean": 0.6074105527950451, + "kd/rkl_advantage_p95": 4.779417123397192, + "kd/rkl_advantage_std": 2.213130287329356, + "kd/ssd_loss": 0.0, + "learning_rate": 8.394030304181065e-07, + "loss": 0.033058623472849526, + "num_tokens": 138500719.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7991666714350383, + "rewards/gt_logging_reward/std": 0.39380384186903633, + "sampling/importance_sampling_ratio/max": 1.9179837147394816, + "sampling/importance_sampling_ratio/mean": 0.9942876120408376, + "sampling/importance_sampling_ratio/min": 0.3885172242919604, + "sampling/sampling_logp_difference/max": 0.48046698570251467, + "sampling/sampling_logp_difference/mean": 0.004306765731113652, + "step": 4230, + "step_time": 7.8829748247362055, + "student/entropy": 0.12642300768444936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555684491992, + "completions/max_length": 488.2, + "completions/max_terminated_length": 447.8, + "completions/mean_length": 191.7305643717448, + "completions/mean_terminated_length": 185.01238403320312, + "completions/min_length": 54.2, + "completions/min_terminated_length": 54.2, + "entropy": 0.12226870047549407, + "epoch": 0.16177419967341358, + "eval/gt_acc_batch": 0.8033333599567414, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23413728177547455, + "kd/policy_loss": 0.008420986034131299, + "kd/rkl_advantage_mean": 0.5809727735662212, + "kd/rkl_advantage_p95": 4.705149080355962, + "kd/rkl_advantage_std": 2.1833930631478626, + "kd/ssd_loss": 0.0, + "learning_rate": 8.38263775490829e-07, + "loss": 0.03368393977483113, + "num_tokens": 139496805.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8033333361148834, + "rewards/gt_logging_reward/std": 0.3885658378402392, + "sampling/importance_sampling_ratio/max": 1.9447064081827798, + "sampling/importance_sampling_ratio/mean": 1.0011164784431457, + "sampling/importance_sampling_ratio/min": 0.4261832709113757, + "sampling/sampling_logp_difference/max": 0.4207845350106557, + "sampling/sampling_logp_difference/mean": 0.004083939998721083, + "step": 4260, + "step_time": 8.008160975850963, + "student/entropy": 0.12226870047549407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02277777905886372, + "completions/max_length": 494.1333333333333, + "completions/max_terminated_length": 448.43333333333334, + "completions/mean_length": 186.33889872233073, + "completions/mean_terminated_length": 178.69863637288412, + "completions/min_length": 55.8, + "completions/min_terminated_length": 55.8, + "entropy": 0.13278882515927157, + "epoch": 0.16291345460069115, + "eval/gt_acc_batch": 0.7769444783528646, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25176528096199036, + "kd/policy_loss": 0.009169622797829409, + "kd/rkl_advantage_mean": 0.7123029805719853, + "kd/rkl_advantage_p95": 5.012585739294688, + "kd/rkl_advantage_std": 2.2778198520342507, + "kd/ssd_loss": 0.0, + "learning_rate": 8.371245205635515e-07, + "loss": 0.036678496996561685, + "num_tokens": 140462601.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7769444465637207, + "rewards/gt_logging_reward/std": 0.41087177296479543, + "sampling/importance_sampling_ratio/max": 1.9165485938390097, + "sampling/importance_sampling_ratio/mean": 0.9935587366422017, + "sampling/importance_sampling_ratio/min": 0.3750995397567749, + "sampling/sampling_logp_difference/max": 0.43510931730270386, + "sampling/sampling_logp_difference/mean": 0.004375011147931218, + "step": 4290, + "step_time": 7.892824964035147, + "student/entropy": 0.13278882515927157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445837289096, + "completions/max_length": 477.6, + "completions/max_terminated_length": 446.3, + "completions/mean_length": 187.93251037597656, + "completions/mean_terminated_length": 180.8996129353841, + "completions/min_length": 50.766666666666666, + "completions/min_terminated_length": 50.766666666666666, + "entropy": 0.12988774714370568, + "epoch": 0.16405270952796872, + "eval/gt_acc_batch": 0.7933333436648051, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22875063121318817, + "kd/policy_loss": 0.008804961365725224, + "kd/rkl_advantage_mean": 0.6138166089231769, + "kd/rkl_advantage_p95": 4.789567377169927, + "kd/rkl_advantage_std": 2.200960459311803, + "kd/ssd_loss": 0.0, + "learning_rate": 8.359852656362738e-07, + "loss": 0.03521984418233236, + "num_tokens": 141446630.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7933333297570546, + "rewards/gt_logging_reward/std": 0.3907527590791384, + "sampling/importance_sampling_ratio/max": 1.9711983283360799, + "sampling/importance_sampling_ratio/mean": 0.9954597314198812, + "sampling/importance_sampling_ratio/min": 0.3752659817536672, + "sampling/sampling_logp_difference/max": 0.4387907862663269, + "sampling/sampling_logp_difference/mean": 0.004335878975689411, + "step": 4320, + "step_time": 7.991025264386553, + "student/entropy": 0.12988774714370568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001459072035, + "completions/max_length": 491.06666666666666, + "completions/max_terminated_length": 458.06666666666666, + "completions/mean_length": 189.28084208170574, + "completions/mean_terminated_length": 180.9113774617513, + "completions/min_length": 47.6, + "completions/min_terminated_length": 47.6, + "entropy": 0.12740826072792213, + "epoch": 0.16519196445524628, + "eval/gt_acc_batch": 0.7855555752913157, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2615356743335724, + "kd/policy_loss": 0.00887118478034002, + "kd/rkl_advantage_mean": 0.6155977550971632, + "kd/rkl_advantage_p95": 4.726803463697434, + "kd/rkl_advantage_std": 2.1859089503685634, + "kd/ssd_loss": 0.0, + "learning_rate": 8.348460107089963e-07, + "loss": 0.03548474709192912, + "num_tokens": 142428105.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555534362793, + "rewards/gt_logging_reward/std": 0.4063624233007431, + "sampling/importance_sampling_ratio/max": 2.0196414868036907, + "sampling/importance_sampling_ratio/mean": 1.0054883539676667, + "sampling/importance_sampling_ratio/min": 0.4323948656519254, + "sampling/sampling_logp_difference/max": 0.4197516103585561, + "sampling/sampling_logp_difference/mean": 0.004196122894063592, + "step": 4350, + "step_time": 7.913747670012526, + "student/entropy": 0.12740826072792213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.010000000645716985, + "completions/max_length": 457.5, + "completions/max_terminated_length": 433.1666666666667, + "completions/mean_length": 177.16917622884114, + "completions/mean_terminated_length": 173.8779037475586, + "completions/min_length": 56.53333333333333, + "completions/min_terminated_length": 56.53333333333333, + "entropy": 0.13573877370605866, + "epoch": 0.16633121938252382, + "eval/gt_acc_batch": 0.812500019868215, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22958894073963165, + "kd/policy_loss": 0.008992421204069009, + "kd/rkl_advantage_mean": 0.5670546897066137, + "kd/rkl_advantage_p95": 4.600766197840373, + "kd/rkl_advantage_std": 2.1719899584849673, + "kd/ssd_loss": 0.0, + "learning_rate": 8.337067557817187e-07, + "loss": 0.03596968650817871, + "num_tokens": 143354218.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8125000019868215, + "rewards/gt_logging_reward/std": 0.38306012799342476, + "sampling/importance_sampling_ratio/max": 1.9170852541923522, + "sampling/importance_sampling_ratio/mean": 0.9930353462696075, + "sampling/importance_sampling_ratio/min": 0.4210829511284828, + "sampling/sampling_logp_difference/max": 0.4191339612007141, + "sampling/sampling_logp_difference/mean": 0.004544446376773218, + "step": 4380, + "step_time": 7.561804490520929, + "student/entropy": 0.13573877370605866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02944444613531232, + "completions/max_length": 492.73333333333335, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 193.28695475260417, + "completions/mean_terminated_length": 183.55145517985025, + "completions/min_length": 53.833333333333336, + "completions/min_terminated_length": 53.833333333333336, + "entropy": 0.13779268302023412, + "epoch": 0.1674704743098014, + "eval/gt_acc_batch": 0.770000018676122, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3444743752479553, + "kd/policy_loss": 0.009203816397348418, + "kd/rkl_advantage_mean": 0.6024982201556365, + "kd/rkl_advantage_p95": 4.741541596253713, + "kd/rkl_advantage_std": 2.183046145240466, + "kd/ssd_loss": 0.0, + "learning_rate": 8.325675008544411e-07, + "loss": 0.03681526978810628, + "num_tokens": 144355939.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7700000007947286, + "rewards/gt_logging_reward/std": 0.4161013712485631, + "sampling/importance_sampling_ratio/max": 1.9570293307304383, + "sampling/importance_sampling_ratio/mean": 0.9965164522329967, + "sampling/importance_sampling_ratio/min": 0.4027052313089371, + "sampling/sampling_logp_difference/max": 0.44581857323646545, + "sampling/sampling_logp_difference/mean": 0.0043686490428323545, + "step": 4410, + "step_time": 8.040070042076211, + "student/entropy": 0.13779268302023412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944445880750813, + "completions/max_length": 494.3333333333333, + "completions/max_terminated_length": 453.1, + "completions/mean_length": 191.8630635579427, + "completions/mean_terminated_length": 183.09041697184244, + "completions/min_length": 51.96666666666667, + "completions/min_terminated_length": 51.96666666666667, + "entropy": 0.13141766488552092, + "epoch": 0.16860972923707895, + "eval/gt_acc_batch": 0.7813889145851135, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26691102981567383, + "kd/policy_loss": 0.008583428520554055, + "kd/rkl_advantage_mean": 0.6079949674468177, + "kd/rkl_advantage_p95": 4.76937315662702, + "kd/rkl_advantage_std": 2.1792185723781587, + "kd/ssd_loss": 0.0, + "learning_rate": 8.314282459271636e-07, + "loss": 0.03433371384938558, + "num_tokens": 145344862.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7813888927300771, + "rewards/gt_logging_reward/std": 0.40287031133969625, + "sampling/importance_sampling_ratio/max": 1.9123770952224732, + "sampling/importance_sampling_ratio/mean": 0.9939290483792623, + "sampling/importance_sampling_ratio/min": 0.39903106689453127, + "sampling/sampling_logp_difference/max": 0.4310739199320475, + "sampling/sampling_logp_difference/mean": 0.004372928656327228, + "step": 4440, + "step_time": 8.000834671012125, + "student/entropy": 0.13141766488552092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033611113329728444, + "completions/max_length": 492.8333333333333, + "completions/max_terminated_length": 450.1, + "completions/mean_length": 194.15056610107422, + "completions/mean_terminated_length": 183.2870641072591, + "completions/min_length": 55.53333333333333, + "completions/min_terminated_length": 55.53333333333333, + "entropy": 0.12776162804414828, + "epoch": 0.16974898416435652, + "eval/gt_acc_batch": 0.7922222554683686, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3164127469062805, + "kd/policy_loss": 0.008474199530125286, + "kd/rkl_advantage_mean": 0.5693696306397518, + "kd/rkl_advantage_p95": 4.676901431878408, + "kd/rkl_advantage_std": 2.188996205727259, + "kd/ssd_loss": 0.0, + "learning_rate": 8.302889909998861e-07, + "loss": 0.03389679988225301, + "num_tokens": 146339292.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222216924032, + "rewards/gt_logging_reward/std": 0.3943330317735672, + "sampling/importance_sampling_ratio/max": 1.9623850226402282, + "sampling/importance_sampling_ratio/mean": 0.9955548346042633, + "sampling/importance_sampling_ratio/min": 0.4032886683940887, + "sampling/sampling_logp_difference/max": 0.43698521653811134, + "sampling/sampling_logp_difference/mean": 0.004246901821655532, + "step": 4470, + "step_time": 7.984031063651977, + "student/entropy": 0.12776162804414828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778934687375, + "completions/max_length": 485.7, + "completions/max_terminated_length": 454.5, + "completions/mean_length": 189.89195404052734, + "completions/mean_terminated_length": 182.57731119791666, + "completions/min_length": 56.266666666666666, + "completions/min_terminated_length": 56.266666666666666, + "entropy": 0.1351021594678362, + "epoch": 0.17088823909163406, + "eval/gt_acc_batch": 0.8066666781902313, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.19093075394630432, + "kd/policy_loss": 0.009064448933349923, + "kd/rkl_advantage_mean": 0.5618911135088032, + "kd/rkl_advantage_p95": 4.63934986392657, + "kd/rkl_advantage_std": 2.1560906539360682, + "kd/ssd_loss": 0.0, + "learning_rate": 8.291497360726085e-07, + "loss": 0.03625779946645101, + "num_tokens": 147328519.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8066666642824809, + "rewards/gt_logging_reward/std": 0.3878040427962939, + "sampling/importance_sampling_ratio/max": 1.9465405106544496, + "sampling/importance_sampling_ratio/mean": 0.9980142474174499, + "sampling/importance_sampling_ratio/min": 0.40284324834744134, + "sampling/sampling_logp_difference/max": 0.43215983907381694, + "sampling/sampling_logp_difference/mean": 0.0043777971994131805, + "step": 4500, + "step_time": 7.985646306048148, + "student/entropy": 0.1351021594678362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02000000113621354, + "completions/max_length": 485.6333333333333, + "completions/max_terminated_length": 438.93333333333334, + "completions/mean_length": 182.3072311401367, + "completions/mean_terminated_length": 175.67564697265624, + "completions/min_length": 52.1, + "completions/min_terminated_length": 52.1, + "entropy": 0.12640664496769508, + "epoch": 0.17202749401891163, + "eval/gt_acc_batch": 0.8147222518920898, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28761154413223267, + "kd/policy_loss": 0.008413757100546112, + "kd/rkl_advantage_mean": 0.6445614118284235, + "kd/rkl_advantage_p95": 4.846730122963588, + "kd/rkl_advantage_std": 2.2158480187257132, + "kd/ssd_loss": 0.0, + "learning_rate": 8.280104811453309e-07, + "loss": 0.0336550235748291, + "num_tokens": 148288633.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8147222201029459, + "rewards/gt_logging_reward/std": 0.37879129548867546, + "sampling/importance_sampling_ratio/max": 1.935811444123586, + "sampling/importance_sampling_ratio/mean": 1.0003867129484811, + "sampling/importance_sampling_ratio/min": 0.44180815716584526, + "sampling/sampling_logp_difference/max": 0.41475554505983986, + "sampling/sampling_logp_difference/mean": 0.004239704886761804, + "step": 4530, + "step_time": 7.797652539199529, + "student/entropy": 0.12640664496769508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0233333347675701, + "completions/max_length": 483.8, + "completions/max_terminated_length": 432.56666666666666, + "completions/mean_length": 181.18473103841146, + "completions/mean_terminated_length": 173.3445841471354, + "completions/min_length": 51.6, + "completions/min_terminated_length": 51.6, + "entropy": 0.13005811826636393, + "epoch": 0.1731667489461892, + "eval/gt_acc_batch": 0.7952778061230977, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30359604954719543, + "kd/policy_loss": 0.008982894301880151, + "kd/rkl_advantage_mean": 0.5897338472306728, + "kd/rkl_advantage_p95": 4.751080181201299, + "kd/rkl_advantage_std": 2.2319200158119203, + "kd/ssd_loss": 0.0, + "learning_rate": 8.268712262180534e-07, + "loss": 0.03593157927195231, + "num_tokens": 149239658.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777743339539, + "rewards/gt_logging_reward/std": 0.39392486413319905, + "sampling/importance_sampling_ratio/max": 1.9605566104253134, + "sampling/importance_sampling_ratio/mean": 1.0029195149739583, + "sampling/importance_sampling_ratio/min": 0.37723441670338315, + "sampling/sampling_logp_difference/max": 0.39552374680836994, + "sampling/sampling_logp_difference/mean": 0.004370854158575336, + "step": 4560, + "step_time": 7.811583073999888, + "student/entropy": 0.13005811826636393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001390775046, + "completions/max_length": 488.26666666666665, + "completions/max_terminated_length": 455.73333333333335, + "completions/mean_length": 186.7961217244466, + "completions/mean_terminated_length": 179.4837855021159, + "completions/min_length": 50.13333333333333, + "completions/min_terminated_length": 50.13333333333333, + "entropy": 0.12826346643269063, + "epoch": 0.17430600387346676, + "eval/gt_acc_batch": 0.7963888982931773, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3308059275150299, + "kd/policy_loss": 0.008675741654587909, + "kd/rkl_advantage_mean": 0.5662101903949709, + "kd/rkl_advantage_p95": 4.757735512653986, + "kd/rkl_advantage_std": 2.1863811870416003, + "kd/ssd_loss": 0.0, + "learning_rate": 8.257319712907757e-07, + "loss": 0.03470296462376912, + "num_tokens": 150218876.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888903458913, + "rewards/gt_logging_reward/std": 0.39548701445261636, + "sampling/importance_sampling_ratio/max": 1.978613305091858, + "sampling/importance_sampling_ratio/mean": 1.001912667353948, + "sampling/importance_sampling_ratio/min": 0.4289726952711741, + "sampling/sampling_logp_difference/max": 0.3773029585679372, + "sampling/sampling_logp_difference/mean": 0.0043018937731782595, + "step": 4590, + "step_time": 7.9483325641679885, + "student/entropy": 0.12826346643269063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333460614085, + "completions/max_length": 497.3, + "completions/max_terminated_length": 462.3333333333333, + "completions/mean_length": 185.18917643229167, + "completions/mean_terminated_length": 178.24716186523438, + "completions/min_length": 49.86666666666667, + "completions/min_terminated_length": 49.86666666666667, + "entropy": 0.12332344210396211, + "epoch": 0.17544525880074432, + "eval/gt_acc_batch": 0.818888908624649, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2777160108089447, + "kd/policy_loss": 0.007984639207522074, + "kd/rkl_advantage_mean": 0.5584135146966825, + "kd/rkl_advantage_p95": 4.640872142712275, + "kd/rkl_advantage_std": 2.1811570515235266, + "kd/ssd_loss": 0.0, + "learning_rate": 8.245927163634982e-07, + "loss": 0.03193855285644531, + "num_tokens": 151194301.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8188888847827911, + "rewards/gt_logging_reward/std": 0.37655321657657626, + "sampling/importance_sampling_ratio/max": 1.9578959067662558, + "sampling/importance_sampling_ratio/mean": 1.003688915570577, + "sampling/importance_sampling_ratio/min": 0.40716680934031807, + "sampling/sampling_logp_difference/max": 0.4396363536516825, + "sampling/sampling_logp_difference/mean": 0.004185476053195695, + "step": 4620, + "step_time": 8.16773157971523, + "student/entropy": 0.12332344210396211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01583333412806193, + "completions/max_length": 471.43333333333334, + "completions/max_terminated_length": 445.3666666666667, + "completions/mean_length": 186.65750986735026, + "completions/mean_terminated_length": 181.48958943684895, + "completions/min_length": 55.9, + "completions/min_terminated_length": 55.9, + "entropy": 0.13256355784833432, + "epoch": 0.17658451372802186, + "eval/gt_acc_batch": 0.8116666853427887, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2945660948753357, + "kd/policy_loss": 0.008556259871693329, + "kd/rkl_advantage_mean": 0.5929759226273745, + "kd/rkl_advantage_p95": 4.641309656699499, + "kd/rkl_advantage_std": 2.1446397483348845, + "kd/ssd_loss": 0.0, + "learning_rate": 8.234534614362207e-07, + "loss": 0.034225038687388104, + "num_tokens": 152164468.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8116666694482167, + "rewards/gt_logging_reward/std": 0.38389231959978737, + "sampling/importance_sampling_ratio/max": 1.9554205338160198, + "sampling/importance_sampling_ratio/mean": 1.0028269708156585, + "sampling/importance_sampling_ratio/min": 0.4107947478691737, + "sampling/sampling_logp_difference/max": 0.3855579455693563, + "sampling/sampling_logp_difference/mean": 0.0042993779371803004, + "step": 4650, + "step_time": 7.799338809668552, + "student/entropy": 0.13256355784833432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000116104881, + "completions/max_length": 482.6666666666667, + "completions/max_terminated_length": 449.4, + "completions/mean_length": 179.59223225911458, + "completions/mean_terminated_length": 173.6505579630534, + "completions/min_length": 52.03333333333333, + "completions/min_terminated_length": 52.03333333333333, + "entropy": 0.13241675111154716, + "epoch": 0.17772376865529943, + "eval/gt_acc_batch": 0.8013889054457347, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3724587857723236, + "kd/policy_loss": 0.008851472563886393, + "kd/rkl_advantage_mean": 0.7153036874408524, + "kd/rkl_advantage_p95": 4.879216810067494, + "kd/rkl_advantage_std": 2.219032023350398, + "kd/ssd_loss": 0.0, + "learning_rate": 8.223142065089432e-07, + "loss": 0.03540589014689128, + "num_tokens": 153106456.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888935248057, + "rewards/gt_logging_reward/std": 0.39207198719183606, + "sampling/importance_sampling_ratio/max": 1.9501164158185322, + "sampling/importance_sampling_ratio/mean": 1.0024066289265952, + "sampling/importance_sampling_ratio/min": 0.4250838026404381, + "sampling/sampling_logp_difference/max": 0.43630882700284324, + "sampling/sampling_logp_difference/mean": 0.004398288282876213, + "step": 4680, + "step_time": 7.852808898683482, + "student/entropy": 0.13241675111154716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334382623433, + "completions/max_length": 474.96666666666664, + "completions/max_terminated_length": 422.9, + "completions/mean_length": 178.6702891031901, + "completions/mean_terminated_length": 172.5624547322591, + "completions/min_length": 52.5, + "completions/min_terminated_length": 52.5, + "entropy": 0.12636554582665363, + "epoch": 0.178863023582577, + "eval/gt_acc_batch": 0.80916668176651, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2682987451553345, + "kd/policy_loss": 0.008390730277945598, + "kd/rkl_advantage_mean": 0.6116872578820524, + "kd/rkl_advantage_p95": 4.742968197663625, + "kd/rkl_advantage_std": 2.1827247778574628, + "kd/ssd_loss": 0.0, + "learning_rate": 8.211749515816655e-07, + "loss": 0.03356292247772217, + "num_tokens": 154051573.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8091666658719381, + "rewards/gt_logging_reward/std": 0.38704575300216676, + "sampling/importance_sampling_ratio/max": 1.899699910481771, + "sampling/importance_sampling_ratio/mean": 0.9954484442869822, + "sampling/importance_sampling_ratio/min": 0.40983395278453827, + "sampling/sampling_logp_difference/max": 0.41518606344858805, + "sampling/sampling_logp_difference/mean": 0.004248332441784441, + "step": 4710, + "step_time": 7.796969199638503, + "student/entropy": 0.12636554582665363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500000881652038, + "completions/max_length": 481.1333333333333, + "completions/max_terminated_length": 451.03333333333336, + "completions/mean_length": 187.83417561848958, + "completions/mean_terminated_length": 182.183327738444, + "completions/min_length": 60.36666666666667, + "completions/min_terminated_length": 60.36666666666667, + "entropy": 0.12451310840745768, + "epoch": 0.18000227850985456, + "eval/gt_acc_batch": 0.8002777973810832, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22770988941192627, + "kd/policy_loss": 0.00824097120591129, + "kd/rkl_advantage_mean": 0.6182008556555957, + "kd/rkl_advantage_p95": 4.792526137828827, + "kd/rkl_advantage_std": 2.1992188721895216, + "kd/ssd_loss": 0.0, + "learning_rate": 8.20035696654388e-07, + "loss": 0.032963887850443525, + "num_tokens": 155027120.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8002777794996897, + "rewards/gt_logging_reward/std": 0.38704984535773596, + "sampling/importance_sampling_ratio/max": 1.9299711863199869, + "sampling/importance_sampling_ratio/mean": 1.0019956847031912, + "sampling/importance_sampling_ratio/min": 0.4091160347064336, + "sampling/sampling_logp_difference/max": 0.4355523645877838, + "sampling/sampling_logp_difference/mean": 0.004151805676519871, + "step": 4740, + "step_time": 7.793366843543481, + "student/entropy": 0.12451310840745768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668020188808, + "completions/max_length": 495.0, + "completions/max_terminated_length": 458.1, + "completions/mean_length": 191.9569549560547, + "completions/mean_terminated_length": 184.2638147989909, + "completions/min_length": 61.36666666666667, + "completions/min_terminated_length": 61.36666666666667, + "entropy": 0.12749640084803104, + "epoch": 0.1811415334371321, + "eval/gt_acc_batch": 0.787500015894572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23528142273426056, + "kd/policy_loss": 0.008920859351443747, + "kd/rkl_advantage_mean": 0.5840230805566534, + "kd/rkl_advantage_p95": 4.680854390064876, + "kd/rkl_advantage_std": 2.173820400238037, + "kd/ssd_loss": 0.0, + "learning_rate": 8.188964417271105e-07, + "loss": 0.0356834332148234, + "num_tokens": 156021021.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7874999980131785, + "rewards/gt_logging_reward/std": 0.39909369697173436, + "sampling/importance_sampling_ratio/max": 2.029611035188039, + "sampling/importance_sampling_ratio/mean": 0.9984595775604248, + "sampling/importance_sampling_ratio/min": 0.43955254058043164, + "sampling/sampling_logp_difference/max": 0.4505197246869405, + "sampling/sampling_logp_difference/mean": 0.0042495976124579705, + "step": 4770, + "step_time": 7.964652934538511, + "student/entropy": 0.12749640084803104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029166668467223645, + "completions/max_length": 495.5, + "completions/max_terminated_length": 463.46666666666664, + "completions/mean_length": 193.05111948649088, + "completions/mean_terminated_length": 183.62896118164062, + "completions/min_length": 49.63333333333333, + "completions/min_terminated_length": 49.63333333333333, + "entropy": 0.13271870849033196, + "epoch": 0.18228078836440967, + "eval/gt_acc_batch": 0.7855555653572083, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2192717343568802, + "kd/policy_loss": 0.008937719250873973, + "kd/rkl_advantage_mean": 0.6457948190470536, + "kd/rkl_advantage_p95": 4.806369322538376, + "kd/rkl_advantage_std": 2.207584433754285, + "kd/ssd_loss": 0.0, + "learning_rate": 8.177571867998328e-07, + "loss": 0.03575087785720825, + "num_tokens": 157014637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555534362793, + "rewards/gt_logging_reward/std": 0.4032997796932856, + "sampling/importance_sampling_ratio/max": 1.9682828823725382, + "sampling/importance_sampling_ratio/mean": 0.9949515104293823, + "sampling/importance_sampling_ratio/min": 0.3712858721613884, + "sampling/sampling_logp_difference/max": 0.4728123962879181, + "sampling/sampling_logp_difference/mean": 0.004397773587455352, + "step": 4800, + "step_time": 8.16429967811564, + "student/entropy": 0.13271870849033196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01944444558272759, + "completions/max_length": 482.56666666666666, + "completions/max_terminated_length": 448.7, + "completions/mean_length": 186.10223236083985, + "completions/mean_terminated_length": 179.69716135660806, + "completions/min_length": 48.8, + "completions/min_terminated_length": 48.8, + "entropy": 0.12820262523988882, + "epoch": 0.18342004329168723, + "eval/gt_acc_batch": 0.7880555709203084, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24675236642360687, + "kd/policy_loss": 0.008679888884459312, + "kd/rkl_advantage_mean": 0.6667423952370882, + "kd/rkl_advantage_p95": 4.81252782146136, + "kd/rkl_advantage_std": 2.1807959894339244, + "kd/ssd_loss": 0.0, + "learning_rate": 8.166179318725553e-07, + "loss": 0.03471955458323161, + "num_tokens": 157979365.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.788055557012558, + "rewards/gt_logging_reward/std": 0.40147166748841606, + "sampling/importance_sampling_ratio/max": 1.9819732308387756, + "sampling/importance_sampling_ratio/mean": 0.9982752879460652, + "sampling/importance_sampling_ratio/min": 0.3733826662103335, + "sampling/sampling_logp_difference/max": 0.44332319299379985, + "sampling/sampling_logp_difference/mean": 0.004285374369161824, + "step": 4830, + "step_time": 7.907448230842904, + "student/entropy": 0.12820262523988882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02750000124797225, + "completions/max_length": 491.3666666666667, + "completions/max_terminated_length": 447.1, + "completions/mean_length": 193.73084564208983, + "completions/mean_terminated_length": 184.77484792073568, + "completions/min_length": 54.7, + "completions/min_terminated_length": 54.7, + "entropy": 0.13906484823673965, + "epoch": 0.1845592982189648, + "eval/gt_acc_batch": 0.7638888895511627, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32641205191612244, + "kd/policy_loss": 0.009824709431268275, + "kd/rkl_advantage_mean": 0.5822771724623939, + "kd/rkl_advantage_p95": 4.74753347436587, + "kd/rkl_advantage_std": 2.206132439772288, + "kd/ssd_loss": 0.0, + "learning_rate": 8.154786769452778e-07, + "loss": 0.03929883639017741, + "num_tokens": 158980868.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7638888895511627, + "rewards/gt_logging_reward/std": 0.413290203611056, + "sampling/importance_sampling_ratio/max": 2.044093358516693, + "sampling/importance_sampling_ratio/mean": 0.9971038361390432, + "sampling/importance_sampling_ratio/min": 0.35653166522582375, + "sampling/sampling_logp_difference/max": 0.4385534107685089, + "sampling/sampling_logp_difference/mean": 0.004586326191201806, + "step": 4860, + "step_time": 8.134320569430322, + "student/entropy": 0.13906484823673965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445775200924, + "completions/max_length": 499.8333333333333, + "completions/max_terminated_length": 465.06666666666666, + "completions/mean_length": 192.41556549072266, + "completions/mean_terminated_length": 185.13451588948567, + "completions/min_length": 53.63333333333333, + "completions/min_terminated_length": 53.63333333333333, + "entropy": 0.12990116843332847, + "epoch": 0.18569855314624237, + "eval/gt_acc_batch": 0.7800000190734864, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2687862813472748, + "kd/policy_loss": 0.009216914876985054, + "kd/rkl_advantage_mean": 0.6565036421467084, + "kd/rkl_advantage_p95": 4.883481299877166, + "kd/rkl_advantage_std": 2.2266299734512964, + "kd/ssd_loss": 0.0, + "learning_rate": 8.143394220180002e-07, + "loss": 0.03686766624450684, + "num_tokens": 159981644.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7800000011920929, + "rewards/gt_logging_reward/std": 0.4075392936666807, + "sampling/importance_sampling_ratio/max": 1.9859662214914957, + "sampling/importance_sampling_ratio/mean": 1.0052743216355642, + "sampling/importance_sampling_ratio/min": 0.394437434275945, + "sampling/sampling_logp_difference/max": 0.44280216892560326, + "sampling/sampling_logp_difference/mean": 0.0043404339347034694, + "step": 4890, + "step_time": 8.200424632627982, + "student/entropy": 0.12990116843332847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01694444545234243, + "completions/max_length": 480.96666666666664, + "completions/max_terminated_length": 452.8666666666667, + "completions/mean_length": 185.49445343017578, + "completions/mean_terminated_length": 179.85411580403647, + "completions/min_length": 45.06666666666667, + "completions/min_terminated_length": 45.06666666666667, + "entropy": 0.13092610612511635, + "epoch": 0.1868378080735199, + "eval/gt_acc_batch": 0.7972222447395325, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29051917791366577, + "kd/policy_loss": 0.009051901889809718, + "kd/rkl_advantage_mean": 0.6398770049369583, + "kd/rkl_advantage_p95": 4.769393529494604, + "kd/rkl_advantage_std": 2.1895749241113664, + "kd/ssd_loss": 0.0, + "learning_rate": 8.132001670907226e-07, + "loss": 0.03620761235555013, + "num_tokens": 160962136.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7972222189108531, + "rewards/gt_logging_reward/std": 0.3965260634819667, + "sampling/importance_sampling_ratio/max": 1.9661730527877808, + "sampling/importance_sampling_ratio/mean": 0.9977195878823598, + "sampling/importance_sampling_ratio/min": 0.42481893400351206, + "sampling/sampling_logp_difference/max": 0.4476340413093567, + "sampling/sampling_logp_difference/mean": 0.004376055602915585, + "step": 4920, + "step_time": 8.137886932783294, + "student/entropy": 0.13092610612511635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01638889005407691, + "completions/max_length": 483.5, + "completions/max_terminated_length": 463.03333333333336, + "completions/mean_length": 186.76278737386068, + "completions/mean_terminated_length": 181.33468424479167, + "completions/min_length": 52.03333333333333, + "completions/min_terminated_length": 52.03333333333333, + "entropy": 0.12739101592451335, + "epoch": 0.18797706300079747, + "eval/gt_acc_batch": 0.8158333539962769, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3056957721710205, + "kd/policy_loss": 0.008308556980531042, + "kd/rkl_advantage_mean": 0.5943945596305032, + "kd/rkl_advantage_p95": 4.767317426204682, + "kd/rkl_advantage_std": 2.2104207346836726, + "kd/ssd_loss": 0.0, + "learning_rate": 8.120609121634451e-07, + "loss": 0.033234230677286786, + "num_tokens": 161942434.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8158333400885264, + "rewards/gt_logging_reward/std": 0.37926527559757234, + "sampling/importance_sampling_ratio/max": 1.8978254556655885, + "sampling/importance_sampling_ratio/mean": 0.9990658899148305, + "sampling/importance_sampling_ratio/min": 0.4368948757648468, + "sampling/sampling_logp_difference/max": 0.3956498364607493, + "sampling/sampling_logp_difference/mean": 0.004294725328994294, + "step": 4950, + "step_time": 8.144437074377977, + "student/entropy": 0.12739101592451335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556658655404, + "completions/max_length": 476.1666666666667, + "completions/max_terminated_length": 434.8333333333333, + "completions/mean_length": 187.70223185221354, + "completions/mean_terminated_length": 180.94827372233073, + "completions/min_length": 54.63333333333333, + "completions/min_terminated_length": 54.63333333333333, + "entropy": 0.1277453636129697, + "epoch": 0.18911631792807504, + "eval/gt_acc_batch": 0.8047222435474396, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23080161213874817, + "kd/policy_loss": 0.008772418657705809, + "kd/rkl_advantage_mean": 0.6119360719031344, + "kd/rkl_advantage_p95": 4.730212795734405, + "kd/rkl_advantage_std": 2.1742894967397053, + "kd/ssd_loss": 0.0, + "learning_rate": 8.109216572361674e-07, + "loss": 0.03508967955907186, + "num_tokens": 162913658.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222197055817, + "rewards/gt_logging_reward/std": 0.384920242925485, + "sampling/importance_sampling_ratio/max": 2.0114851593971252, + "sampling/importance_sampling_ratio/mean": 1.0040387193361917, + "sampling/importance_sampling_ratio/min": 0.44306305249532063, + "sampling/sampling_logp_difference/max": 0.45012377897898354, + "sampling/sampling_logp_difference/mean": 0.004320841197234889, + "step": 4980, + "step_time": 7.891175898998821, + "student/entropy": 0.1277453636129697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334326744078, + "completions/max_length": 481.4, + "completions/max_terminated_length": 440.6666666666667, + "completions/mean_length": 187.43695475260418, + "completions/mean_terminated_length": 180.63911997477214, + "completions/min_length": 52.6, + "completions/min_terminated_length": 52.6, + "entropy": 0.1292789799471696, + "epoch": 0.1902555728553526, + "eval/gt_acc_batch": 0.801944476366043, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29492053389549255, + "kd/policy_loss": 0.008603862760355695, + "kd/rkl_advantage_mean": 0.6031553730756666, + "kd/rkl_advantage_p95": 4.743026771148046, + "kd/rkl_advantage_std": 2.1745767345031104, + "kd/ssd_loss": 0.0, + "learning_rate": 8.097824023088899e-07, + "loss": 0.034415451685587566, + "num_tokens": 163908775.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444425900777, + "rewards/gt_logging_reward/std": 0.38749160220225654, + "sampling/importance_sampling_ratio/max": 1.940081258614858, + "sampling/importance_sampling_ratio/mean": 1.0053059120972951, + "sampling/importance_sampling_ratio/min": 0.4202765787641207, + "sampling/sampling_logp_difference/max": 0.4033089299996694, + "sampling/sampling_logp_difference/mean": 0.004302563704550266, + "step": 5010, + "step_time": 8.305452114871393, + "student/entropy": 0.1292789799471696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779282381136, + "completions/max_length": 490.5, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 187.22278645833333, + "completions/mean_terminated_length": 178.78634440104167, + "completions/min_length": 45.4, + "completions/min_terminated_length": 45.4, + "entropy": 0.12703756699338556, + "epoch": 0.19139482778263017, + "eval/gt_acc_batch": 0.7916666865348816, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2517487406730652, + "kd/policy_loss": 0.008873986780721073, + "kd/rkl_advantage_mean": 0.627296318858862, + "kd/rkl_advantage_p95": 4.721908416350683, + "kd/rkl_advantage_std": 2.168072532614072, + "kd/ssd_loss": 0.0, + "learning_rate": 8.086431473816125e-07, + "loss": 0.035495952765146895, + "num_tokens": 164883609.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7916666686534881, + "rewards/gt_logging_reward/std": 0.398375674088796, + "sampling/importance_sampling_ratio/max": 1.880922516187032, + "sampling/importance_sampling_ratio/mean": 1.001639034350713, + "sampling/importance_sampling_ratio/min": 0.4364289810260137, + "sampling/sampling_logp_difference/max": 0.3999603629112244, + "sampling/sampling_logp_difference/mean": 0.004204002728996178, + "step": 5040, + "step_time": 8.080870116921142, + "student/entropy": 0.12703756699338556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778866390386, + "completions/max_length": 477.3333333333333, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 189.91112060546874, + "completions/mean_terminated_length": 183.48850911458334, + "completions/min_length": 53.266666666666666, + "completions/min_terminated_length": 53.266666666666666, + "entropy": 0.12819868040581545, + "epoch": 0.1925340827099077, + "eval/gt_acc_batch": 0.8019444644451141, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.301169753074646, + "kd/policy_loss": 0.008471824407267074, + "kd/rkl_advantage_mean": 0.6075117065571248, + "kd/rkl_advantage_p95": 4.729077784220378, + "kd/rkl_advantage_std": 2.1599731942017875, + "kd/ssd_loss": 0.0, + "learning_rate": 8.075038924543348e-07, + "loss": 0.0338873028755188, + "num_tokens": 165878561.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444386164347, + "rewards/gt_logging_reward/std": 0.39135581254959106, + "sampling/importance_sampling_ratio/max": 1.8973191181818645, + "sampling/importance_sampling_ratio/mean": 0.9952382663885753, + "sampling/importance_sampling_ratio/min": 0.41198920061190925, + "sampling/sampling_logp_difference/max": 0.4686159054438273, + "sampling/sampling_logp_difference/mean": 0.004321890999563038, + "step": 5070, + "step_time": 8.086159905372188, + "student/entropy": 0.12819868040581545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223381201427, + "completions/max_length": 488.73333333333335, + "completions/max_terminated_length": 447.3, + "completions/mean_length": 185.92445373535156, + "completions/mean_terminated_length": 178.7709701538086, + "completions/min_length": 58.3, + "completions/min_terminated_length": 58.3, + "entropy": 0.12574077397584915, + "epoch": 0.19367333763718528, + "eval/gt_acc_batch": 0.7988889058430989, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.397256076335907, + "kd/policy_loss": 0.00864977237652056, + "kd/rkl_advantage_mean": 0.6163639348388339, + "kd/rkl_advantage_p95": 4.708701378107071, + "kd/rkl_advantage_std": 2.175132308403651, + "kd/ssd_loss": 0.0, + "learning_rate": 8.063646375270573e-07, + "loss": 0.03459908962249756, + "num_tokens": 166852305.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7988888879617055, + "rewards/gt_logging_reward/std": 0.3913717826207479, + "sampling/importance_sampling_ratio/max": 2.023026971022288, + "sampling/importance_sampling_ratio/mean": 1.0017088691393534, + "sampling/importance_sampling_ratio/min": 0.3961103657881419, + "sampling/sampling_logp_difference/max": 0.4183955013751984, + "sampling/sampling_logp_difference/mean": 0.004254672094248235, + "step": 5100, + "step_time": 8.027666551379177, + "student/entropy": 0.12574077397584915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0200000012293458, + "completions/max_length": 487.46666666666664, + "completions/max_terminated_length": 466.3666666666667, + "completions/mean_length": 187.02306569417317, + "completions/mean_terminated_length": 180.5190668741862, + "completions/min_length": 51.56666666666667, + "completions/min_terminated_length": 51.56666666666667, + "entropy": 0.1283335148667296, + "epoch": 0.19481259256446284, + "eval/gt_acc_batch": 0.7955555776755016, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2991281747817993, + "kd/policy_loss": 0.008480795009139305, + "kd/rkl_advantage_mean": 0.6773272602508466, + "kd/rkl_advantage_p95": 4.959339034557343, + "kd/rkl_advantage_std": 2.241229225198428, + "kd/ssd_loss": 0.0, + "learning_rate": 8.052253825997797e-07, + "loss": 0.0339231808980306, + "num_tokens": 167825572.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555558204651, + "rewards/gt_logging_reward/std": 0.3988847315311432, + "sampling/importance_sampling_ratio/max": 1.8942663153012593, + "sampling/importance_sampling_ratio/mean": 0.9986085593700409, + "sampling/importance_sampling_ratio/min": 0.42663000027338666, + "sampling/sampling_logp_difference/max": 0.4068620602289836, + "sampling/sampling_logp_difference/mean": 0.004280555450047056, + "step": 5130, + "step_time": 8.023011575367612, + "student/entropy": 0.1283335148667296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.035000002042700845, + "completions/max_length": 486.3666666666667, + "completions/max_terminated_length": 455.96666666666664, + "completions/mean_length": 196.36223246256512, + "completions/mean_terminated_length": 185.0490976969401, + "completions/min_length": 53.3, + "completions/min_terminated_length": 53.3, + "entropy": 0.13094600594292086, + "epoch": 0.1959518474917404, + "eval/gt_acc_batch": 0.7791666766007741, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.20581738650798798, + "kd/policy_loss": 0.008756941882893443, + "kd/rkl_advantage_mean": 0.6540660596452653, + "kd/rkl_advantage_p95": 4.8508529424667355, + "kd/rkl_advantage_std": 2.200080918272336, + "kd/ssd_loss": 0.0, + "learning_rate": 8.040861276725022e-07, + "loss": 0.03502777417500814, + "num_tokens": 168839564.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7791666666666667, + "rewards/gt_logging_reward/std": 0.4110870490471522, + "sampling/importance_sampling_ratio/max": 2.022777565320333, + "sampling/importance_sampling_ratio/mean": 1.0010766545931498, + "sampling/importance_sampling_ratio/min": 0.3326531062523524, + "sampling/sampling_logp_difference/max": 0.4323797384897868, + "sampling/sampling_logp_difference/mean": 0.004378063588713606, + "step": 5160, + "step_time": 8.149499145289884, + "student/entropy": 0.13094600594292086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02356321978029506, + "completions/max_length": 501.7931034482759, + "completions/max_terminated_length": 457.0689655172414, + "completions/mean_length": 191.34253666318696, + "completions/mean_terminated_length": 183.60046597184805, + "completions/min_length": 56.06896551724138, + "completions/min_terminated_length": 56.06896551724138, + "entropy": 0.13048019484970078, + "epoch": 0.19709110241901795, + "eval/gt_acc_batch": 0.7916666885902142, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3537169396877289, + "kd/policy_loss": 0.008676694882712487, + "kd/rkl_advantage_mean": 0.7002256462543175, + "kd/rkl_advantage_p95": 4.935907851005423, + "kd/rkl_advantage_std": 2.2219893387679397, + "kd/ssd_loss": 0.0, + "learning_rate": 8.029468727452245e-07, + "loss": 0.03354988892873128, + "num_tokens": 169796484.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7916666639262232, + "rewards/gt_logging_reward/std": 0.39545962163086595, + "sampling/importance_sampling_ratio/max": 1.9990886532027146, + "sampling/importance_sampling_ratio/mean": 0.9966847896575928, + "sampling/importance_sampling_ratio/min": 0.42183478690426923, + "sampling/sampling_logp_difference/max": 0.3994228141061191, + "sampling/sampling_logp_difference/mean": 0.004324385205861823, + "step": 5190, + "step_time": 7.840925910071625, + "student/entropy": 0.13048019484970078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223530213037, + "completions/max_length": 487.76666666666665, + "completions/max_terminated_length": 459.43333333333334, + "completions/mean_length": 185.7936223347982, + "completions/mean_terminated_length": 179.30196584065754, + "completions/min_length": 54.233333333333334, + "completions/min_terminated_length": 54.233333333333334, + "entropy": 0.1265678278480967, + "epoch": 0.19823035734629552, + "eval/gt_acc_batch": 0.8041666785875956, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22883690893650055, + "kd/policy_loss": 0.008932129476064195, + "kd/rkl_advantage_mean": 0.6151348318671808, + "kd/rkl_advantage_p95": 4.764021501938502, + "kd/rkl_advantage_std": 2.20256006916364, + "kd/ssd_loss": 0.0, + "learning_rate": 8.01807617817947e-07, + "loss": 0.035728514194488525, + "num_tokens": 170770701.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666666666667, + "rewards/gt_logging_reward/std": 0.3862100834647814, + "sampling/importance_sampling_ratio/max": 1.9690973877906799, + "sampling/importance_sampling_ratio/mean": 1.006190800666809, + "sampling/importance_sampling_ratio/min": 0.4551168272892634, + "sampling/sampling_logp_difference/max": 0.43445527950922647, + "sampling/sampling_logp_difference/mean": 0.004255938918019334, + "step": 5220, + "step_time": 8.000387248342546, + "student/entropy": 0.1265678278480967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0222222234432896, + "completions/max_length": 485.6666666666667, + "completions/max_terminated_length": 446.1, + "completions/mean_length": 187.16528778076173, + "completions/mean_terminated_length": 179.80814361572266, + "completions/min_length": 54.766666666666666, + "completions/min_terminated_length": 54.766666666666666, + "entropy": 0.12948254483441513, + "epoch": 0.19936961227357308, + "eval/gt_acc_batch": 0.7894444684187572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3559996485710144, + "kd/policy_loss": 0.009033587885399659, + "kd/rkl_advantage_mean": 0.5340243935410399, + "kd/rkl_advantage_p95": 4.595965071519216, + "kd/rkl_advantage_std": 2.1638169447580973, + "kd/ssd_loss": 0.0, + "learning_rate": 8.006683628906695e-07, + "loss": 0.03613435029983521, + "num_tokens": 171764464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7894444445768992, + "rewards/gt_logging_reward/std": 0.4005243529876073, + "sampling/importance_sampling_ratio/max": 2.0004639585812884, + "sampling/importance_sampling_ratio/mean": 1.0050809939702352, + "sampling/importance_sampling_ratio/min": 0.42858825822671254, + "sampling/sampling_logp_difference/max": 0.3965487778186798, + "sampling/sampling_logp_difference/mean": 0.004340576683171093, + "step": 5250, + "step_time": 8.222769333243681, + "student/entropy": 0.12948254483441513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112305273613, + "completions/max_length": 475.76666666666665, + "completions/max_terminated_length": 437.6666666666667, + "completions/mean_length": 190.09334462483724, + "completions/mean_terminated_length": 183.29903767903645, + "completions/min_length": 56.43333333333333, + "completions/min_terminated_length": 56.43333333333333, + "entropy": 0.12669192937513193, + "epoch": 0.20050886720085065, + "eval/gt_acc_batch": 0.7938889125982921, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23240606486797333, + "kd/policy_loss": 0.00856164278035673, + "kd/rkl_advantage_mean": 0.6027971886253606, + "kd/rkl_advantage_p95": 4.661067821582159, + "kd/rkl_advantage_std": 2.156881257891655, + "kd/ssd_loss": 0.0, + "learning_rate": 7.995291079633919e-07, + "loss": 0.03424656788508097, + "num_tokens": 172752912.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7938888927300771, + "rewards/gt_logging_reward/std": 0.3966137280066808, + "sampling/importance_sampling_ratio/max": 1.9265421311060587, + "sampling/importance_sampling_ratio/mean": 0.9947320242722829, + "sampling/importance_sampling_ratio/min": 0.42185368786255517, + "sampling/sampling_logp_difference/max": 0.45608983337879183, + "sampling/sampling_logp_difference/mean": 0.004258148170386751, + "step": 5280, + "step_time": 8.025129339339522, + "student/entropy": 0.12669192937513193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02888889064391454, + "completions/max_length": 476.3, + "completions/max_terminated_length": 430.8, + "completions/mean_length": 190.5291758219401, + "completions/mean_terminated_length": 181.04527842203777, + "completions/min_length": 50.36666666666667, + "completions/min_terminated_length": 50.36666666666667, + "entropy": 0.1415137327586611, + "epoch": 0.20164812212812822, + "eval/gt_acc_batch": 0.7747222403685252, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3147912323474884, + "kd/policy_loss": 0.00929154625434118, + "kd/rkl_advantage_mean": 0.5417258228641003, + "kd/rkl_advantage_p95": 4.54262808561325, + "kd/rkl_advantage_std": 2.1196047266324363, + "kd/ssd_loss": 0.0, + "learning_rate": 7.983898530361144e-07, + "loss": 0.03716618220011393, + "num_tokens": 173742713.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7747222244739532, + "rewards/gt_logging_reward/std": 0.4078964153925578, + "sampling/importance_sampling_ratio/max": 2.061245242754618, + "sampling/importance_sampling_ratio/mean": 1.0026904304822286, + "sampling/importance_sampling_ratio/min": 0.40884735584259035, + "sampling/sampling_logp_difference/max": 0.4097396651903788, + "sampling/sampling_logp_difference/mean": 0.0044444727788989745, + "step": 5310, + "step_time": 7.993606386139678, + "student/entropy": 0.1415137327586611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028611112448076407, + "completions/max_length": 477.56666666666666, + "completions/max_terminated_length": 449.7, + "completions/mean_length": 190.17056477864583, + "completions/mean_terminated_length": 180.9695805867513, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.12938817447672288, + "epoch": 0.20278737705540575, + "eval/gt_acc_batch": 0.7919444640477499, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3003069758415222, + "kd/policy_loss": 0.008795870268174136, + "kd/rkl_advantage_mean": 0.6206797004677356, + "kd/rkl_advantage_p95": 4.788314775625865, + "kd/rkl_advantage_std": 2.1878727465867995, + "kd/ssd_loss": 0.0, + "learning_rate": 7.972505981088368e-07, + "loss": 0.03518348534901937, + "num_tokens": 174727943.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7919444461663564, + "rewards/gt_logging_reward/std": 0.39309439311424893, + "sampling/importance_sampling_ratio/max": 1.8762022852897644, + "sampling/importance_sampling_ratio/mean": 0.9934730947017669, + "sampling/importance_sampling_ratio/min": 0.43314648866653443, + "sampling/sampling_logp_difference/max": 0.42374132871627807, + "sampling/sampling_logp_difference/mean": 0.004241814968797068, + "step": 5340, + "step_time": 8.006405598910836, + "student/entropy": 0.12938817447672288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029444446414709093, + "completions/max_length": 501.3333333333333, + "completions/max_terminated_length": 465.4, + "completions/mean_length": 192.84139862060547, + "completions/mean_terminated_length": 183.28834838867186, + "completions/min_length": 58.56666666666667, + "completions/min_terminated_length": 58.56666666666667, + "entropy": 0.13499051611870527, + "epoch": 0.20392663198268332, + "eval/gt_acc_batch": 0.7866666913032532, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.260213166475296, + "kd/policy_loss": 0.00901132765187261, + "kd/rkl_advantage_mean": 0.7174080243334174, + "kd/rkl_advantage_p95": 4.978605473041535, + "kd/rkl_advantage_std": 2.242840287089348, + "kd/ssd_loss": 0.0, + "learning_rate": 7.961113431815592e-07, + "loss": 0.03604531288146973, + "num_tokens": 175732116.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666634877523, + "rewards/gt_logging_reward/std": 0.40042660335699715, + "sampling/importance_sampling_ratio/max": 1.976708217461904, + "sampling/importance_sampling_ratio/mean": 0.9894313891728719, + "sampling/importance_sampling_ratio/min": 0.3614824761946996, + "sampling/sampling_logp_difference/max": 0.4407424529393514, + "sampling/sampling_logp_difference/mean": 0.004449075429389874, + "step": 5370, + "step_time": 8.251546183294462, + "student/entropy": 0.13499051611870527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02972222389653325, + "completions/max_length": 489.0, + "completions/max_terminated_length": 454.56666666666666, + "completions/mean_length": 195.61112009684246, + "completions/mean_terminated_length": 186.0920415242513, + "completions/min_length": 49.86666666666667, + "completions/min_terminated_length": 49.86666666666667, + "entropy": 0.13337129422773916, + "epoch": 0.2050658869099609, + "eval/gt_acc_batch": 0.7741666932900747, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32319706678390503, + "kd/policy_loss": 0.009309611310406278, + "kd/rkl_advantage_mean": 0.6194044188033634, + "kd/rkl_advantage_p95": 4.796221750974655, + "kd/rkl_advantage_std": 2.1968612293402354, + "kd/ssd_loss": 0.0, + "learning_rate": 7.949720882542816e-07, + "loss": 0.037238446871439616, + "num_tokens": 176745684.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7741666694482168, + "rewards/gt_logging_reward/std": 0.4075356274843216, + "sampling/importance_sampling_ratio/max": 2.051927149295807, + "sampling/importance_sampling_ratio/mean": 1.001786035299301, + "sampling/importance_sampling_ratio/min": 0.38589755396048225, + "sampling/sampling_logp_difference/max": 0.4028500219186147, + "sampling/sampling_logp_difference/mean": 0.004370854290512701, + "step": 5400, + "step_time": 8.194908946795234, + "student/entropy": 0.13337129422773916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021839081727225203, + "completions/max_length": 469.41379310344826, + "completions/max_terminated_length": 428.0, + "completions/mean_length": 180.7502978094693, + "completions/mean_terminated_length": 173.53646166571255, + "completions/min_length": 53.48275862068966, + "completions/min_terminated_length": 53.48275862068966, + "entropy": 0.12902775766520663, + "epoch": 0.20620514183723845, + "eval/gt_acc_batch": 0.8198276038827568, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2562881410121918, + "kd/policy_loss": 0.008586089999880642, + "kd/rkl_advantage_mean": 0.49612416630482364, + "kd/rkl_advantage_p95": 4.4460516366465335, + "kd/rkl_advantage_std": 2.103765595575859, + "kd/ssd_loss": 0.0, + "learning_rate": 7.938328333270042e-07, + "loss": 0.03319954872131348, + "num_tokens": 177663631.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8198275853847635, + "rewards/gt_logging_reward/std": 0.3731264240782836, + "sampling/importance_sampling_ratio/max": 1.9136873976937656, + "sampling/importance_sampling_ratio/mean": 0.9994074928349462, + "sampling/importance_sampling_ratio/min": 0.4185659263668389, + "sampling/sampling_logp_difference/max": 0.41368394679036635, + "sampling/sampling_logp_difference/mean": 0.004347394204473701, + "step": 5430, + "step_time": 7.587093648842226, + "student/entropy": 0.12902775766520663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667945683002, + "completions/max_length": 472.6333333333333, + "completions/max_terminated_length": 439.9, + "completions/mean_length": 182.5577860514323, + "completions/mean_terminated_length": 176.2287582397461, + "completions/min_length": 48.53333333333333, + "completions/min_terminated_length": 48.53333333333333, + "entropy": 0.125232717829446, + "epoch": 0.20734439676451602, + "eval/gt_acc_batch": 0.8066666920979818, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2500506043434143, + "kd/policy_loss": 0.008496791384338091, + "kd/rkl_advantage_mean": 0.6938993069032828, + "kd/rkl_advantage_p95": 4.873514552911122, + "kd/rkl_advantage_std": 2.2173260579506557, + "kd/ssd_loss": 0.0, + "learning_rate": 7.926935783997265e-07, + "loss": 0.03398716847101847, + "num_tokens": 178628487.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8066666642824809, + "rewards/gt_logging_reward/std": 0.3899006962776184, + "sampling/importance_sampling_ratio/max": 1.9564496914545695, + "sampling/importance_sampling_ratio/mean": 0.9964917778968811, + "sampling/importance_sampling_ratio/min": 0.435832021633784, + "sampling/sampling_logp_difference/max": 0.47459025382995607, + "sampling/sampling_logp_difference/mean": 0.00423008860864987, + "step": 5460, + "step_time": 8.107880292429279, + "student/entropy": 0.125232717829446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01888888996715347, + "completions/max_length": 457.1333333333333, + "completions/max_terminated_length": 422.26666666666665, + "completions/mean_length": 179.6688975016276, + "completions/mean_terminated_length": 173.43395538330077, + "completions/min_length": 55.36666666666667, + "completions/min_terminated_length": 55.36666666666667, + "entropy": 0.12709176043669382, + "epoch": 0.20848365169179356, + "eval/gt_acc_batch": 0.8152777989705403, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2315036505460739, + "kd/policy_loss": 0.00822673201134118, + "kd/rkl_advantage_mean": 0.5791924027338003, + "kd/rkl_advantage_p95": 4.521941963831583, + "kd/rkl_advantage_std": 2.088363617658615, + "kd/ssd_loss": 0.0, + "learning_rate": 7.91554323472449e-07, + "loss": 0.03290692567825317, + "num_tokens": 179567119.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8152777791023255, + "rewards/gt_logging_reward/std": 0.3801032821337382, + "sampling/importance_sampling_ratio/max": 1.8747817079226177, + "sampling/importance_sampling_ratio/mean": 0.9970693548520406, + "sampling/importance_sampling_ratio/min": 0.438566625614961, + "sampling/sampling_logp_difference/max": 0.42173524498939513, + "sampling/sampling_logp_difference/mean": 0.004261413309723139, + "step": 5490, + "step_time": 7.836766444030218, + "student/entropy": 0.12709176043669382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890078912177, + "completions/max_length": 499.4, + "completions/max_terminated_length": 445.73333333333335, + "completions/mean_length": 182.31889750162762, + "completions/mean_terminated_length": 173.25070393880208, + "completions/min_length": 49.5, + "completions/min_terminated_length": 49.5, + "entropy": 0.13152502570301294, + "epoch": 0.20962290661907113, + "eval/gt_acc_batch": 0.7872222522894542, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34442585706710815, + "kd/policy_loss": 0.00889303203051289, + "kd/rkl_advantage_mean": 0.64286714073581, + "kd/rkl_advantage_p95": 4.745522250731786, + "kd/rkl_advantage_std": 2.208394727110863, + "kd/ssd_loss": 0.0, + "learning_rate": 7.904150685451714e-07, + "loss": 0.03557213147481283, + "num_tokens": 180528259.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7872222205003102, + "rewards/gt_logging_reward/std": 0.40022837817668916, + "sampling/importance_sampling_ratio/max": 1.8933269262313843, + "sampling/importance_sampling_ratio/mean": 0.9930647174517314, + "sampling/importance_sampling_ratio/min": 0.41604195858041443, + "sampling/sampling_logp_difference/max": 0.46777871052424114, + "sampling/sampling_logp_difference/mean": 0.0044169434113428, + "step": 5520, + "step_time": 8.185300577261176, + "student/entropy": 0.13152502570301294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02138889003545046, + "completions/max_length": 481.5, + "completions/max_terminated_length": 447.2, + "completions/mean_length": 185.97834218343098, + "completions/mean_terminated_length": 178.99661610921223, + "completions/min_length": 49.3, + "completions/min_terminated_length": 49.3, + "entropy": 0.13055832802007597, + "epoch": 0.2107621615463487, + "eval/gt_acc_batch": 0.7947222431500752, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24751830101013184, + "kd/policy_loss": 0.008780611907908072, + "kd/rkl_advantage_mean": 0.6928253055550158, + "kd/rkl_advantage_p95": 4.90375883380572, + "kd/rkl_advantage_std": 2.2180306990941365, + "kd/ssd_loss": 0.0, + "learning_rate": 7.892758136178938e-07, + "loss": 0.035122446219126385, + "num_tokens": 181499733.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222252686819, + "rewards/gt_logging_reward/std": 0.3969564184546471, + "sampling/importance_sampling_ratio/max": 1.946396783987681, + "sampling/importance_sampling_ratio/mean": 0.9970524112383524, + "sampling/importance_sampling_ratio/min": 0.4360778411229452, + "sampling/sampling_logp_difference/max": 0.4017364263534546, + "sampling/sampling_logp_difference/mean": 0.0043715935898944736, + "step": 5550, + "step_time": 7.931471060790742, + "student/entropy": 0.13055832802007597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000109896064, + "completions/max_length": 473.0, + "completions/max_terminated_length": 448.3, + "completions/mean_length": 191.0466771443685, + "completions/mean_terminated_length": 185.58940633138022, + "completions/min_length": 55.46666666666667, + "completions/min_terminated_length": 55.46666666666667, + "entropy": 0.1250668091699481, + "epoch": 0.21190141647362626, + "eval/gt_acc_batch": 0.7822222530841827, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23923687636852264, + "kd/policy_loss": 0.008436628081835807, + "kd/rkl_advantage_mean": 0.5996271646115929, + "kd/rkl_advantage_p95": 4.706129682064057, + "kd/rkl_advantage_std": 2.1609189033508303, + "kd/ssd_loss": 0.0, + "learning_rate": 7.881365586906163e-07, + "loss": 0.03374650875727336, + "num_tokens": 182489789.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7822222272555034, + "rewards/gt_logging_reward/std": 0.40455165406068166, + "sampling/importance_sampling_ratio/max": 1.8589762568473815, + "sampling/importance_sampling_ratio/mean": 0.9921265800793966, + "sampling/importance_sampling_ratio/min": 0.44579907357692716, + "sampling/sampling_logp_difference/max": 0.4165138840675354, + "sampling/sampling_logp_difference/mean": 0.004173792052703599, + "step": 5580, + "step_time": 7.886463295835226, + "student/entropy": 0.1250668091699481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223629554113, + "completions/max_length": 497.23333333333335, + "completions/max_terminated_length": 459.03333333333336, + "completions/mean_length": 189.0438980102539, + "completions/mean_terminated_length": 181.76290893554688, + "completions/min_length": 51.56666666666667, + "completions/min_terminated_length": 51.56666666666667, + "entropy": 0.12730340926597516, + "epoch": 0.2130406714009038, + "eval/gt_acc_batch": 0.7813889046510061, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2746838331222534, + "kd/policy_loss": 0.008674928439237799, + "kd/rkl_advantage_mean": 0.5897966586713058, + "kd/rkl_advantage_p95": 4.702760471900304, + "kd/rkl_advantage_std": 2.1689340581496555, + "kd/ssd_loss": 0.0, + "learning_rate": 7.869973037633387e-07, + "loss": 0.03469971418380737, + "num_tokens": 183479027.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7813888887564341, + "rewards/gt_logging_reward/std": 0.40876571337382, + "sampling/importance_sampling_ratio/max": 1.8567088921864827, + "sampling/importance_sampling_ratio/mean": 0.9960138281186421, + "sampling/importance_sampling_ratio/min": 0.3820172796646754, + "sampling/sampling_logp_difference/max": 0.3804553061723709, + "sampling/sampling_logp_difference/mean": 0.0042133899948870145, + "step": 5610, + "step_time": 8.163071076389558, + "student/entropy": 0.12730340926597516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667920847735, + "completions/max_length": 488.9, + "completions/max_terminated_length": 453.9, + "completions/mean_length": 193.83473205566406, + "completions/mean_terminated_length": 187.0249008178711, + "completions/min_length": 51.53333333333333, + "completions/min_terminated_length": 51.53333333333333, + "entropy": 0.12789043970406055, + "epoch": 0.21417992632818136, + "eval/gt_acc_batch": 0.7900000214576721, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34008148312568665, + "kd/policy_loss": 0.008629230262401203, + "kd/rkl_advantage_mean": 0.6330489767715335, + "kd/rkl_advantage_p95": 4.772487719853719, + "kd/rkl_advantage_std": 2.178160043557485, + "kd/ssd_loss": 0.0, + "learning_rate": 7.858580488360613e-07, + "loss": 0.03451692263285319, + "num_tokens": 184496928.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7899999996026357, + "rewards/gt_logging_reward/std": 0.39965747197469076, + "sampling/importance_sampling_ratio/max": 1.9363914529482524, + "sampling/importance_sampling_ratio/mean": 0.9969874779383342, + "sampling/importance_sampling_ratio/min": 0.39826582024494805, + "sampling/sampling_logp_difference/max": 0.45875527858734133, + "sampling/sampling_logp_difference/mean": 0.004241201366918783, + "step": 5640, + "step_time": 8.381786244517814, + "student/entropy": 0.12789043970406055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03083333515872558, + "completions/max_length": 503.5, + "completions/max_terminated_length": 459.1666666666667, + "completions/mean_length": 197.1408447265625, + "completions/mean_terminated_length": 187.2719477335612, + "completions/min_length": 59.3, + "completions/min_terminated_length": 59.3, + "entropy": 0.13283644306162992, + "epoch": 0.21531918125545893, + "eval/gt_acc_batch": 0.7733333528041839, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.188642218708992, + "kd/policy_loss": 0.009595150810976822, + "kd/rkl_advantage_mean": 0.679622879313926, + "kd/rkl_advantage_p95": 4.931788782278697, + "kd/rkl_advantage_std": 2.232171208659808, + "kd/ssd_loss": 0.0, + "learning_rate": 7.847187939087836e-07, + "loss": 0.03838060696919759, + "num_tokens": 185515003.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7733333329359691, + "rewards/gt_logging_reward/std": 0.41041067739327747, + "sampling/importance_sampling_ratio/max": 2.1348331610361737, + "sampling/importance_sampling_ratio/mean": 1.0055423299471538, + "sampling/importance_sampling_ratio/min": 0.3753095120191574, + "sampling/sampling_logp_difference/max": 0.3968024333318075, + "sampling/sampling_logp_difference/mean": 0.004346788814291358, + "step": 5670, + "step_time": 8.326860562207488, + "student/entropy": 0.13283644306162992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001390775046, + "completions/max_length": 486.8666666666667, + "completions/max_terminated_length": 459.53333333333336, + "completions/mean_length": 191.12223205566406, + "completions/mean_terminated_length": 183.7868682861328, + "completions/min_length": 53.86666666666667, + "completions/min_terminated_length": 53.86666666666667, + "entropy": 0.12788576477517685, + "epoch": 0.2164584361827365, + "eval/gt_acc_batch": 0.7952777862548828, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28096887469291687, + "kd/policy_loss": 0.008937225226933758, + "kd/rkl_advantage_mean": 0.6630268151561419, + "kd/rkl_advantage_p95": 4.852271602551142, + "kd/rkl_advantage_std": 2.2115610977013906, + "kd/ssd_loss": 0.0, + "learning_rate": 7.835795389815061e-07, + "loss": 0.035748895009358725, + "num_tokens": 186529203.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777743339539, + "rewards/gt_logging_reward/std": 0.3970170249541601, + "sampling/importance_sampling_ratio/max": 1.9355895876884461, + "sampling/importance_sampling_ratio/mean": 1.001184888680776, + "sampling/importance_sampling_ratio/min": 0.37919475038846334, + "sampling/sampling_logp_difference/max": 0.42830678820610046, + "sampling/sampling_logp_difference/mean": 0.0042645724800725775, + "step": 5700, + "step_time": 8.304077066869164, + "student/entropy": 0.12788576477517685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890221714973, + "completions/max_length": 492.93333333333334, + "completions/max_terminated_length": 443.6333333333333, + "completions/mean_length": 189.16112060546874, + "completions/mean_terminated_length": 182.12786966959635, + "completions/min_length": 57.1, + "completions/min_terminated_length": 57.1, + "entropy": 0.12831549756228924, + "epoch": 0.21759769111001406, + "eval/gt_acc_batch": 0.8127777953942616, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30303895473480225, + "kd/policy_loss": 0.008849725604522973, + "kd/rkl_advantage_mean": 0.7632906638396283, + "kd/rkl_advantage_p95": 4.994319760799408, + "kd/rkl_advantage_std": 2.2230107605457308, + "kd/ssd_loss": 0.0, + "learning_rate": 7.824402840542285e-07, + "loss": 0.0353989044825236, + "num_tokens": 187511343.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8127777755260468, + "rewards/gt_logging_reward/std": 0.3808879022796949, + "sampling/importance_sampling_ratio/max": 2.060555680592855, + "sampling/importance_sampling_ratio/mean": 1.0089763919512431, + "sampling/importance_sampling_ratio/min": 0.45285008996725085, + "sampling/sampling_logp_difference/max": 0.4202309330304464, + "sampling/sampling_logp_difference/mean": 0.00427888873188446, + "step": 5730, + "step_time": 8.076755245414097, + "student/entropy": 0.12831549756228924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03000000144044558, + "completions/max_length": 490.1, + "completions/max_terminated_length": 466.76666666666665, + "completions/mean_length": 193.43584289550782, + "completions/mean_terminated_length": 183.68018900553386, + "completions/min_length": 42.06666666666667, + "completions/min_terminated_length": 42.06666666666667, + "entropy": 0.13391668715824684, + "epoch": 0.2187369460372916, + "eval/gt_acc_batch": 0.7730555792649587, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3678651750087738, + "kd/policy_loss": 0.00943789459997788, + "kd/rkl_advantage_mean": 0.6817300123473008, + "kd/rkl_advantage_p95": 4.9047264417012535, + "kd/rkl_advantage_std": 2.2196541875600815, + "kd/ssd_loss": 0.0, + "learning_rate": 7.813010291269509e-07, + "loss": 0.03775157928466797, + "num_tokens": 188514200.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7730555574099223, + "rewards/gt_logging_reward/std": 0.4085520620147387, + "sampling/importance_sampling_ratio/max": 2.041526945432027, + "sampling/importance_sampling_ratio/mean": 1.0028298477331796, + "sampling/importance_sampling_ratio/min": 0.4376339763402939, + "sampling/sampling_logp_difference/max": 0.41535300215085347, + "sampling/sampling_logp_difference/mean": 0.004383208137005568, + "step": 5760, + "step_time": 8.222226228266178, + "student/entropy": 0.13391668715824684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556491017341, + "completions/max_length": 478.1, + "completions/max_terminated_length": 442.56666666666666, + "completions/mean_length": 181.13251037597655, + "completions/mean_terminated_length": 175.98905436197916, + "completions/min_length": 48.36666666666667, + "completions/min_terminated_length": 48.36666666666667, + "entropy": 0.12858205481121937, + "epoch": 0.21987620096456917, + "eval/gt_acc_batch": 0.8066666881243388, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3343258202075958, + "kd/policy_loss": 0.008838289661798625, + "kd/rkl_advantage_mean": 0.6625545490843554, + "kd/rkl_advantage_p95": 4.888026799758276, + "kd/rkl_advantage_std": 2.2402245422204334, + "kd/ssd_loss": 0.0, + "learning_rate": 7.801617741996733e-07, + "loss": 0.03535315990447998, + "num_tokens": 189465613.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8066666642824809, + "rewards/gt_logging_reward/std": 0.3908297528823217, + "sampling/importance_sampling_ratio/max": 1.9224931875864664, + "sampling/importance_sampling_ratio/mean": 1.0016495029131571, + "sampling/importance_sampling_ratio/min": 0.44221571187178293, + "sampling/sampling_logp_difference/max": 0.4424039125442505, + "sampling/sampling_logp_difference/mean": 0.004322101408615708, + "step": 5790, + "step_time": 7.943245146784466, + "student/entropy": 0.12858205481121937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.012777778475234905, + "completions/max_length": 476.1666666666667, + "completions/max_terminated_length": 431.46666666666664, + "completions/mean_length": 183.53695424397787, + "completions/mean_terminated_length": 179.37187652587892, + "completions/min_length": 54.233333333333334, + "completions/min_terminated_length": 54.233333333333334, + "entropy": 0.12296071797609329, + "epoch": 0.22101545589184673, + "eval/gt_acc_batch": 0.8019444664319356, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21569356322288513, + "kd/policy_loss": 0.00799797362803171, + "kd/rkl_advantage_mean": 0.6836033448576927, + "kd/rkl_advantage_p95": 4.846912217140198, + "kd/rkl_advantage_std": 2.173480122288068, + "kd/ssd_loss": 0.0, + "learning_rate": 7.790225192723958e-07, + "loss": 0.0319918950398763, + "num_tokens": 190417418.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444465637207, + "rewards/gt_logging_reward/std": 0.3880488167206446, + "sampling/importance_sampling_ratio/max": 1.8490684946378073, + "sampling/importance_sampling_ratio/mean": 1.0028865834077199, + "sampling/importance_sampling_ratio/min": 0.4574317678809166, + "sampling/sampling_logp_difference/max": 0.3523656706015269, + "sampling/sampling_logp_difference/mean": 0.0040835080901160834, + "step": 5820, + "step_time": 7.809561456561399, + "student/entropy": 0.12296071797609329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0244444456572334, + "completions/max_length": 489.2, + "completions/max_terminated_length": 448.03333333333336, + "completions/mean_length": 191.64945576985676, + "completions/mean_terminated_length": 183.8726557413737, + "completions/min_length": 52.766666666666666, + "completions/min_terminated_length": 52.766666666666666, + "entropy": 0.13544726489732664, + "epoch": 0.2221547108191243, + "eval/gt_acc_batch": 0.7725000242392223, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28815704584121704, + "kd/policy_loss": 0.009313079774923001, + "kd/rkl_advantage_mean": 0.6865894771491488, + "kd/rkl_advantage_p95": 4.964809554815292, + "kd/rkl_advantage_std": 2.238858007391294, + "kd/ssd_loss": 0.0, + "learning_rate": 7.778832643451183e-07, + "loss": 0.03725232283274333, + "num_tokens": 191413148.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7724999984105428, + "rewards/gt_logging_reward/std": 0.4087403694788615, + "sampling/importance_sampling_ratio/max": 1.8854342063268026, + "sampling/importance_sampling_ratio/mean": 0.9945890764395396, + "sampling/importance_sampling_ratio/min": 0.43529290954271954, + "sampling/sampling_logp_difference/max": 0.3980265339215597, + "sampling/sampling_logp_difference/mean": 0.004457079316489398, + "step": 5850, + "step_time": 8.036277526799434, + "student/entropy": 0.13544726489732664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890221714973, + "completions/max_length": 475.3, + "completions/max_terminated_length": 431.7, + "completions/mean_length": 183.9769536336263, + "completions/mean_terminated_length": 176.9757044474284, + "completions/min_length": 44.2, + "completions/min_terminated_length": 44.2, + "entropy": 0.12993071439365547, + "epoch": 0.22329396574640184, + "eval/gt_acc_batch": 0.8066666781902313, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29278483986854553, + "kd/policy_loss": 0.008625036644904565, + "kd/rkl_advantage_mean": 0.6887329362643262, + "kd/rkl_advantage_p95": 4.913207626342773, + "kd/rkl_advantage_std": 2.2223395725091297, + "kd/ssd_loss": 0.0, + "learning_rate": 7.767440094178407e-07, + "loss": 0.03450014591217041, + "num_tokens": 192378105.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8066666662693024, + "rewards/gt_logging_reward/std": 0.3845016111930211, + "sampling/importance_sampling_ratio/max": 1.9385174711545308, + "sampling/importance_sampling_ratio/mean": 0.9991305847962697, + "sampling/importance_sampling_ratio/min": 0.452227528889974, + "sampling/sampling_logp_difference/max": 0.40471582214037577, + "sampling/sampling_logp_difference/mean": 0.0043562284205108884, + "step": 5880, + "step_time": 7.974808844792036, + "student/entropy": 0.12993071439365547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778804302215, + "completions/max_length": 500.96666666666664, + "completions/max_terminated_length": 451.6333333333333, + "completions/mean_length": 186.37000986735026, + "completions/mean_terminated_length": 179.6831319173177, + "completions/min_length": 48.13333333333333, + "completions/min_terminated_length": 48.13333333333333, + "entropy": 0.12896088728060326, + "epoch": 0.2244332206736794, + "eval/gt_acc_batch": 0.798333364725113, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24819177389144897, + "kd/policy_loss": 0.008702729817014188, + "kd/rkl_advantage_mean": 0.6213313197096189, + "kd/rkl_advantage_p95": 4.733808664480845, + "kd/rkl_advantage_std": 2.162604829668999, + "kd/ssd_loss": 0.0, + "learning_rate": 7.756047544905632e-07, + "loss": 0.03481091658274333, + "num_tokens": 193353013.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.798333328962326, + "rewards/gt_logging_reward/std": 0.39396425187587736, + "sampling/importance_sampling_ratio/max": 2.0117841323216754, + "sampling/importance_sampling_ratio/mean": 1.00590673883756, + "sampling/importance_sampling_ratio/min": 0.421308138469855, + "sampling/sampling_logp_difference/max": 0.40876758893330895, + "sampling/sampling_logp_difference/mean": 0.004307455491895477, + "step": 5910, + "step_time": 8.192492198028292, + "student/entropy": 0.12896088728060326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556882172822, + "completions/max_length": 476.23333333333335, + "completions/max_terminated_length": 447.4, + "completions/mean_length": 188.5438985188802, + "completions/mean_terminated_length": 181.27578786214193, + "completions/min_length": 50.56666666666667, + "completions/min_terminated_length": 50.56666666666667, + "entropy": 0.13853238094598055, + "epoch": 0.22557247560095697, + "eval/gt_acc_batch": 0.7897222419579824, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27892500162124634, + "kd/policy_loss": 0.009205534724363437, + "kd/rkl_advantage_mean": 0.6743422707853218, + "kd/rkl_advantage_p95": 4.869641643762589, + "kd/rkl_advantage_std": 2.205375549197197, + "kd/ssd_loss": 0.0, + "learning_rate": 7.744654995632855e-07, + "loss": 0.03682214021682739, + "num_tokens": 194333467.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.789722220102946, + "rewards/gt_logging_reward/std": 0.39648331999778746, + "sampling/importance_sampling_ratio/max": 1.878988258043925, + "sampling/importance_sampling_ratio/mean": 0.9979678650697072, + "sampling/importance_sampling_ratio/min": 0.4188748821616173, + "sampling/sampling_logp_difference/max": 0.39626409610112506, + "sampling/sampling_logp_difference/mean": 0.004361696417133013, + "step": 5940, + "step_time": 7.935669683668918, + "student/entropy": 0.13853238094598055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223822027444, + "completions/max_length": 483.46666666666664, + "completions/max_terminated_length": 445.93333333333334, + "completions/mean_length": 185.52500864664714, + "completions/mean_terminated_length": 177.2699737548828, + "completions/min_length": 52.9, + "completions/min_terminated_length": 52.9, + "entropy": 0.12607061869154373, + "epoch": 0.22671173052823454, + "eval/gt_acc_batch": 0.8116666833559673, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28922736644744873, + "kd/policy_loss": 0.008378829122132931, + "kd/rkl_advantage_mean": 0.6698283930541947, + "kd/rkl_advantage_p95": 4.85495662689209, + "kd/rkl_advantage_std": 2.2240504691998164, + "kd/ssd_loss": 0.0, + "learning_rate": 7.73326244636008e-07, + "loss": 0.033515310287475585, + "num_tokens": 195296557.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8116666654745738, + "rewards/gt_logging_reward/std": 0.3806858465075493, + "sampling/importance_sampling_ratio/max": 1.942570964495341, + "sampling/importance_sampling_ratio/mean": 0.9995030244191487, + "sampling/importance_sampling_ratio/min": 0.4062091747919718, + "sampling/sampling_logp_difference/max": 0.4503404418627421, + "sampling/sampling_logp_difference/mean": 0.004235495355290672, + "step": 5970, + "step_time": 7.9241529256648695, + "student/entropy": 0.12607061869154373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001235554616, + "completions/max_length": 498.3333333333333, + "completions/max_terminated_length": 455.6333333333333, + "completions/mean_length": 189.9730651855469, + "completions/mean_terminated_length": 182.57928161621095, + "completions/min_length": 52.03333333333333, + "completions/min_terminated_length": 52.03333333333333, + "entropy": 0.13104532758394877, + "epoch": 0.2278509854555121, + "eval/gt_acc_batch": 0.7958333452542623, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22975988686084747, + "kd/policy_loss": 0.009143940704719473, + "kd/rkl_advantage_mean": 0.6065754485627015, + "kd/rkl_advantage_p95": 4.670005939404169, + "kd/rkl_advantage_std": 2.151283989350001, + "kd/ssd_loss": 0.0, + "learning_rate": 7.721869897087304e-07, + "loss": 0.03657575845718384, + "num_tokens": 196284268.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7958333373069764, + "rewards/gt_logging_reward/std": 0.3970336039861043, + "sampling/importance_sampling_ratio/max": 1.9158796389897665, + "sampling/importance_sampling_ratio/mean": 1.0000604828198751, + "sampling/importance_sampling_ratio/min": 0.4204437827070554, + "sampling/sampling_logp_difference/max": 0.353840716679891, + "sampling/sampling_logp_difference/mean": 0.0043151885891954105, + "step": 6000, + "step_time": 8.087329691345804, + "student/entropy": 0.13104532758394877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444558893641, + "completions/max_length": 504.6666666666667, + "completions/max_terminated_length": 467.23333333333335, + "completions/mean_length": 190.24278666178387, + "completions/mean_terminated_length": 183.09772186279298, + "completions/min_length": 48.46666666666667, + "completions/min_terminated_length": 48.46666666666667, + "entropy": 0.13260588670770326, + "epoch": 0.22899024038278964, + "eval/gt_acc_batch": 0.7736111402511596, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27430352568626404, + "kd/policy_loss": 0.009140055797373255, + "kd/rkl_advantage_mean": 0.63308949538817, + "kd/rkl_advantage_p95": 4.786375486850739, + "kd/rkl_advantage_std": 2.1824997772773105, + "kd/ssd_loss": 0.0, + "learning_rate": 7.71047734781453e-07, + "loss": 0.036560225486755374, + "num_tokens": 197274390.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7736111104488372, + "rewards/gt_logging_reward/std": 0.4075977742671967, + "sampling/importance_sampling_ratio/max": 1.9368695259094237, + "sampling/importance_sampling_ratio/mean": 0.9889886975288391, + "sampling/importance_sampling_ratio/min": 0.3951011781891187, + "sampling/sampling_logp_difference/max": 0.44110994736353554, + "sampling/sampling_logp_difference/mean": 0.004443427136478325, + "step": 6030, + "step_time": 8.315528792398982, + "student/entropy": 0.13260588670770326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444596146544, + "completions/max_length": 476.2, + "completions/max_terminated_length": 454.73333333333335, + "completions/mean_length": 189.7800099690755, + "completions/mean_terminated_length": 182.65061899820964, + "completions/min_length": 59.166666666666664, + "completions/min_terminated_length": 59.166666666666664, + "entropy": 0.12737384450932343, + "epoch": 0.2301294953100672, + "eval/gt_acc_batch": 0.7997222383817036, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23629285395145416, + "kd/policy_loss": 0.008441779058193788, + "kd/rkl_advantage_mean": 0.6419032241528233, + "kd/rkl_advantage_p95": 4.866001093387604, + "kd/rkl_advantage_std": 2.2196168065071107, + "kd/ssd_loss": 0.0, + "learning_rate": 7.699084798541753e-07, + "loss": 0.03376712004343669, + "num_tokens": 198259958.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7997222264607747, + "rewards/gt_logging_reward/std": 0.3867082769672076, + "sampling/importance_sampling_ratio/max": 1.882184370358785, + "sampling/importance_sampling_ratio/mean": 1.0009948054949442, + "sampling/importance_sampling_ratio/min": 0.40929896434148155, + "sampling/sampling_logp_difference/max": 0.43364791870117186, + "sampling/sampling_logp_difference/mean": 0.004253823085067173, + "step": 6060, + "step_time": 7.995410609742006, + "student/entropy": 0.12737384450932343 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890103747448, + "completions/max_length": 494.26666666666665, + "completions/max_terminated_length": 441.06666666666666, + "completions/mean_length": 191.42084248860678, + "completions/mean_terminated_length": 183.6046900431315, + "completions/min_length": 48.96666666666667, + "completions/min_terminated_length": 48.96666666666667, + "entropy": 0.13534168048451345, + "epoch": 0.23126875023734478, + "eval/gt_acc_batch": 0.7702777981758118, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3110235631465912, + "kd/policy_loss": 0.009326867169390123, + "kd/rkl_advantage_mean": 0.655918958162268, + "kd/rkl_advantage_p95": 4.913002063830693, + "kd/rkl_advantage_std": 2.2106439491113026, + "kd/ssd_loss": 0.0, + "learning_rate": 7.687692249268978e-07, + "loss": 0.03730746905008952, + "num_tokens": 199251641.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7702777743339538, + "rewards/gt_logging_reward/std": 0.4103247116009394, + "sampling/importance_sampling_ratio/max": 2.0024213910102846, + "sampling/importance_sampling_ratio/mean": 0.9991628030935923, + "sampling/importance_sampling_ratio/min": 0.3916111042102178, + "sampling/sampling_logp_difference/max": 0.40241243243217467, + "sampling/sampling_logp_difference/mean": 0.0044058912356073655, + "step": 6090, + "step_time": 8.227268930318921, + "student/entropy": 0.13534168048451345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01611111213763555, + "completions/max_length": 491.3666666666667, + "completions/max_terminated_length": 452.76666666666665, + "completions/mean_length": 183.8480656941732, + "completions/mean_terminated_length": 178.61597951253256, + "completions/min_length": 55.46666666666667, + "completions/min_terminated_length": 55.46666666666667, + "entropy": 0.13214289546012878, + "epoch": 0.23240800516462234, + "eval/gt_acc_batch": 0.809166685740153, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2709805369377136, + "kd/policy_loss": 0.008674674958456308, + "kd/rkl_advantage_mean": 0.6511059725036223, + "kd/rkl_advantage_p95": 4.827355154355367, + "kd/rkl_advantage_std": 2.1967110057671864, + "kd/ssd_loss": 0.0, + "learning_rate": 7.676299699996203e-07, + "loss": 0.03469870090484619, + "num_tokens": 200209886.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8091666658719381, + "rewards/gt_logging_reward/std": 0.38665656248728436, + "sampling/importance_sampling_ratio/max": 1.8723712046941121, + "sampling/importance_sampling_ratio/mean": 0.9903748591740926, + "sampling/importance_sampling_ratio/min": 0.42082366247971853, + "sampling/sampling_logp_difference/max": 0.3971408446629842, + "sampling/sampling_logp_difference/mean": 0.00438405170571059, + "step": 6120, + "step_time": 8.095425741401657, + "student/entropy": 0.13214289546012878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001111378273, + "completions/max_length": 488.3333333333333, + "completions/max_terminated_length": 457.26666666666665, + "completions/mean_length": 191.24473215738934, + "completions/mean_terminated_length": 184.0130599975586, + "completions/min_length": 56.53333333333333, + "completions/min_terminated_length": 56.53333333333333, + "entropy": 0.12994815253963074, + "epoch": 0.2335472600918999, + "eval/gt_acc_batch": 0.795000014702479, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34392327070236206, + "kd/policy_loss": 0.009036428155377507, + "kd/rkl_advantage_mean": 0.610159932806467, + "kd/rkl_advantage_p95": 4.792581123113632, + "kd/rkl_advantage_std": 2.176404374837875, + "kd/ssd_loss": 0.0, + "learning_rate": 7.664907150723426e-07, + "loss": 0.036145710945129396, + "num_tokens": 201197231.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7950000007947285, + "rewards/gt_logging_reward/std": 0.3875947813193003, + "sampling/importance_sampling_ratio/max": 1.9073782364527385, + "sampling/importance_sampling_ratio/mean": 1.0088681022326151, + "sampling/importance_sampling_ratio/min": 0.45198328197002413, + "sampling/sampling_logp_difference/max": 0.41112184325853984, + "sampling/sampling_logp_difference/mean": 0.004254262164855997, + "step": 6150, + "step_time": 8.101859429955947, + "student/entropy": 0.12994815253963074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028611112479120494, + "completions/max_length": 488.43333333333334, + "completions/max_terminated_length": 433.2, + "completions/mean_length": 186.45889841715496, + "completions/mean_terminated_length": 177.04283040364584, + "completions/min_length": 56.266666666666666, + "completions/min_terminated_length": 56.266666666666666, + "entropy": 0.12880239176253477, + "epoch": 0.23468651501917745, + "eval/gt_acc_batch": 0.7927778005599976, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32142725586891174, + "kd/policy_loss": 0.008768509357469156, + "kd/rkl_advantage_mean": 0.688782718560348, + "kd/rkl_advantage_p95": 4.932652097940445, + "kd/rkl_advantage_std": 2.247202789783478, + "kd/ssd_loss": 0.0, + "learning_rate": 7.653514601450651e-07, + "loss": 0.03507403930028279, + "num_tokens": 202176435.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7927777767181396, + "rewards/gt_logging_reward/std": 0.40062175691127777, + "sampling/importance_sampling_ratio/max": 1.8828734040260315, + "sampling/importance_sampling_ratio/mean": 0.9986003637313843, + "sampling/importance_sampling_ratio/min": 0.4192064791917801, + "sampling/sampling_logp_difference/max": 0.4846411069234212, + "sampling/sampling_logp_difference/mean": 0.0042101180413737895, + "step": 6180, + "step_time": 8.13547533043505, + "student/entropy": 0.12880239176253477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01472222302109003, + "completions/max_length": 463.3, + "completions/max_terminated_length": 441.43333333333334, + "completions/mean_length": 175.86139780680338, + "completions/mean_terminated_length": 171.03020884195965, + "completions/min_length": 59.2, + "completions/min_terminated_length": 59.2, + "entropy": 0.1299978608265519, + "epoch": 0.23582576994645502, + "eval/gt_acc_batch": 0.8186111251513163, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3235873878002167, + "kd/policy_loss": 0.008322320242101947, + "kd/rkl_advantage_mean": 0.6483541022947369, + "kd/rkl_advantage_p95": 4.797852172454198, + "kd/rkl_advantage_std": 2.194501131772995, + "kd/ssd_loss": 0.0, + "learning_rate": 7.642122052177875e-07, + "loss": 0.03328927954037984, + "num_tokens": 203106048.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8186111132303874, + "rewards/gt_logging_reward/std": 0.37935285915931066, + "sampling/importance_sampling_ratio/max": 1.8870131333669027, + "sampling/importance_sampling_ratio/mean": 0.9963464637597402, + "sampling/importance_sampling_ratio/min": 0.41821932395299277, + "sampling/sampling_logp_difference/max": 0.42629339297612506, + "sampling/sampling_logp_difference/mean": 0.004366864954742293, + "step": 6210, + "step_time": 7.833092168206349, + "student/entropy": 0.1299978608265519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445619980494, + "completions/max_length": 491.8333333333333, + "completions/max_terminated_length": 453.3, + "completions/mean_length": 186.61028747558595, + "completions/mean_terminated_length": 179.38998006184895, + "completions/min_length": 54.53333333333333, + "completions/min_terminated_length": 54.53333333333333, + "entropy": 0.12331887781620025, + "epoch": 0.23696502487373258, + "eval/gt_acc_batch": 0.8130555768807729, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27219873666763306, + "kd/policy_loss": 0.008462951225616659, + "kd/rkl_advantage_mean": 0.7024667259926597, + "kd/rkl_advantage_p95": 4.9172854125499725, + "kd/rkl_advantage_std": 2.2145619869232176, + "kd/ssd_loss": 0.0, + "learning_rate": 7.6307295029051e-07, + "loss": 0.03385180632273356, + "num_tokens": 204082253.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8130555589993794, + "rewards/gt_logging_reward/std": 0.38183150192101795, + "sampling/importance_sampling_ratio/max": 1.8632219115893045, + "sampling/importance_sampling_ratio/mean": 0.9992580831050872, + "sampling/importance_sampling_ratio/min": 0.44758213063081104, + "sampling/sampling_logp_difference/max": 0.4689243157704671, + "sampling/sampling_logp_difference/mean": 0.004147490758138399, + "step": 6240, + "step_time": 8.21339652123473, + "student/entropy": 0.12331887781620025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016111112075547378, + "completions/max_length": 476.0, + "completions/max_terminated_length": 441.46666666666664, + "completions/mean_length": 185.91917775472004, + "completions/mean_terminated_length": 180.76614634195963, + "completions/min_length": 52.46666666666667, + "completions/min_terminated_length": 52.46666666666667, + "entropy": 0.13043399980912607, + "epoch": 0.23810427980101015, + "eval/gt_acc_batch": 0.8119444807370504, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23726686835289001, + "kd/policy_loss": 0.008217162999790162, + "kd/rkl_advantage_mean": 0.5797906673513353, + "kd/rkl_advantage_p95": 4.708313755194346, + "kd/rkl_advantage_std": 2.1769098550081254, + "kd/ssd_loss": 0.0, + "learning_rate": 7.619336953632324e-07, + "loss": 0.03286864558855693, + "num_tokens": 205063010.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8119444449742635, + "rewards/gt_logging_reward/std": 0.38343019286791485, + "sampling/importance_sampling_ratio/max": 1.845543372631073, + "sampling/importance_sampling_ratio/mean": 0.9952125469843547, + "sampling/importance_sampling_ratio/min": 0.4258480389912923, + "sampling/sampling_logp_difference/max": 0.4127208431561788, + "sampling/sampling_logp_difference/mean": 0.004199273030584057, + "step": 6270, + "step_time": 8.093902048631572, + "student/entropy": 0.13043399980912607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030555556993931532, + "completions/max_length": 475.6, + "completions/max_terminated_length": 446.9, + "completions/mean_length": 190.7813975016276, + "completions/mean_terminated_length": 180.85533294677734, + "completions/min_length": 49.9, + "completions/min_terminated_length": 49.9, + "entropy": 0.12496696536739667, + "epoch": 0.2392435347282877, + "eval/gt_acc_batch": 0.8097222407658895, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24982069432735443, + "kd/policy_loss": 0.008748501294758172, + "kd/rkl_advantage_mean": 0.6741078318251917, + "kd/rkl_advantage_p95": 4.787641861041387, + "kd/rkl_advantage_std": 2.17306227684021, + "kd/ssd_loss": 0.0, + "learning_rate": 7.607944404359549e-07, + "loss": 0.03499400615692139, + "num_tokens": 206057623.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.809722222884496, + "rewards/gt_logging_reward/std": 0.3808087254563967, + "sampling/importance_sampling_ratio/max": 2.011062236626943, + "sampling/importance_sampling_ratio/mean": 1.0031454384326934, + "sampling/importance_sampling_ratio/min": 0.4314197281996409, + "sampling/sampling_logp_difference/max": 0.4187386751174927, + "sampling/sampling_logp_difference/mean": 0.004139232930416862, + "step": 6300, + "step_time": 8.099489710200578, + "student/entropy": 0.12496696536739667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779381722212, + "completions/max_length": 498.56666666666666, + "completions/max_terminated_length": 463.6666666666667, + "completions/mean_length": 192.26584218343098, + "completions/mean_terminated_length": 183.15040486653646, + "completions/min_length": 48.96666666666667, + "completions/min_terminated_length": 48.96666666666667, + "entropy": 0.12913793232291937, + "epoch": 0.24038278965556525, + "eval/gt_acc_batch": 0.8002778013547261, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3103610575199127, + "kd/policy_loss": 0.008718625331918398, + "kd/rkl_advantage_mean": 0.7567344908912976, + "kd/rkl_advantage_p95": 5.039508686463038, + "kd/rkl_advantage_std": 2.2230162034432093, + "kd/ssd_loss": 0.0, + "learning_rate": 7.596551855086772e-07, + "loss": 0.03487450281778971, + "num_tokens": 207049772.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8002777755260467, + "rewards/gt_logging_reward/std": 0.3912219415108363, + "sampling/importance_sampling_ratio/max": 1.9777502139409384, + "sampling/importance_sampling_ratio/mean": 0.995680965979894, + "sampling/importance_sampling_ratio/min": 0.4151444842418035, + "sampling/sampling_logp_difference/max": 0.4296450783809026, + "sampling/sampling_logp_difference/mean": 0.004319851961918175, + "step": 6330, + "step_time": 8.138093703538955, + "student/entropy": 0.12913793232291937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556720743575, + "completions/max_length": 485.9, + "completions/max_terminated_length": 451.6666666666667, + "completions/mean_length": 186.33167572021483, + "completions/mean_terminated_length": 179.57020416259766, + "completions/min_length": 52.733333333333334, + "completions/min_terminated_length": 52.733333333333334, + "entropy": 0.1327652420848608, + "epoch": 0.24152204458284282, + "eval/gt_acc_batch": 0.783611132701238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28265801072120667, + "kd/policy_loss": 0.008860272904469942, + "kd/rkl_advantage_mean": 0.6699662565719336, + "kd/rkl_advantage_p95": 4.84623194138209, + "kd/rkl_advantage_std": 2.212257832288742, + "kd/ssd_loss": 0.0, + "learning_rate": 7.585159305813997e-07, + "loss": 0.035441092650095624, + "num_tokens": 208023062.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7836111128330231, + "rewards/gt_logging_reward/std": 0.4034515793124835, + "sampling/importance_sampling_ratio/max": 1.9859682321548462, + "sampling/importance_sampling_ratio/mean": 1.001838900645574, + "sampling/importance_sampling_ratio/min": 0.40199041962623594, + "sampling/sampling_logp_difference/max": 0.42122057477633157, + "sampling/sampling_logp_difference/mean": 0.004427524105024835, + "step": 6360, + "step_time": 8.023761424177792, + "student/entropy": 0.1327652420848608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556640028954, + "completions/max_length": 480.1, + "completions/max_terminated_length": 452.03333333333336, + "completions/mean_length": 188.52806549072267, + "completions/mean_terminated_length": 180.02882486979166, + "completions/min_length": 62.733333333333334, + "completions/min_terminated_length": 62.733333333333334, + "entropy": 0.13002879321575164, + "epoch": 0.2426612995101204, + "eval/gt_acc_batch": 0.7816666881243388, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2970639765262604, + "kd/policy_loss": 0.009217195356419931, + "kd/rkl_advantage_mean": 0.7006503107647101, + "kd/rkl_advantage_p95": 4.91127490401268, + "kd/rkl_advantage_std": 2.235979250073433, + "kd/ssd_loss": 0.0, + "learning_rate": 7.573766756541222e-07, + "loss": 0.036868778864542644, + "num_tokens": 208999827.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7816666662693024, + "rewards/gt_logging_reward/std": 0.40385886033376056, + "sampling/importance_sampling_ratio/max": 2.0565109650293985, + "sampling/importance_sampling_ratio/mean": 1.0089419225851695, + "sampling/importance_sampling_ratio/min": 0.43466459810733793, + "sampling/sampling_logp_difference/max": 0.4010078251361847, + "sampling/sampling_logp_difference/mean": 0.004328290419653058, + "step": 6390, + "step_time": 7.967229126777966, + "student/entropy": 0.13002879321575164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944445818662642, + "completions/max_length": 491.56666666666666, + "completions/max_terminated_length": 458.8, + "completions/mean_length": 185.486675008138, + "completions/mean_terminated_length": 176.42356872558594, + "completions/min_length": 49.166666666666664, + "completions/min_terminated_length": 49.166666666666664, + "entropy": 0.13192854622999828, + "epoch": 0.24380055443739795, + "eval/gt_acc_batch": 0.7997222443421682, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24751074612140656, + "kd/policy_loss": 0.008718036629337197, + "kd/rkl_advantage_mean": 0.6847877079310517, + "kd/rkl_advantage_p95": 4.862166025241216, + "kd/rkl_advantage_std": 2.2022369503974915, + "kd/ssd_loss": 0.0, + "learning_rate": 7.562374207268446e-07, + "loss": 0.034872158368428545, + "num_tokens": 209971371.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7997222205003103, + "rewards/gt_logging_reward/std": 0.3935115804274877, + "sampling/importance_sampling_ratio/max": 1.9359491189320883, + "sampling/importance_sampling_ratio/mean": 0.9943469723065694, + "sampling/importance_sampling_ratio/min": 0.40121693710486095, + "sampling/sampling_logp_difference/max": 0.4213083803653717, + "sampling/sampling_logp_difference/mean": 0.004418986073384683, + "step": 6420, + "step_time": 8.150758177247674, + "student/entropy": 0.13192854622999828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112336317697, + "completions/max_length": 473.73333333333335, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 183.6369524637858, + "completions/mean_terminated_length": 176.74135818481446, + "completions/min_length": 53.46666666666667, + "completions/min_terminated_length": 53.46666666666667, + "entropy": 0.1297277480363846, + "epoch": 0.2449398093646755, + "eval/gt_acc_batch": 0.8111111263434092, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3232152760028839, + "kd/policy_loss": 0.008284555396918828, + "kd/rkl_advantage_mean": 0.5611398845522975, + "kd/rkl_advantage_p95": 4.650587461392084, + "kd/rkl_advantage_std": 2.177451287706693, + "kd/ssd_loss": 0.0, + "learning_rate": 7.550981657995671e-07, + "loss": 0.03313822348912557, + "num_tokens": 210930864.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8111111164093018, + "rewards/gt_logging_reward/std": 0.3862161229054133, + "sampling/importance_sampling_ratio/max": 1.8320292313893636, + "sampling/importance_sampling_ratio/mean": 0.9973168015480042, + "sampling/importance_sampling_ratio/min": 0.4025011857350667, + "sampling/sampling_logp_difference/max": 0.39460267424583434, + "sampling/sampling_logp_difference/mean": 0.004344195515538256, + "step": 6450, + "step_time": 7.838175813570463, + "student/entropy": 0.1297277480363846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02416666814436515, + "completions/max_length": 490.73333333333335, + "completions/max_terminated_length": 456.76666666666665, + "completions/mean_length": 188.2527877807617, + "completions/mean_terminated_length": 180.5466049194336, + "completions/min_length": 55.1, + "completions/min_terminated_length": 55.1, + "entropy": 0.13442746407041947, + "epoch": 0.24607906429195306, + "eval/gt_acc_batch": 0.7913889050483703, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24111120402812958, + "kd/policy_loss": 0.009167794570870077, + "kd/rkl_advantage_mean": 0.7579849595824878, + "kd/rkl_advantage_p95": 5.011103264490763, + "kd/rkl_advantage_std": 2.2465117196242015, + "kd/ssd_loss": 0.0, + "learning_rate": 7.539589108722895e-07, + "loss": 0.036671189467112224, + "num_tokens": 211910646.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7913888812065124, + "rewards/gt_logging_reward/std": 0.3950616712371508, + "sampling/importance_sampling_ratio/max": 1.85566113392512, + "sampling/importance_sampling_ratio/mean": 0.996570869286855, + "sampling/importance_sampling_ratio/min": 0.4182213837901751, + "sampling/sampling_logp_difference/max": 0.47024984161059064, + "sampling/sampling_logp_difference/mean": 0.004407738917507231, + "step": 6480, + "step_time": 8.058678201582127, + "student/entropy": 0.13442746407041947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030000001409401497, + "completions/max_length": 479.1333333333333, + "completions/max_terminated_length": 446.46666666666664, + "completions/mean_length": 186.81112009684244, + "completions/mean_terminated_length": 177.01209309895833, + "completions/min_length": 56.666666666666664, + "completions/min_terminated_length": 56.666666666666664, + "entropy": 0.1317155930524071, + "epoch": 0.24721831921923063, + "eval/gt_acc_batch": 0.7902777989705404, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25881513953208923, + "kd/policy_loss": 0.009148378877822931, + "kd/rkl_advantage_mean": 0.6807209809931616, + "kd/rkl_advantage_p95": 4.806604232390722, + "kd/rkl_advantage_std": 2.1716851701339084, + "kd/ssd_loss": 0.0, + "learning_rate": 7.52819655945012e-07, + "loss": 0.03659351666768392, + "num_tokens": 212880982.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7902777830759684, + "rewards/gt_logging_reward/std": 0.3958704407016436, + "sampling/importance_sampling_ratio/max": 1.938192371527354, + "sampling/importance_sampling_ratio/mean": 0.9952379405498505, + "sampling/importance_sampling_ratio/min": 0.4255646526813507, + "sampling/sampling_logp_difference/max": 0.4084997693697611, + "sampling/sampling_logp_difference/mean": 0.004432754870504141, + "step": 6510, + "step_time": 7.9764029064021695, + "student/entropy": 0.1317155930524071 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334581305584, + "completions/max_length": 481.46666666666664, + "completions/max_terminated_length": 444.8, + "completions/mean_length": 190.84639841715494, + "completions/mean_terminated_length": 183.44420420328777, + "completions/min_length": 58.233333333333334, + "completions/min_terminated_length": 58.233333333333334, + "entropy": 0.1307209754983584, + "epoch": 0.2483575741465082, + "eval/gt_acc_batch": 0.7791666984558105, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2319793403148651, + "kd/policy_loss": 0.009327594250983869, + "kd/rkl_advantage_mean": 0.6250596908542018, + "kd/rkl_advantage_p95": 4.798074014981588, + "kd/rkl_advantage_std": 2.1958048224449156, + "kd/ssd_loss": 0.0, + "learning_rate": 7.516804010177343e-07, + "loss": 0.03731037775675456, + "num_tokens": 213879413.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7791666686534882, + "rewards/gt_logging_reward/std": 0.4056950479745865, + "sampling/importance_sampling_ratio/max": 1.9356088439623516, + "sampling/importance_sampling_ratio/mean": 1.0063813308874765, + "sampling/importance_sampling_ratio/min": 0.44139294723669686, + "sampling/sampling_logp_difference/max": 0.40160841941833497, + "sampling/sampling_logp_difference/mean": 0.0042986598253871005, + "step": 6540, + "step_time": 8.149581079778727, + "student/entropy": 0.1307209754983584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556751787663, + "completions/max_length": 468.3, + "completions/max_terminated_length": 435.1, + "completions/mean_length": 185.22778778076173, + "completions/mean_terminated_length": 178.52667592366535, + "completions/min_length": 55.06666666666667, + "completions/min_terminated_length": 55.06666666666667, + "entropy": 0.13345299810171127, + "epoch": 0.24949682907378573, + "eval/gt_acc_batch": 0.7919444600741069, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2400812804698944, + "kd/policy_loss": 0.009333005623193457, + "kd/rkl_advantage_mean": 0.5901699352854242, + "kd/rkl_advantage_p95": 4.581990871826807, + "kd/rkl_advantage_std": 2.115486776828766, + "kd/ssd_loss": 0.0, + "learning_rate": 7.505411460904568e-07, + "loss": 0.037332022190093996, + "num_tokens": 214848073.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7919444461663564, + "rewards/gt_logging_reward/std": 0.3982202430566152, + "sampling/importance_sampling_ratio/max": 2.0125489314397176, + "sampling/importance_sampling_ratio/mean": 1.0062717696030934, + "sampling/importance_sampling_ratio/min": 0.4366688181956609, + "sampling/sampling_logp_difference/max": 0.39994831879933673, + "sampling/sampling_logp_difference/mean": 0.004463640429700415, + "step": 6570, + "step_time": 7.946685146885769, + "student/entropy": 0.13345299810171127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556919425727, + "completions/max_length": 473.2, + "completions/max_terminated_length": 451.06666666666666, + "completions/mean_length": 193.67862243652343, + "completions/mean_terminated_length": 185.421001180013, + "completions/min_length": 59.1, + "completions/min_terminated_length": 59.1, + "entropy": 0.12893886485447487, + "epoch": 0.2506360840010633, + "eval/gt_acc_batch": 0.7733333647251129, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2646295130252838, + "kd/policy_loss": 0.008955428580520675, + "kd/rkl_advantage_mean": 0.6700256830023136, + "kd/rkl_advantage_p95": 4.750998731454214, + "kd/rkl_advantage_std": 2.14154637157917, + "kd/ssd_loss": 0.0, + "learning_rate": 7.494018911631792e-07, + "loss": 0.03582171599070231, + "num_tokens": 215849692.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7733333329359691, + "rewards/gt_logging_reward/std": 0.4087906301021576, + "sampling/importance_sampling_ratio/max": 1.9598654786745706, + "sampling/importance_sampling_ratio/mean": 0.9995945533116658, + "sampling/importance_sampling_ratio/min": 0.42593573182821276, + "sampling/sampling_logp_difference/max": 0.4021790583928426, + "sampling/sampling_logp_difference/mean": 0.004294938625146945, + "step": 6600, + "step_time": 8.025439435092267, + "student/entropy": 0.12893886485447487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667920847735, + "completions/max_length": 484.06666666666666, + "completions/max_terminated_length": 441.3666666666667, + "completions/mean_length": 184.8466766357422, + "completions/mean_terminated_length": 177.56533915201823, + "completions/min_length": 53.266666666666666, + "completions/min_terminated_length": 53.266666666666666, + "entropy": 0.12881126534193754, + "epoch": 0.25177533892834086, + "eval/gt_acc_batch": 0.7897222340106964, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3228004276752472, + "kd/policy_loss": 0.0086019679515933, + "kd/rkl_advantage_mean": 0.6352578810949733, + "kd/rkl_advantage_p95": 4.7387493332227075, + "kd/rkl_advantage_std": 2.17497730354468, + "kd/ssd_loss": 0.0, + "learning_rate": 7.482626362359017e-07, + "loss": 0.03440786997477214, + "num_tokens": 216806052.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7897222220897675, + "rewards/gt_logging_reward/std": 0.40149479707082114, + "sampling/importance_sampling_ratio/max": 1.9305628577868144, + "sampling/importance_sampling_ratio/mean": 0.9963873545328776, + "sampling/importance_sampling_ratio/min": 0.4073688745498657, + "sampling/sampling_logp_difference/max": 0.4689078430334727, + "sampling/sampling_logp_difference/mean": 0.004301401708895962, + "step": 6630, + "step_time": 7.864458196129029, + "student/entropy": 0.12881126534193754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014166667684912681, + "completions/max_length": 478.0, + "completions/max_terminated_length": 455.6666666666667, + "completions/mean_length": 182.60973205566407, + "completions/mean_terminated_length": 177.8578130086263, + "completions/min_length": 52.06666666666667, + "completions/min_terminated_length": 52.06666666666667, + "entropy": 0.12573809946576756, + "epoch": 0.25291459385561843, + "eval/gt_acc_batch": 0.8016666869322459, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2257617563009262, + "kd/policy_loss": 0.008368931417741502, + "kd/rkl_advantage_mean": 0.7087296484038234, + "kd/rkl_advantage_p95": 4.909722928206126, + "kd/rkl_advantage_std": 2.220593562722206, + "kd/ssd_loss": 0.0, + "learning_rate": 7.471233813086242e-07, + "loss": 0.03347572485605876, + "num_tokens": 217762599.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.801666667064031, + "rewards/gt_logging_reward/std": 0.38974318106969197, + "sampling/importance_sampling_ratio/max": 1.8878875891367595, + "sampling/importance_sampling_ratio/mean": 0.998793379465739, + "sampling/importance_sampling_ratio/min": 0.4301188935836156, + "sampling/sampling_logp_difference/max": 0.41356134017308555, + "sampling/sampling_logp_difference/mean": 0.004266316164284944, + "step": 6660, + "step_time": 7.945062089591132, + "student/entropy": 0.12573809946576756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0188888899050653, + "completions/max_length": 481.2, + "completions/max_terminated_length": 450.7, + "completions/mean_length": 181.994731648763, + "completions/mean_terminated_length": 175.68707733154298, + "completions/min_length": 60.43333333333333, + "completions/min_terminated_length": 60.43333333333333, + "entropy": 0.12240205487857263, + "epoch": 0.254053848782896, + "eval/gt_acc_batch": 0.820555571715037, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2635529339313507, + "kd/policy_loss": 0.00791559810944212, + "kd/rkl_advantage_mean": 0.737532316086193, + "kd/rkl_advantage_p95": 4.912347207466762, + "kd/rkl_advantage_std": 2.195744394262632, + "kd/ssd_loss": 0.0, + "learning_rate": 7.459841263813466e-07, + "loss": 0.03166239658991496, + "num_tokens": 218715980.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8205555578072866, + "rewards/gt_logging_reward/std": 0.3758873715996742, + "sampling/importance_sampling_ratio/max": 1.9192961613337198, + "sampling/importance_sampling_ratio/mean": 1.0019152363141377, + "sampling/importance_sampling_ratio/min": 0.439955398440361, + "sampling/sampling_logp_difference/max": 0.41455527742703757, + "sampling/sampling_logp_difference/mean": 0.004177936585620046, + "step": 6690, + "step_time": 7.9371159586861415, + "student/entropy": 0.12240205487857263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667827715477, + "completions/max_length": 486.6666666666667, + "completions/max_terminated_length": 449.73333333333335, + "completions/mean_length": 189.77334340413412, + "completions/mean_terminated_length": 182.6819859822591, + "completions/min_length": 61.166666666666664, + "completions/min_terminated_length": 61.166666666666664, + "entropy": 0.11928754728287458, + "epoch": 0.25519310371017356, + "eval/gt_acc_batch": 0.8069444537162781, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34236061573028564, + "kd/policy_loss": 0.008116432700383787, + "kd/rkl_advantage_mean": 0.661946140229702, + "kd/rkl_advantage_p95": 4.835807512203853, + "kd/rkl_advantage_std": 2.178380723794301, + "kd/ssd_loss": 0.0, + "learning_rate": 7.44844871454069e-07, + "loss": 0.03246572812398275, + "num_tokens": 219699204.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8069444457689922, + "rewards/gt_logging_reward/std": 0.3882801706592242, + "sampling/importance_sampling_ratio/max": 1.8033640702565512, + "sampling/importance_sampling_ratio/mean": 1.0015222907066346, + "sampling/importance_sampling_ratio/min": 0.4643369361758232, + "sampling/sampling_logp_difference/max": 0.3749047636985779, + "sampling/sampling_logp_difference/mean": 0.004006231296807527, + "step": 6720, + "step_time": 8.023343049207082, + "student/entropy": 0.11928754728287458 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02722222382823626, + "completions/max_length": 484.3333333333333, + "completions/max_terminated_length": 452.8, + "completions/mean_length": 192.28112335205077, + "completions/mean_terminated_length": 183.23143056233724, + "completions/min_length": 62.06666666666667, + "completions/min_terminated_length": 62.06666666666667, + "entropy": 0.1325053472071886, + "epoch": 0.25633235863745113, + "eval/gt_acc_batch": 0.7963889102141063, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2888845205307007, + "kd/policy_loss": 0.009105391229968517, + "kd/rkl_advantage_mean": 0.6889169408008456, + "kd/rkl_advantage_p95": 4.872095646460851, + "kd/rkl_advantage_std": 2.1957727124293647, + "kd/ssd_loss": 0.0, + "learning_rate": 7.437056165267914e-07, + "loss": 0.03642156521479289, + "num_tokens": 220694480.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888903458913, + "rewards/gt_logging_reward/std": 0.3958563576141993, + "sampling/importance_sampling_ratio/max": 1.9580243508021036, + "sampling/importance_sampling_ratio/mean": 1.0023346523443857, + "sampling/importance_sampling_ratio/min": 0.4149521400531133, + "sampling/sampling_logp_difference/max": 0.41691152652104696, + "sampling/sampling_logp_difference/mean": 0.004414762471181651, + "step": 6750, + "step_time": 7.930047904727204, + "student/entropy": 0.1325053472071886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889060666164, + "completions/max_length": 499.3666666666667, + "completions/max_terminated_length": 469.43333333333334, + "completions/mean_length": 191.4836217244466, + "completions/mean_terminated_length": 182.9739242553711, + "completions/min_length": 59.13333333333333, + "completions/min_terminated_length": 59.13333333333333, + "entropy": 0.12661677629997334, + "epoch": 0.25747161356472864, + "eval/gt_acc_batch": 0.783055579662323, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2637076675891876, + "kd/policy_loss": 0.009063981263898313, + "kd/rkl_advantage_mean": 0.6177466770478834, + "kd/rkl_advantage_p95": 4.769889424244563, + "kd/rkl_advantage_std": 2.1949843148390453, + "kd/ssd_loss": 0.0, + "learning_rate": 7.425663615995139e-07, + "loss": 0.03625592788060506, + "num_tokens": 221680421.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7830555498600006, + "rewards/gt_logging_reward/std": 0.40643223822116853, + "sampling/importance_sampling_ratio/max": 2.007291229565938, + "sampling/importance_sampling_ratio/mean": 0.9946927607059479, + "sampling/importance_sampling_ratio/min": 0.43872424215078354, + "sampling/sampling_logp_difference/max": 0.423018886645635, + "sampling/sampling_logp_difference/mean": 0.00422237473540008, + "step": 6780, + "step_time": 8.102001418848522, + "student/entropy": 0.12661677629997334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030555557552725078, + "completions/max_length": 492.76666666666665, + "completions/max_terminated_length": 457.03333333333336, + "completions/mean_length": 195.664453125, + "completions/mean_terminated_length": 185.8007044474284, + "completions/min_length": 53.2, + "completions/min_terminated_length": 53.2, + "entropy": 0.13151656972865264, + "epoch": 0.2586108684920062, + "eval/gt_acc_batch": 0.7702777942021688, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2575989067554474, + "kd/policy_loss": 0.009302911534905433, + "kd/rkl_advantage_mean": 0.6222567290067673, + "kd/rkl_advantage_p95": 4.763335400819779, + "kd/rkl_advantage_std": 2.174221494793892, + "kd/ssd_loss": 0.0, + "learning_rate": 7.414271066722363e-07, + "loss": 0.037211652596791586, + "num_tokens": 222701549.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7702777763207753, + "rewards/gt_logging_reward/std": 0.41113004287083943, + "sampling/importance_sampling_ratio/max": 1.975506556034088, + "sampling/importance_sampling_ratio/mean": 0.998224671681722, + "sampling/importance_sampling_ratio/min": 0.4338963061571121, + "sampling/sampling_logp_difference/max": 0.39469812711079916, + "sampling/sampling_logp_difference/mean": 0.0043824572348967195, + "step": 6810, + "step_time": 8.274163667031099, + "student/entropy": 0.13151656972865264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02722222385928035, + "completions/max_length": 493.3333333333333, + "completions/max_terminated_length": 457.6, + "completions/mean_length": 192.0694549560547, + "completions/mean_terminated_length": 183.44447224934896, + "completions/min_length": 54.4, + "completions/min_terminated_length": 54.4, + "entropy": 0.12972833973666031, + "epoch": 0.2597501234192838, + "eval/gt_acc_batch": 0.7844444692134858, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31347623467445374, + "kd/policy_loss": 0.008824968349654228, + "kd/rkl_advantage_mean": 0.7010617584921419, + "kd/rkl_advantage_p95": 4.939160501956939, + "kd/rkl_advantage_std": 2.2384482741355898, + "kd/ssd_loss": 0.0, + "learning_rate": 7.402878517449588e-07, + "loss": 0.03529987335205078, + "num_tokens": 223695503.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7844444453716278, + "rewards/gt_logging_reward/std": 0.40485833187898, + "sampling/importance_sampling_ratio/max": 1.9926565011342368, + "sampling/importance_sampling_ratio/mean": 1.0009188214937845, + "sampling/importance_sampling_ratio/min": 0.4117788036664327, + "sampling/sampling_logp_difference/max": 0.40339021682739257, + "sampling/sampling_logp_difference/mean": 0.004281430559543272, + "step": 6840, + "step_time": 8.092743775981944, + "student/entropy": 0.12972833973666031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779220292965, + "completions/max_length": 480.1333333333333, + "completions/max_terminated_length": 436.73333333333335, + "completions/mean_length": 188.49195353190103, + "completions/mean_terminated_length": 180.44965311686198, + "completions/min_length": 58.8, + "completions/min_terminated_length": 58.8, + "entropy": 0.12720373688886563, + "epoch": 0.26088937834656134, + "eval/gt_acc_batch": 0.7763888994852702, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27323204278945923, + "kd/policy_loss": 0.008373104158090427, + "kd/rkl_advantage_mean": 0.7390439737277726, + "kd/rkl_advantage_p95": 4.948255254824956, + "kd/rkl_advantage_std": 2.2000600516796114, + "kd/ssd_loss": 0.0, + "learning_rate": 7.391485968176812e-07, + "loss": 0.03349241813023885, + "num_tokens": 224672466.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7763888875643412, + "rewards/gt_logging_reward/std": 0.4053095986445745, + "sampling/importance_sampling_ratio/max": 1.832367436091105, + "sampling/importance_sampling_ratio/mean": 0.9945196191469828, + "sampling/importance_sampling_ratio/min": 0.411879796286424, + "sampling/sampling_logp_difference/max": 0.39355329672495526, + "sampling/sampling_logp_difference/mean": 0.00426455324050039, + "step": 6870, + "step_time": 8.098365814813102, + "student/entropy": 0.12720373688886563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445874541997, + "completions/max_length": 488.23333333333335, + "completions/max_terminated_length": 443.93333333333334, + "completions/mean_length": 195.02278900146484, + "completions/mean_terminated_length": 187.10968780517578, + "completions/min_length": 55.4, + "completions/min_terminated_length": 55.4, + "entropy": 0.1266275266185403, + "epoch": 0.2620286332738389, + "eval/gt_acc_batch": 0.7902777830759684, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28177309036254883, + "kd/policy_loss": 0.009025950568805759, + "kd/rkl_advantage_mean": 0.6612976046899954, + "kd/rkl_advantage_p95": 4.786916029453278, + "kd/rkl_advantage_std": 2.1466736018657686, + "kd/ssd_loss": 0.0, + "learning_rate": 7.380093418904036e-07, + "loss": 0.03610380490620931, + "num_tokens": 225682740.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7902777830759684, + "rewards/gt_logging_reward/std": 0.4004335562388102, + "sampling/importance_sampling_ratio/max": 2.1212918559710183, + "sampling/importance_sampling_ratio/mean": 1.0057276030381521, + "sampling/importance_sampling_ratio/min": 0.4316197156906128, + "sampling/sampling_logp_difference/max": 0.3992898722489675, + "sampling/sampling_logp_difference/mean": 0.004189158207736909, + "step": 6900, + "step_time": 8.199730098616177, + "student/entropy": 0.1266275266185403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556658655404, + "completions/max_length": 489.76666666666665, + "completions/max_terminated_length": 454.6666666666667, + "completions/mean_length": 189.16528727213543, + "completions/mean_terminated_length": 182.4216049194336, + "completions/min_length": 52.03333333333333, + "completions/min_terminated_length": 52.03333333333333, + "entropy": 0.12821080069988966, + "epoch": 0.2631678882011165, + "eval/gt_acc_batch": 0.811388905843099, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2238578200340271, + "kd/policy_loss": 0.008730334241408854, + "kd/rkl_advantage_mean": 0.663798309272776, + "kd/rkl_advantage_p95": 4.841734168926875, + "kd/rkl_advantage_std": 2.2082900096972784, + "kd/ssd_loss": 0.0, + "learning_rate": 7.368700869631261e-07, + "loss": 0.0349213441212972, + "num_tokens": 226673655.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8113888919353485, + "rewards/gt_logging_reward/std": 0.387039249142011, + "sampling/importance_sampling_ratio/max": 2.1290335416793824, + "sampling/importance_sampling_ratio/mean": 0.9974997460842132, + "sampling/importance_sampling_ratio/min": 0.43055165658394495, + "sampling/sampling_logp_difference/max": 0.40321135719617207, + "sampling/sampling_logp_difference/mean": 0.0042367249572028715, + "step": 6930, + "step_time": 8.189786452997941, + "student/entropy": 0.12821080069988966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.031666668690741064, + "completions/max_length": 494.56666666666666, + "completions/max_terminated_length": 462.1, + "completions/mean_length": 192.49667714436848, + "completions/mean_terminated_length": 182.14723052978516, + "completions/min_length": 49.13333333333333, + "completions/min_terminated_length": 49.13333333333333, + "entropy": 0.13307219793399175, + "epoch": 0.26430714312839404, + "eval/gt_acc_batch": 0.7766666909058889, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2867799699306488, + "kd/policy_loss": 0.009403167595155536, + "kd/rkl_advantage_mean": 0.6886448294545213, + "kd/rkl_advantage_p95": 4.91659289598465, + "kd/rkl_advantage_std": 2.213735107580821, + "kd/ssd_loss": 0.0, + "learning_rate": 7.357308320358485e-07, + "loss": 0.037612672646840414, + "num_tokens": 227667539.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.776666663090388, + "rewards/gt_logging_reward/std": 0.40818662544091544, + "sampling/importance_sampling_ratio/max": 1.926716697216034, + "sampling/importance_sampling_ratio/mean": 1.0026012301445006, + "sampling/importance_sampling_ratio/min": 0.4004194806019465, + "sampling/sampling_logp_difference/max": 0.42244641383488973, + "sampling/sampling_logp_difference/mean": 0.004397509875707329, + "step": 6960, + "step_time": 8.1596174281983, + "student/entropy": 0.13307219793399175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02305555685112874, + "completions/max_length": 492.43333333333334, + "completions/max_terminated_length": 441.8, + "completions/mean_length": 188.94306640625, + "completions/mean_terminated_length": 181.394411722819, + "completions/min_length": 53.833333333333336, + "completions/min_terminated_length": 53.833333333333336, + "entropy": 0.13091566003859043, + "epoch": 0.2654463980556716, + "eval/gt_acc_batch": 0.7855555812517802, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2603132128715515, + "kd/policy_loss": 0.009042247613736738, + "kd/rkl_advantage_mean": 0.7262087893749898, + "kd/rkl_advantage_p95": 5.0299200514952345, + "kd/rkl_advantage_std": 2.2588981946309405, + "kd/ssd_loss": 0.0, + "learning_rate": 7.34591577108571e-07, + "loss": 0.036168996493021646, + "num_tokens": 228661230.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555514494578, + "rewards/gt_logging_reward/std": 0.40349901616573336, + "sampling/importance_sampling_ratio/max": 1.995185093084971, + "sampling/importance_sampling_ratio/mean": 1.006988956530889, + "sampling/importance_sampling_ratio/min": 0.45723100900650027, + "sampling/sampling_logp_difference/max": 0.3838215818007787, + "sampling/sampling_logp_difference/mean": 0.00437279900846382, + "step": 6990, + "step_time": 8.180050293639457, + "student/entropy": 0.13091566003859043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556813875834, + "completions/max_length": 495.03333333333336, + "completions/max_terminated_length": 457.56666666666666, + "completions/mean_length": 190.94223225911458, + "completions/mean_terminated_length": 184.3092030843099, + "completions/min_length": 53.36666666666667, + "completions/min_terminated_length": 53.36666666666667, + "entropy": 0.1255717379351457, + "epoch": 0.2665856529829492, + "eval/gt_acc_batch": 0.7977778077125549, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.20469558238983154, + "kd/policy_loss": 0.00831234985186408, + "kd/rkl_advantage_mean": 0.7257567259172598, + "kd/rkl_advantage_p95": 4.922374834616979, + "kd/rkl_advantage_std": 2.196360992391904, + "kd/ssd_loss": 0.0, + "learning_rate": 7.334523221812934e-07, + "loss": 0.03324940005938212, + "num_tokens": 229645862.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.797777779897054, + "rewards/gt_logging_reward/std": 0.39365256975094476, + "sampling/importance_sampling_ratio/max": 1.8745338598887125, + "sampling/importance_sampling_ratio/mean": 0.9993794580300649, + "sampling/importance_sampling_ratio/min": 0.4187064816554387, + "sampling/sampling_logp_difference/max": 0.39551650285720824, + "sampling/sampling_logp_difference/mean": 0.004188337169277172, + "step": 7020, + "step_time": 8.080893895879854, + "student/entropy": 0.1255717379351457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556857337555, + "completions/max_length": 482.03333333333336, + "completions/max_terminated_length": 458.56666666666666, + "completions/mean_length": 188.92889760335285, + "completions/mean_terminated_length": 180.48760274251302, + "completions/min_length": 60.86666666666667, + "completions/min_terminated_length": 60.86666666666667, + "entropy": 0.1270712294926246, + "epoch": 0.26772490791022674, + "eval/gt_acc_batch": 0.8113889177640279, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2673044800758362, + "kd/policy_loss": 0.008358221322608491, + "kd/rkl_advantage_mean": 0.7365761488676071, + "kd/rkl_advantage_p95": 4.984905964136123, + "kd/rkl_advantage_std": 2.2323709189891816, + "kd/ssd_loss": 0.0, + "learning_rate": 7.323130672540159e-07, + "loss": 0.03343288898468018, + "num_tokens": 230627654.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8113888839880625, + "rewards/gt_logging_reward/std": 0.38493500848611195, + "sampling/importance_sampling_ratio/max": 1.9668651938438415, + "sampling/importance_sampling_ratio/mean": 0.9977368811766306, + "sampling/importance_sampling_ratio/min": 0.4035266732176145, + "sampling/sampling_logp_difference/max": 0.40533265670140584, + "sampling/sampling_logp_difference/mean": 0.004242446009690563, + "step": 7050, + "step_time": 8.021556702989619, + "student/entropy": 0.1270712294926246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03111111291994651, + "completions/max_length": 484.26666666666665, + "completions/max_terminated_length": 452.3, + "completions/mean_length": 191.30334345499674, + "completions/mean_terminated_length": 181.0300615946452, + "completions/min_length": 51.166666666666664, + "completions/min_terminated_length": 51.166666666666664, + "entropy": 0.12857031021267176, + "epoch": 0.26886416283750425, + "eval/gt_acc_batch": 0.8055555721124014, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3902371823787689, + "kd/policy_loss": 0.008661734559185182, + "kd/rkl_advantage_mean": 0.7123775197193026, + "kd/rkl_advantage_p95": 4.882406133413315, + "kd/rkl_advantage_std": 2.2138723770777387, + "kd/ssd_loss": 0.0, + "learning_rate": 7.311738123267383e-07, + "loss": 0.034646936257680255, + "num_tokens": 231618258.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8055555522441864, + "rewards/gt_logging_reward/std": 0.39114587704340614, + "sampling/importance_sampling_ratio/max": 1.9993839820226034, + "sampling/importance_sampling_ratio/mean": 0.9966413577397665, + "sampling/importance_sampling_ratio/min": 0.41114200750986735, + "sampling/sampling_logp_difference/max": 0.45211298068364464, + "sampling/sampling_logp_difference/mean": 0.0042783253903811175, + "step": 7080, + "step_time": 8.08023926265693, + "student/entropy": 0.12857031021267176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055557049810886, + "completions/max_length": 491.06666666666666, + "completions/max_terminated_length": 447.26666666666665, + "completions/mean_length": 193.54806569417318, + "completions/mean_terminated_length": 184.5179463704427, + "completions/min_length": 50.56666666666667, + "completions/min_terminated_length": 50.56666666666667, + "entropy": 0.13106240443885325, + "epoch": 0.2700034177647818, + "eval/gt_acc_batch": 0.7863889137903849, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2919648587703705, + "kd/policy_loss": 0.008959093854840224, + "kd/rkl_advantage_mean": 0.6709450084716082, + "kd/rkl_advantage_p95": 4.825467518965403, + "kd/rkl_advantage_std": 2.185019408663114, + "kd/ssd_loss": 0.0, + "learning_rate": 7.300345573994607e-07, + "loss": 0.035836378733317055, + "num_tokens": 232627087.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7863888919353486, + "rewards/gt_logging_reward/std": 0.4013235032558441, + "sampling/importance_sampling_ratio/max": 1.9113041043281556, + "sampling/importance_sampling_ratio/mean": 0.9942569812138875, + "sampling/importance_sampling_ratio/min": 0.41415459165970486, + "sampling/sampling_logp_difference/max": 0.3969536532958349, + "sampling/sampling_logp_difference/mean": 0.004340172636633118, + "step": 7110, + "step_time": 8.28920640903137, + "student/entropy": 0.13106240443885325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001452863218, + "completions/max_length": 491.1, + "completions/max_terminated_length": 464.6666666666667, + "completions/mean_length": 186.26639912923176, + "completions/mean_terminated_length": 178.84396107991537, + "completions/min_length": 53.833333333333336, + "completions/min_terminated_length": 53.833333333333336, + "entropy": 0.12979006450623273, + "epoch": 0.2711426726920594, + "eval/gt_acc_batch": 0.785833348830541, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.35547754168510437, + "kd/policy_loss": 0.009000564616871997, + "kd/rkl_advantage_mean": 0.6805640195806821, + "kd/rkl_advantage_p95": 4.892049342393875, + "kd/rkl_advantage_std": 2.230516183376312, + "kd/ssd_loss": 0.0, + "learning_rate": 7.288953024721831e-07, + "loss": 0.03600225448608398, + "num_tokens": 233603910.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.785833336909612, + "rewards/gt_logging_reward/std": 0.4035194824139277, + "sampling/importance_sampling_ratio/max": 1.9517660895983378, + "sampling/importance_sampling_ratio/mean": 0.9995789388815562, + "sampling/importance_sampling_ratio/min": 0.4216391801834106, + "sampling/sampling_logp_difference/max": 0.3953616182009379, + "sampling/sampling_logp_difference/mean": 0.004260512984668215, + "step": 7140, + "step_time": 8.109515184594784, + "student/entropy": 0.12979006450623273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001459072035, + "completions/max_length": 485.6666666666667, + "completions/max_terminated_length": 433.9, + "completions/mean_length": 183.98306528727213, + "completions/mean_terminated_length": 175.77496999104818, + "completions/min_length": 50.53333333333333, + "completions/min_terminated_length": 50.53333333333333, + "entropy": 0.12913555689156056, + "epoch": 0.27228192761933695, + "eval/gt_acc_batch": 0.8030555744965872, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.320027619600296, + "kd/policy_loss": 0.008827545577272151, + "kd/rkl_advantage_mean": 0.650126864258588, + "kd/rkl_advantage_p95": 4.842671497662862, + "kd/rkl_advantage_std": 2.216081448396047, + "kd/ssd_loss": 0.0, + "learning_rate": 7.277560475449056e-07, + "loss": 0.03531018098195394, + "num_tokens": 234575753.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555566151937, + "rewards/gt_logging_reward/std": 0.3885568579037984, + "sampling/importance_sampling_ratio/max": 1.9812754273414612, + "sampling/importance_sampling_ratio/mean": 0.9992644190788269, + "sampling/importance_sampling_ratio/min": 0.4359300305445989, + "sampling/sampling_logp_difference/max": 0.374162882566452, + "sampling/sampling_logp_difference/mean": 0.0043146274906272685, + "step": 7170, + "step_time": 8.13958897848691, + "student/entropy": 0.12913555689156056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888889842977128, + "completions/max_length": 482.7, + "completions/max_terminated_length": 451.56666666666666, + "completions/mean_length": 184.91528828938803, + "completions/mean_terminated_length": 178.75211130777996, + "completions/min_length": 53.3, + "completions/min_terminated_length": 53.3, + "entropy": 0.13244923390448093, + "epoch": 0.2734211825466145, + "eval/gt_acc_batch": 0.8022222479184469, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31756749749183655, + "kd/policy_loss": 0.00868869288630473, + "kd/rkl_advantage_mean": 0.6775181104894727, + "kd/rkl_advantage_p95": 4.850653626521429, + "kd/rkl_advantage_std": 2.2005867153406142, + "kd/ssd_loss": 0.0, + "learning_rate": 7.266167926176281e-07, + "loss": 0.03475476900736491, + "num_tokens": 235543216.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.802222220102946, + "rewards/gt_logging_reward/std": 0.3916413774092992, + "sampling/importance_sampling_ratio/max": 1.8890231649080913, + "sampling/importance_sampling_ratio/mean": 0.9985008060932159, + "sampling/importance_sampling_ratio/min": 0.42834109465281167, + "sampling/sampling_logp_difference/max": 0.3872853934764862, + "sampling/sampling_logp_difference/mean": 0.004358966369181871, + "step": 7200, + "step_time": 8.033701635780744, + "student/entropy": 0.13244923390448093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0336111129882435, + "completions/max_length": 493.3, + "completions/max_terminated_length": 446.46666666666664, + "completions/mean_length": 191.88806610107423, + "completions/mean_terminated_length": 180.7874959309896, + "completions/min_length": 51.86666666666667, + "completions/min_terminated_length": 51.86666666666667, + "entropy": 0.13195215879629055, + "epoch": 0.2745604374738921, + "eval/gt_acc_batch": 0.7922222534815471, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2795392870903015, + "kd/policy_loss": 0.008633831812767312, + "kd/rkl_advantage_mean": 0.5973191520587231, + "kd/rkl_advantage_p95": 4.7056278149286905, + "kd/rkl_advantage_std": 2.1815122971932093, + "kd/ssd_loss": 0.0, + "learning_rate": 7.254775376903505e-07, + "loss": 0.03453532854715983, + "num_tokens": 236554029.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222256660462, + "rewards/gt_logging_reward/std": 0.3966766794522603, + "sampling/importance_sampling_ratio/max": 1.8418158928553263, + "sampling/importance_sampling_ratio/mean": 0.9981395463148753, + "sampling/importance_sampling_ratio/min": 0.4349077920118968, + "sampling/sampling_logp_difference/max": 0.41477184891700747, + "sampling/sampling_logp_difference/mean": 0.004349384177476168, + "step": 7230, + "step_time": 8.328828496458785, + "student/entropy": 0.13195215879629055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001260389883, + "completions/max_length": 496.03333333333336, + "completions/max_terminated_length": 454.93333333333334, + "completions/mean_length": 193.3747319539388, + "completions/mean_terminated_length": 186.90281473795574, + "completions/min_length": 60.93333333333333, + "completions/min_terminated_length": 60.93333333333333, + "entropy": 0.12653256058692933, + "epoch": 0.27569969240116965, + "eval/gt_acc_batch": 0.820555579662323, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29712221026420593, + "kd/policy_loss": 0.00855220912101989, + "kd/rkl_advantage_mean": 0.667461157652239, + "kd/rkl_advantage_p95": 4.884639322757721, + "kd/rkl_advantage_std": 2.2042930165926617, + "kd/ssd_loss": 0.0, + "learning_rate": 7.24338282763073e-07, + "loss": 0.034208842118581134, + "num_tokens": 237548714.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.820555551846822, + "rewards/gt_logging_reward/std": 0.3738074233134588, + "sampling/importance_sampling_ratio/max": 1.9909626205762228, + "sampling/importance_sampling_ratio/mean": 1.0061950266361237, + "sampling/importance_sampling_ratio/min": 0.423416376610597, + "sampling/sampling_logp_difference/max": 0.4035428563753764, + "sampling/sampling_logp_difference/mean": 0.004203322754862408, + "step": 7260, + "step_time": 8.156133976695127, + "student/entropy": 0.12653256058692933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112491538127, + "completions/max_length": 489.8, + "completions/max_terminated_length": 450.03333333333336, + "completions/mean_length": 187.83167877197266, + "completions/mean_terminated_length": 181.01654205322265, + "completions/min_length": 54.333333333333336, + "completions/min_terminated_length": 54.333333333333336, + "entropy": 0.12847090512514114, + "epoch": 0.2768389473284472, + "eval/gt_acc_batch": 0.806944461663564, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22150930762290955, + "kd/policy_loss": 0.008667389988355959, + "kd/rkl_advantage_mean": 0.7316719402869543, + "kd/rkl_advantage_p95": 5.000080126523971, + "kd/rkl_advantage_std": 2.232233554124832, + "kd/ssd_loss": 0.0, + "learning_rate": 7.231990278357953e-07, + "loss": 0.03466955820719401, + "num_tokens": 238526476.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8069444457689922, + "rewards/gt_logging_reward/std": 0.3811695019404093, + "sampling/importance_sampling_ratio/max": 1.8845135569572449, + "sampling/importance_sampling_ratio/mean": 1.0062776704629262, + "sampling/importance_sampling_ratio/min": 0.4106526712576548, + "sampling/sampling_logp_difference/max": 0.37148555914560955, + "sampling/sampling_logp_difference/mean": 0.004256860058133801, + "step": 7290, + "step_time": 8.128393263334875, + "student/entropy": 0.12847090512514114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01888888996715347, + "completions/max_length": 493.93333333333334, + "completions/max_terminated_length": 454.1, + "completions/mean_length": 182.88167572021484, + "completions/mean_terminated_length": 176.5390833536784, + "completions/min_length": 51.63333333333333, + "completions/min_terminated_length": 51.63333333333333, + "entropy": 0.1317394000167648, + "epoch": 0.2779782022557248, + "eval/gt_acc_batch": 0.7908333738644918, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21553803980350494, + "kd/policy_loss": 0.009112790285144001, + "kd/rkl_advantage_mean": 0.62514662258327, + "kd/rkl_advantage_p95": 4.819916617870331, + "kd/rkl_advantage_std": 2.2130232016245523, + "kd/ssd_loss": 0.0, + "learning_rate": 7.220597729085178e-07, + "loss": 0.036451164881388345, + "num_tokens": 239494826.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7908333321412404, + "rewards/gt_logging_reward/std": 0.3985697249571482, + "sampling/importance_sampling_ratio/max": 1.8780595302581786, + "sampling/importance_sampling_ratio/mean": 1.0039331833521525, + "sampling/importance_sampling_ratio/min": 0.4182099297642708, + "sampling/sampling_logp_difference/max": 0.4154897650082906, + "sampling/sampling_logp_difference/mean": 0.004377705844429632, + "step": 7320, + "step_time": 8.279885052916748, + "student/entropy": 0.1317394000167648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778829137485, + "completions/max_length": 473.43333333333334, + "completions/max_terminated_length": 441.3, + "completions/mean_length": 189.6658442179362, + "completions/mean_terminated_length": 184.02222290039063, + "completions/min_length": 58.96666666666667, + "completions/min_terminated_length": 58.96666666666667, + "entropy": 0.11890160012990236, + "epoch": 0.2791174571830023, + "eval/gt_acc_batch": 0.8130555729071299, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2616545259952545, + "kd/policy_loss": 0.007866497751092539, + "kd/rkl_advantage_mean": 0.6495922372986873, + "kd/rkl_advantage_p95": 4.831064222256343, + "kd/rkl_advantage_std": 2.193978323539098, + "kd/ssd_loss": 0.0, + "learning_rate": 7.209205179812402e-07, + "loss": 0.03146599133809407, + "num_tokens": 240479431.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8130555550257365, + "rewards/gt_logging_reward/std": 0.3771941527724266, + "sampling/importance_sampling_ratio/max": 1.9455264488855997, + "sampling/importance_sampling_ratio/mean": 0.9973666826883952, + "sampling/importance_sampling_ratio/min": 0.4294523626565933, + "sampling/sampling_logp_difference/max": 0.39302099744478863, + "sampling/sampling_logp_difference/mean": 0.003974258697902163, + "step": 7350, + "step_time": 7.888823867736694, + "student/entropy": 0.11890160012990236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334773778915, + "completions/max_length": 488.3, + "completions/max_terminated_length": 459.1333333333333, + "completions/mean_length": 188.53306477864584, + "completions/mean_terminated_length": 180.14781850179037, + "completions/min_length": 60.2, + "completions/min_terminated_length": 60.2, + "entropy": 0.13104477574427922, + "epoch": 0.28025671211027986, + "eval/gt_acc_batch": 0.7866666972637176, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2555178701877594, + "kd/policy_loss": 0.00885378066644383, + "kd/rkl_advantage_mean": 0.7146113731898367, + "kd/rkl_advantage_p95": 4.894355609019597, + "kd/rkl_advantage_std": 2.187800592184067, + "kd/ssd_loss": 0.0, + "learning_rate": 7.197812630539626e-07, + "loss": 0.035415124893188474, + "num_tokens": 241457710.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666674613952, + "rewards/gt_logging_reward/std": 0.4010653465986252, + "sampling/importance_sampling_ratio/max": 1.9251043200492859, + "sampling/importance_sampling_ratio/mean": 0.9978241821130117, + "sampling/importance_sampling_ratio/min": 0.44129207531611125, + "sampling/sampling_logp_difference/max": 0.42639047304789224, + "sampling/sampling_logp_difference/mean": 0.0043221672686437765, + "step": 7380, + "step_time": 8.088440095757445, + "student/entropy": 0.13104477574427922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018611112081756195, + "completions/max_length": 474.9, + "completions/max_terminated_length": 437.6333333333333, + "completions/mean_length": 187.0247319539388, + "completions/mean_terminated_length": 180.97616678873698, + "completions/min_length": 57.666666666666664, + "completions/min_terminated_length": 57.666666666666664, + "entropy": 0.12927573726822933, + "epoch": 0.2813959670375574, + "eval/gt_acc_batch": 0.7902777969837189, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3095906972885132, + "kd/policy_loss": 0.00896775655176801, + "kd/rkl_advantage_mean": 0.6248621855241557, + "kd/rkl_advantage_p95": 4.730440032482147, + "kd/rkl_advantage_std": 2.170497782031695, + "kd/ssd_loss": 0.0, + "learning_rate": 7.186420081266851e-07, + "loss": 0.035871028900146484, + "num_tokens": 242433759.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7902777791023254, + "rewards/gt_logging_reward/std": 0.399994333088398, + "sampling/importance_sampling_ratio/max": 1.9782690167427064, + "sampling/importance_sampling_ratio/mean": 1.00160311460495, + "sampling/importance_sampling_ratio/min": 0.41860113143920896, + "sampling/sampling_logp_difference/max": 0.3959237615267436, + "sampling/sampling_logp_difference/mean": 0.0043514217172438896, + "step": 7410, + "step_time": 7.998041561160547, + "student/entropy": 0.12927573726822933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223604718842, + "completions/max_length": 495.73333333333335, + "completions/max_terminated_length": 460.23333333333335, + "completions/mean_length": 188.46945343017578, + "completions/mean_terminated_length": 180.50446217854818, + "completions/min_length": 52.86666666666667, + "completions/min_terminated_length": 52.86666666666667, + "entropy": 0.12449201655884584, + "epoch": 0.282535221964835, + "eval/gt_acc_batch": 0.8194444676240286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24058017134666443, + "kd/policy_loss": 0.008094085634608443, + "kd/rkl_advantage_mean": 0.6648565145209432, + "kd/rkl_advantage_p95": 4.889782045284907, + "kd/rkl_advantage_std": 2.2213917315006255, + "kd/ssd_loss": 0.0, + "learning_rate": 7.175027531994076e-07, + "loss": 0.03237634499867757, + "num_tokens": 243418561.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8194444398085277, + "rewards/gt_logging_reward/std": 0.37773456076780954, + "sampling/importance_sampling_ratio/max": 1.8634984135627746, + "sampling/importance_sampling_ratio/mean": 0.9950605074564616, + "sampling/importance_sampling_ratio/min": 0.39722345074017845, + "sampling/sampling_logp_difference/max": 0.36180466413497925, + "sampling/sampling_logp_difference/mean": 0.004165480192750692, + "step": 7440, + "step_time": 8.13850879422777, + "student/entropy": 0.12449201655884584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223188728093, + "completions/max_length": 480.3666666666667, + "completions/max_terminated_length": 452.6666666666667, + "completions/mean_length": 181.80084330240885, + "completions/mean_terminated_length": 175.26335754394532, + "completions/min_length": 53.8, + "completions/min_terminated_length": 53.8, + "entropy": 0.1290100621059537, + "epoch": 0.28367447689211256, + "eval/gt_acc_batch": 0.8011111239592235, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30264878273010254, + "kd/policy_loss": 0.008407128961213555, + "kd/rkl_advantage_mean": 0.6937856125334899, + "kd/rkl_advantage_p95": 4.926431512832641, + "kd/rkl_advantage_std": 2.2306670884291333, + "kd/ssd_loss": 0.0, + "learning_rate": 7.1636349827213e-07, + "loss": 0.03362851937611898, + "num_tokens": 244379356.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8011111140251159, + "rewards/gt_logging_reward/std": 0.3908787359793981, + "sampling/importance_sampling_ratio/max": 1.8494520982106526, + "sampling/importance_sampling_ratio/mean": 0.9922216852506002, + "sampling/importance_sampling_ratio/min": 0.4032910118500392, + "sampling/sampling_logp_difference/max": 0.40983497301737465, + "sampling/sampling_logp_difference/mean": 0.004308179983248313, + "step": 7470, + "step_time": 8.049019245733508, + "student/entropy": 0.1290100621059537 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01954023083993073, + "completions/max_length": 482.7586206896552, + "completions/max_terminated_length": 441.6896551724138, + "completions/mean_length": 187.14397404111665, + "completions/mean_terminated_length": 180.80667429956895, + "completions/min_length": 54.172413793103445, + "completions/min_terminated_length": 54.172413793103445, + "entropy": 0.12974398114301008, + "epoch": 0.2848137318193901, + "eval/gt_acc_batch": 0.8112069162829169, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25777268409729004, + "kd/policy_loss": 0.008724484102928946, + "kd/rkl_advantage_mean": 0.6844629948509151, + "kd/rkl_advantage_p95": 4.8789235394576504, + "kd/rkl_advantage_std": 2.2019814293960045, + "kd/ssd_loss": 0.0, + "learning_rate": 7.152242433448524e-07, + "loss": 0.033734679222106934, + "num_tokens": 245314345.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8112068957295911, + "rewards/gt_logging_reward/std": 0.37981374073645163, + "sampling/importance_sampling_ratio/max": 1.9253342069428543, + "sampling/importance_sampling_ratio/mean": 0.9967258367045172, + "sampling/importance_sampling_ratio/min": 0.4212525199199545, + "sampling/sampling_logp_difference/max": 0.4117835862883206, + "sampling/sampling_logp_difference/mean": 0.004299978926328236, + "step": 7500, + "step_time": 7.802094062033575, + "student/entropy": 0.12974398114301008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445775200924, + "completions/max_length": 477.6333333333333, + "completions/max_terminated_length": 441.6333333333333, + "completions/mean_length": 191.60501047770182, + "completions/mean_terminated_length": 184.62628326416015, + "completions/min_length": 52.266666666666666, + "completions/min_terminated_length": 52.266666666666666, + "entropy": 0.12551198303699493, + "epoch": 0.2859529867466677, + "eval/gt_acc_batch": 0.7858333508173625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32289934158325195, + "kd/policy_loss": 0.00851394320682933, + "kd/rkl_advantage_mean": 0.6326868486280243, + "kd/rkl_advantage_p95": 4.793203415473302, + "kd/rkl_advantage_std": 2.175092860062917, + "kd/ssd_loss": 0.0, + "learning_rate": 7.140849884175749e-07, + "loss": 0.03405577739079793, + "num_tokens": 246300931.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7858333349227905, + "rewards/gt_logging_reward/std": 0.40166600545247394, + "sampling/importance_sampling_ratio/max": 1.902752161026001, + "sampling/importance_sampling_ratio/mean": 0.9973043084144593, + "sampling/importance_sampling_ratio/min": 0.4102367252111435, + "sampling/sampling_logp_difference/max": 0.37177858750025433, + "sampling/sampling_logp_difference/mean": 0.0041945129089678325, + "step": 7530, + "step_time": 8.027995697952186, + "student/entropy": 0.12551198303699493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02666666836788257, + "completions/max_length": 500.4, + "completions/max_terminated_length": 456.96666666666664, + "completions/mean_length": 192.5469548543294, + "completions/mean_terminated_length": 183.86858723958332, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.13432998632391294, + "epoch": 0.28709224167394526, + "eval/gt_acc_batch": 0.7869444568951924, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2551116347312927, + "kd/policy_loss": 0.009300643752794713, + "kd/rkl_advantage_mean": 0.6909839316251843, + "kd/rkl_advantage_p95": 4.9054478724797566, + "kd/rkl_advantage_std": 2.2254164030154544, + "kd/ssd_loss": 0.0, + "learning_rate": 7.129457334902973e-07, + "loss": 0.0372025728225708, + "num_tokens": 247299316.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444449742635, + "rewards/gt_logging_reward/std": 0.4019670362273852, + "sampling/importance_sampling_ratio/max": 2.0223362684249877, + "sampling/importance_sampling_ratio/mean": 1.0082279165585837, + "sampling/importance_sampling_ratio/min": 0.4176556602120399, + "sampling/sampling_logp_difference/max": 0.4476656913757324, + "sampling/sampling_logp_difference/mean": 0.004463448965301116, + "step": 7560, + "step_time": 8.372763120798238, + "student/entropy": 0.13432998632391294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02694444600492716, + "completions/max_length": 489.8, + "completions/max_terminated_length": 442.1, + "completions/mean_length": 194.866677347819, + "completions/mean_terminated_length": 186.37992350260416, + "completions/min_length": 56.733333333333334, + "completions/min_terminated_length": 56.733333333333334, + "entropy": 0.13449308909475804, + "epoch": 0.2882314966012228, + "eval/gt_acc_batch": 0.771666693687439, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2850792706012726, + "kd/policy_loss": 0.009342372687145446, + "kd/rkl_advantage_mean": 0.6919974890848001, + "kd/rkl_advantage_p95": 4.904853465159734, + "kd/rkl_advantage_std": 2.1925798912843066, + "kd/ssd_loss": 0.0, + "learning_rate": 7.118064785630198e-07, + "loss": 0.03736949364344279, + "num_tokens": 248305508.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7716666618982951, + "rewards/gt_logging_reward/std": 0.41171499093373615, + "sampling/importance_sampling_ratio/max": 1.8823186755180359, + "sampling/importance_sampling_ratio/mean": 0.9978046496709188, + "sampling/importance_sampling_ratio/min": 0.3852042456467946, + "sampling/sampling_logp_difference/max": 0.36237057447433474, + "sampling/sampling_logp_difference/mean": 0.004371510089064638, + "step": 7590, + "step_time": 8.20140072212865, + "student/entropy": 0.13449308909475804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223437080782, + "completions/max_length": 485.9, + "completions/max_terminated_length": 456.93333333333334, + "completions/mean_length": 181.93584442138672, + "completions/mean_terminated_length": 175.43326873779296, + "completions/min_length": 56.56666666666667, + "completions/min_terminated_length": 56.56666666666667, + "entropy": 0.13278786235799392, + "epoch": 0.28937075152850034, + "eval/gt_acc_batch": 0.7930555661519368, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2813803255558014, + "kd/policy_loss": 0.00862608606306215, + "kd/rkl_advantage_mean": 0.722017059638165, + "kd/rkl_advantage_p95": 4.981088976065318, + "kd/rkl_advantage_std": 2.2586400995651883, + "kd/ssd_loss": 0.0, + "learning_rate": 7.106672236357422e-07, + "loss": 0.0345043420791626, + "num_tokens": 249252445.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555601914724, + "rewards/gt_logging_reward/std": 0.3994779924551646, + "sampling/importance_sampling_ratio/max": 1.8742060701052348, + "sampling/importance_sampling_ratio/mean": 0.9938172340393067, + "sampling/importance_sampling_ratio/min": 0.4252035493652026, + "sampling/sampling_logp_difference/max": 0.4132633924484253, + "sampling/sampling_logp_difference/mean": 0.0044174450837696595, + "step": 7620, + "step_time": 7.9955654951899, + "student/entropy": 0.13278786235799392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223784774543, + "completions/max_length": 475.8666666666667, + "completions/max_terminated_length": 452.06666666666666, + "completions/mean_length": 188.2452875773112, + "completions/mean_terminated_length": 180.88937428792318, + "completions/min_length": 51.46666666666667, + "completions/min_terminated_length": 51.46666666666667, + "entropy": 0.12492223164687553, + "epoch": 0.2905100064557779, + "eval/gt_acc_batch": 0.8138889094193776, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1973373293876648, + "kd/policy_loss": 0.008381914508451398, + "kd/rkl_advantage_mean": 0.676210269642373, + "kd/rkl_advantage_p95": 4.913306049505869, + "kd/rkl_advantage_std": 2.2185465345780053, + "kd/ssd_loss": 0.0, + "learning_rate": 7.095279687084647e-07, + "loss": 0.033527652422587075, + "num_tokens": 250241840.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8138888895511627, + "rewards/gt_logging_reward/std": 0.38042057305574417, + "sampling/importance_sampling_ratio/max": 1.9463424563407898, + "sampling/importance_sampling_ratio/mean": 0.9944198906421662, + "sampling/importance_sampling_ratio/min": 0.42315922379493714, + "sampling/sampling_logp_difference/max": 0.37017068862915037, + "sampling/sampling_logp_difference/mean": 0.004117939559121927, + "step": 7650, + "step_time": 8.127120964054484, + "student/entropy": 0.12492223164687553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333469927311, + "completions/max_length": 494.7, + "completions/max_terminated_length": 464.96666666666664, + "completions/mean_length": 193.02806599934897, + "completions/mean_terminated_length": 186.34808197021485, + "completions/min_length": 55.36666666666667, + "completions/min_terminated_length": 55.36666666666667, + "entropy": 0.12791847232729198, + "epoch": 0.29164926138305547, + "eval/gt_acc_batch": 0.7844444731871287, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30657410621643066, + "kd/policy_loss": 0.008794536468728135, + "kd/rkl_advantage_mean": 0.6878189840043585, + "kd/rkl_advantage_p95": 4.888978544871012, + "kd/rkl_advantage_std": 2.1933288226524987, + "kd/ssd_loss": 0.0, + "learning_rate": 7.08388713781187e-07, + "loss": 0.035178144772847496, + "num_tokens": 251248117.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7844444453716278, + "rewards/gt_logging_reward/std": 0.40195291737715405, + "sampling/importance_sampling_ratio/max": 1.9462915937105814, + "sampling/importance_sampling_ratio/mean": 1.0002789874871572, + "sampling/importance_sampling_ratio/min": 0.4279101143280665, + "sampling/sampling_logp_difference/max": 0.3927750507990519, + "sampling/sampling_logp_difference/mean": 0.0041761129628866914, + "step": 7680, + "step_time": 8.354513695770098, + "student/entropy": 0.12791847232729198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556596567232, + "completions/max_length": 495.2, + "completions/max_terminated_length": 463.1333333333333, + "completions/mean_length": 185.58306477864582, + "completions/mean_terminated_length": 178.80243225097655, + "completions/min_length": 53.7, + "completions/min_terminated_length": 53.7, + "entropy": 0.13234983968238037, + "epoch": 0.29278851631033304, + "eval/gt_acc_batch": 0.8133333563804627, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1916031539440155, + "kd/policy_loss": 0.008344037287558118, + "kd/rkl_advantage_mean": 0.719113462670551, + "kd/rkl_advantage_p95": 4.960255457957586, + "kd/rkl_advantage_std": 2.2320378263791403, + "kd/ssd_loss": 0.0, + "learning_rate": 7.072494588539095e-07, + "loss": 0.033376153310139975, + "num_tokens": 252210184.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8133333325386047, + "rewards/gt_logging_reward/std": 0.38271885812282563, + "sampling/importance_sampling_ratio/max": 1.9597482244173685, + "sampling/importance_sampling_ratio/mean": 0.9950135747591654, + "sampling/importance_sampling_ratio/min": 0.4136969188849131, + "sampling/sampling_logp_difference/max": 0.3874598741531372, + "sampling/sampling_logp_difference/mean": 0.004173777477505306, + "step": 7710, + "step_time": 8.03531077800629, + "student/entropy": 0.13234983968238037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777940655748, + "completions/max_length": 484.06666666666666, + "completions/max_terminated_length": 452.53333333333336, + "completions/mean_length": 189.4819544474284, + "completions/mean_terminated_length": 181.40251922607422, + "completions/min_length": 47.46666666666667, + "completions/min_terminated_length": 47.46666666666667, + "entropy": 0.13221267834305764, + "epoch": 0.2939277712376106, + "eval/gt_acc_batch": 0.7791666805744171, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2826479375362396, + "kd/policy_loss": 0.009280706817905108, + "kd/rkl_advantage_mean": 0.6660173285752535, + "kd/rkl_advantage_p95": 4.80612416168054, + "kd/rkl_advantage_std": 2.183683686455091, + "kd/ssd_loss": 0.0, + "learning_rate": 7.061102039266319e-07, + "loss": 0.037122829755147295, + "num_tokens": 253202255.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7791666666666667, + "rewards/gt_logging_reward/std": 0.4074968273440997, + "sampling/importance_sampling_ratio/max": 1.9393693486849466, + "sampling/importance_sampling_ratio/mean": 1.0003304700056712, + "sampling/importance_sampling_ratio/min": 0.4589948852856954, + "sampling/sampling_logp_difference/max": 0.3769507606824239, + "sampling/sampling_logp_difference/mean": 0.004351368996625146, + "step": 7740, + "step_time": 8.202671994284417, + "student/entropy": 0.13221267834305764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778717378776, + "completions/max_length": 477.0, + "completions/max_terminated_length": 447.76666666666665, + "completions/mean_length": 186.90445556640626, + "completions/mean_terminated_length": 179.49115702311198, + "completions/min_length": 50.06666666666667, + "completions/min_terminated_length": 50.06666666666667, + "entropy": 0.12832566568007073, + "epoch": 0.29506702616488817, + "eval/gt_acc_batch": 0.8000000238418579, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26327311992645264, + "kd/policy_loss": 0.0085234900994692, + "kd/rkl_advantage_mean": 0.7063379841235776, + "kd/rkl_advantage_p95": 4.975066602230072, + "kd/rkl_advantage_std": 2.2228442390759784, + "kd/ssd_loss": 0.0, + "learning_rate": 7.049709489993543e-07, + "loss": 0.034093964099884036, + "num_tokens": 254184943.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7999999980131786, + "rewards/gt_logging_reward/std": 0.38735744506120684, + "sampling/importance_sampling_ratio/max": 1.9168116807937623, + "sampling/importance_sampling_ratio/mean": 0.9992313047250112, + "sampling/importance_sampling_ratio/min": 0.40390019367138547, + "sampling/sampling_logp_difference/max": 0.4154171387354533, + "sampling/sampling_logp_difference/mean": 0.004274530002536873, + "step": 7770, + "step_time": 8.147790236837075, + "student/entropy": 0.12832566568007073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112367361785, + "completions/max_length": 471.23333333333335, + "completions/max_terminated_length": 446.6333333333333, + "completions/mean_length": 189.66389872233074, + "completions/mean_terminated_length": 182.97745513916016, + "completions/min_length": 52.733333333333334, + "completions/min_terminated_length": 52.733333333333334, + "entropy": 0.12946879186977941, + "epoch": 0.29620628109216574, + "eval/gt_acc_batch": 0.7977777977784475, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29174333810806274, + "kd/policy_loss": 0.008949945903926467, + "kd/rkl_advantage_mean": 0.5659873115151034, + "kd/rkl_advantage_p95": 4.618185230096182, + "kd/rkl_advantage_std": 2.1430855482816695, + "kd/ssd_loss": 0.0, + "learning_rate": 7.038316940720769e-07, + "loss": 0.03579978942871094, + "num_tokens": 255173189.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7977777779102325, + "rewards/gt_logging_reward/std": 0.39027885695298514, + "sampling/importance_sampling_ratio/max": 1.884654426574707, + "sampling/importance_sampling_ratio/mean": 0.9992243687311808, + "sampling/importance_sampling_ratio/min": 0.4084655463695526, + "sampling/sampling_logp_difference/max": 0.47365596691767375, + "sampling/sampling_logp_difference/mean": 0.004216033150441945, + "step": 7800, + "step_time": 8.037247585625543, + "student/entropy": 0.12946879186977941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779381722212, + "completions/max_length": 491.43333333333334, + "completions/max_terminated_length": 441.8, + "completions/mean_length": 190.92001088460287, + "completions/mean_terminated_length": 181.79068145751953, + "completions/min_length": 57.333333333333336, + "completions/min_terminated_length": 57.333333333333336, + "entropy": 0.12996432402481636, + "epoch": 0.2973455360194433, + "eval/gt_acc_batch": 0.7794444600741068, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28397703170776367, + "kd/policy_loss": 0.008921714971074834, + "kd/rkl_advantage_mean": 0.6956824646020929, + "kd/rkl_advantage_p95": 4.90191034078598, + "kd/rkl_advantage_std": 2.2564422369003294, + "kd/ssd_loss": 0.0, + "learning_rate": 7.026924391447993e-07, + "loss": 0.03568686644236247, + "num_tokens": 256159637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.779444444179535, + "rewards/gt_logging_reward/std": 0.40902405778566997, + "sampling/importance_sampling_ratio/max": 2.0407096266746523, + "sampling/importance_sampling_ratio/mean": 1.0010959188143411, + "sampling/importance_sampling_ratio/min": 0.42104661067326865, + "sampling/sampling_logp_difference/max": 0.41013426780700685, + "sampling/sampling_logp_difference/mean": 0.0042616487558310235, + "step": 7830, + "step_time": 8.06226682367657, + "student/entropy": 0.12996432402481636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667945683002, + "completions/max_length": 480.6333333333333, + "completions/max_terminated_length": 448.43333333333334, + "completions/mean_length": 184.67528788248697, + "completions/mean_terminated_length": 178.30347239176433, + "completions/min_length": 48.9, + "completions/min_terminated_length": 48.9, + "entropy": 0.12672165986150502, + "epoch": 0.29848479094672087, + "eval/gt_acc_batch": 0.8019444723924001, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3114898204803467, + "kd/policy_loss": 0.008144811327413966, + "kd/rkl_advantage_mean": 0.7411577735717098, + "kd/rkl_advantage_p95": 4.976274504264196, + "kd/rkl_advantage_std": 2.2099788675705594, + "kd/ssd_loss": 0.0, + "learning_rate": 7.015531842175218e-07, + "loss": 0.032579243183135986, + "num_tokens": 257125636.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444425900777, + "rewards/gt_logging_reward/std": 0.3912709911664327, + "sampling/importance_sampling_ratio/max": 1.948303226629893, + "sampling/importance_sampling_ratio/mean": 0.9968331297238667, + "sampling/importance_sampling_ratio/min": 0.42082385619481405, + "sampling/sampling_logp_difference/max": 0.40383689999580386, + "sampling/sampling_logp_difference/mean": 0.004239057330414653, + "step": 7860, + "step_time": 7.923556204411822, + "student/entropy": 0.12672165986150502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013611111727853616, + "completions/max_length": 484.3666666666667, + "completions/max_terminated_length": 455.6, + "completions/mean_length": 187.53667653401692, + "completions/mean_terminated_length": 183.1054453531901, + "completions/min_length": 57.233333333333334, + "completions/min_terminated_length": 57.233333333333334, + "entropy": 0.12915795811762412, + "epoch": 0.2996240458739984, + "eval/gt_acc_batch": 0.7911111275355022, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27148157358169556, + "kd/policy_loss": 0.009231099726942677, + "kd/rkl_advantage_mean": 0.7701348431486016, + "kd/rkl_advantage_p95": 5.098598366975784, + "kd/rkl_advantage_std": 2.2579104880491894, + "kd/ssd_loss": 0.0, + "learning_rate": 7.004139292902441e-07, + "loss": 0.03692439397176107, + "num_tokens": 258096856.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7911111056804657, + "rewards/gt_logging_reward/std": 0.3998505746324857, + "sampling/importance_sampling_ratio/max": 2.0587182799975077, + "sampling/importance_sampling_ratio/mean": 1.000919508934021, + "sampling/importance_sampling_ratio/min": 0.44793639580408734, + "sampling/sampling_logp_difference/max": 0.3850701113541921, + "sampling/sampling_logp_difference/mean": 0.004254226906535526, + "step": 7890, + "step_time": 8.084433331685917, + "student/entropy": 0.12915795811762412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112280438343, + "completions/max_length": 499.46666666666664, + "completions/max_terminated_length": 451.6333333333333, + "completions/mean_length": 188.5541763305664, + "completions/mean_terminated_length": 180.78846740722656, + "completions/min_length": 52.266666666666666, + "completions/min_terminated_length": 52.266666666666666, + "entropy": 0.12820197536299627, + "epoch": 0.30076330080127595, + "eval/gt_acc_batch": 0.8072222272555033, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29439711570739746, + "kd/policy_loss": 0.008424010098678991, + "kd/rkl_advantage_mean": 0.7260900138023619, + "kd/rkl_advantage_p95": 4.9656450311342875, + "kd/rkl_advantage_std": 2.2236838678518933, + "kd/ssd_loss": 0.0, + "learning_rate": 6.992746743629666e-07, + "loss": 0.03369603951772054, + "num_tokens": 259068715.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222232818603, + "rewards/gt_logging_reward/std": 0.385924868285656, + "sampling/importance_sampling_ratio/max": 1.9858551740646362, + "sampling/importance_sampling_ratio/mean": 1.000299878915151, + "sampling/importance_sampling_ratio/min": 0.4306361754735311, + "sampling/sampling_logp_difference/max": 0.3604607343673706, + "sampling/sampling_logp_difference/mean": 0.004226866651636859, + "step": 7920, + "step_time": 8.095093808676271, + "student/entropy": 0.12820197536299627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028611112696429093, + "completions/max_length": 489.53333333333336, + "completions/max_terminated_length": 454.93333333333334, + "completions/mean_length": 189.22445526123047, + "completions/mean_terminated_length": 179.97029520670574, + "completions/min_length": 54.13333333333333, + "completions/min_terminated_length": 54.13333333333333, + "entropy": 0.12671863163510957, + "epoch": 0.3019025557285535, + "eval/gt_acc_batch": 0.7836111426353455, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2785177528858185, + "kd/policy_loss": 0.008541210826175908, + "kd/rkl_advantage_mean": 0.7461035484292855, + "kd/rkl_advantage_p95": 5.028793956836065, + "kd/rkl_advantage_std": 2.2635088751713437, + "kd/ssd_loss": 0.0, + "learning_rate": 6.98135419435689e-07, + "loss": 0.034164841969807944, + "num_tokens": 260060531.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7836111148198446, + "rewards/gt_logging_reward/std": 0.40433193842569987, + "sampling/importance_sampling_ratio/max": 1.9323698242505392, + "sampling/importance_sampling_ratio/mean": 0.9928284029165904, + "sampling/importance_sampling_ratio/min": 0.38991453299919765, + "sampling/sampling_logp_difference/max": 0.4393004635969798, + "sampling/sampling_logp_difference/mean": 0.004181418009102345, + "step": 7950, + "step_time": 8.178795074737476, + "student/entropy": 0.12671863163510957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890414188305, + "completions/max_length": 496.03333333333336, + "completions/max_terminated_length": 455.8, + "completions/mean_length": 187.12501017252603, + "completions/mean_terminated_length": 179.17007904052736, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.13045757524669171, + "epoch": 0.3030418106558311, + "eval/gt_acc_batch": 0.7916666785875957, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3191779851913452, + "kd/policy_loss": 0.008630218651766578, + "kd/rkl_advantage_mean": 0.7810533531631033, + "kd/rkl_advantage_p95": 5.025513519843419, + "kd/rkl_advantage_std": 2.240301220615705, + "kd/ssd_loss": 0.0, + "learning_rate": 6.969961645084114e-07, + "loss": 0.03452087640762329, + "num_tokens": 261035469.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7916666686534881, + "rewards/gt_logging_reward/std": 0.3992473170161247, + "sampling/importance_sampling_ratio/max": 1.8990725914637248, + "sampling/importance_sampling_ratio/mean": 0.9935619751612346, + "sampling/importance_sampling_ratio/min": 0.4321083347002665, + "sampling/sampling_logp_difference/max": 0.3817866325378418, + "sampling/sampling_logp_difference/mean": 0.004284323728643358, + "step": 7980, + "step_time": 8.140066279924941, + "student/entropy": 0.13045757524669171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667765627305, + "completions/max_length": 471.6, + "completions/max_terminated_length": 439.8, + "completions/mean_length": 193.52639821370443, + "completions/mean_terminated_length": 186.6849802652995, + "completions/min_length": 56.8, + "completions/min_terminated_length": 56.8, + "entropy": 0.127671993834277, + "epoch": 0.30418106558310865, + "eval/gt_acc_batch": 0.7986111402511596, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3080930709838867, + "kd/policy_loss": 0.008873973111622035, + "kd/rkl_advantage_mean": 0.6301079919716964, + "kd/rkl_advantage_p95": 4.7425407310326895, + "kd/rkl_advantage_std": 2.172306857506434, + "kd/ssd_loss": 0.0, + "learning_rate": 6.958569095811339e-07, + "loss": 0.03549589316050212, + "num_tokens": 262037884.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111084620158, + "rewards/gt_logging_reward/std": 0.39281623462835946, + "sampling/importance_sampling_ratio/max": 2.023660441239675, + "sampling/importance_sampling_ratio/mean": 0.9990261316299438, + "sampling/importance_sampling_ratio/min": 0.42471478978792826, + "sampling/sampling_logp_difference/max": 0.3664470911026001, + "sampling/sampling_logp_difference/mean": 0.004173655831255019, + "step": 8010, + "step_time": 8.005879651029439, + "student/entropy": 0.127671993834277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334773778915, + "completions/max_length": 494.6, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 190.5358434041341, + "completions/mean_terminated_length": 182.1780990600586, + "completions/min_length": 51.9, + "completions/min_terminated_length": 51.9, + "entropy": 0.13477400820702315, + "epoch": 0.3053203205103862, + "eval/gt_acc_batch": 0.7725000242392223, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2903474271297455, + "kd/policy_loss": 0.00920351585179257, + "kd/rkl_advantage_mean": 0.7555451489984989, + "kd/rkl_advantage_p95": 5.020200232664744, + "kd/rkl_advantage_std": 2.278938070933024, + "kd/ssd_loss": 0.0, + "learning_rate": 6.947176546538564e-07, + "loss": 0.03681406577428182, + "num_tokens": 263041437.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7725000023841858, + "rewards/gt_logging_reward/std": 0.41078338225682576, + "sampling/importance_sampling_ratio/max": 1.9809740344683329, + "sampling/importance_sampling_ratio/mean": 0.996174430847168, + "sampling/importance_sampling_ratio/min": 0.40766132523616155, + "sampling/sampling_logp_difference/max": 0.4593360821406047, + "sampling/sampling_logp_difference/mean": 0.004428394588952263, + "step": 8040, + "step_time": 8.51191537710062, + "student/entropy": 0.13477400820702315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02305555697530508, + "completions/max_length": 495.5, + "completions/max_terminated_length": 455.3666666666667, + "completions/mean_length": 193.46334177652994, + "completions/mean_terminated_length": 185.95989227294922, + "completions/min_length": 59.93333333333333, + "completions/min_terminated_length": 59.93333333333333, + "entropy": 0.12734902128577233, + "epoch": 0.3064595754376638, + "eval/gt_acc_batch": 0.7911111374696096, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22734984755516052, + "kd/policy_loss": 0.008530391209448377, + "kd/rkl_advantage_mean": 0.6589152462780475, + "kd/rkl_advantage_p95": 4.838178829352061, + "kd/rkl_advantage_std": 2.1820967386166257, + "kd/ssd_loss": 0.0, + "learning_rate": 6.935783997265788e-07, + "loss": 0.03412156899770101, + "num_tokens": 264034881.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7911111156145731, + "rewards/gt_logging_reward/std": 0.3981965839862823, + "sampling/importance_sampling_ratio/max": 1.9726892232894897, + "sampling/importance_sampling_ratio/mean": 0.9993841032187144, + "sampling/importance_sampling_ratio/min": 0.45649208426475524, + "sampling/sampling_logp_difference/max": 0.41073160568873085, + "sampling/sampling_logp_difference/mean": 0.004234331636689603, + "step": 8070, + "step_time": 8.1781967502359, + "student/entropy": 0.12734902128577233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02388889013479153, + "completions/max_length": 483.06666666666666, + "completions/max_terminated_length": 456.6, + "completions/mean_length": 192.9100102742513, + "completions/mean_terminated_length": 185.313818359375, + "completions/min_length": 52.233333333333334, + "completions/min_terminated_length": 52.233333333333334, + "entropy": 0.13224350636204082, + "epoch": 0.30759883036494134, + "eval/gt_acc_batch": 0.766111147403717, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.253036230802536, + "kd/policy_loss": 0.009226441546343268, + "kd/rkl_advantage_mean": 0.7030738583455484, + "kd/rkl_advantage_p95": 4.915995558102925, + "kd/rkl_advantage_std": 2.183627490202586, + "kd/ssd_loss": 0.0, + "learning_rate": 6.924391447993012e-07, + "loss": 0.036905765533447266, + "num_tokens": 265032901.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7661111136277516, + "rewards/gt_logging_reward/std": 0.416210866967837, + "sampling/importance_sampling_ratio/max": 2.024342075983683, + "sampling/importance_sampling_ratio/mean": 1.0014472345511118, + "sampling/importance_sampling_ratio/min": 0.3877014418443044, + "sampling/sampling_logp_difference/max": 0.4013754228750865, + "sampling/sampling_logp_difference/mean": 0.004328109960382184, + "step": 8100, + "step_time": 8.255580194457433, + "student/entropy": 0.13224350636204082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028611112603296837, + "completions/max_length": 498.96666666666664, + "completions/max_terminated_length": 464.6333333333333, + "completions/mean_length": 191.4966781616211, + "completions/mean_terminated_length": 182.22972310384114, + "completions/min_length": 56.766666666666666, + "completions/min_terminated_length": 56.766666666666666, + "entropy": 0.13312652961661417, + "epoch": 0.3087380852922189, + "eval/gt_acc_batch": 0.7669444719950358, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2479233592748642, + "kd/policy_loss": 0.0094405393251994, + "kd/rkl_advantage_mean": 0.7439281607667605, + "kd/rkl_advantage_p95": 5.002383544047674, + "kd/rkl_advantage_std": 2.2426028827826183, + "kd/ssd_loss": 0.0, + "learning_rate": 6.912998898720237e-07, + "loss": 0.037762161095937094, + "num_tokens": 266022913.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7669444421927134, + "rewards/gt_logging_reward/std": 0.41500459412733715, + "sampling/importance_sampling_ratio/max": 1.9273448864618936, + "sampling/importance_sampling_ratio/mean": 0.9959462583065033, + "sampling/importance_sampling_ratio/min": 0.40525284508864085, + "sampling/sampling_logp_difference/max": 0.43919384280840557, + "sampling/sampling_logp_difference/mean": 0.004366057319566607, + "step": 8130, + "step_time": 8.405102116540851, + "student/entropy": 0.13312652961661417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026666668088485796, + "completions/max_length": 498.1666666666667, + "completions/max_terminated_length": 461.03333333333336, + "completions/mean_length": 191.2344538370768, + "completions/mean_terminated_length": 182.50704905192057, + "completions/min_length": 51.233333333333334, + "completions/min_terminated_length": 51.233333333333334, + "entropy": 0.13743435287227232, + "epoch": 0.3098773402194964, + "eval/gt_acc_batch": 0.7830555617809296, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2827863097190857, + "kd/policy_loss": 0.009014608097883563, + "kd/rkl_advantage_mean": 0.7831722415750846, + "kd/rkl_advantage_p95": 5.051604572931925, + "kd/rkl_advantage_std": 2.241032110651334, + "kd/ssd_loss": 0.0, + "learning_rate": 6.90160634944746e-07, + "loss": 0.0360584298769633, + "num_tokens": 267009469.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.783055555820465, + "rewards/gt_logging_reward/std": 0.4067437688509623, + "sampling/importance_sampling_ratio/max": 1.912741482257843, + "sampling/importance_sampling_ratio/mean": 0.9999624331792195, + "sampling/importance_sampling_ratio/min": 0.4051282371083895, + "sampling/sampling_logp_difference/max": 0.3907126565774282, + "sampling/sampling_logp_difference/mean": 0.0043076858234902225, + "step": 8160, + "step_time": 8.202666431777955, + "student/entropy": 0.13743435287227232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001297642788, + "completions/max_length": 493.6333333333333, + "completions/max_terminated_length": 450.9, + "completions/mean_length": 189.31334482828777, + "completions/mean_terminated_length": 181.97473551432293, + "completions/min_length": 54.666666666666664, + "completions/min_terminated_length": 54.666666666666664, + "entropy": 0.1276719604929288, + "epoch": 0.311016595146774, + "eval/gt_acc_batch": 0.7936111311117808, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2598976790904999, + "kd/policy_loss": 0.00841502429684624, + "kd/rkl_advantage_mean": 0.6778800476963321, + "kd/rkl_advantage_p95": 4.9318534553051, + "kd/rkl_advantage_std": 2.234111000100772, + "kd/ssd_loss": 0.0, + "learning_rate": 6.890213800174686e-07, + "loss": 0.03366009394327799, + "num_tokens": 268002389.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111052831014, + "rewards/gt_logging_reward/std": 0.396973118185997, + "sampling/importance_sampling_ratio/max": 1.9005735874176026, + "sampling/importance_sampling_ratio/mean": 0.9972155114014943, + "sampling/importance_sampling_ratio/min": 0.45818411409854887, + "sampling/sampling_logp_difference/max": 0.39282062848409016, + "sampling/sampling_logp_difference/mean": 0.004195075559740265, + "step": 8190, + "step_time": 8.307744815022064, + "student/entropy": 0.1276719604929288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055556956678628, + "completions/max_length": 505.3333333333333, + "completions/max_terminated_length": 459.23333333333335, + "completions/mean_length": 193.1975102742513, + "completions/mean_terminated_length": 184.11436716715494, + "completions/min_length": 61.4, + "completions/min_terminated_length": 61.4, + "entropy": 0.12783271130174398, + "epoch": 0.31215585007405156, + "eval/gt_acc_batch": 0.7933333535989125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2338627725839615, + "kd/policy_loss": 0.008694928539140771, + "kd/rkl_advantage_mean": 0.7189088577792669, + "kd/rkl_advantage_p95": 4.94693039059639, + "kd/rkl_advantage_std": 2.213687347372373, + "kd/ssd_loss": 0.0, + "learning_rate": 6.87882125090191e-07, + "loss": 0.034779715538024905, + "num_tokens": 269014724.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7933333317438761, + "rewards/gt_logging_reward/std": 0.3931067168712616, + "sampling/importance_sampling_ratio/max": 1.988636883099874, + "sampling/importance_sampling_ratio/mean": 1.0064445932706196, + "sampling/importance_sampling_ratio/min": 0.39764509797872355, + "sampling/sampling_logp_difference/max": 0.5656359553337097, + "sampling/sampling_logp_difference/mean": 0.004217798379249871, + "step": 8220, + "step_time": 8.490134261389418, + "student/entropy": 0.12783271130174398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223449498416, + "completions/max_length": 478.6, + "completions/max_terminated_length": 444.26666666666665, + "completions/mean_length": 189.33028666178384, + "completions/mean_terminated_length": 181.47399495442707, + "completions/min_length": 60.53333333333333, + "completions/min_terminated_length": 60.53333333333333, + "entropy": 0.1246159682671229, + "epoch": 0.3132951050013291, + "eval/gt_acc_batch": 0.7883333543936412, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30631735920906067, + "kd/policy_loss": 0.008417430040814603, + "kd/rkl_advantage_mean": 0.6260349647918095, + "kd/rkl_advantage_p95": 4.71088000535965, + "kd/rkl_advantage_std": 2.151095740000407, + "kd/ssd_loss": 0.0, + "learning_rate": 6.867428701629134e-07, + "loss": 0.033669718106587726, + "num_tokens": 269994065.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7883333365122477, + "rewards/gt_logging_reward/std": 0.3969871600468953, + "sampling/importance_sampling_ratio/max": 1.9124387184778848, + "sampling/importance_sampling_ratio/mean": 1.001412703593572, + "sampling/importance_sampling_ratio/min": 0.44709332287311554, + "sampling/sampling_logp_difference/max": 0.4296522696812948, + "sampling/sampling_logp_difference/mean": 0.004147849534638226, + "step": 8250, + "step_time": 8.130510303401389, + "student/entropy": 0.1246159682671229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778996775546, + "completions/max_length": 482.4, + "completions/max_terminated_length": 436.8666666666667, + "completions/mean_length": 186.02223205566406, + "completions/mean_terminated_length": 178.4942189534505, + "completions/min_length": 44.93333333333333, + "completions/min_terminated_length": 44.93333333333333, + "entropy": 0.13258436204244692, + "epoch": 0.3144343599286067, + "eval/gt_acc_batch": 0.7988889078299205, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2419353723526001, + "kd/policy_loss": 0.008726734506975238, + "kd/rkl_advantage_mean": 0.7162716191262006, + "kd/rkl_advantage_p95": 4.933476175864538, + "kd/rkl_advantage_std": 2.2069155822197595, + "kd/ssd_loss": 0.0, + "learning_rate": 6.856036152356358e-07, + "loss": 0.034906939665476484, + "num_tokens": 270963393.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.79888889392217, + "rewards/gt_logging_reward/std": 0.3957697411378225, + "sampling/importance_sampling_ratio/max": 2.057222068309784, + "sampling/importance_sampling_ratio/mean": 0.9967463751633961, + "sampling/importance_sampling_ratio/min": 0.41508087317148845, + "sampling/sampling_logp_difference/max": 0.39457394580046334, + "sampling/sampling_logp_difference/mean": 0.004321820506205162, + "step": 8280, + "step_time": 8.14594660057531, + "student/entropy": 0.13258436204244692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029166668467223645, + "completions/max_length": 501.23333333333335, + "completions/max_terminated_length": 455.26666666666665, + "completions/mean_length": 190.74556579589844, + "completions/mean_terminated_length": 181.19400126139323, + "completions/min_length": 48.2, + "completions/min_terminated_length": 48.2, + "entropy": 0.13169103444864352, + "epoch": 0.31557361485588425, + "eval/gt_acc_batch": 0.7838889042536418, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2878350019454956, + "kd/policy_loss": 0.008935515367193147, + "kd/rkl_advantage_mean": 0.7181084876259168, + "kd/rkl_advantage_p95": 4.957957132657369, + "kd/rkl_advantage_std": 2.2619640678167343, + "kd/ssd_loss": 0.0, + "learning_rate": 6.844643603083583e-07, + "loss": 0.03574206034342448, + "num_tokens": 271951997.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7838888883590698, + "rewards/gt_logging_reward/std": 0.40466095209121705, + "sampling/importance_sampling_ratio/max": 1.9629389286041259, + "sampling/importance_sampling_ratio/mean": 0.9942821621894836, + "sampling/importance_sampling_ratio/min": 0.41048687100410464, + "sampling/sampling_logp_difference/max": 0.36530070304870604, + "sampling/sampling_logp_difference/mean": 0.004328181517, + "step": 8310, + "step_time": 8.415632698615081, + "student/entropy": 0.13169103444864352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016111112168679633, + "completions/max_length": 471.56666666666666, + "completions/max_terminated_length": 443.03333333333336, + "completions/mean_length": 182.46834259033204, + "completions/mean_terminated_length": 177.42541961669923, + "completions/min_length": 50.93333333333333, + "completions/min_terminated_length": 50.93333333333333, + "entropy": 0.1259428852548202, + "epoch": 0.3167128697831618, + "eval/gt_acc_batch": 0.8177778005599976, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27428439259529114, + "kd/policy_loss": 0.00825489557852658, + "kd/rkl_advantage_mean": 0.6506200798631956, + "kd/rkl_advantage_p95": 4.795858526229859, + "kd/rkl_advantage_std": 2.182651687661807, + "kd/ssd_loss": 0.0, + "learning_rate": 6.833251053810808e-07, + "loss": 0.03301958640416463, + "num_tokens": 272910931.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8177777767181397, + "rewards/gt_logging_reward/std": 0.37430750479300817, + "sampling/importance_sampling_ratio/max": 1.9026568015416463, + "sampling/importance_sampling_ratio/mean": 0.9962528447310129, + "sampling/importance_sampling_ratio/min": 0.45954305281241736, + "sampling/sampling_logp_difference/max": 0.37396459182103475, + "sampling/sampling_logp_difference/mean": 0.0041381387660900755, + "step": 8340, + "step_time": 8.14997990312598, + "student/entropy": 0.1259428852548202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668020188808, + "completions/max_length": 501.26666666666665, + "completions/max_terminated_length": 457.73333333333335, + "completions/mean_length": 188.49000854492186, + "completions/mean_terminated_length": 180.44159138997395, + "completions/min_length": 60.333333333333336, + "completions/min_terminated_length": 60.333333333333336, + "entropy": 0.12785292336096366, + "epoch": 0.3178521247104394, + "eval/gt_acc_batch": 0.8016666869322459, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1784721463918686, + "kd/policy_loss": 0.00863059035424764, + "kd/rkl_advantage_mean": 0.6254451589658856, + "kd/rkl_advantage_p95": 4.760281370083491, + "kd/rkl_advantage_std": 2.1916444500287375, + "kd/ssd_loss": 0.0, + "learning_rate": 6.821858504538031e-07, + "loss": 0.034522358576456705, + "num_tokens": 273889695.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8016666690508525, + "rewards/gt_logging_reward/std": 0.3935589780410131, + "sampling/importance_sampling_ratio/max": 1.9925379077593486, + "sampling/importance_sampling_ratio/mean": 0.997292673587799, + "sampling/importance_sampling_ratio/min": 0.4320015569527944, + "sampling/sampling_logp_difference/max": 0.38558813333511355, + "sampling/sampling_logp_difference/mean": 0.0041940762506177025, + "step": 8370, + "step_time": 8.192613052867818, + "student/entropy": 0.12785292336096366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333482965827, + "completions/max_length": 462.03333333333336, + "completions/max_terminated_length": 428.6333333333333, + "completions/mean_length": 178.7136210123698, + "completions/mean_terminated_length": 170.9363260904948, + "completions/min_length": 50.666666666666664, + "completions/min_terminated_length": 50.666666666666664, + "entropy": 0.12815146911889314, + "epoch": 0.31899137963771695, + "eval/gt_acc_batch": 0.8200000206629435, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30205821990966797, + "kd/policy_loss": 0.00842558623990044, + "kd/rkl_advantage_mean": 0.6699591166650255, + "kd/rkl_advantage_p95": 4.9575323839982355, + "kd/rkl_advantage_std": 2.2477012405792873, + "kd/ssd_loss": 0.0, + "learning_rate": 6.810465955265257e-07, + "loss": 0.03370234966278076, + "num_tokens": 274848720.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.82000000278155, + "rewards/gt_logging_reward/std": 0.37882622281710304, + "sampling/importance_sampling_ratio/max": 1.8505788127581277, + "sampling/importance_sampling_ratio/mean": 0.9951942483584086, + "sampling/importance_sampling_ratio/min": 0.43621045102675754, + "sampling/sampling_logp_difference/max": 0.4792909324169159, + "sampling/sampling_logp_difference/mean": 0.004283950167397658, + "step": 8400, + "step_time": 8.25512477181231, + "student/entropy": 0.12815146911889314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02000000113621354, + "completions/max_length": 488.8666666666667, + "completions/max_terminated_length": 444.43333333333334, + "completions/mean_length": 185.8372319539388, + "completions/mean_terminated_length": 179.2657435099284, + "completions/min_length": 55.96666666666667, + "completions/min_terminated_length": 55.96666666666667, + "entropy": 0.12987772847215334, + "epoch": 0.3201306345649945, + "eval/gt_acc_batch": 0.798333348830541, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3064788579940796, + "kd/policy_loss": 0.008578827158392718, + "kd/rkl_advantage_mean": 0.6666143011301756, + "kd/rkl_advantage_p95": 4.855276914437612, + "kd/rkl_advantage_std": 2.1976909418900807, + "kd/ssd_loss": 0.0, + "learning_rate": 6.799073405992481e-07, + "loss": 0.034315311908721925, + "num_tokens": 275818270.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7983333349227906, + "rewards/gt_logging_reward/std": 0.39602545102437336, + "sampling/importance_sampling_ratio/max": 1.9320125261942545, + "sampling/importance_sampling_ratio/mean": 0.9922675212224324, + "sampling/importance_sampling_ratio/min": 0.39782090733448666, + "sampling/sampling_logp_difference/max": 0.38894654512405397, + "sampling/sampling_logp_difference/mean": 0.004279561868558327, + "step": 8430, + "step_time": 8.026568830866987, + "student/entropy": 0.12987772847215334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223641971747, + "completions/max_length": 489.46666666666664, + "completions/max_terminated_length": 455.46666666666664, + "completions/mean_length": 194.21278737386066, + "completions/mean_terminated_length": 185.42237904866536, + "completions/min_length": 59.86666666666667, + "completions/min_terminated_length": 59.86666666666667, + "entropy": 0.1275697650387883, + "epoch": 0.32126988949227203, + "eval/gt_acc_batch": 0.7855555733044942, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2833205759525299, + "kd/policy_loss": 0.00856663526695532, + "kd/rkl_advantage_mean": 0.652627089060843, + "kd/rkl_advantage_p95": 4.738772612810135, + "kd/rkl_advantage_std": 2.128523947795232, + "kd/ssd_loss": 0.0, + "learning_rate": 6.787680856719705e-07, + "loss": 0.03426653544108073, + "num_tokens": 276813228.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555514494578, + "rewards/gt_logging_reward/std": 0.4048611670732498, + "sampling/importance_sampling_ratio/max": 1.9869576573371888, + "sampling/importance_sampling_ratio/mean": 0.99285413424174, + "sampling/importance_sampling_ratio/min": 0.3901004955172539, + "sampling/sampling_logp_difference/max": 0.388815446694692, + "sampling/sampling_logp_difference/mean": 0.0041831432453667125, + "step": 8460, + "step_time": 8.019526541062321, + "student/entropy": 0.1275697650387883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026666668119529884, + "completions/max_length": 466.6666666666667, + "completions/max_terminated_length": 439.3666666666667, + "completions/mean_length": 182.00362040201824, + "completions/mean_terminated_length": 173.09803822835286, + "completions/min_length": 53.43333333333333, + "completions/min_terminated_length": 53.43333333333333, + "entropy": 0.12936283126473427, + "epoch": 0.3224091444195496, + "eval/gt_acc_batch": 0.8183333476384481, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2698460817337036, + "kd/policy_loss": 0.008577545986433203, + "kd/rkl_advantage_mean": 0.6268694808522317, + "kd/rkl_advantage_p95": 4.743229977289835, + "kd/rkl_advantage_std": 2.172880221406619, + "kd/ssd_loss": 0.0, + "learning_rate": 6.776288307446929e-07, + "loss": 0.03431018590927124, + "num_tokens": 277770489.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8183333337306976, + "rewards/gt_logging_reward/std": 0.37817918409903845, + "sampling/importance_sampling_ratio/max": 1.9058789451917013, + "sampling/importance_sampling_ratio/mean": 0.9927881062030792, + "sampling/importance_sampling_ratio/min": 0.44554262856642407, + "sampling/sampling_logp_difference/max": 0.4148217936356862, + "sampling/sampling_logp_difference/mean": 0.004285782909331222, + "step": 8490, + "step_time": 7.914119776672063, + "student/entropy": 0.12936283126473427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223511586587, + "completions/max_length": 501.93333333333334, + "completions/max_terminated_length": 468.3666666666667, + "completions/mean_length": 192.94862111409506, + "completions/mean_terminated_length": 184.9907480875651, + "completions/min_length": 53.63333333333333, + "completions/min_terminated_length": 53.63333333333333, + "entropy": 0.12570547225574652, + "epoch": 0.32354839934682716, + "eval/gt_acc_batch": 0.7875000139077505, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24622979760169983, + "kd/policy_loss": 0.00862590159716395, + "kd/rkl_advantage_mean": 0.7666273268638178, + "kd/rkl_advantage_p95": 5.038853277762731, + "kd/rkl_advantage_std": 2.236299588282903, + "kd/ssd_loss": 0.0, + "learning_rate": 6.764895758174154e-07, + "loss": 0.03450361092885335, + "num_tokens": 278771800.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.787500003973643, + "rewards/gt_logging_reward/std": 0.40542584160963696, + "sampling/importance_sampling_ratio/max": 1.978078297773997, + "sampling/importance_sampling_ratio/mean": 1.0015506168206534, + "sampling/importance_sampling_ratio/min": 0.4456667939821879, + "sampling/sampling_logp_difference/max": 0.36251864234606423, + "sampling/sampling_logp_difference/mean": 0.00413245406622688, + "step": 8520, + "step_time": 8.29042280811506, + "student/entropy": 0.12570547225574652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444558893641, + "completions/max_length": 482.93333333333334, + "completions/max_terminated_length": 444.9, + "completions/mean_length": 187.05084279378255, + "completions/mean_terminated_length": 179.85850830078124, + "completions/min_length": 49.733333333333334, + "completions/min_terminated_length": 49.733333333333334, + "entropy": 0.12555604142447313, + "epoch": 0.32468765427410473, + "eval/gt_acc_batch": 0.7886111319065094, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2899806797504425, + "kd/policy_loss": 0.008494836480046312, + "kd/rkl_advantage_mean": 0.710896366473753, + "kd/rkl_advantage_p95": 4.920590837796529, + "kd/rkl_advantage_std": 2.2340145309766135, + "kd/ssd_loss": 0.0, + "learning_rate": 6.753503208901377e-07, + "loss": 0.03397934436798096, + "num_tokens": 279745247.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7886111080646515, + "rewards/gt_logging_reward/std": 0.4017337441444397, + "sampling/importance_sampling_ratio/max": 1.8342518091201783, + "sampling/importance_sampling_ratio/mean": 0.9968911170959472, + "sampling/importance_sampling_ratio/min": 0.4346864109237989, + "sampling/sampling_logp_difference/max": 0.3759603301684062, + "sampling/sampling_logp_difference/mean": 0.004122713720425964, + "step": 8550, + "step_time": 8.048140358917104, + "student/entropy": 0.12555604142447313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500001285225154, + "completions/max_length": 475.3, + "completions/max_terminated_length": 444.43333333333334, + "completions/mean_length": 181.6952880859375, + "completions/mean_terminated_length": 175.9866185506185, + "completions/min_length": 46.96666666666667, + "completions/min_terminated_length": 46.96666666666667, + "entropy": 0.1266918442522486, + "epoch": 0.3258269092013823, + "eval/gt_acc_batch": 0.8202778061230978, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2735084891319275, + "kd/policy_loss": 0.00836192957746486, + "kd/rkl_advantage_mean": 0.6579931124734382, + "kd/rkl_advantage_p95": 4.769901090860367, + "kd/rkl_advantage_std": 2.19208063185215, + "kd/ssd_loss": 0.0, + "learning_rate": 6.742110659628603e-07, + "loss": 0.03344771862030029, + "num_tokens": 280705534.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8202777783075968, + "rewards/gt_logging_reward/std": 0.37307341198126476, + "sampling/importance_sampling_ratio/max": 1.9243560274442038, + "sampling/importance_sampling_ratio/mean": 1.002505115667979, + "sampling/importance_sampling_ratio/min": 0.43364440302054086, + "sampling/sampling_logp_difference/max": 0.4582654396692912, + "sampling/sampling_logp_difference/mean": 0.0041228556695083775, + "step": 8580, + "step_time": 7.901649216030879, + "student/entropy": 0.1266918442522486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668051232895, + "completions/max_length": 500.7, + "completions/max_terminated_length": 460.56666666666666, + "completions/mean_length": 188.99028727213542, + "completions/mean_terminated_length": 181.12115681966145, + "completions/min_length": 53.733333333333334, + "completions/min_terminated_length": 53.733333333333334, + "entropy": 0.1317533024897178, + "epoch": 0.32696616412865986, + "eval/gt_acc_batch": 0.7955555776755016, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34527796506881714, + "kd/policy_loss": 0.008734636439476163, + "kd/rkl_advantage_mean": 0.6765692373427252, + "kd/rkl_advantage_p95": 4.828026390075683, + "kd/rkl_advantage_std": 2.2019779562950133, + "kd/ssd_loss": 0.0, + "learning_rate": 6.730718110355828e-07, + "loss": 0.034938542048136394, + "num_tokens": 281693611.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555578072866, + "rewards/gt_logging_reward/std": 0.39107387165228524, + "sampling/importance_sampling_ratio/max": 2.0069327394167584, + "sampling/importance_sampling_ratio/mean": 1.0024435599644979, + "sampling/importance_sampling_ratio/min": 0.3937139679988225, + "sampling/sampling_logp_difference/max": 0.39501031835873923, + "sampling/sampling_logp_difference/mean": 0.00435396859732767, + "step": 8610, + "step_time": 8.335681490010272, + "student/entropy": 0.1317533024897178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001396983863, + "completions/max_length": 483.23333333333335, + "completions/max_terminated_length": 437.3666666666667, + "completions/mean_length": 184.03306528727214, + "completions/mean_terminated_length": 175.75660909016926, + "completions/min_length": 53.46666666666667, + "completions/min_terminated_length": 53.46666666666667, + "entropy": 0.12758857781688374, + "epoch": 0.32810541905593743, + "eval/gt_acc_batch": 0.8061111251513163, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31180712580680847, + "kd/policy_loss": 0.008596957839714984, + "kd/rkl_advantage_mean": 0.6594313100601236, + "kd/rkl_advantage_p95": 4.750219132502874, + "kd/rkl_advantage_std": 2.1639523377021153, + "kd/ssd_loss": 0.0, + "learning_rate": 6.719325561083051e-07, + "loss": 0.03438783089319865, + "num_tokens": 282650266.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8061111132303874, + "rewards/gt_logging_reward/std": 0.38705718914667764, + "sampling/importance_sampling_ratio/max": 1.9615538040796916, + "sampling/importance_sampling_ratio/mean": 1.0011113663514455, + "sampling/importance_sampling_ratio/min": 0.4335600634415944, + "sampling/sampling_logp_difference/max": 0.42324379086494446, + "sampling/sampling_logp_difference/mean": 0.004143522493541241, + "step": 8640, + "step_time": 7.904211141654135, + "student/entropy": 0.12758857781688374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01465517327446362, + "completions/max_length": 475.7241379310345, + "completions/max_terminated_length": 443.2413793103448, + "completions/mean_length": 185.0603537723936, + "completions/mean_terminated_length": 180.2525661073882, + "completions/min_length": 56.206896551724135, + "completions/min_terminated_length": 56.206896551724135, + "entropy": 0.12215300488831668, + "epoch": 0.329244673983215, + "eval/gt_acc_batch": 0.815517261110503, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2989735007286072, + "kd/policy_loss": 0.008225711560326403, + "kd/rkl_advantage_mean": 0.6570834801773188, + "kd/rkl_advantage_p95": 4.815799877561372, + "kd/rkl_advantage_std": 2.185256608601274, + "kd/ssd_loss": 0.0, + "learning_rate": 6.707933011810276e-07, + "loss": 0.03180609146753947, + "num_tokens": 283587804.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8155172364465122, + "rewards/gt_logging_reward/std": 0.38034029202214603, + "sampling/importance_sampling_ratio/max": 1.9751177697346127, + "sampling/importance_sampling_ratio/mean": 1.0028353682879745, + "sampling/importance_sampling_ratio/min": 0.44089852427614146, + "sampling/sampling_logp_difference/max": 0.3634093177729639, + "sampling/sampling_logp_difference/mean": 0.004027233980917211, + "step": 8670, + "step_time": 7.733067771679877, + "student/entropy": 0.12215300488831668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016944445421298346, + "completions/max_length": 488.1333333333333, + "completions/max_terminated_length": 446.8, + "completions/mean_length": 185.5886210123698, + "completions/mean_terminated_length": 179.977663675944, + "completions/min_length": 53.3, + "completions/min_terminated_length": 53.3, + "entropy": 0.12636293284595013, + "epoch": 0.33038392891049256, + "eval/gt_acc_batch": 0.8047222395737966, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28756144642829895, + "kd/policy_loss": 0.007992266789854815, + "kd/rkl_advantage_mean": 0.7485401768547794, + "kd/rkl_advantage_p95": 5.063990251223246, + "kd/rkl_advantage_std": 2.2321324408054353, + "kd/ssd_loss": 0.0, + "learning_rate": 6.6965404625375e-07, + "loss": 0.03196906646092733, + "num_tokens": 284554987.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222177187602, + "rewards/gt_logging_reward/std": 0.38999810020128883, + "sampling/importance_sampling_ratio/max": 1.8561342517534891, + "sampling/importance_sampling_ratio/mean": 0.9944567700227102, + "sampling/importance_sampling_ratio/min": 0.4003780464331309, + "sampling/sampling_logp_difference/max": 0.3593200186888377, + "sampling/sampling_logp_difference/mean": 0.004189534190421303, + "step": 8700, + "step_time": 8.081669338210485, + "student/entropy": 0.12636293284595013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778965731462, + "completions/max_length": 483.8666666666667, + "completions/max_terminated_length": 446.7, + "completions/mean_length": 189.24501037597656, + "completions/mean_terminated_length": 181.92221425374348, + "completions/min_length": 62.2, + "completions/min_terminated_length": 62.2, + "entropy": 0.12661002191404502, + "epoch": 0.3315231838377701, + "eval/gt_acc_batch": 0.7969444731871287, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33669647574424744, + "kd/policy_loss": 0.008446681421870987, + "kd/rkl_advantage_mean": 0.6703882876162728, + "kd/rkl_advantage_p95": 4.841607226928075, + "kd/rkl_advantage_std": 2.1675153464078902, + "kd/ssd_loss": 0.0, + "learning_rate": 6.685147913264724e-07, + "loss": 0.03378672997156779, + "num_tokens": 285539877.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444473584493, + "rewards/gt_logging_reward/std": 0.39350444823503494, + "sampling/importance_sampling_ratio/max": 1.8871628085772196, + "sampling/importance_sampling_ratio/mean": 1.0006260097026825, + "sampling/importance_sampling_ratio/min": 0.440630312760671, + "sampling/sampling_logp_difference/max": 0.341574572523435, + "sampling/sampling_logp_difference/mean": 0.004156845008643965, + "step": 8730, + "step_time": 8.06389281974795, + "student/entropy": 0.12661002191404502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013333334183941285, + "completions/max_length": 456.1666666666667, + "completions/max_terminated_length": 415.96666666666664, + "completions/mean_length": 177.18889973958332, + "completions/mean_terminated_length": 172.8654551188151, + "completions/min_length": 58.8, + "completions/min_terminated_length": 58.8, + "entropy": 0.12926998740683, + "epoch": 0.33266243876504764, + "eval/gt_acc_batch": 0.8202778021494548, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2306824028491974, + "kd/policy_loss": 0.008299718197667972, + "kd/rkl_advantage_mean": 0.5694848079505997, + "kd/rkl_advantage_p95": 4.630170973141988, + "kd/rkl_advantage_std": 2.1634669651587806, + "kd/ssd_loss": 0.0, + "learning_rate": 6.673755363991948e-07, + "loss": 0.03319887121518453, + "num_tokens": 286475157.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8202777723471324, + "rewards/gt_logging_reward/std": 0.3751339892546336, + "sampling/importance_sampling_ratio/max": 1.8881141583124796, + "sampling/importance_sampling_ratio/mean": 0.99896293481191, + "sampling/importance_sampling_ratio/min": 0.4474107414484024, + "sampling/sampling_logp_difference/max": 0.35313992500305175, + "sampling/sampling_logp_difference/mean": 0.004293617179306845, + "step": 8760, + "step_time": 7.851646132945704, + "student/entropy": 0.12926998740683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223641971747, + "completions/max_length": 482.23333333333335, + "completions/max_terminated_length": 451.1666666666667, + "completions/mean_length": 188.8858444213867, + "completions/mean_terminated_length": 180.0051254272461, + "completions/min_length": 56.266666666666666, + "completions/min_terminated_length": 56.266666666666666, + "entropy": 0.12959283174326022, + "epoch": 0.3338016936923252, + "eval/gt_acc_batch": 0.7905555764834086, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2782401442527771, + "kd/policy_loss": 0.00859983594273217, + "kd/rkl_advantage_mean": 0.6542566049844026, + "kd/rkl_advantage_p95": 4.800155123074849, + "kd/rkl_advantage_std": 2.1831458896398543, + "kd/ssd_loss": 0.0, + "learning_rate": 6.662362814719174e-07, + "loss": 0.03439934253692627, + "num_tokens": 287456706.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7905555526415508, + "rewards/gt_logging_reward/std": 0.39903866350650785, + "sampling/importance_sampling_ratio/max": 2.022561824321747, + "sampling/importance_sampling_ratio/mean": 0.9967817405859629, + "sampling/importance_sampling_ratio/min": 0.3830740670363108, + "sampling/sampling_logp_difference/max": 0.45671876668930056, + "sampling/sampling_logp_difference/mean": 0.004218928104576965, + "step": 8790, + "step_time": 8.139063032270254, + "student/entropy": 0.12959283174326022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02416666829958558, + "completions/max_length": 480.8333333333333, + "completions/max_terminated_length": 447.3333333333333, + "completions/mean_length": 189.6797317504883, + "completions/mean_terminated_length": 181.63480072021486, + "completions/min_length": 62.06666666666667, + "completions/min_terminated_length": 62.06666666666667, + "entropy": 0.12951488538334768, + "epoch": 0.3349409486196028, + "eval/gt_acc_batch": 0.7869444668293, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3576202392578125, + "kd/policy_loss": 0.008692857139976695, + "kd/rkl_advantage_mean": 0.7121804726426489, + "kd/rkl_advantage_p95": 4.885150104761124, + "kd/rkl_advantage_std": 2.1809640884399415, + "kd/ssd_loss": 0.0, + "learning_rate": 6.650970265446398e-07, + "loss": 0.034771426518758135, + "num_tokens": 288441561.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444449742635, + "rewards/gt_logging_reward/std": 0.39771625846624375, + "sampling/importance_sampling_ratio/max": 1.886244567235311, + "sampling/importance_sampling_ratio/mean": 0.9938910722732544, + "sampling/importance_sampling_ratio/min": 0.4463439474503199, + "sampling/sampling_logp_difference/max": 0.37362276315689086, + "sampling/sampling_logp_difference/mean": 0.004255933687090874, + "step": 8820, + "step_time": 8.231736096103365, + "student/entropy": 0.12951488538334768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03000000159566601, + "completions/max_length": 502.5, + "completions/max_terminated_length": 455.06666666666666, + "completions/mean_length": 192.93750966389973, + "completions/mean_terminated_length": 183.137717183431, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.12689653125902017, + "epoch": 0.33608020354688034, + "eval/gt_acc_batch": 0.7866666932900747, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22728019952774048, + "kd/policy_loss": 0.008711807429790497, + "kd/rkl_advantage_mean": 0.7152943244514366, + "kd/rkl_advantage_p95": 4.9652286489804585, + "kd/rkl_advantage_std": 2.230519098043442, + "kd/ssd_loss": 0.0, + "learning_rate": 6.639577716173622e-07, + "loss": 0.034847227732340495, + "num_tokens": 289446464.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666674613952, + "rewards/gt_logging_reward/std": 0.3998132675886154, + "sampling/importance_sampling_ratio/max": 1.9588260610898336, + "sampling/importance_sampling_ratio/mean": 0.9983718554178874, + "sampling/importance_sampling_ratio/min": 0.42282937467098236, + "sampling/sampling_logp_difference/max": 0.3648748795191447, + "sampling/sampling_logp_difference/mean": 0.004169857882273694, + "step": 8850, + "step_time": 8.323017490162359, + "student/entropy": 0.12689653125902017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890389353038, + "completions/max_length": 484.3666666666667, + "completions/max_terminated_length": 455.56666666666666, + "completions/mean_length": 191.95612131754558, + "completions/mean_terminated_length": 183.4303009033203, + "completions/min_length": 59.266666666666666, + "completions/min_terminated_length": 59.266666666666666, + "entropy": 0.1319762883707881, + "epoch": 0.3372194584741579, + "eval/gt_acc_batch": 0.7763889173666636, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3590720295906067, + "kd/policy_loss": 0.008954666772236426, + "kd/rkl_advantage_mean": 0.7488782301545143, + "kd/rkl_advantage_p95": 4.980200443665186, + "kd/rkl_advantage_std": 2.2150216897328696, + "kd/ssd_loss": 0.0, + "learning_rate": 6.628185166900847e-07, + "loss": 0.03581867218017578, + "num_tokens": 290440786.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7763888855775197, + "rewards/gt_logging_reward/std": 0.4092646916707357, + "sampling/importance_sampling_ratio/max": 1.897650178273519, + "sampling/importance_sampling_ratio/mean": 0.9989227732022603, + "sampling/importance_sampling_ratio/min": 0.40374892999728523, + "sampling/sampling_logp_difference/max": 0.3965316285689672, + "sampling/sampling_logp_difference/mean": 0.004295167629607022, + "step": 8880, + "step_time": 8.243943615832055, + "student/entropy": 0.1319762883707881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445713112753, + "completions/max_length": 488.06666666666666, + "completions/max_terminated_length": 459.8666666666667, + "completions/mean_length": 190.010009765625, + "completions/mean_terminated_length": 182.87838541666667, + "completions/min_length": 56.46666666666667, + "completions/min_terminated_length": 56.46666666666667, + "entropy": 0.1271217186624805, + "epoch": 0.3383587134014355, + "eval/gt_acc_batch": 0.8075000226497651, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2668098509311676, + "kd/policy_loss": 0.008471001511982953, + "kd/rkl_advantage_mean": 0.6084519537786642, + "kd/rkl_advantage_p95": 4.685510146617889, + "kd/rkl_advantage_std": 2.159306467572848, + "kd/ssd_loss": 0.0, + "learning_rate": 6.616792617628071e-07, + "loss": 0.0338840126991272, + "num_tokens": 291428710.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8075000027815501, + "rewards/gt_logging_reward/std": 0.3905651867389679, + "sampling/importance_sampling_ratio/max": 2.0461269418398538, + "sampling/importance_sampling_ratio/mean": 1.003346081574758, + "sampling/importance_sampling_ratio/min": 0.45734696586926776, + "sampling/sampling_logp_difference/max": 0.4023272136847178, + "sampling/sampling_logp_difference/mean": 0.0041735503744954865, + "step": 8910, + "step_time": 8.177752814996833, + "student/entropy": 0.1271217186624805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001390775046, + "completions/max_length": 482.76666666666665, + "completions/max_terminated_length": 445.6333333333333, + "completions/mean_length": 185.96417643229168, + "completions/mean_terminated_length": 178.544921875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.13381755575537682, + "epoch": 0.33949796832871304, + "eval/gt_acc_batch": 0.8008333563804626, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3079426884651184, + "kd/policy_loss": 0.009124745948550601, + "kd/rkl_advantage_mean": 0.6921505567928156, + "kd/rkl_advantage_p95": 4.874881263573965, + "kd/rkl_advantage_std": 2.2090082238117854, + "kd/ssd_loss": 0.0, + "learning_rate": 6.605400068355295e-07, + "loss": 0.03649898370107015, + "num_tokens": 292392669.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333265781402, + "rewards/gt_logging_reward/std": 0.39618745843569436, + "sampling/importance_sampling_ratio/max": 1.921631133556366, + "sampling/importance_sampling_ratio/mean": 1.0023756702740987, + "sampling/importance_sampling_ratio/min": 0.40795307358105976, + "sampling/sampling_logp_difference/max": 0.4085219661394755, + "sampling/sampling_logp_difference/mean": 0.004373661599432429, + "step": 8940, + "step_time": 8.169943514233456, + "student/entropy": 0.13381755575537682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01500000103066365, + "completions/max_length": 468.2, + "completions/max_terminated_length": 439.3666666666667, + "completions/mean_length": 186.9555643717448, + "completions/mean_terminated_length": 182.2327082316081, + "completions/min_length": 50.766666666666666, + "completions/min_terminated_length": 50.766666666666666, + "entropy": 0.1293987731138865, + "epoch": 0.3406372232559906, + "eval/gt_acc_batch": 0.8072222491105397, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23846212029457092, + "kd/policy_loss": 0.00873971525967742, + "kd/rkl_advantage_mean": 0.6573196187304954, + "kd/rkl_advantage_p95": 4.785433608293533, + "kd/rkl_advantage_std": 2.156586045026779, + "kd/ssd_loss": 0.0, + "learning_rate": 6.59400751908252e-07, + "loss": 0.03495886325836182, + "num_tokens": 293362421.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222262620926, + "rewards/gt_logging_reward/std": 0.38333106438318887, + "sampling/importance_sampling_ratio/max": 1.934975000222524, + "sampling/importance_sampling_ratio/mean": 1.000257913271586, + "sampling/importance_sampling_ratio/min": 0.4318603148063024, + "sampling/sampling_logp_difference/max": 0.4488362669944763, + "sampling/sampling_logp_difference/mean": 0.004218335457456608, + "step": 8970, + "step_time": 7.994321068314215, + "student/entropy": 0.1293987731138865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667765627305, + "completions/max_length": 479.3333333333333, + "completions/max_terminated_length": 427.1666666666667, + "completions/mean_length": 185.44945373535157, + "completions/mean_terminated_length": 178.3060267130534, + "completions/min_length": 52.56666666666667, + "completions/min_terminated_length": 52.56666666666667, + "entropy": 0.13491631646951038, + "epoch": 0.3417764781832681, + "eval/gt_acc_batch": 0.8197222471237182, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.46091240644454956, + "kd/policy_loss": 0.008325056741402175, + "kd/rkl_advantage_mean": 0.6438771753261486, + "kd/rkl_advantage_p95": 4.855372665325801, + "kd/rkl_advantage_std": 2.1953291376431783, + "kd/ssd_loss": 0.0, + "learning_rate": 6.582614969809745e-07, + "loss": 0.033300224939982095, + "num_tokens": 294331399.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8197222212950389, + "rewards/gt_logging_reward/std": 0.3737477863828341, + "sampling/importance_sampling_ratio/max": 1.9432935078938802, + "sampling/importance_sampling_ratio/mean": 1.0001396576563517, + "sampling/importance_sampling_ratio/min": 0.45445378025372823, + "sampling/sampling_logp_difference/max": 0.4072263975938161, + "sampling/sampling_logp_difference/mean": 0.004199628330146273, + "step": 9000, + "step_time": 8.10061858277768, + "student/entropy": 0.13491631646951038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778953313828, + "completions/max_length": 482.23333333333335, + "completions/max_terminated_length": 428.53333333333336, + "completions/mean_length": 189.89362131754558, + "completions/mean_terminated_length": 184.15562896728517, + "completions/min_length": 52.63333333333333, + "completions/min_terminated_length": 52.63333333333333, + "entropy": 0.1254212811589241, + "epoch": 0.3429157331105457, + "eval/gt_acc_batch": 0.7838889221350352, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3086441457271576, + "kd/policy_loss": 0.008148523980829244, + "kd/rkl_advantage_mean": 0.6280489933085239, + "kd/rkl_advantage_p95": 4.83648251692454, + "kd/rkl_advantage_std": 2.2041015048821766, + "kd/ssd_loss": 0.0, + "learning_rate": 6.571222420536968e-07, + "loss": 0.032594090700149535, + "num_tokens": 295327144.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7838888903458913, + "rewards/gt_logging_reward/std": 0.40477747122446694, + "sampling/importance_sampling_ratio/max": 1.8843031843503317, + "sampling/importance_sampling_ratio/mean": 0.9921175003051758, + "sampling/importance_sampling_ratio/min": 0.44654210408528644, + "sampling/sampling_logp_difference/max": 0.4032543659210205, + "sampling/sampling_logp_difference/mean": 0.00413291829948624, + "step": 9030, + "step_time": 8.363437591737602, + "student/entropy": 0.1254212811589241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01861111239219705, + "completions/max_length": 491.3333333333333, + "completions/max_terminated_length": 456.6, + "completions/mean_length": 183.68417561848958, + "completions/mean_terminated_length": 177.5930658976237, + "completions/min_length": 47.03333333333333, + "completions/min_terminated_length": 47.03333333333333, + "entropy": 0.13182379926244417, + "epoch": 0.34405498803782325, + "eval/gt_acc_batch": 0.8011111358801524, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2587132155895233, + "kd/policy_loss": 0.009006756231732046, + "kd/rkl_advantage_mean": 0.6711975915997754, + "kd/rkl_advantage_p95": 4.796699341138204, + "kd/rkl_advantage_std": 2.188013333082199, + "kd/ssd_loss": 0.0, + "learning_rate": 6.559829871264193e-07, + "loss": 0.036027026176452634, + "num_tokens": 296299791.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8011111080646515, + "rewards/gt_logging_reward/std": 0.38936307430267336, + "sampling/importance_sampling_ratio/max": 1.887475355466207, + "sampling/importance_sampling_ratio/mean": 1.0012987474600474, + "sampling/importance_sampling_ratio/min": 0.4040193140506744, + "sampling/sampling_logp_difference/max": 0.38825971881548565, + "sampling/sampling_logp_difference/mean": 0.0043161343472699325, + "step": 9060, + "step_time": 8.320210897736251, + "student/entropy": 0.13182379926244417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01916666766628623, + "completions/max_length": 475.46666666666664, + "completions/max_terminated_length": 444.56666666666666, + "completions/mean_length": 184.8322326660156, + "completions/mean_terminated_length": 178.61439005533853, + "completions/min_length": 52.3, + "completions/min_terminated_length": 52.3, + "entropy": 0.13092005519817274, + "epoch": 0.3451942429651008, + "eval/gt_acc_batch": 0.80055557290713, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2776658535003662, + "kd/policy_loss": 0.00885129823582247, + "kd/rkl_advantage_mean": 0.6356165125655632, + "kd/rkl_advantage_p95": 4.761682732899984, + "kd/rkl_advantage_std": 2.183447990814845, + "kd/ssd_loss": 0.0, + "learning_rate": 6.548437321991417e-07, + "loss": 0.035405194759368895, + "num_tokens": 297274795.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.800555557012558, + "rewards/gt_logging_reward/std": 0.3911796083052953, + "sampling/importance_sampling_ratio/max": 1.9102790236473084, + "sampling/importance_sampling_ratio/mean": 1.0014369825522105, + "sampling/importance_sampling_ratio/min": 0.4164133980870247, + "sampling/sampling_logp_difference/max": 0.37316996455192564, + "sampling/sampling_logp_difference/mean": 0.0042652916784087815, + "step": 9090, + "step_time": 8.245698942181965, + "student/entropy": 0.13092005519817274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668051232895, + "completions/max_length": 499.3, + "completions/max_terminated_length": 443.26666666666665, + "completions/mean_length": 188.71556498209637, + "completions/mean_terminated_length": 180.861865234375, + "completions/min_length": 55.266666666666666, + "completions/min_terminated_length": 55.266666666666666, + "entropy": 0.12890561819076538, + "epoch": 0.3463334978923784, + "eval/gt_acc_batch": 0.8041666865348815, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.287801593542099, + "kd/policy_loss": 0.008728590194368734, + "kd/rkl_advantage_mean": 0.6650229601189495, + "kd/rkl_advantage_p95": 4.870977316300074, + "kd/rkl_advantage_std": 2.1969819257656735, + "kd/ssd_loss": 0.0, + "learning_rate": 6.537044772718641e-07, + "loss": 0.03491436243057251, + "num_tokens": 298259963.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666686534882, + "rewards/gt_logging_reward/std": 0.3914580777287483, + "sampling/importance_sampling_ratio/max": 1.8678146282831827, + "sampling/importance_sampling_ratio/mean": 0.9956495920817058, + "sampling/importance_sampling_ratio/min": 0.43996299505233766, + "sampling/sampling_logp_difference/max": 0.37470380067825315, + "sampling/sampling_logp_difference/mean": 0.004261499332884947, + "step": 9120, + "step_time": 8.391516423415547, + "student/entropy": 0.12890561819076538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334798614182, + "completions/max_length": 484.76666666666665, + "completions/max_terminated_length": 462.46666666666664, + "completions/mean_length": 188.17501017252604, + "completions/mean_terminated_length": 180.4718022664388, + "completions/min_length": 52.06666666666667, + "completions/min_terminated_length": 52.06666666666667, + "entropy": 0.13524126249055068, + "epoch": 0.34747275281965595, + "eval/gt_acc_batch": 0.7822222451368968, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3709876537322998, + "kd/policy_loss": 0.00920374914809751, + "kd/rkl_advantage_mean": 0.76872334900933, + "kd/rkl_advantage_p95": 5.057345676422119, + "kd/rkl_advantage_std": 2.2645326435565947, + "kd/ssd_loss": 0.0, + "learning_rate": 6.525652223445866e-07, + "loss": 0.03681499163309733, + "num_tokens": 299236313.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7822222232818603, + "rewards/gt_logging_reward/std": 0.40362928559382755, + "sampling/importance_sampling_ratio/max": 1.8849472085634866, + "sampling/importance_sampling_ratio/mean": 0.9995686014493307, + "sampling/importance_sampling_ratio/min": 0.4413154751062393, + "sampling/sampling_logp_difference/max": 0.3650614023208618, + "sampling/sampling_logp_difference/mean": 0.004365714422116677, + "step": 9150, + "step_time": 8.18719573849424, + "student/entropy": 0.13524126249055068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333423982064, + "completions/max_length": 485.03333333333336, + "completions/max_terminated_length": 454.6, + "completions/mean_length": 185.55973256429036, + "completions/mean_terminated_length": 177.80599110921224, + "completions/min_length": 52.56666666666667, + "completions/min_terminated_length": 52.56666666666667, + "entropy": 0.1345903476079305, + "epoch": 0.3486120077469335, + "eval/gt_acc_batch": 0.7961111227671306, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31358328461647034, + "kd/policy_loss": 0.008836147404508665, + "kd/rkl_advantage_mean": 0.6459995245716225, + "kd/rkl_advantage_p95": 4.792645688851675, + "kd/rkl_advantage_std": 2.2186480422814685, + "kd/ssd_loss": 0.0, + "learning_rate": 6.514259674173091e-07, + "loss": 0.035344592730204266, + "num_tokens": 300201248.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111148198445, + "rewards/gt_logging_reward/std": 0.3925876274704933, + "sampling/importance_sampling_ratio/max": 1.9299854358037314, + "sampling/importance_sampling_ratio/mean": 0.9989327768484751, + "sampling/importance_sampling_ratio/min": 0.41303906639417015, + "sampling/sampling_logp_difference/max": 0.37383585969607036, + "sampling/sampling_logp_difference/mean": 0.004358282700801889, + "step": 9180, + "step_time": 8.219485089858063, + "student/entropy": 0.1345903476079305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556789040567, + "completions/max_length": 494.1333333333333, + "completions/max_terminated_length": 459.6333333333333, + "completions/mean_length": 184.59334208170574, + "completions/mean_terminated_length": 177.01032155354818, + "completions/min_length": 48.166666666666664, + "completions/min_terminated_length": 48.166666666666664, + "entropy": 0.1298273166641593, + "epoch": 0.3497512626742111, + "eval/gt_acc_batch": 0.7894444584846496, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2816925048828125, + "kd/policy_loss": 0.008748850511619822, + "kd/rkl_advantage_mean": 0.6325677173600222, + "kd/rkl_advantage_p95": 4.716289190451304, + "kd/rkl_advantage_std": 2.1376844227313994, + "kd/ssd_loss": 0.0, + "learning_rate": 6.502867124900316e-07, + "loss": 0.034995404879252116, + "num_tokens": 301165712.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7894444445768992, + "rewards/gt_logging_reward/std": 0.3980559875567754, + "sampling/importance_sampling_ratio/max": 1.9008180816968283, + "sampling/importance_sampling_ratio/mean": 1.0034698784351348, + "sampling/importance_sampling_ratio/min": 0.41833346436421076, + "sampling/sampling_logp_difference/max": 0.36410555243492126, + "sampling/sampling_logp_difference/mean": 0.004256947905135652, + "step": 9210, + "step_time": 8.329688590369187, + "student/entropy": 0.1298273166641593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890004406372, + "completions/max_length": 468.26666666666665, + "completions/max_terminated_length": 428.8333333333333, + "completions/mean_length": 186.89139760335286, + "completions/mean_terminated_length": 179.79624837239584, + "completions/min_length": 49.1, + "completions/min_terminated_length": 49.1, + "entropy": 0.12749013031522433, + "epoch": 0.35089051760148865, + "eval/gt_acc_batch": 0.8008333444595337, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31007659435272217, + "kd/policy_loss": 0.008355037652654574, + "kd/rkl_advantage_mean": 0.657732292295744, + "kd/rkl_advantage_p95": 4.776065043608347, + "kd/rkl_advantage_std": 2.1498479783535003, + "kd/ssd_loss": 0.0, + "learning_rate": 6.491474575627539e-07, + "loss": 0.03342014948527018, + "num_tokens": 302146049.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333345254263, + "rewards/gt_logging_reward/std": 0.39380994339783987, + "sampling/importance_sampling_ratio/max": 1.8663376291592917, + "sampling/importance_sampling_ratio/mean": 1.000386267900467, + "sampling/importance_sampling_ratio/min": 0.41665874073902764, + "sampling/sampling_logp_difference/max": 0.35977547566095985, + "sampling/sampling_logp_difference/mean": 0.004130745042736332, + "step": 9240, + "step_time": 8.197469069669022, + "student/entropy": 0.12749013031522433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166667989144724, + "completions/max_length": 475.3666666666667, + "completions/max_terminated_length": 447.06666666666666, + "completions/mean_length": 185.19056549072266, + "completions/mean_terminated_length": 177.26368357340496, + "completions/min_length": 53.7, + "completions/min_terminated_length": 53.7, + "entropy": 0.133661797704796, + "epoch": 0.35202977252876616, + "eval/gt_acc_batch": 0.7777778009573618, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2714308202266693, + "kd/policy_loss": 0.009325351705774665, + "kd/rkl_advantage_mean": 0.6939681196119636, + "kd/rkl_advantage_p95": 4.942061366637548, + "kd/rkl_advantage_std": 2.2540754874547324, + "kd/ssd_loss": 0.0, + "learning_rate": 6.480082026354764e-07, + "loss": 0.03730141321818034, + "num_tokens": 303112191.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.777777777115504, + "rewards/gt_logging_reward/std": 0.4081997801860174, + "sampling/importance_sampling_ratio/max": 1.9407877643903098, + "sampling/importance_sampling_ratio/mean": 1.0079192340373992, + "sampling/importance_sampling_ratio/min": 0.44201667110125226, + "sampling/sampling_logp_difference/max": 0.35082286794980366, + "sampling/sampling_logp_difference/mean": 0.004375100694596767, + "step": 9270, + "step_time": 8.121871920869065, + "student/entropy": 0.133661797704796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333445712924, + "completions/max_length": 478.6333333333333, + "completions/max_terminated_length": 438.46666666666664, + "completions/mean_length": 189.10139872233074, + "completions/mean_terminated_length": 181.68348236083983, + "completions/min_length": 59.4, + "completions/min_terminated_length": 59.4, + "entropy": 0.13047293461859227, + "epoch": 0.3531690274560437, + "eval/gt_acc_batch": 0.8055555760860443, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24611951410770416, + "kd/policy_loss": 0.00853938504587859, + "kd/rkl_advantage_mean": 0.6977147083060118, + "kd/rkl_advantage_p95": 4.8433394948641455, + "kd/rkl_advantage_std": 2.1831346998612084, + "kd/ssd_loss": 0.0, + "learning_rate": 6.468689477081988e-07, + "loss": 0.034157538414001466, + "num_tokens": 304098236.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8055555502573649, + "rewards/gt_logging_reward/std": 0.38850513895352684, + "sampling/importance_sampling_ratio/max": 1.8928034106890361, + "sampling/importance_sampling_ratio/mean": 0.9941564639409383, + "sampling/importance_sampling_ratio/min": 0.4235199640194575, + "sampling/sampling_logp_difference/max": 0.35629377563794457, + "sampling/sampling_logp_difference/mean": 0.004163155130421122, + "step": 9300, + "step_time": 8.10758480411411, + "student/entropy": 0.13047293461859227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02722222392136852, + "completions/max_length": 475.6666666666667, + "completions/max_terminated_length": 446.43333333333334, + "completions/mean_length": 189.8466766357422, + "completions/mean_terminated_length": 180.94353332519532, + "completions/min_length": 55.06666666666667, + "completions/min_terminated_length": 55.06666666666667, + "entropy": 0.12814508496473234, + "epoch": 0.3543082823833213, + "eval/gt_acc_batch": 0.7775000214576722, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3473929166793823, + "kd/policy_loss": 0.008719173081529637, + "kd/rkl_advantage_mean": 0.6683686429013809, + "kd/rkl_advantage_p95": 4.826034231980642, + "kd/rkl_advantage_std": 2.1841811150312425, + "kd/ssd_loss": 0.0, + "learning_rate": 6.457296927809212e-07, + "loss": 0.034876692295074466, + "num_tokens": 305087636.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7775000035762787, + "rewards/gt_logging_reward/std": 0.40620450526475904, + "sampling/importance_sampling_ratio/max": 1.912766933441162, + "sampling/importance_sampling_ratio/mean": 1.0001560906569162, + "sampling/importance_sampling_ratio/min": 0.4233213598529498, + "sampling/sampling_logp_difference/max": 0.41106130679448444, + "sampling/sampling_logp_difference/mean": 0.0042153894901275635, + "step": 9330, + "step_time": 8.281981782704436, + "student/entropy": 0.12814508496473234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0269444456944863, + "completions/max_length": 497.3333333333333, + "completions/max_terminated_length": 466.46666666666664, + "completions/mean_length": 189.01250915527345, + "completions/mean_terminated_length": 180.22821960449218, + "completions/min_length": 54.6, + "completions/min_terminated_length": 54.6, + "entropy": 0.13169871295491856, + "epoch": 0.35544753731059886, + "eval/gt_acc_batch": 0.795555579662323, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.38635683059692383, + "kd/policy_loss": 0.009149243445911755, + "kd/rkl_advantage_mean": 0.7161016612003247, + "kd/rkl_advantage_p95": 4.974159638086955, + "kd/rkl_advantage_std": 2.2399903506040575, + "kd/ssd_loss": 0.0, + "learning_rate": 6.445904378536436e-07, + "loss": 0.036596973737080894, + "num_tokens": 306079145.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555558204651, + "rewards/gt_logging_reward/std": 0.3988635162512461, + "sampling/importance_sampling_ratio/max": 1.9010874191919962, + "sampling/importance_sampling_ratio/mean": 0.999707160393397, + "sampling/importance_sampling_ratio/min": 0.38848243554433187, + "sampling/sampling_logp_difference/max": 0.3742696662743886, + "sampling/sampling_logp_difference/mean": 0.004331180298080047, + "step": 9360, + "step_time": 8.500752118125092, + "student/entropy": 0.13169871295491856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001428027947, + "completions/max_length": 493.96666666666664, + "completions/max_terminated_length": 462.8666666666667, + "completions/mean_length": 194.90528767903646, + "completions/mean_terminated_length": 186.95475209554036, + "completions/min_length": 55.2, + "completions/min_terminated_length": 55.2, + "entropy": 0.12660903359452882, + "epoch": 0.3565867922378764, + "eval/gt_acc_batch": 0.7916666825612386, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3097769021987915, + "kd/policy_loss": 0.008473035041242838, + "kd/rkl_advantage_mean": 0.5819644222036005, + "kd/rkl_advantage_p95": 4.695436841249466, + "kd/rkl_advantage_std": 2.1663837254047396, + "kd/ssd_loss": 0.0, + "learning_rate": 6.434511829263662e-07, + "loss": 0.03389214674631755, + "num_tokens": 307085340.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7916666646798451, + "rewards/gt_logging_reward/std": 0.3986374964316686, + "sampling/importance_sampling_ratio/max": 1.937364363670349, + "sampling/importance_sampling_ratio/mean": 0.9974105854829153, + "sampling/importance_sampling_ratio/min": 0.42100464502970375, + "sampling/sampling_logp_difference/max": 0.38091036876042683, + "sampling/sampling_logp_difference/mean": 0.004104954570842286, + "step": 9390, + "step_time": 8.426375470974017, + "student/entropy": 0.12660903359452882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166667989144724, + "completions/max_length": 493.4, + "completions/max_terminated_length": 454.0, + "completions/mean_length": 187.19473215738932, + "completions/mean_terminated_length": 179.3586451212565, + "completions/min_length": 54.8, + "completions/min_terminated_length": 54.8, + "entropy": 0.1332708147043983, + "epoch": 0.357726047165154, + "eval/gt_acc_batch": 0.8019444564978282, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27597299218177795, + "kd/policy_loss": 0.008820178626532045, + "kd/rkl_advantage_mean": 0.7215134064046045, + "kd/rkl_advantage_p95": 4.935378134250641, + "kd/rkl_advantage_std": 2.221444809436798, + "kd/ssd_loss": 0.0, + "learning_rate": 6.423119279990886e-07, + "loss": 0.035280712445576984, + "num_tokens": 308065897.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444406032562, + "rewards/gt_logging_reward/std": 0.3845115855336189, + "sampling/importance_sampling_ratio/max": 1.9273186564445495, + "sampling/importance_sampling_ratio/mean": 0.9955881555875142, + "sampling/importance_sampling_ratio/min": 0.38406532605489097, + "sampling/sampling_logp_difference/max": 0.411193044980367, + "sampling/sampling_logp_difference/mean": 0.004280142361919085, + "step": 9420, + "step_time": 8.406993264394503, + "student/entropy": 0.1332708147043983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02583333505317569, + "completions/max_length": 484.53333333333336, + "completions/max_terminated_length": 428.8666666666667, + "completions/mean_length": 183.7336201985677, + "completions/mean_terminated_length": 175.10066680908204, + "completions/min_length": 52.96666666666667, + "completions/min_terminated_length": 52.96666666666667, + "entropy": 0.1259868914571901, + "epoch": 0.35886530209243156, + "eval/gt_acc_batch": 0.8047222415606181, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23642845451831818, + "kd/policy_loss": 0.008446283609373495, + "kd/rkl_advantage_mean": 0.6746477019662659, + "kd/rkl_advantage_p95": 4.860642115275065, + "kd/rkl_advantage_std": 2.1904714703559875, + "kd/ssd_loss": 0.0, + "learning_rate": 6.41172673071811e-07, + "loss": 0.033785136540730794, + "num_tokens": 309036106.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222256660461, + "rewards/gt_logging_reward/std": 0.388948118686676, + "sampling/importance_sampling_ratio/max": 1.8903741240501404, + "sampling/importance_sampling_ratio/mean": 0.9937155286471049, + "sampling/importance_sampling_ratio/min": 0.4160972754160563, + "sampling/sampling_logp_difference/max": 0.3876836061477661, + "sampling/sampling_logp_difference/mean": 0.004130588060555359, + "step": 9450, + "step_time": 8.398102629549491, + "student/entropy": 0.1259868914571901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028333334904164075, + "completions/max_length": 483.8666666666667, + "completions/max_terminated_length": 445.46666666666664, + "completions/mean_length": 189.77528686523436, + "completions/mean_terminated_length": 180.5810979207357, + "completions/min_length": 58.5, + "completions/min_terminated_length": 58.5, + "entropy": 0.13478964579602082, + "epoch": 0.3600045570197091, + "eval/gt_acc_batch": 0.7866666853427887, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31807762384414673, + "kd/policy_loss": 0.009065636868278186, + "kd/rkl_advantage_mean": 0.7391211165736119, + "kd/rkl_advantage_p95": 4.886943592627843, + "kd/rkl_advantage_std": 2.1811495810747146, + "kd/ssd_loss": 0.0, + "learning_rate": 6.400334181445335e-07, + "loss": 0.03626254399617513, + "num_tokens": 310020593.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666694482167, + "rewards/gt_logging_reward/std": 0.4022507021824519, + "sampling/importance_sampling_ratio/max": 1.8695498029390971, + "sampling/importance_sampling_ratio/mean": 0.9980003078778584, + "sampling/importance_sampling_ratio/min": 0.4240652432044347, + "sampling/sampling_logp_difference/max": 0.37015681465466815, + "sampling/sampling_logp_difference/mean": 0.004379474340627591, + "step": 9480, + "step_time": 8.245613798611641, + "student/entropy": 0.13478964579602082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0233333347675701, + "completions/max_length": 500.26666666666665, + "completions/max_terminated_length": 458.43333333333334, + "completions/mean_length": 186.6872319539388, + "completions/mean_terminated_length": 179.145166015625, + "completions/min_length": 60.53333333333333, + "completions/min_terminated_length": 60.53333333333333, + "entropy": 0.12981611688931782, + "epoch": 0.3611438119469867, + "eval/gt_acc_batch": 0.7866666773955028, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26110631227493286, + "kd/policy_loss": 0.008487485640216619, + "kd/rkl_advantage_mean": 0.8014332528536519, + "kd/rkl_advantage_p95": 5.02556734085083, + "kd/rkl_advantage_std": 2.2485829989115396, + "kd/ssd_loss": 0.0, + "learning_rate": 6.388941632172558e-07, + "loss": 0.03394994338353475, + "num_tokens": 310996747.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666674613952, + "rewards/gt_logging_reward/std": 0.40047278106212614, + "sampling/importance_sampling_ratio/max": 1.9332031965255738, + "sampling/importance_sampling_ratio/mean": 0.993144420782725, + "sampling/importance_sampling_ratio/min": 0.44345046480496725, + "sampling/sampling_logp_difference/max": 0.3874913175900777, + "sampling/sampling_logp_difference/mean": 0.0042376039394487934, + "step": 9510, + "step_time": 8.332020094249552, + "student/entropy": 0.12981611688931782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01861111233010888, + "completions/max_length": 462.8, + "completions/max_terminated_length": 431.53333333333336, + "completions/mean_length": 183.13140004475912, + "completions/mean_terminated_length": 177.01922912597655, + "completions/min_length": 51.9, + "completions/min_terminated_length": 51.9, + "entropy": 0.13161723017692567, + "epoch": 0.3622830668742642, + "eval/gt_acc_batch": 0.7855555772781372, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2264113575220108, + "kd/policy_loss": 0.009053950652014465, + "kd/rkl_advantage_mean": 0.7328890686544279, + "kd/rkl_advantage_p95": 4.945240958531698, + "kd/rkl_advantage_std": 2.206369627515475, + "kd/ssd_loss": 0.0, + "learning_rate": 6.377549082899783e-07, + "loss": 0.036215806007385255, + "num_tokens": 311966428.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555574099223, + "rewards/gt_logging_reward/std": 0.40231440563996634, + "sampling/importance_sampling_ratio/max": 1.9194711168607077, + "sampling/importance_sampling_ratio/mean": 1.0002658228079477, + "sampling/importance_sampling_ratio/min": 0.43808894554773964, + "sampling/sampling_logp_difference/max": 0.3976186235745748, + "sampling/sampling_logp_difference/mean": 0.0042328283889219165, + "step": 9540, + "step_time": 8.350950199237559, + "student/entropy": 0.13161723017692567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888889998197555, + "completions/max_length": 480.0, + "completions/max_terminated_length": 448.6333333333333, + "completions/mean_length": 187.38362147013348, + "completions/mean_terminated_length": 181.3318967183431, + "completions/min_length": 49.56666666666667, + "completions/min_terminated_length": 49.56666666666667, + "entropy": 0.12906674252202113, + "epoch": 0.36342232180154177, + "eval/gt_acc_batch": 0.8025000135103861, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21750858426094055, + "kd/policy_loss": 0.008429230359615758, + "kd/rkl_advantage_mean": 0.7529655702412128, + "kd/rkl_advantage_p95": 4.9720430235068, + "kd/rkl_advantage_std": 2.2233044395844144, + "kd/ssd_loss": 0.0, + "learning_rate": 6.366156533627008e-07, + "loss": 0.033716924985249835, + "num_tokens": 312940321.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8025000015894572, + "rewards/gt_logging_reward/std": 0.38901411394278207, + "sampling/importance_sampling_ratio/max": 1.8935015002886455, + "sampling/importance_sampling_ratio/mean": 0.9970864832401276, + "sampling/importance_sampling_ratio/min": 0.41628940800825753, + "sampling/sampling_logp_difference/max": 0.4016576488812765, + "sampling/sampling_logp_difference/mean": 0.004203814050803582, + "step": 9570, + "step_time": 8.164428117126226, + "student/entropy": 0.12906674252202113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444596146544, + "completions/max_length": 482.3333333333333, + "completions/max_terminated_length": 450.43333333333334, + "completions/mean_length": 187.108620707194, + "completions/mean_terminated_length": 179.8906682332357, + "completions/min_length": 56.733333333333334, + "completions/min_terminated_length": 56.733333333333334, + "entropy": 0.1274147034312288, + "epoch": 0.36456157672881934, + "eval/gt_acc_batch": 0.8100000242392222, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2903934717178345, + "kd/policy_loss": 0.008450969201900686, + "kd/rkl_advantage_mean": 0.6795555861666799, + "kd/rkl_advantage_p95": 4.913840295871099, + "kd/rkl_advantage_std": 2.206829917430878, + "kd/ssd_loss": 0.0, + "learning_rate": 6.354763984354232e-07, + "loss": 0.03380387624104818, + "num_tokens": 313918104.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8099999944368999, + "rewards/gt_logging_reward/std": 0.38297681162754693, + "sampling/importance_sampling_ratio/max": 1.9153131127357483, + "sampling/importance_sampling_ratio/mean": 0.9934942960739136, + "sampling/importance_sampling_ratio/min": 0.4420156409343084, + "sampling/sampling_logp_difference/max": 0.4360279440879822, + "sampling/sampling_logp_difference/mean": 0.004174494370818138, + "step": 9600, + "step_time": 8.206924693771482, + "student/entropy": 0.1274147034312288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02222222350537777, + "completions/max_length": 491.4, + "completions/max_terminated_length": 443.23333333333335, + "completions/mean_length": 187.79056447347006, + "completions/mean_terminated_length": 180.577294921875, + "completions/min_length": 54.6, + "completions/min_terminated_length": 54.6, + "entropy": 0.13150816597044468, + "epoch": 0.3657008316560969, + "eval/gt_acc_batch": 0.8022222419579824, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4527110755443573, + "kd/policy_loss": 0.008835546396827947, + "kd/rkl_advantage_mean": 0.6466934797198822, + "kd/rkl_advantage_p95": 4.84856506784757, + "kd/rkl_advantage_std": 2.212957883874575, + "kd/ssd_loss": 0.0, + "learning_rate": 6.343371435081456e-07, + "loss": 0.03534218470255534, + "num_tokens": 314887486.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8022222240765889, + "rewards/gt_logging_reward/std": 0.3909742956360181, + "sampling/importance_sampling_ratio/max": 1.9889745712280273, + "sampling/importance_sampling_ratio/mean": 0.9998695154984792, + "sampling/importance_sampling_ratio/min": 0.40135740439097084, + "sampling/sampling_logp_difference/max": 0.3845667362213135, + "sampling/sampling_logp_difference/mean": 0.004193580981033544, + "step": 9630, + "step_time": 8.22805117207269, + "student/entropy": 0.13150816597044468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02555555704360207, + "completions/max_length": 486.6, + "completions/max_terminated_length": 449.8666666666667, + "completions/mean_length": 195.80223236083984, + "completions/mean_terminated_length": 187.84988454182943, + "completions/min_length": 49.166666666666664, + "completions/min_terminated_length": 49.166666666666664, + "entropy": 0.12793788177271684, + "epoch": 0.36684008658337447, + "eval/gt_acc_batch": 0.769444465637207, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27880409359931946, + "kd/policy_loss": 0.008468073880067095, + "kd/rkl_advantage_mean": 0.7700247859427084, + "kd/rkl_advantage_p95": 4.972950311501821, + "kd/rkl_advantage_std": 2.202912465731303, + "kd/ssd_loss": 0.0, + "learning_rate": 6.331978885808681e-07, + "loss": 0.03387229442596436, + "num_tokens": 315892030.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7694444398085276, + "rewards/gt_logging_reward/std": 0.41239723761876423, + "sampling/importance_sampling_ratio/max": 1.9893386761347454, + "sampling/importance_sampling_ratio/mean": 0.9970775326093038, + "sampling/importance_sampling_ratio/min": 0.4341450870037079, + "sampling/sampling_logp_difference/max": 0.38411622246106464, + "sampling/sampling_logp_difference/mean": 0.004134661556842427, + "step": 9660, + "step_time": 8.255183985363692, + "student/entropy": 0.12793788177271684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02388889032105605, + "completions/max_length": 472.06666666666666, + "completions/max_terminated_length": 441.6666666666667, + "completions/mean_length": 186.61195576985676, + "completions/mean_terminated_length": 178.65365651448568, + "completions/min_length": 49.5, + "completions/min_terminated_length": 49.5, + "entropy": 0.13046905342489482, + "epoch": 0.36797934151065204, + "eval/gt_acc_batch": 0.7930555681387583, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27260613441467285, + "kd/policy_loss": 0.008860151091357692, + "kd/rkl_advantage_mean": 0.6591877078947922, + "kd/rkl_advantage_p95": 4.845076235135396, + "kd/rkl_advantage_std": 2.2051877230405807, + "kd/ssd_loss": 0.0, + "learning_rate": 6.320586336535906e-07, + "loss": 0.03544060389200846, + "num_tokens": 316859953.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555522441864, + "rewards/gt_logging_reward/std": 0.3932260269920031, + "sampling/importance_sampling_ratio/max": 1.8667312383651733, + "sampling/importance_sampling_ratio/mean": 0.9995024303595225, + "sampling/importance_sampling_ratio/min": 0.42264144817988075, + "sampling/sampling_logp_difference/max": 0.3771143893400828, + "sampling/sampling_logp_difference/mean": 0.004256996512413025, + "step": 9690, + "step_time": 8.02468842509358, + "student/entropy": 0.13046905342489482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02250000135973096, + "completions/max_length": 488.5, + "completions/max_terminated_length": 463.23333333333335, + "completions/mean_length": 191.95445353190104, + "completions/mean_terminated_length": 184.623882039388, + "completions/min_length": 48.6, + "completions/min_terminated_length": 48.6, + "entropy": 0.12544294483959675, + "epoch": 0.3691185964379296, + "eval/gt_acc_batch": 0.7952777981758118, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3248385488986969, + "kd/policy_loss": 0.008660057093948126, + "kd/rkl_advantage_mean": 0.684173948628207, + "kd/rkl_advantage_p95": 4.957046578327815, + "kd/rkl_advantage_std": 2.2364496260881426, + "kd/ssd_loss": 0.0, + "learning_rate": 6.309193787263129e-07, + "loss": 0.034640232721964516, + "num_tokens": 317857757.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777723471324, + "rewards/gt_logging_reward/std": 0.39913299183050793, + "sampling/importance_sampling_ratio/max": 1.9571431477864583, + "sampling/importance_sampling_ratio/mean": 0.9994183123111725, + "sampling/importance_sampling_ratio/min": 0.43334222535292305, + "sampling/sampling_logp_difference/max": 0.37234841187795004, + "sampling/sampling_logp_difference/mean": 0.004089299278954665, + "step": 9720, + "step_time": 8.404127637762576, + "student/entropy": 0.12544294483959675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667641450962, + "completions/max_length": 479.56666666666666, + "completions/max_terminated_length": 447.6, + "completions/mean_length": 182.15695495605468, + "completions/mean_terminated_length": 174.95840199788412, + "completions/min_length": 56.3, + "completions/min_terminated_length": 56.3, + "entropy": 0.13400048421074948, + "epoch": 0.37025785136520717, + "eval/gt_acc_batch": 0.8094444751739502, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25068312883377075, + "kd/policy_loss": 0.008765360026154667, + "kd/rkl_advantage_mean": 0.7229058289900422, + "kd/rkl_advantage_p95": 4.968716514110565, + "kd/rkl_advantage_std": 2.2319500784079236, + "kd/ssd_loss": 0.0, + "learning_rate": 6.297801237990355e-07, + "loss": 0.03506143887837728, + "num_tokens": 318814434.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8094444453716279, + "rewards/gt_logging_reward/std": 0.3846282680829366, + "sampling/importance_sampling_ratio/max": 1.8931588570276896, + "sampling/importance_sampling_ratio/mean": 0.999732498327891, + "sampling/importance_sampling_ratio/min": 0.4198591609795888, + "sampling/sampling_logp_difference/max": 0.37176328102747597, + "sampling/sampling_logp_difference/mean": 0.00431956029497087, + "step": 9750, + "step_time": 8.143168282861977, + "student/entropy": 0.13400048421074948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444558893641, + "completions/max_length": 481.03333333333336, + "completions/max_terminated_length": 435.9, + "completions/mean_length": 184.41778564453125, + "completions/mean_terminated_length": 177.20860290527344, + "completions/min_length": 50.666666666666664, + "completions/min_terminated_length": 50.666666666666664, + "entropy": 0.12405492809290687, + "epoch": 0.37139710629248474, + "eval/gt_acc_batch": 0.8091666877269745, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26208576560020447, + "kd/policy_loss": 0.008014675714851668, + "kd/rkl_advantage_mean": 0.7431144814938306, + "kd/rkl_advantage_p95": 4.996762901544571, + "kd/rkl_advantage_std": 2.2202061623334886, + "kd/ssd_loss": 0.0, + "learning_rate": 6.286408688717579e-07, + "loss": 0.03205870787302653, + "num_tokens": 319774714.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.809166669845581, + "rewards/gt_logging_reward/std": 0.38235337684551873, + "sampling/importance_sampling_ratio/max": 1.9204445997873942, + "sampling/importance_sampling_ratio/mean": 1.00126656293869, + "sampling/importance_sampling_ratio/min": 0.45836518704891205, + "sampling/sampling_logp_difference/max": 0.37431111335754397, + "sampling/sampling_logp_difference/mean": 0.00403997964070489, + "step": 9780, + "step_time": 8.110478018115584, + "student/entropy": 0.12405492809290687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01611111219972372, + "completions/max_length": 458.3666666666667, + "completions/max_terminated_length": 420.93333333333334, + "completions/mean_length": 182.8052874247233, + "completions/mean_terminated_length": 177.46178665161133, + "completions/min_length": 56.7, + "completions/min_terminated_length": 56.7, + "entropy": 0.13145444554587205, + "epoch": 0.3725363612197623, + "eval/gt_acc_batch": 0.8150000254313151, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3131003975868225, + "kd/policy_loss": 0.008640028615870203, + "kd/rkl_advantage_mean": 0.6463978946985056, + "kd/rkl_advantage_p95": 4.7554896434148155, + "kd/rkl_advantage_std": 2.1571982463200885, + "kd/ssd_loss": 0.0, + "learning_rate": 6.275016139444803e-07, + "loss": 0.034560108184814455, + "num_tokens": 320736037.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8149999976158142, + "rewards/gt_logging_reward/std": 0.3770480225483576, + "sampling/importance_sampling_ratio/max": 1.8777260899543762, + "sampling/importance_sampling_ratio/mean": 0.9948222021261851, + "sampling/importance_sampling_ratio/min": 0.47452540894349415, + "sampling/sampling_logp_difference/max": 0.3661513090133667, + "sampling/sampling_logp_difference/mean": 0.004219039025095602, + "step": 9810, + "step_time": 7.889701761169515, + "student/entropy": 0.13145444554587205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030555557397504648, + "completions/max_length": 502.1, + "completions/max_terminated_length": 453.03333333333336, + "completions/mean_length": 196.26028747558593, + "completions/mean_terminated_length": 186.37083536783854, + "completions/min_length": 49.36666666666667, + "completions/min_terminated_length": 49.36666666666667, + "entropy": 0.13344640346864858, + "epoch": 0.3736756161470398, + "eval/gt_acc_batch": 0.7783333480358123, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23301827907562256, + "kd/policy_loss": 0.008788678174217543, + "kd/rkl_advantage_mean": 0.7809964195514719, + "kd/rkl_advantage_p95": 5.111612468957901, + "kd/rkl_advantage_std": 2.247116985917091, + "kd/ssd_loss": 0.0, + "learning_rate": 6.263623590172027e-07, + "loss": 0.03515472014745077, + "num_tokens": 321749894.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7783333321412405, + "rewards/gt_logging_reward/std": 0.407735471924146, + "sampling/importance_sampling_ratio/max": 1.9431749860445657, + "sampling/importance_sampling_ratio/mean": 0.9951545377572377, + "sampling/importance_sampling_ratio/min": 0.4263415902853012, + "sampling/sampling_logp_difference/max": 0.35411285360654193, + "sampling/sampling_logp_difference/mean": 0.004295963848320147, + "step": 9840, + "step_time": 8.266102849540767, + "student/entropy": 0.13344640346864858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334426085154, + "completions/max_length": 491.23333333333335, + "completions/max_terminated_length": 452.1333333333333, + "completions/mean_length": 192.54362030029296, + "completions/mean_terminated_length": 185.11483103434244, + "completions/min_length": 53.03333333333333, + "completions/min_terminated_length": 53.03333333333333, + "entropy": 0.1281501216813922, + "epoch": 0.3748148710743174, + "eval/gt_acc_batch": 0.772777799765269, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32173681259155273, + "kd/policy_loss": 0.00838592100966101, + "kd/rkl_advantage_mean": 0.7101572992901007, + "kd/rkl_advantage_p95": 4.934435325860977, + "kd/rkl_advantage_std": 2.2035057326157887, + "kd/ssd_loss": 0.0, + "learning_rate": 6.252231040899252e-07, + "loss": 0.033543686072031655, + "num_tokens": 322742627.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7727777779102325, + "rewards/gt_logging_reward/std": 0.40975583692391715, + "sampling/importance_sampling_ratio/max": 1.9260729908943177, + "sampling/importance_sampling_ratio/mean": 0.996148532629013, + "sampling/importance_sampling_ratio/min": 0.4186852087577184, + "sampling/sampling_logp_difference/max": 0.38078646461168925, + "sampling/sampling_logp_difference/mean": 0.004149565170519054, + "step": 9870, + "step_time": 8.282067395692382, + "student/entropy": 0.1281501216813922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334798614182, + "completions/max_length": 500.1333333333333, + "completions/max_terminated_length": 458.4, + "completions/mean_length": 194.48861999511718, + "completions/mean_terminated_length": 187.01885325113932, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.13451248041043679, + "epoch": 0.37595412600159495, + "eval/gt_acc_batch": 0.7738889078299205, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31938645243644714, + "kd/policy_loss": 0.00912278784865824, + "kd/rkl_advantage_mean": 0.6920032396912574, + "kd/rkl_advantage_p95": 4.877023730675379, + "kd/rkl_advantage_std": 2.20730052391688, + "kd/ssd_loss": 0.0, + "learning_rate": 6.240838491626475e-07, + "loss": 0.03649114767710368, + "num_tokens": 323742474.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7738888879617055, + "rewards/gt_logging_reward/std": 0.4118642409642537, + "sampling/importance_sampling_ratio/max": 2.007888146241506, + "sampling/importance_sampling_ratio/mean": 0.9968001782894135, + "sampling/importance_sampling_ratio/min": 0.39600727905829747, + "sampling/sampling_logp_difference/max": 0.4670242210229238, + "sampling/sampling_logp_difference/mean": 0.004281714372336864, + "step": 9900, + "step_time": 8.376614935928956, + "student/entropy": 0.13451248041043679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016111111951371035, + "completions/max_length": 474.73333333333335, + "completions/max_terminated_length": 440.1333333333333, + "completions/mean_length": 185.07084147135416, + "completions/mean_terminated_length": 179.7723592122396, + "completions/min_length": 49.46666666666667, + "completions/min_terminated_length": 49.46666666666667, + "entropy": 0.1310834500938654, + "epoch": 0.3770933809288725, + "eval/gt_acc_batch": 0.7922222475210826, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2839756906032562, + "kd/policy_loss": 0.008929014718160034, + "kd/rkl_advantage_mean": 0.7055610371753573, + "kd/rkl_advantage_p95": 4.846905070543289, + "kd/rkl_advantage_std": 2.183122459053993, + "kd/ssd_loss": 0.0, + "learning_rate": 6.2294459423537e-07, + "loss": 0.03571605682373047, + "num_tokens": 324709353.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222216924032, + "rewards/gt_logging_reward/std": 0.40013818244139354, + "sampling/importance_sampling_ratio/max": 1.906126085917155, + "sampling/importance_sampling_ratio/mean": 0.9985102017720541, + "sampling/importance_sampling_ratio/min": 0.4232762138048808, + "sampling/sampling_logp_difference/max": 0.3585298418998718, + "sampling/sampling_logp_difference/mean": 0.004237664025276899, + "step": 9930, + "step_time": 8.154406209317191, + "student/entropy": 0.1310834500938654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030277779325842857, + "completions/max_length": 486.3666666666667, + "completions/max_terminated_length": 444.26666666666665, + "completions/mean_length": 193.0711212158203, + "completions/mean_terminated_length": 183.22935587565104, + "completions/min_length": 46.233333333333334, + "completions/min_terminated_length": 46.233333333333334, + "entropy": 0.13445791322737932, + "epoch": 0.3782326358561501, + "eval/gt_acc_batch": 0.7888889114061991, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3098655343055725, + "kd/policy_loss": 0.009096783812856302, + "kd/rkl_advantage_mean": 0.7015199278694733, + "kd/rkl_advantage_p95": 4.867603911956151, + "kd/rkl_advantage_std": 2.196913417180379, + "kd/ssd_loss": 0.0, + "learning_rate": 6.218053393080925e-07, + "loss": 0.03638713359832764, + "num_tokens": 325709385.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7888888855775197, + "rewards/gt_logging_reward/std": 0.40138972202936807, + "sampling/importance_sampling_ratio/max": 1.9480564792950947, + "sampling/importance_sampling_ratio/mean": 0.9978017389774323, + "sampling/importance_sampling_ratio/min": 0.4050105080008507, + "sampling/sampling_logp_difference/max": 0.36892584959665936, + "sampling/sampling_logp_difference/mean": 0.00428771705677112, + "step": 9960, + "step_time": 8.212174342293292, + "student/entropy": 0.13445791322737932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778996775546, + "completions/max_length": 489.06666666666666, + "completions/max_terminated_length": 459.06666666666666, + "completions/mean_length": 192.6358434041341, + "completions/mean_terminated_length": 185.0674840291341, + "completions/min_length": 54.7, + "completions/min_terminated_length": 54.7, + "entropy": 0.12343849763274192, + "epoch": 0.37937189078342765, + "eval/gt_acc_batch": 0.7947222371896108, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31597113609313965, + "kd/policy_loss": 0.008433795580640436, + "kd/rkl_advantage_mean": 0.7190398591104895, + "kd/rkl_advantage_p95": 4.927952102820078, + "kd/rkl_advantage_std": 2.2250452746947604, + "kd/ssd_loss": 0.0, + "learning_rate": 6.206660843808149e-07, + "loss": 0.03373518387476603, + "num_tokens": 326710082.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222212950389, + "rewards/gt_logging_reward/std": 0.3931838974356651, + "sampling/importance_sampling_ratio/max": 1.940947941939036, + "sampling/importance_sampling_ratio/mean": 0.9977596402168274, + "sampling/importance_sampling_ratio/min": 0.4043866559863091, + "sampling/sampling_logp_difference/max": 0.39347648521264394, + "sampling/sampling_logp_difference/mean": 0.003971061048408349, + "step": 9990, + "step_time": 8.216579726350028, + "student/entropy": 0.12343849763274192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779282381136, + "completions/max_length": 479.76666666666665, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 185.3175084431966, + "completions/mean_terminated_length": 176.9407704671224, + "completions/min_length": 53.6, + "completions/min_terminated_length": 53.6, + "entropy": 0.12489491999149323, + "epoch": 0.3805111457107052, + "eval/gt_acc_batch": 0.7969444672266642, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2321690022945404, + "kd/policy_loss": 0.008339020662242546, + "kd/rkl_advantage_mean": 0.7345149738093217, + "kd/rkl_advantage_p95": 4.980962312221527, + "kd/rkl_advantage_std": 2.2169794837633767, + "kd/ssd_loss": 0.0, + "learning_rate": 6.195268294535374e-07, + "loss": 0.033356086413065596, + "num_tokens": 327684137.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444433848063, + "rewards/gt_logging_reward/std": 0.3933699478705724, + "sampling/importance_sampling_ratio/max": 1.8897527694702148, + "sampling/importance_sampling_ratio/mean": 1.0035442729791006, + "sampling/importance_sampling_ratio/min": 0.4623557021220525, + "sampling/sampling_logp_difference/max": 0.34225456714630126, + "sampling/sampling_logp_difference/mean": 0.004014853178523481, + "step": 10020, + "step_time": 8.270922718724856, + "student/entropy": 0.12489491999149323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223455707233, + "completions/max_length": 492.8, + "completions/max_terminated_length": 446.26666666666665, + "completions/mean_length": 191.29806569417318, + "completions/mean_terminated_length": 182.66209513346354, + "completions/min_length": 53.13333333333333, + "completions/min_terminated_length": 53.13333333333333, + "entropy": 0.1242199797804157, + "epoch": 0.3816504006379828, + "eval/gt_acc_batch": 0.8055555760860443, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25575730204582214, + "kd/policy_loss": 0.00811742366834854, + "kd/rkl_advantage_mean": 0.7123264652987321, + "kd/rkl_advantage_p95": 4.924403454860052, + "kd/rkl_advantage_std": 2.1964634944995245, + "kd/ssd_loss": 0.0, + "learning_rate": 6.183875745262598e-07, + "loss": 0.03246969779332479, + "num_tokens": 328669850.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8055555522441864, + "rewards/gt_logging_reward/std": 0.38956208725770314, + "sampling/importance_sampling_ratio/max": 1.9209080497423807, + "sampling/importance_sampling_ratio/mean": 1.0025264024734497, + "sampling/importance_sampling_ratio/min": 0.4312830130259196, + "sampling/sampling_logp_difference/max": 0.4031056781609853, + "sampling/sampling_logp_difference/mean": 0.004052581319895884, + "step": 10050, + "step_time": 8.19740539599831, + "student/entropy": 0.1242199797804157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556522061427, + "completions/max_length": 477.7, + "completions/max_terminated_length": 441.43333333333334, + "completions/mean_length": 179.40973307291668, + "completions/mean_terminated_length": 174.1570930480957, + "completions/min_length": 52.4, + "completions/min_terminated_length": 52.4, + "entropy": 0.12652920155475536, + "epoch": 0.38278965556526034, + "eval/gt_acc_batch": 0.8147222379843394, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23832790553569794, + "kd/policy_loss": 0.008377545200831567, + "kd/rkl_advantage_mean": 0.6914118158786248, + "kd/rkl_advantage_p95": 4.8867443362871805, + "kd/rkl_advantage_std": 2.2129261285066604, + "kd/ssd_loss": 0.0, + "learning_rate": 6.172483195989822e-07, + "loss": 0.03351018031438192, + "num_tokens": 329611149.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8147222240765889, + "rewards/gt_logging_reward/std": 0.38156190812587737, + "sampling/importance_sampling_ratio/max": 1.930700163046519, + "sampling/importance_sampling_ratio/mean": 0.998485134045283, + "sampling/importance_sampling_ratio/min": 0.4519537885983785, + "sampling/sampling_logp_difference/max": 0.3678448021411896, + "sampling/sampling_logp_difference/mean": 0.0041094814582417405, + "step": 10080, + "step_time": 7.987613461337363, + "student/entropy": 0.12652920155475536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015000000906487307, + "completions/max_length": 477.46666666666664, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 190.323898824056, + "completions/mean_terminated_length": 185.529665629069, + "completions/min_length": 62.766666666666666, + "completions/min_terminated_length": 62.766666666666666, + "entropy": 0.12530008647590876, + "epoch": 0.38392891049253786, + "eval/gt_acc_batch": 0.8169444620609283, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3986154794692993, + "kd/policy_loss": 0.008167816283336531, + "kd/rkl_advantage_mean": 0.5976809051508705, + "kd/rkl_advantage_p95": 4.688899372021357, + "kd/rkl_advantage_std": 2.1312288880348205, + "kd/ssd_loss": 0.0, + "learning_rate": 6.161090646717046e-07, + "loss": 0.03267126679420471, + "num_tokens": 330602107.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8169444461663564, + "rewards/gt_logging_reward/std": 0.38069140712420146, + "sampling/importance_sampling_ratio/max": 1.9561619758605957, + "sampling/importance_sampling_ratio/mean": 0.9918095648288727, + "sampling/importance_sampling_ratio/min": 0.42063508729139965, + "sampling/sampling_logp_difference/max": 0.398466153939565, + "sampling/sampling_logp_difference/mean": 0.0040523367002606395, + "step": 10110, + "step_time": 8.233962846104987, + "student/entropy": 0.12530008647590876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01722222308938702, + "completions/max_length": 472.1333333333333, + "completions/max_terminated_length": 445.8666666666667, + "completions/mean_length": 185.90528767903646, + "completions/mean_terminated_length": 180.39632771809895, + "completions/min_length": 47.2, + "completions/min_terminated_length": 47.2, + "entropy": 0.12697908735523622, + "epoch": 0.3850681654198154, + "eval/gt_acc_batch": 0.7952778081099192, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2450903058052063, + "kd/policy_loss": 0.00861721879725034, + "kd/rkl_advantage_mean": 0.7205763199439389, + "kd/rkl_advantage_p95": 4.880520806709925, + "kd/rkl_advantage_std": 2.181394581993421, + "kd/ssd_loss": 0.0, + "learning_rate": 6.149698097444272e-07, + "loss": 0.03446887334187825, + "num_tokens": 331576494.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777763207753, + "rewards/gt_logging_reward/std": 0.39568873941898347, + "sampling/importance_sampling_ratio/max": 1.9337709625562032, + "sampling/importance_sampling_ratio/mean": 1.0059771756331126, + "sampling/importance_sampling_ratio/min": 0.45979283303022384, + "sampling/sampling_logp_difference/max": 0.34328846335411073, + "sampling/sampling_logp_difference/mean": 0.0041526952292770146, + "step": 10140, + "step_time": 8.267443561627685, + "student/entropy": 0.12697908735523622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02555555695046981, + "completions/max_length": 486.46666666666664, + "completions/max_terminated_length": 444.23333333333335, + "completions/mean_length": 187.49445546468098, + "completions/mean_terminated_length": 179.03060760498047, + "completions/min_length": 51.53333333333333, + "completions/min_terminated_length": 51.53333333333333, + "entropy": 0.12802983671426774, + "epoch": 0.386207420347093, + "eval/gt_acc_batch": 0.7969444553057353, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31947869062423706, + "kd/policy_loss": 0.00867235321784392, + "kd/rkl_advantage_mean": 0.6759752164284388, + "kd/rkl_advantage_p95": 4.906816260019938, + "kd/rkl_advantage_std": 2.2060528606176377, + "kd/ssd_loss": 0.0, + "learning_rate": 6.138305548171496e-07, + "loss": 0.03468941847483317, + "num_tokens": 332557042.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444453716278, + "rewards/gt_logging_reward/std": 0.39634432395299274, + "sampling/importance_sampling_ratio/max": 1.878971564769745, + "sampling/importance_sampling_ratio/mean": 1.001742168267568, + "sampling/importance_sampling_ratio/min": 0.38673013548056284, + "sampling/sampling_logp_difference/max": 0.37305970589319865, + "sampling/sampling_logp_difference/mean": 0.0041400702747826775, + "step": 10170, + "step_time": 8.324218877331198, + "student/entropy": 0.12802983671426774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500001223136982, + "completions/max_length": 482.8666666666667, + "completions/max_terminated_length": 448.96666666666664, + "completions/mean_length": 187.5069544474284, + "completions/mean_terminated_length": 181.83792826334636, + "completions/min_length": 57.06666666666667, + "completions/min_terminated_length": 57.06666666666667, + "entropy": 0.12697829399257898, + "epoch": 0.38734667527437056, + "eval/gt_acc_batch": 0.804166688521703, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2994507849216461, + "kd/policy_loss": 0.008319904703724509, + "kd/rkl_advantage_mean": 0.7224339076628288, + "kd/rkl_advantage_p95": 4.9260926425457, + "kd/rkl_advantage_std": 2.199181161324183, + "kd/ssd_loss": 0.0, + "learning_rate": 6.12691299889872e-07, + "loss": 0.03327962358792623, + "num_tokens": 333534491.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666666666667, + "rewards/gt_logging_reward/std": 0.3889688104391098, + "sampling/importance_sampling_ratio/max": 1.9056219140688577, + "sampling/importance_sampling_ratio/mean": 0.9997601052125294, + "sampling/importance_sampling_ratio/min": 0.42182309925556183, + "sampling/sampling_logp_difference/max": 0.36165018479029337, + "sampling/sampling_logp_difference/mean": 0.00411458876915276, + "step": 10200, + "step_time": 8.131652444709713, + "student/entropy": 0.12697829399257898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556826293468, + "completions/max_length": 492.1333333333333, + "completions/max_terminated_length": 453.43333333333334, + "completions/mean_length": 195.04001057942708, + "completions/mean_terminated_length": 186.84317220052083, + "completions/min_length": 59.666666666666664, + "completions/min_terminated_length": 59.666666666666664, + "entropy": 0.12655156745264928, + "epoch": 0.3884859302016481, + "eval/gt_acc_batch": 0.7855555792649587, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2493259608745575, + "kd/policy_loss": 0.008427743144178141, + "kd/rkl_advantage_mean": 0.6948126898457606, + "kd/rkl_advantage_p95": 4.9485011061032615, + "kd/rkl_advantage_std": 2.2344351450602216, + "kd/ssd_loss": 0.0, + "learning_rate": 6.115520449625944e-07, + "loss": 0.03371097246805827, + "num_tokens": 334541339.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7855555633703868, + "rewards/gt_logging_reward/std": 0.4039842625459035, + "sampling/importance_sampling_ratio/max": 1.8600865443547567, + "sampling/importance_sampling_ratio/mean": 0.9938223004341126, + "sampling/importance_sampling_ratio/min": 0.4558413833379745, + "sampling/sampling_logp_difference/max": 0.3778587241967519, + "sampling/sampling_logp_difference/mean": 0.0040210429656629765, + "step": 10230, + "step_time": 8.250182303999706, + "student/entropy": 0.12655156745264928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03472222428147991, + "completions/max_length": 482.73333333333335, + "completions/max_terminated_length": 444.73333333333335, + "completions/mean_length": 191.9763982137044, + "completions/mean_terminated_length": 180.6290481567383, + "completions/min_length": 54.43333333333333, + "completions/min_terminated_length": 54.43333333333333, + "entropy": 0.12902700199435155, + "epoch": 0.3896251851289257, + "eval/gt_acc_batch": 0.7744444688161214, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2781360447406769, + "kd/policy_loss": 0.008976664431005095, + "kd/rkl_advantage_mean": 0.7903159651129196, + "kd/rkl_advantage_p95": 5.0526085575421655, + "kd/rkl_advantage_std": 2.244813864429792, + "kd/ssd_loss": 0.0, + "learning_rate": 6.104127900353169e-07, + "loss": 0.035906656583150225, + "num_tokens": 335546830.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.774444446961085, + "rewards/gt_logging_reward/std": 0.40676483511924744, + "sampling/importance_sampling_ratio/max": 1.9644323348999024, + "sampling/importance_sampling_ratio/mean": 1.0102351943651835, + "sampling/importance_sampling_ratio/min": 0.44939912259578707, + "sampling/sampling_logp_difference/max": 0.3913691242535909, + "sampling/sampling_logp_difference/mean": 0.004154381869981686, + "step": 10260, + "step_time": 8.354230338754132, + "student/entropy": 0.12902700199435155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334705481927, + "completions/max_length": 489.8333333333333, + "completions/max_terminated_length": 455.8333333333333, + "completions/mean_length": 188.3997319539388, + "completions/mean_terminated_length": 180.69031982421876, + "completions/min_length": 52.8, + "completions/min_terminated_length": 52.8, + "entropy": 0.12628398487965267, + "epoch": 0.39076444005620325, + "eval/gt_acc_batch": 0.7872222542762757, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26478201150894165, + "kd/policy_loss": 0.008441130966336157, + "kd/rkl_advantage_mean": 0.7879686736812194, + "kd/rkl_advantage_p95": 5.058080005645752, + "kd/rkl_advantage_std": 2.245703478654226, + "kd/ssd_loss": 0.0, + "learning_rate": 6.092735351080393e-07, + "loss": 0.03376452922821045, + "num_tokens": 336530941.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7872222224871318, + "rewards/gt_logging_reward/std": 0.40412157972653706, + "sampling/importance_sampling_ratio/max": 1.9847240130106607, + "sampling/importance_sampling_ratio/mean": 0.9955563644568125, + "sampling/importance_sampling_ratio/min": 0.40349572400252026, + "sampling/sampling_logp_difference/max": 0.3871663471062978, + "sampling/sampling_logp_difference/mean": 0.004089852864854038, + "step": 10290, + "step_time": 8.294336703182974, + "student/entropy": 0.12628398487965267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223759939273, + "completions/max_length": 483.8333333333333, + "completions/max_terminated_length": 448.03333333333336, + "completions/mean_length": 189.35139923095704, + "completions/mean_terminated_length": 181.5300715128581, + "completions/min_length": 57.1, + "completions/min_terminated_length": 57.1, + "entropy": 0.1261238032951951, + "epoch": 0.3919036949834808, + "eval/gt_acc_batch": 0.7994444648424784, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4802449643611908, + "kd/policy_loss": 0.008257585397223011, + "kd/rkl_advantage_mean": 0.6362483878697579, + "kd/rkl_advantage_p95": 4.8008205711841585, + "kd/rkl_advantage_std": 2.1885229219992954, + "kd/ssd_loss": 0.0, + "learning_rate": 6.081342801807617e-07, + "loss": 0.033030347029368086, + "num_tokens": 337513766.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.799444442987442, + "rewards/gt_logging_reward/std": 0.3903897196054459, + "sampling/importance_sampling_ratio/max": 1.8780599673589071, + "sampling/importance_sampling_ratio/mean": 0.9952785432338714, + "sampling/importance_sampling_ratio/min": 0.3958033618827661, + "sampling/sampling_logp_difference/max": 0.5941510299841563, + "sampling/sampling_logp_difference/mean": 0.004130534168022375, + "step": 10320, + "step_time": 8.143811307494373, + "student/entropy": 0.1261238032951951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890476276476, + "completions/max_length": 468.43333333333334, + "completions/max_terminated_length": 433.26666666666665, + "completions/mean_length": 187.5100102742513, + "completions/mean_terminated_length": 179.8037338256836, + "completions/min_length": 57.1, + "completions/min_terminated_length": 57.1, + "entropy": 0.13294159881770612, + "epoch": 0.3930429499107584, + "eval/gt_acc_batch": 0.7988889018694559, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2596026360988617, + "kd/policy_loss": 0.009348773337357367, + "kd/rkl_advantage_mean": 0.6829862282727845, + "kd/rkl_advantage_p95": 4.806303509076437, + "kd/rkl_advantage_std": 2.182272347807884, + "kd/ssd_loss": 0.0, + "learning_rate": 6.069950252534843e-07, + "loss": 0.03739509185155233, + "num_tokens": 338486130.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.798888885974884, + "rewards/gt_logging_reward/std": 0.38875184108813604, + "sampling/importance_sampling_ratio/max": 2.041721733411153, + "sampling/importance_sampling_ratio/mean": 0.9983010629812876, + "sampling/importance_sampling_ratio/min": 0.39211993118127186, + "sampling/sampling_logp_difference/max": 0.4021190007527669, + "sampling/sampling_logp_difference/mean": 0.0042422649761041, + "step": 10350, + "step_time": 7.90422823264574, + "student/entropy": 0.13294159881770612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334866911174, + "completions/max_length": 491.96666666666664, + "completions/max_terminated_length": 452.9, + "completions/mean_length": 196.6063990275065, + "completions/mean_terminated_length": 188.25635681152343, + "completions/min_length": 54.53333333333333, + "completions/min_terminated_length": 54.53333333333333, + "entropy": 0.12412393266956011, + "epoch": 0.3941822048380359, + "eval/gt_acc_batch": 0.7980555772781373, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.299937903881073, + "kd/policy_loss": 0.008200121785436446, + "kd/rkl_advantage_mean": 0.7375374026286106, + "kd/rkl_advantage_p95": 4.981474326054255, + "kd/rkl_advantage_std": 2.219531876842181, + "kd/ssd_loss": 0.0, + "learning_rate": 6.058557703262066e-07, + "loss": 0.032800483703613284, + "num_tokens": 339495505.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7980555574099223, + "rewards/gt_logging_reward/std": 0.39063335458437604, + "sampling/importance_sampling_ratio/max": 1.8451335668563842, + "sampling/importance_sampling_ratio/mean": 1.000986115137736, + "sampling/importance_sampling_ratio/min": 0.40850267906983695, + "sampling/sampling_logp_difference/max": 0.3902754247188568, + "sampling/sampling_logp_difference/mean": 0.004027543659321964, + "step": 10380, + "step_time": 8.236227757619538, + "student/entropy": 0.12412393266956011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02888889058182637, + "completions/max_length": 498.1333333333333, + "completions/max_terminated_length": 453.56666666666666, + "completions/mean_length": 193.67084299723308, + "completions/mean_terminated_length": 184.268679300944, + "completions/min_length": 55.8, + "completions/min_terminated_length": 55.8, + "entropy": 0.12769989718993505, + "epoch": 0.39532145976531347, + "eval/gt_acc_batch": 0.7922222356001536, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2254512906074524, + "kd/policy_loss": 0.008641013136366383, + "kd/rkl_advantage_mean": 0.7566630424155544, + "kd/rkl_advantage_p95": 5.049440878629684, + "kd/rkl_advantage_std": 2.2569124668836595, + "kd/ssd_loss": 0.0, + "learning_rate": 6.047165153989291e-07, + "loss": 0.03456405798594157, + "num_tokens": 340499280.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222256660462, + "rewards/gt_logging_reward/std": 0.39891810715198517, + "sampling/importance_sampling_ratio/max": 1.9558592240015666, + "sampling/importance_sampling_ratio/mean": 0.9976052006085714, + "sampling/importance_sampling_ratio/min": 0.43653740907708805, + "sampling/sampling_logp_difference/max": 0.33924348950386046, + "sampling/sampling_logp_difference/mean": 0.004150114712926248, + "step": 10410, + "step_time": 8.418215082407308, + "student/entropy": 0.12769989718993505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019444445644815764, + "completions/max_length": 484.1666666666667, + "completions/max_terminated_length": 447.6333333333333, + "completions/mean_length": 193.13167826334634, + "completions/mean_terminated_length": 186.9132512410482, + "completions/min_length": 58.36666666666667, + "completions/min_terminated_length": 58.36666666666667, + "entropy": 0.13044219029446444, + "epoch": 0.39646071469259103, + "eval/gt_acc_batch": 0.7852777977784474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.260187029838562, + "kd/policy_loss": 0.008849827916128561, + "kd/rkl_advantage_mean": 0.7837284609675408, + "kd/rkl_advantage_p95": 5.084891470273336, + "kd/rkl_advantage_std": 2.237830387552579, + "kd/ssd_loss": 0.0, + "learning_rate": 6.035772604716515e-07, + "loss": 0.03539931376775106, + "num_tokens": 341505858.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7852777739365896, + "rewards/gt_logging_reward/std": 0.40534499088923137, + "sampling/importance_sampling_ratio/max": 1.9373135050137837, + "sampling/importance_sampling_ratio/mean": 1.0003085672855376, + "sampling/importance_sampling_ratio/min": 0.4221147398153941, + "sampling/sampling_logp_difference/max": 0.3763553977012634, + "sampling/sampling_logp_difference/mean": 0.004201953198450307, + "step": 10440, + "step_time": 8.218302009592298, + "student/entropy": 0.13044219029446444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779505898555, + "completions/max_length": 495.3333333333333, + "completions/max_terminated_length": 455.8333333333333, + "completions/mean_length": 188.92084299723308, + "completions/mean_terminated_length": 179.72516784667968, + "completions/min_length": 46.13333333333333, + "completions/min_terminated_length": 46.13333333333333, + "entropy": 0.12828426553557318, + "epoch": 0.3975999696198686, + "eval/gt_acc_batch": 0.8025000294049581, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3363610506057739, + "kd/policy_loss": 0.008382120749835546, + "kd/rkl_advantage_mean": 0.7918566590795915, + "kd/rkl_advantage_p95": 5.148655641078949, + "kd/rkl_advantage_std": 2.2626444419225056, + "kd/ssd_loss": 0.0, + "learning_rate": 6.024380055443739e-07, + "loss": 0.03352848688761393, + "num_tokens": 342500141.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8024999976158143, + "rewards/gt_logging_reward/std": 0.38850411723057426, + "sampling/importance_sampling_ratio/max": 1.8759971817334493, + "sampling/importance_sampling_ratio/mean": 0.994582058986028, + "sampling/importance_sampling_ratio/min": 0.41367944578329724, + "sampling/sampling_logp_difference/max": 0.37943705519040427, + "sampling/sampling_logp_difference/mean": 0.004206137157355746, + "step": 10470, + "step_time": 8.471915021845295, + "student/entropy": 0.12828426553557318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667734583218, + "completions/max_length": 479.8, + "completions/max_terminated_length": 446.03333333333336, + "completions/mean_length": 187.4488983154297, + "completions/mean_terminated_length": 180.25680643717448, + "completions/min_length": 58.766666666666666, + "completions/min_terminated_length": 58.766666666666666, + "entropy": 0.12673202647517126, + "epoch": 0.39873922454714616, + "eval/gt_acc_batch": 0.8125000278155009, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25298401713371277, + "kd/policy_loss": 0.008066516858525575, + "kd/rkl_advantage_mean": 0.7252732593566179, + "kd/rkl_advantage_p95": 4.921247969071071, + "kd/rkl_advantage_std": 2.2072092284758886, + "kd/ssd_loss": 0.0, + "learning_rate": 6.012987506170963e-07, + "loss": 0.03226606647173564, + "num_tokens": 343476861.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8125000079472859, + "rewards/gt_logging_reward/std": 0.3833318630854289, + "sampling/importance_sampling_ratio/max": 1.935193125406901, + "sampling/importance_sampling_ratio/mean": 0.9993380347887675, + "sampling/importance_sampling_ratio/min": 0.42956757446130117, + "sampling/sampling_logp_difference/max": 0.38223926424980165, + "sampling/sampling_logp_difference/mean": 0.00408207622046272, + "step": 10500, + "step_time": 8.151610345432225, + "student/entropy": 0.12673202647517126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890041659276, + "completions/max_length": 497.1333333333333, + "completions/max_terminated_length": 459.43333333333334, + "completions/mean_length": 193.55278828938802, + "completions/mean_terminated_length": 185.99297993977865, + "completions/min_length": 58.733333333333334, + "completions/min_terminated_length": 58.733333333333334, + "entropy": 0.12207784838974475, + "epoch": 0.39987847947442373, + "eval/gt_acc_batch": 0.8055555820465088, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2624225318431854, + "kd/policy_loss": 0.008701883456281697, + "kd/rkl_advantage_mean": 0.6952139757573604, + "kd/rkl_advantage_p95": 4.951360183954239, + "kd/rkl_advantage_std": 2.238209065794945, + "kd/ssd_loss": 0.0, + "learning_rate": 6.001594956898188e-07, + "loss": 0.03480753898620605, + "num_tokens": 344494907.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8055555502573649, + "rewards/gt_logging_reward/std": 0.3899929761886597, + "sampling/importance_sampling_ratio/max": 1.897826651732127, + "sampling/importance_sampling_ratio/mean": 0.9988108396530151, + "sampling/importance_sampling_ratio/min": 0.444990145166715, + "sampling/sampling_logp_difference/max": 0.353996217250824, + "sampling/sampling_logp_difference/mean": 0.003978785833654305, + "step": 10530, + "step_time": 8.241240713705094, + "student/entropy": 0.12207784838974475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016944445421298346, + "completions/max_length": 477.6, + "completions/max_terminated_length": 443.23333333333335, + "completions/mean_length": 183.74778645833334, + "completions/mean_terminated_length": 178.13280080159504, + "completions/min_length": 52.4, + "completions/min_terminated_length": 52.4, + "entropy": 0.12364342007786036, + "epoch": 0.4010177344017013, + "eval/gt_acc_batch": 0.7994444747765859, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2695307731628418, + "kd/policy_loss": 0.00850772473301428, + "kd/rkl_advantage_mean": 0.8033940463637312, + "kd/rkl_advantage_p95": 5.1964630762736, + "kd/rkl_advantage_std": 2.2766631583372754, + "kd/ssd_loss": 0.0, + "learning_rate": 5.990202407625414e-07, + "loss": 0.0340308944384257, + "num_tokens": 345466167.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.799444439013799, + "rewards/gt_logging_reward/std": 0.3941947663823763, + "sampling/importance_sampling_ratio/max": 1.95120876232783, + "sampling/importance_sampling_ratio/mean": 0.9986362914244334, + "sampling/importance_sampling_ratio/min": 0.4449803670247396, + "sampling/sampling_logp_difference/max": 0.3559811015923818, + "sampling/sampling_logp_difference/mean": 0.004017813402848939, + "step": 10560, + "step_time": 7.899725150633215, + "student/entropy": 0.12364342007786036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334742734828, + "completions/max_length": 498.73333333333335, + "completions/max_terminated_length": 454.7, + "completions/mean_length": 191.54278869628905, + "completions/mean_terminated_length": 183.08368835449218, + "completions/min_length": 58.13333333333333, + "completions/min_terminated_length": 58.13333333333333, + "entropy": 0.12120778858661652, + "epoch": 0.40215698932897886, + "eval/gt_acc_batch": 0.8055555721124014, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2632123529911041, + "kd/policy_loss": 0.008499011216918007, + "kd/rkl_advantage_mean": 0.8394763531784216, + "kd/rkl_advantage_p95": 5.201586111386617, + "kd/rkl_advantage_std": 2.2697713593641917, + "kd/ssd_loss": 0.0, + "learning_rate": 5.978809858352637e-07, + "loss": 0.03399604558944702, + "num_tokens": 346463369.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8055555522441864, + "rewards/gt_logging_reward/std": 0.39117661317189534, + "sampling/importance_sampling_ratio/max": 1.9139292081197103, + "sampling/importance_sampling_ratio/mean": 0.9941296319166819, + "sampling/importance_sampling_ratio/min": 0.43435709476470946, + "sampling/sampling_logp_difference/max": 0.3893790364265442, + "sampling/sampling_logp_difference/mean": 0.003904475651991864, + "step": 10590, + "step_time": 7.9608642486666215, + "student/entropy": 0.12120778858661652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777918924888, + "completions/max_length": 490.6333333333333, + "completions/max_terminated_length": 449.8666666666667, + "completions/mean_length": 187.89362030029298, + "completions/mean_terminated_length": 179.5716807047526, + "completions/min_length": 59.53333333333333, + "completions/min_terminated_length": 59.53333333333333, + "entropy": 0.11787495004634062, + "epoch": 0.40329624425625643, + "eval/gt_acc_batch": 0.8155555685361227, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23196366429328918, + "kd/policy_loss": 0.00792016582369494, + "kd/rkl_advantage_mean": 0.7885824842999379, + "kd/rkl_advantage_p95": 5.133045214414596, + "kd/rkl_advantage_std": 2.2537056396404904, + "kd/ssd_loss": 0.0, + "learning_rate": 5.967417309079862e-07, + "loss": 0.03168066740036011, + "num_tokens": 347450946.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8155555526415507, + "rewards/gt_logging_reward/std": 0.38325621088345846, + "sampling/importance_sampling_ratio/max": 1.851384131113688, + "sampling/importance_sampling_ratio/mean": 0.9941918273766835, + "sampling/importance_sampling_ratio/min": 0.43764520188172656, + "sampling/sampling_logp_difference/max": 0.3574194570382436, + "sampling/sampling_logp_difference/mean": 0.0038523649408792457, + "step": 10620, + "step_time": 7.895587878096558, + "student/entropy": 0.11787495004634062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0211111121190091, + "completions/max_length": 486.3, + "completions/max_terminated_length": 436.26666666666665, + "completions/mean_length": 180.9225112915039, + "completions/mean_terminated_length": 173.75101064046223, + "completions/min_length": 55.03333333333333, + "completions/min_terminated_length": 55.03333333333333, + "entropy": 0.12748178448528052, + "epoch": 0.40443549918353394, + "eval/gt_acc_batch": 0.7977777938048045, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31685635447502136, + "kd/policy_loss": 0.00867722161541072, + "kd/rkl_advantage_mean": 0.8010423277815183, + "kd/rkl_advantage_p95": 5.068478218714396, + "kd/rkl_advantage_std": 2.2548077086607616, + "kd/ssd_loss": 0.0, + "learning_rate": 5.956024759807086e-07, + "loss": 0.0347088893254598, + "num_tokens": 348391091.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.797777775923411, + "rewards/gt_logging_reward/std": 0.39485761324564617, + "sampling/importance_sampling_ratio/max": 1.865190533796946, + "sampling/importance_sampling_ratio/mean": 0.9953875561555227, + "sampling/importance_sampling_ratio/min": 0.4089632083972295, + "sampling/sampling_logp_difference/max": 0.4063802202542623, + "sampling/sampling_logp_difference/mean": 0.004106242287283142, + "step": 10650, + "step_time": 7.649718028967133, + "student/entropy": 0.12748178448528052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112435658773, + "completions/max_length": 500.8, + "completions/max_terminated_length": 462.43333333333334, + "completions/mean_length": 190.5261220296224, + "completions/mean_terminated_length": 182.68347727457683, + "completions/min_length": 54.86666666666667, + "completions/min_terminated_length": 54.86666666666667, + "entropy": 0.12614295004556578, + "epoch": 0.4055747541108115, + "eval/gt_acc_batch": 0.7866666952768961, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2542363107204437, + "kd/policy_loss": 0.008874607998101662, + "kd/rkl_advantage_mean": 0.8650987508396307, + "kd/rkl_advantage_p95": 5.276357622941335, + "kd/rkl_advantage_std": 2.2966013381878536, + "kd/ssd_loss": 0.0, + "learning_rate": 5.94463221053431e-07, + "loss": 0.035498428344726565, + "num_tokens": 349378937.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666575272878, + "rewards/gt_logging_reward/std": 0.40179196894168856, + "sampling/importance_sampling_ratio/max": 1.9289963046709697, + "sampling/importance_sampling_ratio/mean": 0.9995276510715485, + "sampling/importance_sampling_ratio/min": 0.455810709297657, + "sampling/sampling_logp_difference/max": 0.38887224594751996, + "sampling/sampling_logp_difference/mean": 0.004029872027846674, + "step": 10680, + "step_time": 7.942622533199028, + "student/entropy": 0.12614295004556578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02472222385307153, + "completions/max_length": 472.76666666666665, + "completions/max_terminated_length": 436.53333333333336, + "completions/mean_length": 186.0519536336263, + "completions/mean_terminated_length": 178.06328684488932, + "completions/min_length": 54.166666666666664, + "completions/min_terminated_length": 54.166666666666664, + "entropy": 0.12194116935133933, + "epoch": 0.4067140090380891, + "eval/gt_acc_batch": 0.8063889046510061, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.5285079479217529, + "kd/policy_loss": 0.008404361054999754, + "kd/rkl_advantage_mean": 0.750483313932394, + "kd/rkl_advantage_p95": 4.960639995336533, + "kd/rkl_advantage_std": 2.2089276144901913, + "kd/ssd_loss": 0.0, + "learning_rate": 5.933239661261534e-07, + "loss": 0.03361744483311971, + "num_tokens": 350351924.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8063888867696126, + "rewards/gt_logging_reward/std": 0.3829671124617259, + "sampling/importance_sampling_ratio/max": 2.0044728914896646, + "sampling/importance_sampling_ratio/mean": 1.003524480263392, + "sampling/importance_sampling_ratio/min": 0.47735979755719504, + "sampling/sampling_logp_difference/max": 0.349388853708903, + "sampling/sampling_logp_difference/mean": 0.003918835047322015, + "step": 10710, + "step_time": 7.777415638530511, + "student/entropy": 0.12194116935133933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02816092142642572, + "completions/max_length": 505.0344827586207, + "completions/max_terminated_length": 452.8965517241379, + "completions/mean_length": 187.26064168995825, + "completions/mean_terminated_length": 177.90686193005791, + "completions/min_length": 57.03448275862069, + "completions/min_terminated_length": 57.03448275862069, + "entropy": 0.12918645650919142, + "epoch": 0.40785326396536664, + "eval/gt_acc_batch": 0.790517256177705, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.38703030347824097, + "kd/policy_loss": 0.009054176336764518, + "kd/rkl_advantage_mean": 0.8436282056158987, + "kd/rkl_advantage_p95": 5.169765945138602, + "kd/rkl_advantage_std": 2.2698032033854516, + "kd/ssd_loss": 0.0, + "learning_rate": 5.92184711198876e-07, + "loss": 0.03500947952270508, + "num_tokens": 351300695.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7905172479563746, + "rewards/gt_logging_reward/std": 0.4029906587354068, + "sampling/importance_sampling_ratio/max": 1.9759930290024856, + "sampling/importance_sampling_ratio/mean": 1.00705441113176, + "sampling/importance_sampling_ratio/min": 0.43564349104618205, + "sampling/sampling_logp_difference/max": 0.3534643444521674, + "sampling/sampling_logp_difference/mean": 0.004125198453729008, + "step": 10740, + "step_time": 7.7762239007007645, + "student/entropy": 0.12918645650919142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166667989144724, + "completions/max_length": 486.06666666666666, + "completions/max_terminated_length": 437.2, + "completions/mean_length": 185.3975102742513, + "completions/mean_terminated_length": 177.42224629720053, + "completions/min_length": 49.3, + "completions/min_terminated_length": 49.3, + "entropy": 0.125607464214166, + "epoch": 0.4089925188926442, + "eval/gt_acc_batch": 0.7930555760860443, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33320990204811096, + "kd/policy_loss": 0.00856847232595707, + "kd/rkl_advantage_mean": 0.7880529242567718, + "kd/rkl_advantage_p95": 5.120797036091487, + "kd/rkl_advantage_std": 2.2659881502389907, + "kd/ssd_loss": 0.0, + "learning_rate": 5.910454562715983e-07, + "loss": 0.03427388668060303, + "num_tokens": 352261886.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555542310079, + "rewards/gt_logging_reward/std": 0.3989254966378212, + "sampling/importance_sampling_ratio/max": 1.9457431435585022, + "sampling/importance_sampling_ratio/mean": 0.9940714100996654, + "sampling/importance_sampling_ratio/min": 0.3950306514898936, + "sampling/sampling_logp_difference/max": 0.3592452963193258, + "sampling/sampling_logp_difference/mean": 0.004083046449037889, + "step": 10770, + "step_time": 7.764885103199534, + "student/entropy": 0.125607464214166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445936630168, + "completions/max_length": 481.56666666666666, + "completions/max_terminated_length": 439.26666666666665, + "completions/mean_length": 193.4327885945638, + "completions/mean_terminated_length": 185.64923400878905, + "completions/min_length": 63.13333333333333, + "completions/min_terminated_length": 63.13333333333333, + "entropy": 0.12752354020873705, + "epoch": 0.4101317738199218, + "eval/gt_acc_batch": 0.8077777961889903, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2653803527355194, + "kd/policy_loss": 0.008474512726146107, + "kd/rkl_advantage_mean": 0.7582023064916332, + "kd/rkl_advantage_p95": 4.966840287049611, + "kd/rkl_advantage_std": 2.2277325659990312, + "kd/ssd_loss": 0.0, + "learning_rate": 5.899062013443208e-07, + "loss": 0.033898051579793295, + "num_tokens": 353264036.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8077777822812399, + "rewards/gt_logging_reward/std": 0.3846756408611933, + "sampling/importance_sampling_ratio/max": 1.9891152262687684, + "sampling/importance_sampling_ratio/mean": 0.993836905558904, + "sampling/importance_sampling_ratio/min": 0.41395831455787024, + "sampling/sampling_logp_difference/max": 0.3352519909540812, + "sampling/sampling_logp_difference/mean": 0.004051566830215355, + "step": 10800, + "step_time": 7.886331143337399, + "student/entropy": 0.12752354020873705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223343948523, + "completions/max_length": 480.8666666666667, + "completions/max_terminated_length": 435.76666666666665, + "completions/mean_length": 188.80306498209634, + "completions/mean_terminated_length": 182.4061731974284, + "completions/min_length": 52.766666666666666, + "completions/min_terminated_length": 52.766666666666666, + "entropy": 0.12422056545813878, + "epoch": 0.41127102874719934, + "eval/gt_acc_batch": 0.7963889102141063, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3042498528957367, + "kd/policy_loss": 0.00866517920124655, + "kd/rkl_advantage_mean": 0.7662867549263562, + "kd/rkl_advantage_p95": 5.042865946888924, + "kd/rkl_advantage_std": 2.236620851357778, + "kd/ssd_loss": 0.0, + "learning_rate": 5.887669464170433e-07, + "loss": 0.03466071685155233, + "num_tokens": 354249543.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888923327128, + "rewards/gt_logging_reward/std": 0.39596930146217346, + "sampling/importance_sampling_ratio/max": 1.8969589710235595, + "sampling/importance_sampling_ratio/mean": 0.9995357116063436, + "sampling/importance_sampling_ratio/min": 0.44987736344337464, + "sampling/sampling_logp_difference/max": 0.3585905780394872, + "sampling/sampling_logp_difference/mean": 0.003967347244421641, + "step": 10830, + "step_time": 7.764017306226985, + "student/entropy": 0.12422056545813878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500000912696122, + "completions/max_length": 466.23333333333335, + "completions/max_terminated_length": 437.03333333333336, + "completions/mean_length": 186.15945536295573, + "completions/mean_terminated_length": 180.540407816569, + "completions/min_length": 52.9, + "completions/min_terminated_length": 52.9, + "entropy": 0.12542476101468006, + "epoch": 0.4124102836744769, + "eval/gt_acc_batch": 0.8008333524068196, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21073341369628906, + "kd/policy_loss": 0.008808878952792535, + "kd/rkl_advantage_mean": 0.7740348021189372, + "kd/rkl_advantage_p95": 4.9942957878112795, + "kd/rkl_advantage_std": 2.2034556527932483, + "kd/ssd_loss": 0.0, + "learning_rate": 5.876276914897656e-07, + "loss": 0.03523551225662232, + "num_tokens": 355216237.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333384990692, + "rewards/gt_logging_reward/std": 0.3917003264029821, + "sampling/importance_sampling_ratio/max": 2.0196523865063987, + "sampling/importance_sampling_ratio/mean": 1.007257310549418, + "sampling/importance_sampling_ratio/min": 0.4608916292587916, + "sampling/sampling_logp_difference/max": 0.35771257877349855, + "sampling/sampling_logp_difference/mean": 0.004021588790540894, + "step": 10860, + "step_time": 7.641711621867338, + "student/entropy": 0.12542476101468006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02722222382823626, + "completions/max_length": 486.3666666666667, + "completions/max_terminated_length": 449.6333333333333, + "completions/mean_length": 191.60278778076173, + "completions/mean_terminated_length": 182.71871999104818, + "completions/min_length": 60.766666666666666, + "completions/min_terminated_length": 60.766666666666666, + "entropy": 0.12420201965918144, + "epoch": 0.4135495386017545, + "eval/gt_acc_batch": 0.8077777981758117, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4471946656703949, + "kd/policy_loss": 0.008470389169330399, + "kd/rkl_advantage_mean": 0.8011665217578411, + "kd/rkl_advantage_p95": 5.090583336353302, + "kd/rkl_advantage_std": 2.2563922186692555, + "kd/ssd_loss": 0.0, + "learning_rate": 5.864884365624881e-07, + "loss": 0.033881564935048424, + "num_tokens": 356214615.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8077777783075969, + "rewards/gt_logging_reward/std": 0.38770818213621777, + "sampling/importance_sampling_ratio/max": 1.90258998076121, + "sampling/importance_sampling_ratio/mean": 1.0028245568275451, + "sampling/importance_sampling_ratio/min": 0.42885444362958275, + "sampling/sampling_logp_difference/max": 0.3720317800839742, + "sampling/sampling_logp_difference/mean": 0.003941055620089173, + "step": 10890, + "step_time": 7.916933299636487, + "student/entropy": 0.12420201965918144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028888890861223142, + "completions/max_length": 497.6, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 192.14834238688152, + "completions/mean_terminated_length": 182.69435475667316, + "completions/min_length": 60.3, + "completions/min_terminated_length": 60.3, + "entropy": 0.12500317922482887, + "epoch": 0.41468879352903204, + "eval/gt_acc_batch": 0.796666685740153, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2658860385417938, + "kd/policy_loss": 0.008729055353129903, + "kd/rkl_advantage_mean": 0.779737031335632, + "kd/rkl_advantage_p95": 5.099222348133723, + "kd/rkl_advantage_std": 2.256130571166674, + "kd/ssd_loss": 0.0, + "learning_rate": 5.853491816352105e-07, + "loss": 0.0349162220954895, + "num_tokens": 357221573.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7966666678587596, + "rewards/gt_logging_reward/std": 0.3929822579026222, + "sampling/importance_sampling_ratio/max": 1.895244308312734, + "sampling/importance_sampling_ratio/mean": 0.9989401400089264, + "sampling/importance_sampling_ratio/min": 0.4561264644066493, + "sampling/sampling_logp_difference/max": 0.338992178440094, + "sampling/sampling_logp_difference/mean": 0.003946910961531103, + "step": 10920, + "step_time": 8.230200215565855, + "student/entropy": 0.12500317922482887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944446222235758, + "completions/max_length": 488.43333333333334, + "completions/max_terminated_length": 463.9, + "completions/mean_length": 183.4452885945638, + "completions/mean_terminated_length": 174.53964233398438, + "completions/min_length": 55.03333333333333, + "completions/min_terminated_length": 55.03333333333333, + "entropy": 0.129709267988801, + "epoch": 0.41582804845630955, + "eval/gt_acc_batch": 0.8047222336133321, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21270230412483215, + "kd/policy_loss": 0.008817604067735374, + "kd/rkl_advantage_mean": 0.8292535634400944, + "kd/rkl_advantage_p95": 5.188517554601034, + "kd/rkl_advantage_std": 2.2989805420239766, + "kd/ssd_loss": 0.0, + "learning_rate": 5.84209926707933e-07, + "loss": 0.0352704127629598, + "num_tokens": 358178408.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222197055817, + "rewards/gt_logging_reward/std": 0.3882374922434489, + "sampling/importance_sampling_ratio/max": 1.8938692172368368, + "sampling/importance_sampling_ratio/mean": 1.000975110133489, + "sampling/importance_sampling_ratio/min": 0.44805833995342254, + "sampling/sampling_logp_difference/max": 0.338328750928243, + "sampling/sampling_logp_difference/mean": 0.004136632126756012, + "step": 10950, + "step_time": 7.780633494335052, + "student/entropy": 0.129709267988801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223517795404, + "completions/max_length": 489.03333333333336, + "completions/max_terminated_length": 444.2, + "completions/mean_length": 184.31556498209636, + "completions/mean_terminated_length": 175.30699106852214, + "completions/min_length": 44.7, + "completions/min_terminated_length": 44.7, + "entropy": 0.12932452528427044, + "epoch": 0.4169673033835871, + "eval/gt_acc_batch": 0.8050000250339509, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2591937482357025, + "kd/policy_loss": 0.008842953960023199, + "kd/rkl_advantage_mean": 0.8290664931914459, + "kd/rkl_advantage_p95": 5.167586690187454, + "kd/rkl_advantage_std": 2.2737001279989877, + "kd/ssd_loss": 0.0, + "learning_rate": 5.830706717806554e-07, + "loss": 0.03537181218465169, + "num_tokens": 359158608.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8049999952316285, + "rewards/gt_logging_reward/std": 0.3874802549680074, + "sampling/importance_sampling_ratio/max": 1.9606038252512614, + "sampling/importance_sampling_ratio/mean": 0.9952649215857188, + "sampling/importance_sampling_ratio/min": 0.44397147595882414, + "sampling/sampling_logp_difference/max": 0.3612501780192057, + "sampling/sampling_logp_difference/mean": 0.0040940668511514865, + "step": 10980, + "step_time": 8.248861019739222, + "student/entropy": 0.12932452528427044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500001192092895, + "completions/max_length": 488.73333333333335, + "completions/max_terminated_length": 449.56666666666666, + "completions/mean_length": 184.8725092569987, + "completions/mean_terminated_length": 179.0597712198893, + "completions/min_length": 50.2, + "completions/min_terminated_length": 50.2, + "entropy": 0.12394448798149824, + "epoch": 0.4181065583108647, + "eval/gt_acc_batch": 0.8172222435474396, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25163689255714417, + "kd/policy_loss": 0.008220065449131652, + "kd/rkl_advantage_mean": 0.7826869182288647, + "kd/rkl_advantage_p95": 5.085982749859492, + "kd/rkl_advantage_std": 2.2580818285544715, + "kd/ssd_loss": 0.0, + "learning_rate": 5.819314168533779e-07, + "loss": 0.032880264520645144, + "num_tokens": 360121261.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8172222197055816, + "rewards/gt_logging_reward/std": 0.37853196561336516, + "sampling/importance_sampling_ratio/max": 1.8593367298444112, + "sampling/importance_sampling_ratio/mean": 1.0078315118948618, + "sampling/importance_sampling_ratio/min": 0.4691827744245529, + "sampling/sampling_logp_difference/max": 0.3082729816436768, + "sampling/sampling_logp_difference/mean": 0.003924928815104067, + "step": 11010, + "step_time": 7.925345746831348, + "student/entropy": 0.12394448798149824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02000000113621354, + "completions/max_length": 477.5, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 187.02528686523436, + "completions/mean_terminated_length": 180.523823038737, + "completions/min_length": 48.43333333333333, + "completions/min_terminated_length": 48.43333333333333, + "entropy": 0.12993011064827442, + "epoch": 0.41924581323814225, + "eval/gt_acc_batch": 0.7872222542762757, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24298010766506195, + "kd/policy_loss": 0.009076951690561448, + "kd/rkl_advantage_mean": 0.7553342174777451, + "kd/rkl_advantage_p95": 5.012013920148214, + "kd/rkl_advantage_std": 2.220237065354983, + "kd/ssd_loss": 0.0, + "learning_rate": 5.807921619261002e-07, + "loss": 0.03630780776341756, + "num_tokens": 361095336.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7872222224871318, + "rewards/gt_logging_reward/std": 0.3995201165477435, + "sampling/importance_sampling_ratio/max": 1.9528343240420023, + "sampling/importance_sampling_ratio/mean": 0.9985416134198507, + "sampling/importance_sampling_ratio/min": 0.433385768532753, + "sampling/sampling_logp_difference/max": 0.36888420979181924, + "sampling/sampling_logp_difference/mean": 0.004151241636524598, + "step": 11040, + "step_time": 7.904930816731454, + "student/entropy": 0.12993011064827442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667790462572, + "completions/max_length": 471.23333333333335, + "completions/max_terminated_length": 432.46666666666664, + "completions/mean_length": 184.54639739990233, + "completions/mean_terminated_length": 178.18759104410807, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.12880986283222834, + "epoch": 0.4203850681654198, + "eval/gt_acc_batch": 0.8008333563804626, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3085721731185913, + "kd/policy_loss": 0.008803926119192814, + "kd/rkl_advantage_mean": 0.6820449548463027, + "kd/rkl_advantage_p95": 4.915043437480927, + "kd/rkl_advantage_std": 2.1935487429300946, + "kd/ssd_loss": 0.0, + "learning_rate": 5.796529069988227e-07, + "loss": 0.035215707619984944, + "num_tokens": 362068111.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333305517833, + "rewards/gt_logging_reward/std": 0.3921476120750109, + "sampling/importance_sampling_ratio/max": 2.0264683683713276, + "sampling/importance_sampling_ratio/mean": 1.0074478149414063, + "sampling/importance_sampling_ratio/min": 0.4566768676042557, + "sampling/sampling_logp_difference/max": 0.3410379747549693, + "sampling/sampling_logp_difference/mean": 0.004086233008032044, + "step": 11070, + "step_time": 7.999714906234294, + "student/entropy": 0.12880986283222834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444586833318, + "completions/max_length": 478.26666666666665, + "completions/max_terminated_length": 439.2, + "completions/mean_length": 188.33306579589845, + "completions/mean_terminated_length": 181.06346995035807, + "completions/min_length": 57.833333333333336, + "completions/min_terminated_length": 57.833333333333336, + "entropy": 0.12791222501546146, + "epoch": 0.4215243230926974, + "eval/gt_acc_batch": 0.8047222415606181, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25002676248550415, + "kd/policy_loss": 0.008760467489870886, + "kd/rkl_advantage_mean": 0.7697031732027729, + "kd/rkl_advantage_p95": 4.963197908798853, + "kd/rkl_advantage_std": 2.2070022890965144, + "kd/ssd_loss": 0.0, + "learning_rate": 5.785136520715452e-07, + "loss": 0.03504187266031901, + "num_tokens": 363057374.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222256660461, + "rewards/gt_logging_reward/std": 0.38869201838970185, + "sampling/importance_sampling_ratio/max": 1.8413901646931967, + "sampling/importance_sampling_ratio/mean": 0.9972897211710612, + "sampling/importance_sampling_ratio/min": 0.4511428048213323, + "sampling/sampling_logp_difference/max": 0.33121751944224037, + "sampling/sampling_logp_difference/mean": 0.0040814243334655964, + "step": 11100, + "step_time": 8.089216584036087, + "student/entropy": 0.12791222501546146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556857337555, + "completions/max_length": 474.43333333333334, + "completions/max_terminated_length": 436.1, + "completions/mean_length": 188.72834116617838, + "completions/mean_terminated_length": 180.58513488769532, + "completions/min_length": 50.233333333333334, + "completions/min_terminated_length": 50.233333333333334, + "entropy": 0.11866806540638208, + "epoch": 0.42266357801997495, + "eval/gt_acc_batch": 0.8138889034589132, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2723023593425751, + "kd/policy_loss": 0.00798421764629893, + "kd/rkl_advantage_mean": 0.7725955202554663, + "kd/rkl_advantage_p95": 5.09555814464887, + "kd/rkl_advantage_std": 2.262621342142423, + "kd/ssd_loss": 0.0, + "learning_rate": 5.773743971442677e-07, + "loss": 0.03193687399228414, + "num_tokens": 364049372.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8138888875643412, + "rewards/gt_logging_reward/std": 0.37864691615104673, + "sampling/importance_sampling_ratio/max": 1.8349785168965658, + "sampling/importance_sampling_ratio/mean": 0.9984891553719838, + "sampling/importance_sampling_ratio/min": 0.47732709447542826, + "sampling/sampling_logp_difference/max": 0.34683728714783985, + "sampling/sampling_logp_difference/mean": 0.0037664261103297274, + "step": 11130, + "step_time": 8.02442361832994, + "student/entropy": 0.11866806540638208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0247222236978511, + "completions/max_length": 491.06666666666666, + "completions/max_terminated_length": 445.06666666666666, + "completions/mean_length": 186.99556528727214, + "completions/mean_terminated_length": 178.92196197509764, + "completions/min_length": 47.766666666666666, + "completions/min_terminated_length": 47.766666666666666, + "entropy": 0.13067898588875929, + "epoch": 0.4238028329472525, + "eval/gt_acc_batch": 0.797222242752711, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2978976368904114, + "kd/policy_loss": 0.00865006383198003, + "kd/rkl_advantage_mean": 0.7731514656295378, + "kd/rkl_advantage_p95": 5.066205354531606, + "kd/rkl_advantage_std": 2.251801476875941, + "kd/ssd_loss": 0.0, + "learning_rate": 5.762351422169901e-07, + "loss": 0.03460024992624919, + "num_tokens": 365031100.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7972222149372101, + "rewards/gt_logging_reward/std": 0.39352360318104423, + "sampling/importance_sampling_ratio/max": 1.9272628625233967, + "sampling/importance_sampling_ratio/mean": 0.9933995525042216, + "sampling/importance_sampling_ratio/min": 0.4079070970416069, + "sampling/sampling_logp_difference/max": 0.32501941521962485, + "sampling/sampling_logp_difference/mean": 0.00412472162861377, + "step": 11160, + "step_time": 8.166388187632158, + "student/entropy": 0.13067898588875929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111281439662, + "completions/max_length": 490.3666666666667, + "completions/max_terminated_length": 451.7, + "completions/mean_length": 193.73084309895833, + "completions/mean_terminated_length": 185.43362630208333, + "completions/min_length": 57.56666666666667, + "completions/min_terminated_length": 57.56666666666667, + "entropy": 0.13179865976174673, + "epoch": 0.4249420878745301, + "eval/gt_acc_batch": 0.7869444628556569, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3106021285057068, + "kd/policy_loss": 0.009301108994986863, + "kd/rkl_advantage_mean": 0.861307215380172, + "kd/rkl_advantage_p95": 5.2087114453315735, + "kd/rkl_advantage_std": 2.2753982186317443, + "kd/ssd_loss": 0.0, + "learning_rate": 5.750958872897125e-07, + "loss": 0.037204432487487796, + "num_tokens": 366035851.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444410006206, + "rewards/gt_logging_reward/std": 0.4033940056959788, + "sampling/importance_sampling_ratio/max": 2.037217450141907, + "sampling/importance_sampling_ratio/mean": 1.0038142919540405, + "sampling/importance_sampling_ratio/min": 0.4327598164478938, + "sampling/sampling_logp_difference/max": 0.33052988052368165, + "sampling/sampling_logp_difference/mean": 0.004164413960340122, + "step": 11190, + "step_time": 8.056227477396412, + "student/entropy": 0.13179865976174673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112435658773, + "completions/max_length": 483.6333333333333, + "completions/max_terminated_length": 461.23333333333335, + "completions/mean_length": 190.05250905354816, + "completions/mean_terminated_length": 182.40032602945965, + "completions/min_length": 49.53333333333333, + "completions/min_terminated_length": 49.53333333333333, + "entropy": 0.1266085082044204, + "epoch": 0.4260813428018076, + "eval/gt_acc_batch": 0.8019444684187571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2158711552619934, + "kd/policy_loss": 0.008484772534575313, + "kd/rkl_advantage_mean": 0.8479551448641965, + "kd/rkl_advantage_p95": 5.20724185705185, + "kd/rkl_advantage_std": 2.2941170980532966, + "kd/ssd_loss": 0.0, + "learning_rate": 5.73956632362435e-07, + "loss": 0.03393908739089966, + "num_tokens": 367023832.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444485505421, + "rewards/gt_logging_reward/std": 0.39036971429983774, + "sampling/importance_sampling_ratio/max": 1.9822932243347169, + "sampling/importance_sampling_ratio/mean": 0.9976930658022563, + "sampling/importance_sampling_ratio/min": 0.46582283775011696, + "sampling/sampling_logp_difference/max": 0.3527142037947973, + "sampling/sampling_logp_difference/mean": 0.00399091235982875, + "step": 11220, + "step_time": 8.030181897330234, + "student/entropy": 0.1266085082044204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667889803648, + "completions/max_length": 465.46666666666664, + "completions/max_terminated_length": 427.73333333333335, + "completions/mean_length": 186.5983418782552, + "completions/mean_terminated_length": 179.52516428629556, + "completions/min_length": 62.43333333333333, + "completions/min_terminated_length": 62.43333333333333, + "entropy": 0.12533150172481936, + "epoch": 0.42722059772908516, + "eval/gt_acc_batch": 0.8100000301996867, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3758449852466583, + "kd/policy_loss": 0.008399219021278744, + "kd/rkl_advantage_mean": 0.7076372888560096, + "kd/rkl_advantage_p95": 4.872074631849925, + "kd/rkl_advantage_std": 2.1807295362154644, + "kd/ssd_loss": 0.0, + "learning_rate": 5.728173774351573e-07, + "loss": 0.033596877257029215, + "num_tokens": 367994506.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8100000023841858, + "rewards/gt_logging_reward/std": 0.38448666483163835, + "sampling/importance_sampling_ratio/max": 1.9318461855252584, + "sampling/importance_sampling_ratio/mean": 0.999275267124176, + "sampling/importance_sampling_ratio/min": 0.4572328085700671, + "sampling/sampling_logp_difference/max": 0.322148597240448, + "sampling/sampling_logp_difference/mean": 0.003928730318633219, + "step": 11250, + "step_time": 7.854440692966454, + "student/entropy": 0.12533150172481936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001043081285, + "completions/max_length": 480.23333333333335, + "completions/max_terminated_length": 435.8, + "completions/mean_length": 182.83000946044922, + "completions/mean_terminated_length": 176.16265513102215, + "completions/min_length": 49.46666666666667, + "completions/min_terminated_length": 49.46666666666667, + "entropy": 0.1275231911490361, + "epoch": 0.4283598526563627, + "eval/gt_acc_batch": 0.8069444795449575, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34125104546546936, + "kd/policy_loss": 0.008516371071649095, + "kd/rkl_advantage_mean": 0.7540580379466216, + "kd/rkl_advantage_p95": 5.035788347323735, + "kd/rkl_advantage_std": 2.2574027349551518, + "kd/ssd_loss": 0.0, + "learning_rate": 5.716781225078798e-07, + "loss": 0.03406548500061035, + "num_tokens": 368952294.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8069444417953491, + "rewards/gt_logging_reward/std": 0.3903204739093781, + "sampling/importance_sampling_ratio/max": 1.8893377741177877, + "sampling/importance_sampling_ratio/mean": 1.0001621504624685, + "sampling/importance_sampling_ratio/min": 0.4182889983057976, + "sampling/sampling_logp_difference/max": 0.3155474017063777, + "sampling/sampling_logp_difference/mean": 0.004037419753149152, + "step": 11280, + "step_time": 7.84328851623286, + "student/entropy": 0.1275231911490361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112404614686, + "completions/max_length": 480.23333333333335, + "completions/max_terminated_length": 444.43333333333334, + "completions/mean_length": 185.85501047770182, + "completions/mean_terminated_length": 178.03470560709636, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.126002014738818, + "epoch": 0.4294991075836403, + "eval/gt_acc_batch": 0.795000022649765, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2704719305038452, + "kd/policy_loss": 0.008470776128039386, + "kd/rkl_advantage_mean": 0.8179263540854057, + "kd/rkl_advantage_p95": 5.105910396575927, + "kd/rkl_advantage_std": 2.238905473550161, + "kd/ssd_loss": 0.0, + "learning_rate": 5.705388675806022e-07, + "loss": 0.03388310670852661, + "num_tokens": 369927780.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7950000007947285, + "rewards/gt_logging_reward/std": 0.39737036526203157, + "sampling/importance_sampling_ratio/max": 1.9620817144711813, + "sampling/importance_sampling_ratio/mean": 0.9985138674577078, + "sampling/importance_sampling_ratio/min": 0.4493737662831942, + "sampling/sampling_logp_difference/max": 0.36315704782803854, + "sampling/sampling_logp_difference/mean": 0.004015575077695151, + "step": 11310, + "step_time": 8.089973823267792, + "student/entropy": 0.126002014738818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0158333341901501, + "completions/max_length": 481.76666666666665, + "completions/max_terminated_length": 453.43333333333334, + "completions/mean_length": 185.61945393880208, + "completions/mean_terminated_length": 180.47264353434244, + "completions/min_length": 52.8, + "completions/min_terminated_length": 52.8, + "entropy": 0.13106321015705666, + "epoch": 0.43063836251091786, + "eval/gt_acc_batch": 0.7986111422379811, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4565618634223938, + "kd/policy_loss": 0.00892021637992002, + "kd/rkl_advantage_mean": 0.7560482613742352, + "kd/rkl_advantage_p95": 4.945070141553879, + "kd/rkl_advantage_std": 2.19042622645696, + "kd/ssd_loss": 0.0, + "learning_rate": 5.693996126533247e-07, + "loss": 0.03568086624145508, + "num_tokens": 370895338.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111164093017, + "rewards/gt_logging_reward/std": 0.39417697191238404, + "sampling/importance_sampling_ratio/max": 1.9638185302416484, + "sampling/importance_sampling_ratio/mean": 1.0016366561253867, + "sampling/importance_sampling_ratio/min": 0.44007737189531326, + "sampling/sampling_logp_difference/max": 0.3429566383361816, + "sampling/sampling_logp_difference/mean": 0.004193193341294924, + "step": 11340, + "step_time": 7.953461444832889, + "student/entropy": 0.13106321015705666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013333334121853113, + "completions/max_length": 479.6333333333333, + "completions/max_terminated_length": 454.43333333333334, + "completions/mean_length": 184.0666768391927, + "completions/mean_terminated_length": 179.66200002034506, + "completions/min_length": 56.06666666666667, + "completions/min_terminated_length": 56.06666666666667, + "entropy": 0.12782959192991256, + "epoch": 0.4317776174381954, + "eval/gt_acc_batch": 0.8219444553057352, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2516724467277527, + "kd/policy_loss": 0.008234480190246056, + "kd/rkl_advantage_mean": 0.739900568438073, + "kd/rkl_advantage_p95": 5.013181231419245, + "kd/rkl_advantage_std": 2.237910489241282, + "kd/ssd_loss": 0.0, + "learning_rate": 5.682603577260472e-07, + "loss": 0.032937920093536376, + "num_tokens": 371865642.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8219444493452708, + "rewards/gt_logging_reward/std": 0.37703707814216614, + "sampling/importance_sampling_ratio/max": 1.9399539868036906, + "sampling/importance_sampling_ratio/mean": 1.0008496026198068, + "sampling/importance_sampling_ratio/min": 0.45469182829062144, + "sampling/sampling_logp_difference/max": 0.32901699940363566, + "sampling/sampling_logp_difference/mean": 0.003992986655794084, + "step": 11370, + "step_time": 8.11936890656604, + "student/entropy": 0.12782959192991256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0255555571988225, + "completions/max_length": 482.6333333333333, + "completions/max_terminated_length": 447.5, + "completions/mean_length": 193.7613993326823, + "completions/mean_terminated_length": 185.53423207600912, + "completions/min_length": 45.166666666666664, + "completions/min_terminated_length": 45.166666666666664, + "entropy": 0.12734760685513416, + "epoch": 0.432916872365473, + "eval/gt_acc_batch": 0.789722228050232, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34110063314437866, + "kd/policy_loss": 0.008443815752010171, + "kd/rkl_advantage_mean": 0.7463594195743402, + "kd/rkl_advantage_p95": 5.032692964871725, + "kd/rkl_advantage_std": 2.241734658678373, + "kd/ssd_loss": 0.0, + "learning_rate": 5.671211027987696e-07, + "loss": 0.03377526601155599, + "num_tokens": 372879711.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.789722224076589, + "rewards/gt_logging_reward/std": 0.3992016464471817, + "sampling/importance_sampling_ratio/max": 2.037269683678945, + "sampling/importance_sampling_ratio/mean": 0.9982204020023346, + "sampling/importance_sampling_ratio/min": 0.4168927232424418, + "sampling/sampling_logp_difference/max": 0.35298062761624655, + "sampling/sampling_logp_difference/mean": 0.004035789798945188, + "step": 11400, + "step_time": 8.074420509397168, + "student/entropy": 0.12734760685513416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01805555643513799, + "completions/max_length": 478.1333333333333, + "completions/max_terminated_length": 444.73333333333335, + "completions/mean_length": 185.20584309895833, + "completions/mean_terminated_length": 179.3300313313802, + "completions/min_length": 52.4, + "completions/min_terminated_length": 52.4, + "entropy": 0.1289910117785136, + "epoch": 0.43405612729275056, + "eval/gt_acc_batch": 0.8141666928927104, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3170662522315979, + "kd/policy_loss": 0.008488527126610279, + "kd/rkl_advantage_mean": 0.7495379378398259, + "kd/rkl_advantage_p95": 5.026414149999619, + "kd/rkl_advantage_std": 2.2278493573268254, + "kd/ssd_loss": 0.0, + "learning_rate": 5.65981847871492e-07, + "loss": 0.03395411173502604, + "num_tokens": 373859364.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8141666650772095, + "rewards/gt_logging_reward/std": 0.3831405997276306, + "sampling/importance_sampling_ratio/max": 1.8420002897580465, + "sampling/importance_sampling_ratio/mean": 0.9978655020395915, + "sampling/importance_sampling_ratio/min": 0.4287222623825073, + "sampling/sampling_logp_difference/max": 0.37705453832944236, + "sampling/sampling_logp_difference/mean": 0.004106219268093506, + "step": 11430, + "step_time": 8.100613432732159, + "student/entropy": 0.1289910117785136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223424663145, + "completions/max_length": 495.3, + "completions/max_terminated_length": 457.56666666666666, + "completions/mean_length": 188.74695383707683, + "completions/mean_terminated_length": 179.63865458170574, + "completions/min_length": 56.36666666666667, + "completions/min_terminated_length": 56.36666666666667, + "entropy": 0.1274974225088954, + "epoch": 0.4351953822200281, + "eval/gt_acc_batch": 0.8022222439448039, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2787800431251526, + "kd/policy_loss": 0.008797077411630501, + "kd/rkl_advantage_mean": 0.7305536361721655, + "kd/rkl_advantage_p95": 4.981649388869603, + "kd/rkl_advantage_std": 2.204659538467725, + "kd/ssd_loss": 0.0, + "learning_rate": 5.648425929442144e-07, + "loss": 0.03518830935160319, + "num_tokens": 374838765.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8022222260634104, + "rewards/gt_logging_reward/std": 0.3934246381123861, + "sampling/importance_sampling_ratio/max": 1.8553470571835835, + "sampling/importance_sampling_ratio/mean": 0.9984675447146097, + "sampling/importance_sampling_ratio/min": 0.44022238900264105, + "sampling/sampling_logp_difference/max": 0.3477927188078562, + "sampling/sampling_logp_difference/mean": 0.004018687549978495, + "step": 11460, + "step_time": 8.046144979037248, + "student/entropy": 0.1274974225088954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.036388890879849596, + "completions/max_length": 504.3666666666667, + "completions/max_terminated_length": 453.8, + "completions/mean_length": 194.79250895182292, + "completions/mean_terminated_length": 183.3503189086914, + "completions/min_length": 49.1, + "completions/min_terminated_length": 49.1, + "entropy": 0.13075141745309035, + "epoch": 0.43633463714730564, + "eval/gt_acc_batch": 0.7808333516120911, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.19623211026191711, + "kd/policy_loss": 0.008761590915188815, + "kd/rkl_advantage_mean": 0.83263626024127, + "kd/rkl_advantage_p95": 5.21355218688647, + "kd/rkl_advantage_std": 2.289816269278526, + "kd/ssd_loss": 0.0, + "learning_rate": 5.637033380169369e-07, + "loss": 0.035046366850535075, + "num_tokens": 375849330.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7808333337306976, + "rewards/gt_logging_reward/std": 0.40280557026465735, + "sampling/importance_sampling_ratio/max": 1.8930689215660095, + "sampling/importance_sampling_ratio/mean": 0.9942979296048482, + "sampling/importance_sampling_ratio/min": 0.4450619806845983, + "sampling/sampling_logp_difference/max": 0.3481112480163574, + "sampling/sampling_logp_difference/mean": 0.004094026183399061, + "step": 11490, + "step_time": 8.260081543201037, + "student/entropy": 0.13075141745309035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778996775546, + "completions/max_length": 491.3, + "completions/max_terminated_length": 447.46666666666664, + "completions/mean_length": 190.40584208170574, + "completions/mean_terminated_length": 182.98750864664714, + "completions/min_length": 60.93333333333333, + "completions/min_terminated_length": 60.93333333333333, + "entropy": 0.12774408770104248, + "epoch": 0.4374738920745832, + "eval/gt_acc_batch": 0.7897222459316253, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24044856429100037, + "kd/policy_loss": 0.0087102738112056, + "kd/rkl_advantage_mean": 0.8949220889558395, + "kd/rkl_advantage_p95": 5.332709223031998, + "kd/rkl_advantage_std": 2.3312124073505402, + "kd/ssd_loss": 0.0, + "learning_rate": 5.625640830896594e-07, + "loss": 0.034841092427571614, + "num_tokens": 376832951.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.789722224076589, + "rewards/gt_logging_reward/std": 0.39851535658041637, + "sampling/importance_sampling_ratio/max": 1.9364305933316548, + "sampling/importance_sampling_ratio/mean": 0.9997584740320842, + "sampling/importance_sampling_ratio/min": 0.39720841497182846, + "sampling/sampling_logp_difference/max": 0.31929536660512287, + "sampling/sampling_logp_difference/mean": 0.004048191787054141, + "step": 11520, + "step_time": 7.933718773032402, + "student/entropy": 0.12774408770104248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556726952395, + "completions/max_length": 497.3666666666667, + "completions/max_terminated_length": 446.1666666666667, + "completions/mean_length": 188.77223205566406, + "completions/mean_terminated_length": 181.24918416341146, + "completions/min_length": 53.833333333333336, + "completions/min_terminated_length": 53.833333333333336, + "entropy": 0.12390445911635956, + "epoch": 0.43861314700186077, + "eval/gt_acc_batch": 0.8050000150998433, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2510381042957306, + "kd/policy_loss": 0.008350844809319824, + "kd/rkl_advantage_mean": 0.8598504309852918, + "kd/rkl_advantage_p95": 5.234802307685216, + "kd/rkl_advantage_std": 2.2683691531419754, + "kd/ssd_loss": 0.0, + "learning_rate": 5.614248281623818e-07, + "loss": 0.03340338071187337, + "num_tokens": 377816027.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8050000031789144, + "rewards/gt_logging_reward/std": 0.3893147518237432, + "sampling/importance_sampling_ratio/max": 1.9478103399276734, + "sampling/importance_sampling_ratio/mean": 0.9985718846321106, + "sampling/importance_sampling_ratio/min": 0.44102411915858586, + "sampling/sampling_logp_difference/max": 0.3229081074396769, + "sampling/sampling_logp_difference/mean": 0.003876647655852139, + "step": 11550, + "step_time": 7.96979540753479, + "student/entropy": 0.12390445911635956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779344469307, + "completions/max_length": 494.8, + "completions/max_terminated_length": 472.56666666666666, + "completions/mean_length": 192.90056355794272, + "completions/mean_terminated_length": 184.84651947021484, + "completions/min_length": 49.3, + "completions/min_terminated_length": 49.3, + "entropy": 0.12849274203181266, + "epoch": 0.43975240192913834, + "eval/gt_acc_batch": 0.7883333444595337, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22863176465034485, + "kd/policy_loss": 0.008965088120506455, + "kd/rkl_advantage_mean": 0.770871565490961, + "kd/rkl_advantage_p95": 5.060108218590418, + "kd/rkl_advantage_std": 2.224726704756419, + "kd/ssd_loss": 0.0, + "learning_rate": 5.602855732351042e-07, + "loss": 0.035860343774159746, + "num_tokens": 378817797.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7883333345254262, + "rewards/gt_logging_reward/std": 0.4035507599512736, + "sampling/importance_sampling_ratio/max": 2.0174263874689737, + "sampling/importance_sampling_ratio/mean": 1.0029634833335876, + "sampling/importance_sampling_ratio/min": 0.4231244067351023, + "sampling/sampling_logp_difference/max": 0.33953218658765155, + "sampling/sampling_logp_difference/mean": 0.003979000154261788, + "step": 11580, + "step_time": 8.05831890850095, + "student/entropy": 0.12849274203181266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500000881652038, + "completions/max_length": 490.2, + "completions/max_terminated_length": 449.1333333333333, + "completions/mean_length": 188.62723083496093, + "completions/mean_terminated_length": 183.0042953491211, + "completions/min_length": 58.8, + "completions/min_terminated_length": 58.8, + "entropy": 0.12196001832683881, + "epoch": 0.4408916568564159, + "eval/gt_acc_batch": 0.7969444712003072, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.20473287999629974, + "kd/policy_loss": 0.008157000772189348, + "kd/rkl_advantage_mean": 0.8192717425525189, + "kd/rkl_advantage_p95": 5.152626891930898, + "kd/rkl_advantage_std": 2.267831738789876, + "kd/ssd_loss": 0.0, + "learning_rate": 5.591463183078267e-07, + "loss": 0.032628007729848224, + "num_tokens": 379794991.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444433848063, + "rewards/gt_logging_reward/std": 0.39531231224536895, + "sampling/importance_sampling_ratio/max": 2.0035086909929913, + "sampling/importance_sampling_ratio/mean": 1.003898940483729, + "sampling/importance_sampling_ratio/min": 0.46857076982657114, + "sampling/sampling_logp_difference/max": 0.3511040290196737, + "sampling/sampling_logp_difference/mean": 0.0038627872321133814, + "step": 11610, + "step_time": 7.8961040179992175, + "student/entropy": 0.12196001832683881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944446160147587, + "completions/max_length": 479.6666666666667, + "completions/max_terminated_length": 449.96666666666664, + "completions/mean_length": 190.4836217244466, + "completions/mean_terminated_length": 181.6720001220703, + "completions/min_length": 61.2, + "completions/min_terminated_length": 61.2, + "entropy": 0.1270018999154369, + "epoch": 0.44203091178369347, + "eval/gt_acc_batch": 0.7941667000452678, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33135125041007996, + "kd/policy_loss": 0.008627993956906721, + "kd/rkl_advantage_mean": 0.7327132009435445, + "kd/rkl_advantage_p95": 4.954172877470652, + "kd/rkl_advantage_std": 2.2100267390410107, + "kd/ssd_loss": 0.0, + "learning_rate": 5.580070633805491e-07, + "loss": 0.03451197942097982, + "num_tokens": 380790684.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7941666682561238, + "rewards/gt_logging_reward/std": 0.39538770020008085, + "sampling/importance_sampling_ratio/max": 1.9614120244979858, + "sampling/importance_sampling_ratio/mean": 0.9972096582253774, + "sampling/importance_sampling_ratio/min": 0.4698720335960388, + "sampling/sampling_logp_difference/max": 0.3488775432109833, + "sampling/sampling_logp_difference/mean": 0.003992912902807196, + "step": 11640, + "step_time": 8.053600064432734, + "student/entropy": 0.1270018999154369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666801397999, + "completions/max_length": 482.73333333333335, + "completions/max_terminated_length": 448.76666666666665, + "completions/mean_length": 187.06417694091797, + "completions/mean_terminated_length": 179.96803080240886, + "completions/min_length": 55.266666666666666, + "completions/min_terminated_length": 55.266666666666666, + "entropy": 0.12451817175994316, + "epoch": 0.44317016671097104, + "eval/gt_acc_batch": 0.8016666968663534, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30949196219444275, + "kd/policy_loss": 0.00838309078826569, + "kd/rkl_advantage_mean": 0.77105615294228, + "kd/rkl_advantage_p95": 4.976109951734543, + "kd/rkl_advantage_std": 2.210820542772611, + "kd/ssd_loss": 0.0, + "learning_rate": 5.568678084532715e-07, + "loss": 0.033532365163167314, + "num_tokens": 381769611.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8016666650772095, + "rewards/gt_logging_reward/std": 0.38965435574452084, + "sampling/importance_sampling_ratio/max": 1.9081045627593993, + "sampling/importance_sampling_ratio/mean": 0.9979966700077056, + "sampling/importance_sampling_ratio/min": 0.4625123590230942, + "sampling/sampling_logp_difference/max": 0.3474987943967183, + "sampling/sampling_logp_difference/mean": 0.003964844695292413, + "step": 11670, + "step_time": 7.898506471431756, + "student/entropy": 0.12451817175994316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112150053183, + "completions/max_length": 489.73333333333335, + "completions/max_terminated_length": 452.1666666666667, + "completions/mean_length": 189.44250996907553, + "completions/mean_terminated_length": 182.7487782796224, + "completions/min_length": 59.666666666666664, + "completions/min_terminated_length": 59.666666666666664, + "entropy": 0.1273148354763786, + "epoch": 0.4443094216382486, + "eval/gt_acc_batch": 0.7947222471237183, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.38450661301612854, + "kd/policy_loss": 0.008845442135740693, + "kd/rkl_advantage_mean": 0.7522298318644365, + "kd/rkl_advantage_p95": 4.995620630184809, + "kd/rkl_advantage_std": 2.195026456316312, + "kd/ssd_loss": 0.0, + "learning_rate": 5.55728553525994e-07, + "loss": 0.03538177013397217, + "num_tokens": 382760892.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222153345744, + "rewards/gt_logging_reward/std": 0.39516275425752007, + "sampling/importance_sampling_ratio/max": 1.9572381258010865, + "sampling/importance_sampling_ratio/mean": 1.0022704859574636, + "sampling/importance_sampling_ratio/min": 0.42390644947687783, + "sampling/sampling_logp_difference/max": 0.35675330956776935, + "sampling/sampling_logp_difference/mean": 0.0040344038900608815, + "step": 11700, + "step_time": 8.070355656768758, + "student/entropy": 0.1273148354763786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02888889048869411, + "completions/max_length": 484.7, + "completions/max_terminated_length": 447.3333333333333, + "completions/mean_length": 191.41723175048827, + "completions/mean_terminated_length": 182.01618092854818, + "completions/min_length": 50.4, + "completions/min_terminated_length": 50.4, + "entropy": 0.13005729311456282, + "epoch": 0.44544867656552617, + "eval/gt_acc_batch": 0.7858333667119344, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.37692791223526, + "kd/policy_loss": 0.008497272369762261, + "kd/rkl_advantage_mean": 0.8157773998255531, + "kd/rkl_advantage_p95": 5.127555092175801, + "kd/rkl_advantage_std": 2.2356165140867232, + "kd/ssd_loss": 0.0, + "learning_rate": 5.545892985987164e-07, + "loss": 0.03398909171422323, + "num_tokens": 383759482.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.785833328962326, + "rewards/gt_logging_reward/std": 0.3994343648354212, + "sampling/importance_sampling_ratio/max": 1.8703674793243408, + "sampling/importance_sampling_ratio/mean": 0.9955096940199534, + "sampling/importance_sampling_ratio/min": 0.4365615606307983, + "sampling/sampling_logp_difference/max": 0.36432543595631917, + "sampling/sampling_logp_difference/mean": 0.003969570777068536, + "step": 11730, + "step_time": 8.078351310167152, + "student/entropy": 0.13005729311456282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778872599203, + "completions/max_length": 493.3333333333333, + "completions/max_terminated_length": 465.46666666666664, + "completions/mean_length": 187.9247314453125, + "completions/mean_terminated_length": 180.49365743001303, + "completions/min_length": 50.5, + "completions/min_terminated_length": 50.5, + "entropy": 0.12723729057858388, + "epoch": 0.4465879314928037, + "eval/gt_acc_batch": 0.816111131509145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32571759819984436, + "kd/policy_loss": 0.008576208523785074, + "kd/rkl_advantage_mean": 0.8299570726851623, + "kd/rkl_advantage_p95": 5.217686629295349, + "kd/rkl_advantage_std": 2.3087016830841702, + "kd/ssd_loss": 0.0, + "learning_rate": 5.534500436714389e-07, + "loss": 0.03430484135945638, + "num_tokens": 384740115.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8161111136277517, + "rewards/gt_logging_reward/std": 0.3815735220909119, + "sampling/importance_sampling_ratio/max": 1.9349141558011373, + "sampling/importance_sampling_ratio/mean": 1.003368576367696, + "sampling/importance_sampling_ratio/min": 0.4318300182620684, + "sampling/sampling_logp_difference/max": 0.31116458773612976, + "sampling/sampling_logp_difference/mean": 0.003987009970781704, + "step": 11760, + "step_time": 7.9829765179034435, + "student/entropy": 0.12723729057858388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015000000906487307, + "completions/max_length": 462.7, + "completions/max_terminated_length": 433.3333333333333, + "completions/mean_length": 184.37278671264647, + "completions/mean_terminated_length": 179.48554865519205, + "completions/min_length": 57.96666666666667, + "completions/min_terminated_length": 57.96666666666667, + "entropy": 0.11963666311154762, + "epoch": 0.44772718642008125, + "eval/gt_acc_batch": 0.8141666789849599, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23927870392799377, + "kd/policy_loss": 0.007753877302942177, + "kd/rkl_advantage_mean": 0.7611672976820653, + "kd/rkl_advantage_p95": 5.006147203842799, + "kd/rkl_advantage_std": 2.20835799574852, + "kd/ssd_loss": 0.0, + "learning_rate": 5.523107887441613e-07, + "loss": 0.03101550738016764, + "num_tokens": 385717305.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.814166667064031, + "rewards/gt_logging_reward/std": 0.38008989989757536, + "sampling/importance_sampling_ratio/max": 1.8327450275421142, + "sampling/importance_sampling_ratio/mean": 0.9966122527917226, + "sampling/importance_sampling_ratio/min": 0.4442432483037313, + "sampling/sampling_logp_difference/max": 0.33312641779581703, + "sampling/sampling_logp_difference/mean": 0.003806824532027046, + "step": 11790, + "step_time": 7.917652834999414, + "student/entropy": 0.11963666311154762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779251337052, + "completions/max_length": 499.8333333333333, + "completions/max_terminated_length": 444.9, + "completions/mean_length": 192.68306579589844, + "completions/mean_terminated_length": 184.43591868082683, + "completions/min_length": 58.86666666666667, + "completions/min_terminated_length": 58.86666666666667, + "entropy": 0.12605640826125938, + "epoch": 0.4488664413473588, + "eval/gt_acc_batch": 0.7997222522894541, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21553899347782135, + "kd/policy_loss": 0.008403326947397241, + "kd/rkl_advantage_mean": 0.7684778449436028, + "kd/rkl_advantage_p95": 5.01877718369166, + "kd/rkl_advantage_std": 2.24449217915535, + "kd/ssd_loss": 0.0, + "learning_rate": 5.511715338168837e-07, + "loss": 0.03361331621805827, + "num_tokens": 386714196.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7997222224871318, + "rewards/gt_logging_reward/std": 0.38953013916810353, + "sampling/importance_sampling_ratio/max": 1.9053038001060485, + "sampling/importance_sampling_ratio/mean": 1.0007560352484386, + "sampling/importance_sampling_ratio/min": 0.44970961660146713, + "sampling/sampling_logp_difference/max": 0.3371387521425883, + "sampling/sampling_logp_difference/mean": 0.003999390173703432, + "step": 11820, + "step_time": 7.98917300440177, + "student/entropy": 0.12605640826125938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001210719346, + "completions/max_length": 493.4, + "completions/max_terminated_length": 452.9, + "completions/mean_length": 194.28751118977866, + "completions/mean_terminated_length": 186.20176340738934, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.13216456808149815, + "epoch": 0.4500056962746364, + "eval/gt_acc_batch": 0.7719444513320923, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2585335671901703, + "kd/policy_loss": 0.009133713343180716, + "kd/rkl_advantage_mean": 0.7791572157293558, + "kd/rkl_advantage_p95": 5.092261878649394, + "kd/rkl_advantage_std": 2.253809830546379, + "kd/ssd_loss": 0.0, + "learning_rate": 5.500322788896061e-07, + "loss": 0.03653485377629598, + "num_tokens": 387719999.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7719444473584492, + "rewards/gt_logging_reward/std": 0.4070851032932599, + "sampling/importance_sampling_ratio/max": 1.8567302624384563, + "sampling/importance_sampling_ratio/mean": 0.9980574309825897, + "sampling/importance_sampling_ratio/min": 0.4460436473290125, + "sampling/sampling_logp_difference/max": 0.3403144737084707, + "sampling/sampling_logp_difference/mean": 0.004107474737490217, + "step": 11850, + "step_time": 8.086806227334213, + "student/entropy": 0.13216456808149815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444446029762427, + "completions/max_length": 494.3666666666667, + "completions/max_terminated_length": 460.9, + "completions/mean_length": 194.70334167480468, + "completions/mean_terminated_length": 187.00848999023438, + "completions/min_length": 55.3, + "completions/min_terminated_length": 55.3, + "entropy": 0.12698029465973376, + "epoch": 0.45114495120191395, + "eval/gt_acc_batch": 0.7947222491105398, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2579437792301178, + "kd/policy_loss": 0.008462744660209864, + "kd/rkl_advantage_mean": 0.7482010906562209, + "kd/rkl_advantage_p95": 5.019400147596995, + "kd/rkl_advantage_std": 2.2100096782048544, + "kd/ssd_loss": 0.0, + "learning_rate": 5.488930239623286e-07, + "loss": 0.03385097980499267, + "num_tokens": 388727891.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222272555033, + "rewards/gt_logging_reward/std": 0.39679141690333686, + "sampling/importance_sampling_ratio/max": 1.9057703097661336, + "sampling/importance_sampling_ratio/mean": 1.0026654561360677, + "sampling/importance_sampling_ratio/min": 0.4301087309916814, + "sampling/sampling_logp_difference/max": 0.34376593430836994, + "sampling/sampling_logp_difference/mean": 0.00401786375635614, + "step": 11880, + "step_time": 8.136607864434579, + "student/entropy": 0.12698029465973376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0258333345875144, + "completions/max_length": 487.56666666666666, + "completions/max_terminated_length": 447.3, + "completions/mean_length": 195.7136220296224, + "completions/mean_terminated_length": 187.49509023030598, + "completions/min_length": 52.766666666666666, + "completions/min_terminated_length": 52.766666666666666, + "entropy": 0.12829925219217936, + "epoch": 0.4522842061291915, + "eval/gt_acc_batch": 0.8052777965863546, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3354143798351288, + "kd/policy_loss": 0.008491399263342221, + "kd/rkl_advantage_mean": 0.7631049758289009, + "kd/rkl_advantage_p95": 5.010220529635747, + "kd/rkl_advantage_std": 2.2081862916549047, + "kd/ssd_loss": 0.0, + "learning_rate": 5.477537690350512e-07, + "loss": 0.03396560351053874, + "num_tokens": 389729844.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8052777806917827, + "rewards/gt_logging_reward/std": 0.3923739194869995, + "sampling/importance_sampling_ratio/max": 1.8995325922966004, + "sampling/importance_sampling_ratio/mean": 1.0007681449254353, + "sampling/importance_sampling_ratio/min": 0.44639915972948074, + "sampling/sampling_logp_difference/max": 0.31044217745463054, + "sampling/sampling_logp_difference/mean": 0.00396394773075978, + "step": 11910, + "step_time": 8.027570355097607, + "student/entropy": 0.12829925219217936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944445973883072, + "completions/max_length": 469.76666666666665, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 189.8702870686849, + "completions/mean_terminated_length": 181.0322463989258, + "completions/min_length": 52.3, + "completions/min_terminated_length": 52.3, + "entropy": 0.12513675882170597, + "epoch": 0.4534234610564691, + "eval/gt_acc_batch": 0.7925000190734863, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23538579046726227, + "kd/policy_loss": 0.008504804479889572, + "kd/rkl_advantage_mean": 0.7597595894709229, + "kd/rkl_advantage_p95": 4.992324103911717, + "kd/rkl_advantage_std": 2.224011715253194, + "kd/ssd_loss": 0.0, + "learning_rate": 5.466145141077735e-07, + "loss": 0.03401922384897868, + "num_tokens": 390716529.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7924999972184499, + "rewards/gt_logging_reward/std": 0.39806604087352754, + "sampling/importance_sampling_ratio/max": 1.8765013376871744, + "sampling/importance_sampling_ratio/mean": 1.0007257123788198, + "sampling/importance_sampling_ratio/min": 0.4320206219951312, + "sampling/sampling_logp_difference/max": 0.3308928906917572, + "sampling/sampling_logp_difference/mean": 0.003912045410834253, + "step": 11940, + "step_time": 8.000353744334522, + "student/entropy": 0.12513675882170597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03972222491477927, + "completions/max_length": 492.3666666666667, + "completions/max_terminated_length": 473.6, + "completions/mean_length": 199.96056518554687, + "completions/mean_terminated_length": 187.25412241617838, + "completions/min_length": 49.86666666666667, + "completions/min_terminated_length": 49.86666666666667, + "entropy": 0.13511500060558318, + "epoch": 0.45456271598374665, + "eval/gt_acc_batch": 0.7816666901111603, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22071725130081177, + "kd/policy_loss": 0.009446450124960393, + "kd/rkl_advantage_mean": 0.7470880875363946, + "kd/rkl_advantage_p95": 5.028647708892822, + "kd/rkl_advantage_std": 2.2432570358117423, + "kd/ssd_loss": 0.0, + "learning_rate": 5.45475259180496e-07, + "loss": 0.03778580824534098, + "num_tokens": 391738907.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7816666662693024, + "rewards/gt_logging_reward/std": 0.40555890202522277, + "sampling/importance_sampling_ratio/max": 1.9463974237442017, + "sampling/importance_sampling_ratio/mean": 0.9959320863087971, + "sampling/importance_sampling_ratio/min": 0.4032391240199407, + "sampling/sampling_logp_difference/max": 0.3672845125198364, + "sampling/sampling_logp_difference/mean": 0.004156980368619164, + "step": 11970, + "step_time": 8.065991825701591, + "student/entropy": 0.13511500060558318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555668969949, + "completions/max_length": 472.03333333333336, + "completions/max_terminated_length": 428.6, + "completions/mean_length": 180.6336212158203, + "completions/mean_terminated_length": 173.76263732910155, + "completions/min_length": 51.733333333333334, + "completions/min_terminated_length": 51.733333333333334, + "entropy": 0.13032022460053364, + "epoch": 0.4557019709110242, + "eval/gt_acc_batch": 0.8186111489931742, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.279405415058136, + "kd/policy_loss": 0.008192786273624127, + "kd/rkl_advantage_mean": 0.7063602925588687, + "kd/rkl_advantage_p95": 4.922167044878006, + "kd/rkl_advantage_std": 2.1944000830252963, + "kd/ssd_loss": 0.0, + "learning_rate": 5.443360042532184e-07, + "loss": 0.032771148284276325, + "num_tokens": 392691244.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8186111152172089, + "rewards/gt_logging_reward/std": 0.3747957726319631, + "sampling/importance_sampling_ratio/max": 1.7784202496210735, + "sampling/importance_sampling_ratio/mean": 0.9927469909191131, + "sampling/importance_sampling_ratio/min": 0.417162894209226, + "sampling/sampling_logp_difference/max": 0.3315996050834656, + "sampling/sampling_logp_difference/mean": 0.004049071320332586, + "step": 12000, + "step_time": 7.945789945797879, + "student/entropy": 0.13032022460053364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016944445421298346, + "completions/max_length": 486.96666666666664, + "completions/max_terminated_length": 450.23333333333335, + "completions/mean_length": 189.90945587158203, + "completions/mean_terminated_length": 184.41861012776693, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.12799301072955133, + "epoch": 0.4568412258383017, + "eval/gt_acc_batch": 0.8136111219724019, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2825597822666168, + "kd/policy_loss": 0.008338462703007584, + "kd/rkl_advantage_mean": 0.7807475455105305, + "kd/rkl_advantage_p95": 5.135435656706492, + "kd/rkl_advantage_std": 2.264342340826988, + "kd/ssd_loss": 0.0, + "learning_rate": 5.431967493259408e-07, + "loss": 0.03335385322570801, + "num_tokens": 393671838.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813611110051473, + "rewards/gt_logging_reward/std": 0.3768211121360461, + "sampling/importance_sampling_ratio/max": 1.9151448011398315, + "sampling/importance_sampling_ratio/mean": 1.0074583450953165, + "sampling/importance_sampling_ratio/min": 0.42339751273393633, + "sampling/sampling_logp_difference/max": 0.3393979549407959, + "sampling/sampling_logp_difference/mean": 0.004025807557627559, + "step": 12030, + "step_time": 7.904765253470396, + "student/entropy": 0.12799301072955133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223629554113, + "completions/max_length": 472.6, + "completions/max_terminated_length": 439.9, + "completions/mean_length": 188.8072311401367, + "completions/mean_terminated_length": 181.54793345133464, + "completions/min_length": 56.266666666666666, + "completions/min_terminated_length": 56.266666666666666, + "entropy": 0.13337998470912377, + "epoch": 0.4579804807655793, + "eval/gt_acc_batch": 0.7850000123182933, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31098082661628723, + "kd/policy_loss": 0.008980945111640418, + "kd/rkl_advantage_mean": 0.7959753692770998, + "kd/rkl_advantage_p95": 5.065071958303451, + "kd/rkl_advantage_std": 2.2123953054348626, + "kd/ssd_loss": 0.0, + "learning_rate": 5.420574943986632e-07, + "loss": 0.03592378298441569, + "num_tokens": 394653704.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7850000043710073, + "rewards/gt_logging_reward/std": 0.40459794700145724, + "sampling/importance_sampling_ratio/max": 1.9783270716667176, + "sampling/importance_sampling_ratio/mean": 0.9989055732885996, + "sampling/importance_sampling_ratio/min": 0.4522521351774534, + "sampling/sampling_logp_difference/max": 0.30883174141248065, + "sampling/sampling_logp_difference/mean": 0.004110450212222835, + "step": 12060, + "step_time": 7.930055189168585, + "student/entropy": 0.13337998470912377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019444445427507163, + "completions/max_length": 456.96666666666664, + "completions/max_terminated_length": 426.8666666666667, + "completions/mean_length": 182.3011220296224, + "completions/mean_terminated_length": 175.9831278483073, + "completions/min_length": 47.233333333333334, + "completions/min_terminated_length": 47.233333333333334, + "entropy": 0.1281306093558669, + "epoch": 0.45911973569285686, + "eval/gt_acc_batch": 0.8072222451368968, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2872403860092163, + "kd/policy_loss": 0.008477109352437158, + "kd/rkl_advantage_mean": 0.6978572747670114, + "kd/rkl_advantage_p95": 4.835377901792526, + "kd/rkl_advantage_std": 2.160737705230713, + "kd/ssd_loss": 0.0, + "learning_rate": 5.409182394713857e-07, + "loss": 0.03390843868255615, + "num_tokens": 395610044.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222193082174, + "rewards/gt_logging_reward/std": 0.3904307097196579, + "sampling/importance_sampling_ratio/max": 1.9422001123428345, + "sampling/importance_sampling_ratio/mean": 1.0016786257425945, + "sampling/importance_sampling_ratio/min": 0.45843269377946855, + "sampling/sampling_logp_difference/max": 0.37298805713653566, + "sampling/sampling_logp_difference/mean": 0.004057158568563561, + "step": 12090, + "step_time": 7.728222152529634, + "student/entropy": 0.1281306093558669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028611112385988235, + "completions/max_length": 489.1, + "completions/max_terminated_length": 427.4, + "completions/mean_length": 193.21195475260416, + "completions/mean_terminated_length": 184.02374114990235, + "completions/min_length": 52.233333333333334, + "completions/min_terminated_length": 52.233333333333334, + "entropy": 0.12577250252167385, + "epoch": 0.4602589906201344, + "eval/gt_acc_batch": 0.7969444652398427, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30789715051651, + "kd/policy_loss": 0.00849053099906693, + "kd/rkl_advantage_mean": 0.7787505216896534, + "kd/rkl_advantage_p95": 5.1172090371449785, + "kd/rkl_advantage_std": 2.2487810134887694, + "kd/ssd_loss": 0.0, + "learning_rate": 5.397789845441081e-07, + "loss": 0.033962130546569824, + "num_tokens": 396599823.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444473584493, + "rewards/gt_logging_reward/std": 0.38750017384688057, + "sampling/importance_sampling_ratio/max": 2.0049553592999776, + "sampling/importance_sampling_ratio/mean": 1.0072013795375825, + "sampling/importance_sampling_ratio/min": 0.4497729440530141, + "sampling/sampling_logp_difference/max": 0.33171613017718, + "sampling/sampling_logp_difference/mean": 0.003979091175521413, + "step": 12120, + "step_time": 8.030885695067505, + "student/entropy": 0.12577250252167385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777912716071, + "completions/max_length": 487.9, + "completions/max_terminated_length": 453.93333333333334, + "completions/mean_length": 182.65167795817058, + "completions/mean_terminated_length": 174.12720947265626, + "completions/min_length": 50.4, + "completions/min_terminated_length": 50.4, + "entropy": 0.12850449445346993, + "epoch": 0.461398245547412, + "eval/gt_acc_batch": 0.8016666869322459, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3051902651786804, + "kd/policy_loss": 0.008476567268371583, + "kd/rkl_advantage_mean": 0.8039091775504251, + "kd/rkl_advantage_p95": 5.114813641707102, + "kd/rkl_advantage_std": 2.290487782160441, + "kd/ssd_loss": 0.0, + "learning_rate": 5.386397296168306e-07, + "loss": 0.033906273047129315, + "num_tokens": 397555377.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8016666690508525, + "rewards/gt_logging_reward/std": 0.3898630325992902, + "sampling/importance_sampling_ratio/max": 1.925388475259145, + "sampling/importance_sampling_ratio/mean": 1.0013880014419556, + "sampling/importance_sampling_ratio/min": 0.43317556778589883, + "sampling/sampling_logp_difference/max": 0.3799127479394277, + "sampling/sampling_logp_difference/mean": 0.004026151386400064, + "step": 12150, + "step_time": 7.984927275465937, + "student/entropy": 0.12850449445346993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01916666766628623, + "completions/max_length": 483.26666666666665, + "completions/max_terminated_length": 450.9, + "completions/mean_length": 186.55723215738934, + "completions/mean_terminated_length": 180.2538070678711, + "completions/min_length": 54.833333333333336, + "completions/min_terminated_length": 54.833333333333336, + "entropy": 0.12956419413288434, + "epoch": 0.46253750047468956, + "eval/gt_acc_batch": 0.7958333591620127, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2549331486225128, + "kd/policy_loss": 0.008816639733656, + "kd/rkl_advantage_mean": 0.7689006454000871, + "kd/rkl_advantage_p95": 5.07081171075503, + "kd/rkl_advantage_std": 2.2311460425456366, + "kd/ssd_loss": 0.0, + "learning_rate": 5.375004746895531e-07, + "loss": 0.03526655832926432, + "num_tokens": 398530111.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7958333353201549, + "rewards/gt_logging_reward/std": 0.3966590245564779, + "sampling/importance_sampling_ratio/max": 1.9275898615519205, + "sampling/importance_sampling_ratio/mean": 1.0009394566218057, + "sampling/importance_sampling_ratio/min": 0.4812654346227646, + "sampling/sampling_logp_difference/max": 0.3434873064359029, + "sampling/sampling_logp_difference/mean": 0.004052965971641243, + "step": 12180, + "step_time": 7.907386727261959, + "student/entropy": 0.12956419413288434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027500001713633537, + "completions/max_length": 491.8333333333333, + "completions/max_terminated_length": 460.2, + "completions/mean_length": 193.01278635660807, + "completions/mean_terminated_length": 184.16851552327475, + "completions/min_length": 53.9, + "completions/min_terminated_length": 53.9, + "entropy": 0.12869466841220856, + "epoch": 0.4636767554019671, + "eval/gt_acc_batch": 0.7930555760860443, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4258292019367218, + "kd/policy_loss": 0.008373634554057692, + "kd/rkl_advantage_mean": 0.7480928803483645, + "kd/rkl_advantage_p95": 5.043279594182968, + "kd/rkl_advantage_std": 2.2217055052518844, + "kd/ssd_loss": 0.0, + "learning_rate": 5.363612197622754e-07, + "loss": 0.03349453608194987, + "num_tokens": 399532213.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555582046509, + "rewards/gt_logging_reward/std": 0.39734552800655365, + "sampling/importance_sampling_ratio/max": 1.8310639619827271, + "sampling/importance_sampling_ratio/mean": 0.9924701273441314, + "sampling/importance_sampling_ratio/min": 0.41165054539839424, + "sampling/sampling_logp_difference/max": 0.36141204635302226, + "sampling/sampling_logp_difference/mean": 0.0040165397571399804, + "step": 12210, + "step_time": 8.054497408536553, + "student/entropy": 0.12869466841220856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02277777933826049, + "completions/max_length": 493.0, + "completions/max_terminated_length": 445.7, + "completions/mean_length": 189.71639862060547, + "completions/mean_terminated_length": 182.38717651367188, + "completions/min_length": 52.733333333333334, + "completions/min_terminated_length": 52.733333333333334, + "entropy": 0.12905156395087639, + "epoch": 0.4648160103292447, + "eval/gt_acc_batch": 0.7861111442248027, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2826915681362152, + "kd/policy_loss": 0.008705345463628571, + "kd/rkl_advantage_mean": 0.7891308058674137, + "kd/rkl_advantage_p95": 5.112093053261439, + "kd/rkl_advantage_std": 2.278004095951716, + "kd/ssd_loss": 0.0, + "learning_rate": 5.352219648349979e-07, + "loss": 0.03482138315836589, + "num_tokens": 400520304.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111104488373, + "rewards/gt_logging_reward/std": 0.3994013249874115, + "sampling/importance_sampling_ratio/max": 1.9566477855046591, + "sampling/importance_sampling_ratio/mean": 0.9952687402566274, + "sampling/importance_sampling_ratio/min": 0.44381705621878303, + "sampling/sampling_logp_difference/max": 0.3578454474608103, + "sampling/sampling_logp_difference/mean": 0.003978842605526249, + "step": 12240, + "step_time": 7.93886410856503, + "student/entropy": 0.12905156395087639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556944260994, + "completions/max_length": 465.9, + "completions/max_terminated_length": 439.0, + "completions/mean_length": 184.73834330240885, + "completions/mean_terminated_length": 177.3744333902995, + "completions/min_length": 58.266666666666666, + "completions/min_terminated_length": 58.266666666666666, + "entropy": 0.1316041840861241, + "epoch": 0.46595526525652226, + "eval/gt_acc_batch": 0.8002778033415476, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28901320695877075, + "kd/policy_loss": 0.008861186498931299, + "kd/rkl_advantage_mean": 0.8377581735452017, + "kd/rkl_advantage_p95": 5.188427786032359, + "kd/rkl_advantage_std": 2.2697444836298626, + "kd/ssd_loss": 0.0, + "learning_rate": 5.340827099077203e-07, + "loss": 0.035444752375284834, + "num_tokens": 401492346.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8002777775128682, + "rewards/gt_logging_reward/std": 0.3883898377418518, + "sampling/importance_sampling_ratio/max": 1.9238938053448995, + "sampling/importance_sampling_ratio/mean": 1.0013447026411693, + "sampling/importance_sampling_ratio/min": 0.44839653968811033, + "sampling/sampling_logp_difference/max": 0.35615657170613607, + "sampling/sampling_logp_difference/mean": 0.0041023522227381665, + "step": 12270, + "step_time": 7.998448583833912, + "student/entropy": 0.1316041840861241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02583333443229397, + "completions/max_length": 491.9, + "completions/max_terminated_length": 450.76666666666665, + "completions/mean_length": 191.52584279378254, + "completions/mean_terminated_length": 183.18075307210287, + "completions/min_length": 51.6, + "completions/min_terminated_length": 51.6, + "entropy": 0.12544056779394547, + "epoch": 0.4670945201837998, + "eval/gt_acc_batch": 0.7936111330986023, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.35356587171554565, + "kd/policy_loss": 0.00853559344735307, + "kd/rkl_advantage_mean": 0.7803453713655472, + "kd/rkl_advantage_p95": 5.095985708634059, + "kd/rkl_advantage_std": 2.2327536781628927, + "kd/ssd_loss": 0.0, + "learning_rate": 5.329434549804428e-07, + "loss": 0.034142374992370605, + "num_tokens": 402497455.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111152172088, + "rewards/gt_logging_reward/std": 0.39681895474592843, + "sampling/importance_sampling_ratio/max": 1.8700268546740213, + "sampling/importance_sampling_ratio/mean": 1.001883480946223, + "sampling/importance_sampling_ratio/min": 0.43674261420965194, + "sampling/sampling_logp_difference/max": 0.34496406515439354, + "sampling/sampling_logp_difference/mean": 0.003968942631036043, + "step": 12300, + "step_time": 8.146593546066045, + "student/entropy": 0.12544056779394547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334742734828, + "completions/max_length": 505.93333333333334, + "completions/max_terminated_length": 461.4, + "completions/mean_length": 190.9844538370768, + "completions/mean_terminated_length": 182.51958211263022, + "completions/min_length": 57.5, + "completions/min_terminated_length": 57.5, + "entropy": 0.13454602435231208, + "epoch": 0.46823377511107733, + "eval/gt_acc_batch": 0.8005555868148804, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28282517194747925, + "kd/policy_loss": 0.008879554050508887, + "kd/rkl_advantage_mean": 0.8585422556226452, + "kd/rkl_advantage_p95": 5.18667913277944, + "kd/rkl_advantage_std": 2.26518285771211, + "kd/ssd_loss": 0.0, + "learning_rate": 5.318042000531652e-07, + "loss": 0.035518217086791995, + "num_tokens": 403487479.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8005555550257365, + "rewards/gt_logging_reward/std": 0.391682335237662, + "sampling/importance_sampling_ratio/max": 1.9527653972307841, + "sampling/importance_sampling_ratio/mean": 0.9971980492273966, + "sampling/importance_sampling_ratio/min": 0.4198535069823265, + "sampling/sampling_logp_difference/max": 0.37571562131245934, + "sampling/sampling_logp_difference/mean": 0.004155640591246386, + "step": 12330, + "step_time": 8.262695367068712, + "student/entropy": 0.13454602435231208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778953313828, + "completions/max_length": 491.8666666666667, + "completions/max_terminated_length": 447.03333333333336, + "completions/mean_length": 182.05695292154948, + "completions/mean_terminated_length": 176.0688496907552, + "completions/min_length": 51.333333333333336, + "completions/min_terminated_length": 51.333333333333336, + "entropy": 0.12975815553218126, + "epoch": 0.4693730300383549, + "eval/gt_acc_batch": 0.7944444755713145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3199203312397003, + "kd/policy_loss": 0.00869534924859181, + "kd/rkl_advantage_mean": 0.7870756124146283, + "kd/rkl_advantage_p95": 5.114649401108424, + "kd/rkl_advantage_std": 2.2828834851582847, + "kd/ssd_loss": 0.0, + "learning_rate": 5.306649451258877e-07, + "loss": 0.03478139638900757, + "num_tokens": 404446940.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7944444417953491, + "rewards/gt_logging_reward/std": 0.3986867109934489, + "sampling/importance_sampling_ratio/max": 1.8747958064079284, + "sampling/importance_sampling_ratio/mean": 0.9929058909416199, + "sampling/importance_sampling_ratio/min": 0.437885095179081, + "sampling/sampling_logp_difference/max": 0.3231948455174764, + "sampling/sampling_logp_difference/mean": 0.004070127957190076, + "step": 12360, + "step_time": 8.023203751935217, + "student/entropy": 0.12975815553218126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030833334910372892, + "completions/max_length": 494.73333333333335, + "completions/max_terminated_length": 451.1666666666667, + "completions/mean_length": 197.84945576985677, + "completions/mean_terminated_length": 188.0748540242513, + "completions/min_length": 51.56666666666667, + "completions/min_terminated_length": 51.56666666666667, + "entropy": 0.12908708558728296, + "epoch": 0.47051228496563247, + "eval/gt_acc_batch": 0.7930555701255798, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24288931488990784, + "kd/policy_loss": 0.008531666034832597, + "kd/rkl_advantage_mean": 0.8102629373470942, + "kd/rkl_advantage_p95": 5.144208264350891, + "kd/rkl_advantage_std": 2.2631691992282867, + "kd/ssd_loss": 0.0, + "learning_rate": 5.2952569019861e-07, + "loss": 0.03412667115529378, + "num_tokens": 405468046.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555522441864, + "rewards/gt_logging_reward/std": 0.39825192590554553, + "sampling/importance_sampling_ratio/max": 1.9421889185905457, + "sampling/importance_sampling_ratio/mean": 1.0011794507503509, + "sampling/importance_sampling_ratio/min": 0.45422680377960206, + "sampling/sampling_logp_difference/max": 0.31996582746505736, + "sampling/sampling_logp_difference/mean": 0.003985898375200729, + "step": 12390, + "step_time": 8.254475776099328, + "student/entropy": 0.12908708558728296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02833333481103182, + "completions/max_length": 486.6, + "completions/max_terminated_length": 451.3666666666667, + "completions/mean_length": 192.4555648803711, + "completions/mean_terminated_length": 183.28630574544272, + "completions/min_length": 56.36666666666667, + "completions/min_terminated_length": 56.36666666666667, + "entropy": 0.12820224712292352, + "epoch": 0.47165153989291003, + "eval/gt_acc_batch": 0.7944444636503856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2593255937099457, + "kd/policy_loss": 0.008674797390510018, + "kd/rkl_advantage_mean": 0.8075697331999739, + "kd/rkl_advantage_p95": 5.1253375271956125, + "kd/rkl_advantage_std": 2.2392161319653194, + "kd/ssd_loss": 0.0, + "learning_rate": 5.283864352713325e-07, + "loss": 0.034699193636576336, + "num_tokens": 406455750.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7944444437821706, + "rewards/gt_logging_reward/std": 0.39720159272352856, + "sampling/importance_sampling_ratio/max": 1.976891847451528, + "sampling/importance_sampling_ratio/mean": 0.9993873596191406, + "sampling/importance_sampling_ratio/min": 0.42790596683820087, + "sampling/sampling_logp_difference/max": 0.33373506863911945, + "sampling/sampling_logp_difference/mean": 0.003946350538171828, + "step": 12420, + "step_time": 8.017524809301054, + "student/entropy": 0.12820224712292352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112367361785, + "completions/max_length": 491.3666666666667, + "completions/max_terminated_length": 443.56666666666666, + "completions/mean_length": 188.79195454915364, + "completions/mean_terminated_length": 181.9593287150065, + "completions/min_length": 57.2, + "completions/min_terminated_length": 57.2, + "entropy": 0.13498600851744413, + "epoch": 0.4727907948201876, + "eval/gt_acc_batch": 0.7972222487131755, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28816768527030945, + "kd/policy_loss": 0.009159433022917558, + "kd/rkl_advantage_mean": 0.8423999354243279, + "kd/rkl_advantage_p95": 5.189157114426295, + "kd/rkl_advantage_std": 2.2744650334119796, + "kd/ssd_loss": 0.0, + "learning_rate": 5.272471803440549e-07, + "loss": 0.036637735366821286, + "num_tokens": 407430553.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7972222189108531, + "rewards/gt_logging_reward/std": 0.3963650673627853, + "sampling/importance_sampling_ratio/max": 1.837975549697876, + "sampling/importance_sampling_ratio/mean": 0.9944822669029236, + "sampling/importance_sampling_ratio/min": 0.4234780485431353, + "sampling/sampling_logp_difference/max": 0.35751566688219705, + "sampling/sampling_logp_difference/mean": 0.004125487843217949, + "step": 12450, + "step_time": 8.069132091835975, + "student/entropy": 0.13498600851744413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030833334972461064, + "completions/max_length": 495.1, + "completions/max_terminated_length": 452.1333333333333, + "completions/mean_length": 188.45528920491537, + "completions/mean_terminated_length": 178.19663848876954, + "completions/min_length": 55.46666666666667, + "completions/min_terminated_length": 55.46666666666667, + "entropy": 0.13365290326376755, + "epoch": 0.47393004974746517, + "eval/gt_acc_batch": 0.7850000222524007, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25963732600212097, + "kd/policy_loss": 0.008860667108092458, + "kd/rkl_advantage_mean": 0.809007806951801, + "kd/rkl_advantage_p95": 5.1819097936153415, + "kd/rkl_advantage_std": 2.273602482676506, + "kd/ssd_loss": 0.0, + "learning_rate": 5.261079254167774e-07, + "loss": 0.035442666212717695, + "num_tokens": 408409088.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7849999984105428, + "rewards/gt_logging_reward/std": 0.4033943961064021, + "sampling/importance_sampling_ratio/max": 1.9409206748008727, + "sampling/importance_sampling_ratio/mean": 0.9996331989765167, + "sampling/importance_sampling_ratio/min": 0.4205568845073382, + "sampling/sampling_logp_difference/max": 0.34457824428876244, + "sampling/sampling_logp_difference/mean": 0.00412205553924044, + "step": 12480, + "step_time": 8.08000808700114, + "student/entropy": 0.13365290326376755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666751727462, + "completions/max_length": 491.0, + "completions/max_terminated_length": 441.1, + "completions/mean_length": 194.4136215209961, + "completions/mean_terminated_length": 187.2980951944987, + "completions/min_length": 51.63333333333333, + "completions/min_terminated_length": 51.63333333333333, + "entropy": 0.13033027419199547, + "epoch": 0.47506930467474273, + "eval/gt_acc_batch": 0.7777777969837188, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28601422905921936, + "kd/policy_loss": 0.009291781892534346, + "kd/rkl_advantage_mean": 0.8224023768678308, + "kd/rkl_advantage_p95": 5.146378113826116, + "kd/rkl_advantage_std": 2.2531438281138736, + "kd/ssd_loss": 0.0, + "learning_rate": 5.249686704894999e-07, + "loss": 0.03716712792714437, + "num_tokens": 409405441.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7777777791023255, + "rewards/gt_logging_reward/std": 0.40889160335063934, + "sampling/importance_sampling_ratio/max": 1.9376919468243916, + "sampling/importance_sampling_ratio/mean": 1.0036546528339385, + "sampling/importance_sampling_ratio/min": 0.39551488955815634, + "sampling/sampling_logp_difference/max": 0.35029176870981854, + "sampling/sampling_logp_difference/mean": 0.004034199534604947, + "step": 12510, + "step_time": 7.950618540866708, + "student/entropy": 0.13033027419199547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556813875834, + "completions/max_length": 483.3, + "completions/max_terminated_length": 450.1333333333333, + "completions/mean_length": 187.92501068115234, + "completions/mean_terminated_length": 181.3502161661784, + "completions/min_length": 55.9, + "completions/min_terminated_length": 55.9, + "entropy": 0.1250470625857512, + "epoch": 0.4762085596020203, + "eval/gt_acc_batch": 0.8052778164545695, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3301781415939331, + "kd/policy_loss": 0.008609231926190357, + "kd/rkl_advantage_mean": 0.7017073646110172, + "kd/rkl_advantage_p95": 4.862059146165848, + "kd/rkl_advantage_std": 2.1899453987677893, + "kd/ssd_loss": 0.0, + "learning_rate": 5.238294155622223e-07, + "loss": 0.03443692525227864, + "num_tokens": 410387755.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8052777727444966, + "rewards/gt_logging_reward/std": 0.385347855091095, + "sampling/importance_sampling_ratio/max": 1.9171698172887166, + "sampling/importance_sampling_ratio/mean": 0.9953366061051686, + "sampling/importance_sampling_ratio/min": 0.45803446918725965, + "sampling/sampling_logp_difference/max": 0.3533571779727936, + "sampling/sampling_logp_difference/mean": 0.0038906583096832035, + "step": 12540, + "step_time": 7.84950780516762, + "student/entropy": 0.1250470625857512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01944444573794802, + "completions/max_length": 504.56666666666666, + "completions/max_terminated_length": 452.46666666666664, + "completions/mean_length": 183.7888977050781, + "completions/mean_terminated_length": 177.2657679239909, + "completions/min_length": 54.166666666666664, + "completions/min_terminated_length": 54.166666666666664, + "entropy": 0.1373899048815171, + "epoch": 0.47734781452929786, + "eval/gt_acc_batch": 0.7880555689334869, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33327504992485046, + "kd/policy_loss": 0.00919904092637201, + "kd/rkl_advantage_mean": 0.7638780643620218, + "kd/rkl_advantage_p95": 5.089945125579834, + "kd/rkl_advantage_std": 2.272619946797689, + "kd/ssd_loss": 0.0, + "learning_rate": 5.226901606349448e-07, + "loss": 0.036796164512634275, + "num_tokens": 411348083.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7880555589993795, + "rewards/gt_logging_reward/std": 0.3989095409711202, + "sampling/importance_sampling_ratio/max": 1.9794055660565695, + "sampling/importance_sampling_ratio/mean": 1.005633242925008, + "sampling/importance_sampling_ratio/min": 0.4240633398294449, + "sampling/sampling_logp_difference/max": 0.32150013049443565, + "sampling/sampling_logp_difference/mean": 0.00413146597178032, + "step": 12570, + "step_time": 7.923313862433618, + "student/entropy": 0.1373899048815171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778704961143, + "completions/max_length": 457.6666666666667, + "completions/max_terminated_length": 414.8333333333333, + "completions/mean_length": 184.74723256429036, + "completions/mean_terminated_length": 178.9701944986979, + "completions/min_length": 53.766666666666666, + "completions/min_terminated_length": 53.766666666666666, + "entropy": 0.12524750089893738, + "epoch": 0.4784870694565754, + "eval/gt_acc_batch": 0.8036111334959666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30214250087738037, + "kd/policy_loss": 0.00841039073614714, + "kd/rkl_advantage_mean": 0.7116778746208486, + "kd/rkl_advantage_p95": 4.844099666674932, + "kd/rkl_advantage_std": 2.1577556560436886, + "kd/ssd_loss": 0.0, + "learning_rate": 5.215509057076671e-07, + "loss": 0.03364156484603882, + "num_tokens": 412311421.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111096541086, + "rewards/gt_logging_reward/std": 0.38855257133642834, + "sampling/importance_sampling_ratio/max": 1.8663800239562989, + "sampling/importance_sampling_ratio/mean": 1.0016922255357106, + "sampling/importance_sampling_ratio/min": 0.43024966021378835, + "sampling/sampling_logp_difference/max": 0.3467615048090617, + "sampling/sampling_logp_difference/mean": 0.0039003147588421903, + "step": 12600, + "step_time": 7.655710810002347, + "student/entropy": 0.12524750089893738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001297642788, + "completions/max_length": 487.06666666666666, + "completions/max_terminated_length": 456.1333333333333, + "completions/mean_length": 188.10167744954427, + "completions/mean_terminated_length": 180.73096466064453, + "completions/min_length": 47.4, + "completions/min_terminated_length": 47.4, + "entropy": 0.13153769026199977, + "epoch": 0.47962632438385294, + "eval/gt_acc_batch": 0.7980555812517802, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2609873414039612, + "kd/policy_loss": 0.008707795220349606, + "kd/rkl_advantage_mean": 0.7694641329968969, + "kd/rkl_advantage_p95": 5.108043116331101, + "kd/rkl_advantage_std": 2.2635294953982035, + "kd/ssd_loss": 0.0, + "learning_rate": 5.204116507803896e-07, + "loss": 0.03483118216196696, + "num_tokens": 413298467.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7980555593967438, + "rewards/gt_logging_reward/std": 0.3963110496600469, + "sampling/importance_sampling_ratio/max": 1.8643792589505515, + "sampling/importance_sampling_ratio/mean": 0.9989051381746928, + "sampling/importance_sampling_ratio/min": 0.49423596262931824, + "sampling/sampling_logp_difference/max": 0.3765718857447306, + "sampling/sampling_logp_difference/mean": 0.004107233633597692, + "step": 12630, + "step_time": 7.970042690967967, + "student/entropy": 0.13153769026199977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112429449956, + "completions/max_length": 487.1333333333333, + "completions/max_terminated_length": 453.3666666666667, + "completions/mean_length": 186.78834228515626, + "completions/mean_terminated_length": 179.8039764404297, + "completions/min_length": 56.666666666666664, + "completions/min_terminated_length": 56.666666666666664, + "entropy": 0.12940102635572354, + "epoch": 0.4807655793111305, + "eval/gt_acc_batch": 0.8041666825612386, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28625544905662537, + "kd/policy_loss": 0.00821543302348194, + "kd/rkl_advantage_mean": 0.7745011765509844, + "kd/rkl_advantage_p95": 5.061305189132691, + "kd/rkl_advantage_std": 2.2277542104323707, + "kd/ssd_loss": 0.0, + "learning_rate": 5.19272395853112e-07, + "loss": 0.03286173144976298, + "num_tokens": 414264281.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666666666667, + "rewards/gt_logging_reward/std": 0.38842560946941374, + "sampling/importance_sampling_ratio/max": 1.7729122201601664, + "sampling/importance_sampling_ratio/mean": 0.9926212906837464, + "sampling/importance_sampling_ratio/min": 0.43396565119425456, + "sampling/sampling_logp_difference/max": 0.32010950446128844, + "sampling/sampling_logp_difference/mean": 0.004025701456703246, + "step": 12660, + "step_time": 7.752204642337165, + "student/entropy": 0.12940102635572354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01194444519157211, + "completions/max_length": 457.93333333333334, + "completions/max_terminated_length": 430.2, + "completions/mean_length": 179.58778737386066, + "completions/mean_terminated_length": 175.55408274332683, + "completions/min_length": 56.7, + "completions/min_terminated_length": 56.7, + "entropy": 0.1294696072116494, + "epoch": 0.4819048342384081, + "eval/gt_acc_batch": 0.8213889022668203, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25255241990089417, + "kd/policy_loss": 0.008138923702063039, + "kd/rkl_advantage_mean": 0.7223417156065504, + "kd/rkl_advantage_p95": 4.890562552213669, + "kd/rkl_advantage_std": 2.1737030218044917, + "kd/ssd_loss": 0.0, + "learning_rate": 5.181331409258344e-07, + "loss": 0.032555697361628215, + "num_tokens": 415213637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8213888883590699, + "rewards/gt_logging_reward/std": 0.377575746178627, + "sampling/importance_sampling_ratio/max": 1.8993517756462097, + "sampling/importance_sampling_ratio/mean": 1.0006264011065165, + "sampling/importance_sampling_ratio/min": 0.38869684835275015, + "sampling/sampling_logp_difference/max": 0.3556972702344259, + "sampling/sampling_logp_difference/mean": 0.004051536849389473, + "step": 12690, + "step_time": 7.724716821066977, + "student/entropy": 0.1294696072116494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223797192177, + "completions/max_length": 481.43333333333334, + "completions/max_terminated_length": 447.8666666666667, + "completions/mean_length": 189.83000895182292, + "completions/mean_terminated_length": 180.99695892333983, + "completions/min_length": 60.233333333333334, + "completions/min_terminated_length": 60.233333333333334, + "entropy": 0.1336875431239605, + "epoch": 0.48304408916568564, + "eval/gt_acc_batch": 0.7966666917006174, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3290959298610687, + "kd/policy_loss": 0.009111005294835195, + "kd/rkl_advantage_mean": 0.7284327437790732, + "kd/rkl_advantage_p95": 4.907591076691945, + "kd/rkl_advantage_std": 2.1913512726624806, + "kd/ssd_loss": 0.0, + "learning_rate": 5.169938859985569e-07, + "loss": 0.03644402821858724, + "num_tokens": 416202113.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7966666638851165, + "rewards/gt_logging_reward/std": 0.3919711252053579, + "sampling/importance_sampling_ratio/max": 1.9949310580889383, + "sampling/importance_sampling_ratio/mean": 0.9968347609043121, + "sampling/importance_sampling_ratio/min": 0.44508445014556247, + "sampling/sampling_logp_difference/max": 0.3466535607973735, + "sampling/sampling_logp_difference/mean": 0.004015412616233031, + "step": 12720, + "step_time": 7.9249766730650055, + "student/entropy": 0.1336875431239605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500001130004723, + "completions/max_length": 475.3, + "completions/max_terminated_length": 447.8, + "completions/mean_length": 187.96195373535156, + "completions/mean_terminated_length": 182.2034927368164, + "completions/min_length": 54.5, + "completions/min_terminated_length": 54.5, + "entropy": 0.12608816946546236, + "epoch": 0.4841833440929632, + "eval/gt_acc_batch": 0.8080555876096089, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27184510231018066, + "kd/policy_loss": 0.008268644030128296, + "kd/rkl_advantage_mean": 0.8023717172443867, + "kd/rkl_advantage_p95": 5.125519134600958, + "kd/rkl_advantage_std": 2.2660691291093826, + "kd/ssd_loss": 0.0, + "learning_rate": 5.158546310712794e-07, + "loss": 0.03307457566261292, + "num_tokens": 417185480.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8080555578072866, + "rewards/gt_logging_reward/std": 0.38630496760209404, + "sampling/importance_sampling_ratio/max": 1.9522570927937826, + "sampling/importance_sampling_ratio/mean": 1.0008424699306488, + "sampling/importance_sampling_ratio/min": 0.4588800440231959, + "sampling/sampling_logp_difference/max": 0.3224307656288147, + "sampling/sampling_logp_difference/mean": 0.003932757986088594, + "step": 12750, + "step_time": 7.9768145864674205, + "student/entropy": 0.12608816946546236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500001130004723, + "completions/max_length": 480.8666666666667, + "completions/max_terminated_length": 457.03333333333336, + "completions/mean_length": 189.50750885009765, + "completions/mean_terminated_length": 183.7904825846354, + "completions/min_length": 48.56666666666667, + "completions/min_terminated_length": 48.56666666666667, + "entropy": 0.12585870946447056, + "epoch": 0.4853225990202408, + "eval/gt_acc_batch": 0.7869444568951924, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2548651695251465, + "kd/policy_loss": 0.008568632546424244, + "kd/rkl_advantage_mean": 0.7703784157832464, + "kd/rkl_advantage_p95": 4.972911282380422, + "kd/rkl_advantage_std": 2.2032086004813514, + "kd/ssd_loss": 0.0, + "learning_rate": 5.147153761440018e-07, + "loss": 0.03427453438440959, + "num_tokens": 418174051.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444410006206, + "rewards/gt_logging_reward/std": 0.40290601750214894, + "sampling/importance_sampling_ratio/max": 1.9557335098584494, + "sampling/importance_sampling_ratio/mean": 0.9976895689964295, + "sampling/importance_sampling_ratio/min": 0.3921566759546598, + "sampling/sampling_logp_difference/max": 0.3479256014029185, + "sampling/sampling_logp_difference/mean": 0.003929089677209656, + "step": 12780, + "step_time": 7.7869664665020535, + "student/entropy": 0.12585870946447056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02416666786496838, + "completions/max_length": 494.6666666666667, + "completions/max_terminated_length": 454.96666666666664, + "completions/mean_length": 192.06251017252603, + "completions/mean_terminated_length": 184.00249735514322, + "completions/min_length": 56.46666666666667, + "completions/min_terminated_length": 56.46666666666667, + "entropy": 0.12727266885340213, + "epoch": 0.48646185394751834, + "eval/gt_acc_batch": 0.7877777934074401, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3017065227031708, + "kd/policy_loss": 0.008563455217517913, + "kd/rkl_advantage_mean": 0.7222618437682589, + "kd/rkl_advantage_p95": 5.002208779255549, + "kd/rkl_advantage_std": 2.23451785047849, + "kd/ssd_loss": 0.0, + "learning_rate": 5.135761212167242e-07, + "loss": 0.03425381978352864, + "num_tokens": 419174812.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7877777775128683, + "rewards/gt_logging_reward/std": 0.4024914781252543, + "sampling/importance_sampling_ratio/max": 1.9597543239593507, + "sampling/importance_sampling_ratio/mean": 1.0006189147631328, + "sampling/importance_sampling_ratio/min": 0.4346261004606883, + "sampling/sampling_logp_difference/max": 0.33620619773864746, + "sampling/sampling_logp_difference/mean": 0.003976158075965941, + "step": 12810, + "step_time": 7.921080640230502, + "student/entropy": 0.12727266885340213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111253499985, + "completions/max_length": 483.6, + "completions/max_terminated_length": 449.96666666666664, + "completions/mean_length": 193.5452880859375, + "completions/mean_terminated_length": 185.16425323486328, + "completions/min_length": 53.2, + "completions/min_terminated_length": 53.2, + "entropy": 0.12653754372149706, + "epoch": 0.4876011088747959, + "eval/gt_acc_batch": 0.7844444572925567, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27356666326522827, + "kd/policy_loss": 0.008159154884439582, + "kd/rkl_advantage_mean": 0.7619230711211761, + "kd/rkl_advantage_p95": 4.992198308308919, + "kd/rkl_advantage_std": 2.2159988860289257, + "kd/ssd_loss": 0.0, + "learning_rate": 5.124368662894467e-07, + "loss": 0.032636616627375284, + "num_tokens": 420166175.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7844444453716278, + "rewards/gt_logging_reward/std": 0.3987035363912582, + "sampling/importance_sampling_ratio/max": 1.7942757527033488, + "sampling/importance_sampling_ratio/mean": 0.9919310172398885, + "sampling/importance_sampling_ratio/min": 0.4443650404612223, + "sampling/sampling_logp_difference/max": 0.3363239884376526, + "sampling/sampling_logp_difference/mean": 0.003919214537988106, + "step": 12840, + "step_time": 7.862965442899925, + "student/entropy": 0.12653754372149706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334730317194, + "completions/max_length": 487.8666666666667, + "completions/max_terminated_length": 452.2, + "completions/mean_length": 187.97667744954427, + "completions/mean_terminated_length": 181.18448638916016, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "entropy": 0.1314614546795686, + "epoch": 0.4887403638020734, + "eval/gt_acc_batch": 0.7986111283302307, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.283154159784317, + "kd/policy_loss": 0.008840304324015355, + "kd/rkl_advantage_mean": 0.7335600520173708, + "kd/rkl_advantage_p95": 4.930224364995956, + "kd/rkl_advantage_std": 2.2265256037314733, + "kd/ssd_loss": 0.0, + "learning_rate": 5.11297611362169e-07, + "loss": 0.035361218452453616, + "num_tokens": 421138467.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111124356587, + "rewards/gt_logging_reward/std": 0.390252781411012, + "sampling/importance_sampling_ratio/max": 1.9257989843686423, + "sampling/importance_sampling_ratio/mean": 0.9939515431722005, + "sampling/importance_sampling_ratio/min": 0.45312494138876597, + "sampling/sampling_logp_difference/max": 0.3593298157056173, + "sampling/sampling_logp_difference/mean": 0.004034339046726624, + "step": 12870, + "step_time": 7.775239194768559, + "student/entropy": 0.1314614546795686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016944445421298346, + "completions/max_length": 468.96666666666664, + "completions/max_terminated_length": 444.53333333333336, + "completions/mean_length": 182.70223134358724, + "completions/mean_terminated_length": 177.14229532877604, + "completions/min_length": 60.86666666666667, + "completions/min_terminated_length": 60.86666666666667, + "entropy": 0.126068641555806, + "epoch": 0.489879618729351, + "eval/gt_acc_batch": 0.8091666837533315, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24527336657047272, + "kd/policy_loss": 0.008150560296295831, + "kd/rkl_advantage_mean": 0.7703741757199168, + "kd/rkl_advantage_p95": 4.987259052197138, + "kd/rkl_advantage_std": 2.2059757729371388, + "kd/ssd_loss": 0.0, + "learning_rate": 5.101583564348916e-07, + "loss": 0.03260224262873332, + "num_tokens": 422110011.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.809166669845581, + "rewards/gt_logging_reward/std": 0.3830762048562368, + "sampling/importance_sampling_ratio/max": 1.8716726859410604, + "sampling/importance_sampling_ratio/mean": 1.004055960973104, + "sampling/importance_sampling_ratio/min": 0.4763409117857615, + "sampling/sampling_logp_difference/max": 0.3607962926228841, + "sampling/sampling_logp_difference/mean": 0.003923911298625171, + "step": 12900, + "step_time": 7.806473378799759, + "student/entropy": 0.126068641555806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112404614686, + "completions/max_length": 485.06666666666666, + "completions/max_terminated_length": 451.76666666666665, + "completions/mean_length": 190.2894541422526, + "completions/mean_terminated_length": 182.7700190226237, + "completions/min_length": 54.5, + "completions/min_terminated_length": 54.5, + "entropy": 0.12317278763900201, + "epoch": 0.49101887365662855, + "eval/gt_acc_batch": 0.7858333547910055, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.35849878191947937, + "kd/policy_loss": 0.008315546232430885, + "kd/rkl_advantage_mean": 0.7614697139089306, + "kd/rkl_advantage_p95": 4.975421154499054, + "kd/rkl_advantage_std": 2.2033395061890286, + "kd/ssd_loss": 0.0, + "learning_rate": 5.09019101507614e-07, + "loss": 0.03326218128204346, + "num_tokens": 423100533.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.785833336909612, + "rewards/gt_logging_reward/std": 0.39778612355391185, + "sampling/importance_sampling_ratio/max": 1.9797397613525392, + "sampling/importance_sampling_ratio/mean": 1.0031026562054952, + "sampling/importance_sampling_ratio/min": 0.41837929636240007, + "sampling/sampling_logp_difference/max": 0.3370972255865733, + "sampling/sampling_logp_difference/mean": 0.003815776598639786, + "step": 12930, + "step_time": 7.9327052281655295, + "student/entropy": 0.12317278763900201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055557192613683, + "completions/max_length": 491.26666666666665, + "completions/max_terminated_length": 464.3666666666667, + "completions/mean_length": 197.05389963785808, + "completions/mean_terminated_length": 189.74186147054036, + "completions/min_length": 51.166666666666664, + "completions/min_terminated_length": 51.166666666666664, + "entropy": 0.1314670872564117, + "epoch": 0.4921581285839061, + "eval/gt_acc_batch": 0.7683333575725555, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2538837790489197, + "kd/policy_loss": 0.009529152551355462, + "kd/rkl_advantage_mean": 0.7642346562196811, + "kd/rkl_advantage_p95": 5.036253313223521, + "kd/rkl_advantage_std": 2.2237118860085805, + "kd/ssd_loss": 0.0, + "learning_rate": 5.078798465803365e-07, + "loss": 0.0381166140238444, + "num_tokens": 424124703.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7683333337306977, + "rewards/gt_logging_reward/std": 0.41510116557280224, + "sampling/importance_sampling_ratio/max": 2.0355565667152407, + "sampling/importance_sampling_ratio/mean": 1.0035285254319508, + "sampling/importance_sampling_ratio/min": 0.43620601296424866, + "sampling/sampling_logp_difference/max": 0.33650799592336017, + "sampling/sampling_logp_difference/mean": 0.004063276729236047, + "step": 12960, + "step_time": 8.049393393867165, + "student/entropy": 0.1314670872564117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112559835116, + "completions/max_length": 492.5, + "completions/max_terminated_length": 468.4, + "completions/mean_length": 193.4077885945638, + "completions/mean_terminated_length": 185.9240966796875, + "completions/min_length": 62.833333333333336, + "completions/min_terminated_length": 62.833333333333336, + "entropy": 0.1339859005063772, + "epoch": 0.4932973835111837, + "eval/gt_acc_batch": 0.7869444688161215, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2393186241388321, + "kd/policy_loss": 0.008823801311276231, + "kd/rkl_advantage_mean": 0.839279082044959, + "kd/rkl_advantage_p95": 5.164686053991318, + "kd/rkl_advantage_std": 2.234951266646385, + "kd/ssd_loss": 0.0, + "learning_rate": 5.067405916530588e-07, + "loss": 0.03529520829518636, + "num_tokens": 425119947.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.786944446961085, + "rewards/gt_logging_reward/std": 0.39855003356933594, + "sampling/importance_sampling_ratio/max": 1.9424442132314046, + "sampling/importance_sampling_ratio/mean": 1.0008370399475097, + "sampling/importance_sampling_ratio/min": 0.4217667758464813, + "sampling/sampling_logp_difference/max": 0.33361701369285585, + "sampling/sampling_logp_difference/mean": 0.004139342865285774, + "step": 12990, + "step_time": 7.963213203195482, + "student/entropy": 0.1339859005063772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889035830895, + "completions/max_length": 485.53333333333336, + "completions/max_terminated_length": 439.9, + "completions/mean_length": 183.5375089009603, + "completions/mean_terminated_length": 174.85404434204102, + "completions/min_length": 47.766666666666666, + "completions/min_terminated_length": 47.766666666666666, + "entropy": 0.12621417008340358, + "epoch": 0.49443663843846125, + "eval/gt_acc_batch": 0.8102777977784474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26729056239128113, + "kd/policy_loss": 0.00839853349995489, + "kd/rkl_advantage_mean": 0.8338907292112708, + "kd/rkl_advantage_p95": 5.200264549255371, + "kd/rkl_advantage_std": 2.297726969917615, + "kd/ssd_loss": 0.0, + "learning_rate": 5.056013367257813e-07, + "loss": 0.03359413146972656, + "num_tokens": 426084866.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8102777739365895, + "rewards/gt_logging_reward/std": 0.3791096925735474, + "sampling/importance_sampling_ratio/max": 1.8559505621592203, + "sampling/importance_sampling_ratio/mean": 0.9967754940191905, + "sampling/importance_sampling_ratio/min": 0.40957562923431395, + "sampling/sampling_logp_difference/max": 0.3283091644446055, + "sampling/sampling_logp_difference/mean": 0.003932061100689073, + "step": 13020, + "step_time": 7.874777145999057, + "student/entropy": 0.12621417008340358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03583333535740773, + "completions/max_length": 492.06666666666666, + "completions/max_terminated_length": 452.96666666666664, + "completions/mean_length": 192.06612243652344, + "completions/mean_terminated_length": 180.32630055745443, + "completions/min_length": 57.9, + "completions/min_terminated_length": 57.9, + "entropy": 0.13460152813543877, + "epoch": 0.4955758933657388, + "eval/gt_acc_batch": 0.7827777981758117, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24928902089595795, + "kd/policy_loss": 0.009149543100890393, + "kd/rkl_advantage_mean": 0.7775705125648529, + "kd/rkl_advantage_p95": 5.014631113409996, + "kd/rkl_advantage_std": 2.2298164844512938, + "kd/ssd_loss": 0.0, + "learning_rate": 5.044620817985038e-07, + "loss": 0.03659817377726237, + "num_tokens": 427073112.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7827777783075969, + "rewards/gt_logging_reward/std": 0.40819736023743947, + "sampling/importance_sampling_ratio/max": 1.947151792049408, + "sampling/importance_sampling_ratio/mean": 1.0023244043191275, + "sampling/importance_sampling_ratio/min": 0.43849104394515354, + "sampling/sampling_logp_difference/max": 0.3641825954119364, + "sampling/sampling_logp_difference/mean": 0.004113110387697816, + "step": 13050, + "step_time": 7.878095649300667, + "student/entropy": 0.13460152813543877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223834445078, + "completions/max_length": 488.23333333333335, + "completions/max_terminated_length": 448.5, + "completions/mean_length": 190.5452885945638, + "completions/mean_terminated_length": 180.74005076090495, + "completions/min_length": 54.63333333333333, + "completions/min_terminated_length": 54.63333333333333, + "entropy": 0.13053163395573694, + "epoch": 0.4967151482930164, + "eval/gt_acc_batch": 0.7897222379843394, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3087827265262604, + "kd/policy_loss": 0.008969450590666384, + "kd/rkl_advantage_mean": 0.7714304440033932, + "kd/rkl_advantage_p95": 5.002454785505931, + "kd/rkl_advantage_std": 2.229687425494194, + "kd/ssd_loss": 0.0, + "learning_rate": 5.033228268712261e-07, + "loss": 0.03587780396143595, + "num_tokens": 428065531.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7897222161293029, + "rewards/gt_logging_reward/std": 0.3968115518490473, + "sampling/importance_sampling_ratio/max": 1.9843746145566306, + "sampling/importance_sampling_ratio/mean": 1.0075724124908447, + "sampling/importance_sampling_ratio/min": 0.4422375758488973, + "sampling/sampling_logp_difference/max": 0.33616142670313515, + "sampling/sampling_logp_difference/mean": 0.004063787730410695, + "step": 13080, + "step_time": 7.980096963732406, + "student/entropy": 0.13053163395573694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029166668094694613, + "completions/max_length": 478.8333333333333, + "completions/max_terminated_length": 443.53333333333336, + "completions/mean_length": 192.71334330240884, + "completions/mean_terminated_length": 183.4984558105469, + "completions/min_length": 56.7, + "completions/min_terminated_length": 56.7, + "entropy": 0.1282183055455486, + "epoch": 0.49785440322029395, + "eval/gt_acc_batch": 0.7905555705229441, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26518622040748596, + "kd/policy_loss": 0.008455333282472565, + "kd/rkl_advantage_mean": 0.754959103371948, + "kd/rkl_advantage_p95": 5.012832333644231, + "kd/rkl_advantage_std": 2.207040989398956, + "kd/ssd_loss": 0.0, + "learning_rate": 5.021835719439487e-07, + "loss": 0.03382134040196737, + "num_tokens": 429064755.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7905555566151937, + "rewards/gt_logging_reward/std": 0.3973430961370468, + "sampling/importance_sampling_ratio/max": 2.008651773134867, + "sampling/importance_sampling_ratio/mean": 1.0014704843362172, + "sampling/importance_sampling_ratio/min": 0.44071121712525685, + "sampling/sampling_logp_difference/max": 0.3932512352863948, + "sampling/sampling_logp_difference/mean": 0.004019444955823322, + "step": 13110, + "step_time": 7.904125590297918, + "student/entropy": 0.1282183055455486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555557385087014, + "completions/max_length": 487.96666666666664, + "completions/max_terminated_length": 458.93333333333334, + "completions/mean_length": 194.17306467692057, + "completions/mean_terminated_length": 186.03732198079427, + "completions/min_length": 53.7, + "completions/min_terminated_length": 53.7, + "entropy": 0.12837067128469545, + "epoch": 0.49899365814757146, + "eval/gt_acc_batch": 0.7908333480358124, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2535662353038788, + "kd/policy_loss": 0.00853493712687244, + "kd/rkl_advantage_mean": 0.8733937223131458, + "kd/rkl_advantage_p95": 5.2161651968955995, + "kd/rkl_advantage_std": 2.279207627971967, + "kd/ssd_loss": 0.0, + "learning_rate": 5.010443170166711e-07, + "loss": 0.03413975238800049, + "num_tokens": 430071914.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7908333341280619, + "rewards/gt_logging_reward/std": 0.3961788425842921, + "sampling/importance_sampling_ratio/max": 1.8649105827013652, + "sampling/importance_sampling_ratio/mean": 0.995687089363734, + "sampling/importance_sampling_ratio/min": 0.46317173341910045, + "sampling/sampling_logp_difference/max": 0.33434557914733887, + "sampling/sampling_logp_difference/mean": 0.003974920519006749, + "step": 13140, + "step_time": 8.044115194931996, + "student/entropy": 0.12837067128469545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668051232895, + "completions/max_length": 500.26666666666665, + "completions/max_terminated_length": 458.8333333333333, + "completions/mean_length": 191.49417622884116, + "completions/mean_terminated_length": 183.69904022216798, + "completions/min_length": 53.86666666666667, + "completions/min_terminated_length": 53.86666666666667, + "entropy": 0.1276354743167758, + "epoch": 0.5001329130748491, + "eval/gt_acc_batch": 0.7991666793823242, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24526314437389374, + "kd/policy_loss": 0.008484186364027362, + "kd/rkl_advantage_mean": 0.8408616100127498, + "kd/rkl_advantage_p95": 5.179795014858246, + "kd/rkl_advantage_std": 2.2809967319170634, + "kd/ssd_loss": 0.0, + "learning_rate": 4.999050620893935e-07, + "loss": 0.03393675088882446, + "num_tokens": 431072637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7991666634877522, + "rewards/gt_logging_reward/std": 0.3913499022523562, + "sampling/importance_sampling_ratio/max": 1.9556130687395732, + "sampling/importance_sampling_ratio/mean": 0.9997881213823955, + "sampling/importance_sampling_ratio/min": 0.46809071600437163, + "sampling/sampling_logp_difference/max": 0.3230780720710754, + "sampling/sampling_logp_difference/mean": 0.003960460470989346, + "step": 13170, + "step_time": 8.203752201866397, + "student/entropy": 0.1276354743167758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166667740792035, + "completions/max_length": 487.03333333333336, + "completions/max_terminated_length": 446.93333333333334, + "completions/mean_length": 184.72917683919272, + "completions/mean_terminated_length": 176.65811208089193, + "completions/min_length": 54.4, + "completions/min_terminated_length": 54.4, + "entropy": 0.1375619273011883, + "epoch": 0.5012721680021266, + "eval/gt_acc_batch": 0.796111132701238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2962196171283722, + "kd/policy_loss": 0.009256092111657684, + "kd/rkl_advantage_mean": 0.7987864051247016, + "kd/rkl_advantage_p95": 5.174599754810333, + "kd/rkl_advantage_std": 2.296146933237712, + "kd/ssd_loss": 0.0, + "learning_rate": 4.98765807162116e-07, + "loss": 0.03702437480290731, + "num_tokens": 432030750.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111108462016, + "rewards/gt_logging_reward/std": 0.3956479440132777, + "sampling/importance_sampling_ratio/max": 1.8867369532585143, + "sampling/importance_sampling_ratio/mean": 1.009014755487442, + "sampling/importance_sampling_ratio/min": 0.44499262471993767, + "sampling/sampling_logp_difference/max": 0.3494084636370341, + "sampling/sampling_logp_difference/mean": 0.0040081636669735115, + "step": 13200, + "step_time": 7.8094346632647405, + "student/entropy": 0.1375619273011883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334668229023, + "completions/max_length": 475.26666666666665, + "completions/max_terminated_length": 419.8666666666667, + "completions/mean_length": 192.1961196899414, + "completions/mean_terminated_length": 185.57786509195964, + "completions/min_length": 61.7, + "completions/min_terminated_length": 61.7, + "entropy": 0.1264710328852137, + "epoch": 0.5024114229294042, + "eval/gt_acc_batch": 0.7958333412806193, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24649715423583984, + "kd/policy_loss": 0.008655014257722844, + "kd/rkl_advantage_mean": 0.7202112230171527, + "kd/rkl_advantage_p95": 4.943854818741481, + "kd/rkl_advantage_std": 2.1738750924666723, + "kd/ssd_loss": 0.0, + "learning_rate": 4.976265522348384e-07, + "loss": 0.03462006251017253, + "num_tokens": 433034216.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7958333373069764, + "rewards/gt_logging_reward/std": 0.39780582785606383, + "sampling/importance_sampling_ratio/max": 1.9371222297350565, + "sampling/importance_sampling_ratio/mean": 1.0032226105531057, + "sampling/importance_sampling_ratio/min": 0.461353359123071, + "sampling/sampling_logp_difference/max": 0.3145077735185623, + "sampling/sampling_logp_difference/mean": 0.003950933421341081, + "step": 13230, + "step_time": 7.955366395799501, + "student/entropy": 0.1264710328852137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01666666762903333, + "completions/max_length": 466.6666666666667, + "completions/max_terminated_length": 428.1, + "completions/mean_length": 181.7775085449219, + "completions/mean_terminated_length": 176.28002014160157, + "completions/min_length": 56.833333333333336, + "completions/min_terminated_length": 56.833333333333336, + "entropy": 0.12771618279318014, + "epoch": 0.5035506778566817, + "eval/gt_acc_batch": 0.821111136674881, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2551063597202301, + "kd/policy_loss": 0.008151335930839802, + "kd/rkl_advantage_mean": 0.7570568180332581, + "kd/rkl_advantage_p95": 4.934720222155253, + "kd/rkl_advantage_std": 2.173758061726888, + "kd/ssd_loss": 0.0, + "learning_rate": 4.964872973075608e-07, + "loss": 0.03260534604390462, + "num_tokens": 433986207.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8211111108462016, + "rewards/gt_logging_reward/std": 0.3782418469587962, + "sampling/importance_sampling_ratio/max": 1.8684049169222514, + "sampling/importance_sampling_ratio/mean": 1.0016039629777274, + "sampling/importance_sampling_ratio/min": 0.46488766074180604, + "sampling/sampling_logp_difference/max": 0.32926758925120037, + "sampling/sampling_logp_difference/mean": 0.003971799688103298, + "step": 13260, + "step_time": 7.9125686012024135, + "student/entropy": 0.12771618279318014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027500001589457194, + "completions/max_length": 496.3666666666667, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 192.24556630452474, + "completions/mean_terminated_length": 183.40699971516926, + "completions/min_length": 49.93333333333333, + "completions/min_terminated_length": 49.93333333333333, + "entropy": 0.13322918731719255, + "epoch": 0.5046899327839592, + "eval/gt_acc_batch": 0.7805555740992228, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31269320845603943, + "kd/policy_loss": 0.00915673931594938, + "kd/rkl_advantage_mean": 0.8193847555667162, + "kd/rkl_advantage_p95": 5.115215700864792, + "kd/rkl_advantage_std": 2.2608208298683166, + "kd/ssd_loss": 0.0, + "learning_rate": 4.953480423802833e-07, + "loss": 0.036626954873402916, + "num_tokens": 434982739.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7805555522441864, + "rewards/gt_logging_reward/std": 0.407390163342158, + "sampling/importance_sampling_ratio/max": 1.962827738126119, + "sampling/importance_sampling_ratio/mean": 1.0062683562437693, + "sampling/importance_sampling_ratio/min": 0.44755819290876386, + "sampling/sampling_logp_difference/max": 0.368687907854716, + "sampling/sampling_logp_difference/mean": 0.004105556119854251, + "step": 13290, + "step_time": 8.185467554962088, + "student/entropy": 0.13322918731719255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889042039712, + "completions/max_length": 494.8333333333333, + "completions/max_terminated_length": 445.46666666666664, + "completions/mean_length": 187.0783437093099, + "completions/mean_terminated_length": 178.2075393676758, + "completions/min_length": 49.53333333333333, + "completions/min_terminated_length": 49.53333333333333, + "entropy": 0.12515191187461217, + "epoch": 0.5058291877112369, + "eval/gt_acc_batch": 0.8002777894337972, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3359780013561249, + "kd/policy_loss": 0.008154199146277581, + "kd/rkl_advantage_mean": 0.7690290198971828, + "kd/rkl_advantage_p95": 5.0485188364982605, + "kd/rkl_advantage_std": 2.2545389433701835, + "kd/ssd_loss": 0.0, + "learning_rate": 4.942087874530057e-07, + "loss": 0.03261679410934448, + "num_tokens": 435949853.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8002777794996897, + "rewards/gt_logging_reward/std": 0.3923842877149582, + "sampling/importance_sampling_ratio/max": 1.8534108440081278, + "sampling/importance_sampling_ratio/mean": 0.9955401321252187, + "sampling/importance_sampling_ratio/min": 0.4455983271201452, + "sampling/sampling_logp_difference/max": 0.34340945482254026, + "sampling/sampling_logp_difference/mean": 0.003896466890970866, + "step": 13320, + "step_time": 7.997136927200093, + "student/entropy": 0.12515191187461217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666668076068162, + "completions/max_length": 488.1333333333333, + "completions/max_terminated_length": 448.5, + "completions/mean_length": 188.18973185221355, + "completions/mean_terminated_length": 181.19383595784504, + "completions/min_length": 50.86666666666667, + "completions/min_terminated_length": 50.86666666666667, + "entropy": 0.12467981179555257, + "epoch": 0.5069684426385144, + "eval/gt_acc_batch": 0.796111128727595, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2820405662059784, + "kd/policy_loss": 0.008294938578425596, + "kd/rkl_advantage_mean": 0.7662440652338167, + "kd/rkl_advantage_p95": 5.072757263978322, + "kd/rkl_advantage_std": 2.2323935051759083, + "kd/ssd_loss": 0.0, + "learning_rate": 4.930695325257282e-07, + "loss": 0.03317975600560506, + "num_tokens": 436928880.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111148198445, + "rewards/gt_logging_reward/std": 0.39195047368605934, + "sampling/importance_sampling_ratio/max": 1.992165982723236, + "sampling/importance_sampling_ratio/mean": 1.0025160094102223, + "sampling/importance_sampling_ratio/min": 0.4517238607009252, + "sampling/sampling_logp_difference/max": 0.3380700091520945, + "sampling/sampling_logp_difference/mean": 0.003911356379588445, + "step": 13350, + "step_time": 7.983294080166282, + "student/entropy": 0.12467981179555257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02111111208796501, + "completions/max_length": 484.1333333333333, + "completions/max_terminated_length": 455.46666666666664, + "completions/mean_length": 184.7255625406901, + "completions/mean_terminated_length": 177.79320373535157, + "completions/min_length": 59.3, + "completions/min_terminated_length": 59.3, + "entropy": 0.13155973901351292, + "epoch": 0.508107697565792, + "eval/gt_acc_batch": 0.8136111319065094, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3096168339252472, + "kd/policy_loss": 0.008671039207062374, + "kd/rkl_advantage_mean": 0.8796043239533902, + "kd/rkl_advantage_p95": 5.234441208839416, + "kd/rkl_advantage_std": 2.2961909343798954, + "kd/ssd_loss": 0.0, + "learning_rate": 4.919302775984505e-07, + "loss": 0.034684157371520995, + "num_tokens": 437896852.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8136111120382945, + "rewards/gt_logging_reward/std": 0.38488171100616453, + "sampling/importance_sampling_ratio/max": 1.8916655619939169, + "sampling/importance_sampling_ratio/mean": 0.9952857553958893, + "sampling/importance_sampling_ratio/min": 0.4136922722061475, + "sampling/sampling_logp_difference/max": 0.3248528003692627, + "sampling/sampling_logp_difference/mean": 0.004091426737916967, + "step": 13380, + "step_time": 7.948688119998648, + "student/entropy": 0.13155973901351292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027500001123795907, + "completions/max_length": 485.46666666666664, + "completions/max_terminated_length": 457.03333333333336, + "completions/mean_length": 191.8291763305664, + "completions/mean_terminated_length": 182.94952646891275, + "completions/min_length": 55.833333333333336, + "completions/min_terminated_length": 55.833333333333336, + "entropy": 0.1311343631396691, + "epoch": 0.5092469524930695, + "eval/gt_acc_batch": 0.7791666905085246, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28257858753204346, + "kd/policy_loss": 0.008993870050956806, + "kd/rkl_advantage_mean": 0.7776369160662094, + "kd/rkl_advantage_p95": 5.067190390825272, + "kd/rkl_advantage_std": 2.2339003572861356, + "kd/ssd_loss": 0.0, + "learning_rate": 4.90791022671173e-07, + "loss": 0.035975476106007896, + "num_tokens": 438903685.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7791666646798452, + "rewards/gt_logging_reward/std": 0.4039033363262812, + "sampling/importance_sampling_ratio/max": 1.9410647392272948, + "sampling/importance_sampling_ratio/mean": 1.0030520061651866, + "sampling/importance_sampling_ratio/min": 0.4271466612815857, + "sampling/sampling_logp_difference/max": 0.36923479835192363, + "sampling/sampling_logp_difference/mean": 0.004076689442930122, + "step": 13410, + "step_time": 8.302446158399107, + "student/entropy": 0.1311343631396691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021839081952027207, + "completions/max_length": 494.0689655172414, + "completions/max_terminated_length": 460.7586206896552, + "completions/mean_length": 190.31667564655172, + "completions/mean_terminated_length": 183.24531081627154, + "completions/min_length": 61.96551724137931, + "completions/min_terminated_length": 61.96551724137931, + "entropy": 0.12539255574088673, + "epoch": 0.5103862074203471, + "eval/gt_acc_batch": 0.789655181868323, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.268333375453949, + "kd/policy_loss": 0.008740880636996108, + "kd/rkl_advantage_mean": 0.8316077430700434, + "kd/rkl_advantage_p95": 5.138561659845813, + "kd/rkl_advantage_std": 2.2401756623695634, + "kd/ssd_loss": 0.0, + "learning_rate": 4.896517677438955e-07, + "loss": 0.03379807472229004, + "num_tokens": 439854835.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7896551757023252, + "rewards/gt_logging_reward/std": 0.39896534126380395, + "sampling/importance_sampling_ratio/max": 1.9991547814730941, + "sampling/importance_sampling_ratio/mean": 1.001974159273608, + "sampling/importance_sampling_ratio/min": 0.4396031627367283, + "sampling/sampling_logp_difference/max": 0.3317016229547303, + "sampling/sampling_logp_difference/mean": 0.003919753093465135, + "step": 13440, + "step_time": 7.768348437467163, + "student/entropy": 0.12539255574088673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333473652601, + "completions/max_length": 476.73333333333335, + "completions/max_terminated_length": 451.6, + "completions/mean_length": 191.9269561767578, + "completions/mean_terminated_length": 184.54794565836588, + "completions/min_length": 59.4, + "completions/min_terminated_length": 59.4, + "entropy": 0.12801065780222415, + "epoch": 0.5115254623476246, + "eval/gt_acc_batch": 0.7947222510973613, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25952646136283875, + "kd/policy_loss": 0.008443901053396985, + "kd/rkl_advantage_mean": 0.8220498546957969, + "kd/rkl_advantage_p95": 5.13026261528333, + "kd/rkl_advantage_std": 2.2310816903909045, + "kd/ssd_loss": 0.0, + "learning_rate": 4.885125128166179e-07, + "loss": 0.03377559979756673, + "num_tokens": 440850932.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222232818604, + "rewards/gt_logging_reward/std": 0.39274535824855167, + "sampling/importance_sampling_ratio/max": 1.9392152627309163, + "sampling/importance_sampling_ratio/mean": 0.9976689755916596, + "sampling/importance_sampling_ratio/min": 0.41977618684371315, + "sampling/sampling_logp_difference/max": 0.3255950371424357, + "sampling/sampling_logp_difference/mean": 0.003986841967950265, + "step": 13470, + "step_time": 7.9586431915299425, + "student/entropy": 0.12801065780222415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666785875956, + "completions/max_length": 485.23333333333335, + "completions/max_terminated_length": 448.43333333333334, + "completions/mean_length": 189.1005650838216, + "completions/mean_terminated_length": 181.95362141927083, + "completions/min_length": 64.73333333333333, + "completions/min_terminated_length": 64.73333333333333, + "entropy": 0.11984459217637777, + "epoch": 0.5126647172749023, + "eval/gt_acc_batch": 0.8177777985731761, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24430814385414124, + "kd/policy_loss": 0.007897363579832017, + "kd/rkl_advantage_mean": 0.6872668864748751, + "kd/rkl_advantage_p95": 4.864506145318349, + "kd/rkl_advantage_std": 2.2011391202608745, + "kd/ssd_loss": 0.0, + "learning_rate": 4.873732578893404e-07, + "loss": 0.031589452425638834, + "num_tokens": 441846790.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8177777806917826, + "rewards/gt_logging_reward/std": 0.37878380964199704, + "sampling/importance_sampling_ratio/max": 1.893928297360738, + "sampling/importance_sampling_ratio/mean": 0.9958478411038717, + "sampling/importance_sampling_ratio/min": 0.41062778035799663, + "sampling/sampling_logp_difference/max": 0.36681549151738485, + "sampling/sampling_logp_difference/mean": 0.003774503362365067, + "step": 13500, + "step_time": 8.04497896546721, + "student/entropy": 0.11984459217637777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555668969949, + "completions/max_length": 478.1666666666667, + "completions/max_terminated_length": 444.3666666666667, + "completions/mean_length": 187.58250986735027, + "completions/mean_terminated_length": 180.97928059895833, + "completions/min_length": 56.733333333333334, + "completions/min_terminated_length": 56.733333333333334, + "entropy": 0.12203903887420893, + "epoch": 0.5138039722021798, + "eval/gt_acc_batch": 0.8155555705229441, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24134255945682526, + "kd/policy_loss": 0.008116279230065023, + "kd/rkl_advantage_mean": 0.842855021605889, + "kd/rkl_advantage_p95": 5.158920633792877, + "kd/rkl_advantage_std": 2.2467195520798366, + "kd/ssd_loss": 0.0, + "learning_rate": 4.862340029620628e-07, + "loss": 0.03246511419614156, + "num_tokens": 442819583.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8155555605888367, + "rewards/gt_logging_reward/std": 0.37923822154601416, + "sampling/importance_sampling_ratio/max": 1.8642985979715982, + "sampling/importance_sampling_ratio/mean": 1.0044053276379903, + "sampling/importance_sampling_ratio/min": 0.44336141645908356, + "sampling/sampling_logp_difference/max": 0.3295977433522542, + "sampling/sampling_logp_difference/mean": 0.003868533670902252, + "step": 13530, + "step_time": 7.901388114631603, + "student/entropy": 0.12203903887420893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01916666782150666, + "completions/max_length": 481.03333333333336, + "completions/max_terminated_length": 439.8666666666667, + "completions/mean_length": 185.97445424397787, + "completions/mean_terminated_length": 179.71214243570964, + "completions/min_length": 52.36666666666667, + "completions/min_terminated_length": 52.36666666666667, + "entropy": 0.13936131938050192, + "epoch": 0.5149432271294573, + "eval/gt_acc_batch": 0.8097222487131754, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26740798354148865, + "kd/policy_loss": 0.009075922573295732, + "kd/rkl_advantage_mean": 0.8259214186420043, + "kd/rkl_advantage_p95": 5.127994169791539, + "kd/rkl_advantage_std": 2.248959493637085, + "kd/ssd_loss": 0.0, + "learning_rate": 4.850947480347852e-07, + "loss": 0.03630369504292806, + "num_tokens": 443784379.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8097222248713175, + "rewards/gt_logging_reward/std": 0.384277014931043, + "sampling/importance_sampling_ratio/max": 1.8808781981468201, + "sampling/importance_sampling_ratio/mean": 0.9989433268706004, + "sampling/importance_sampling_ratio/min": 0.44252419571081797, + "sampling/sampling_logp_difference/max": 0.3672982692718506, + "sampling/sampling_logp_difference/mean": 0.0041104407981038095, + "step": 13560, + "step_time": 8.117828416832587, + "student/entropy": 0.13936131938050192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030000001937150956, + "completions/max_length": 494.3666666666667, + "completions/max_terminated_length": 458.6, + "completions/mean_length": 192.3291768391927, + "completions/mean_terminated_length": 182.60846048990885, + "completions/min_length": 46.8, + "completions/min_terminated_length": 46.8, + "entropy": 0.13288527689874172, + "epoch": 0.5160824820567349, + "eval/gt_acc_batch": 0.7997222344080607, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.40810221433639526, + "kd/policy_loss": 0.00876244815105262, + "kd/rkl_advantage_mean": 0.849934043114384, + "kd/rkl_advantage_p95": 5.208024096488953, + "kd/rkl_advantage_std": 2.2789222965637843, + "kd/ssd_loss": 0.0, + "learning_rate": 4.839554931075077e-07, + "loss": 0.03504979610443115, + "num_tokens": 444785116.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7997222205003103, + "rewards/gt_logging_reward/std": 0.38855270942052206, + "sampling/importance_sampling_ratio/max": 1.9489187637964884, + "sampling/importance_sampling_ratio/mean": 1.0011720597743987, + "sampling/importance_sampling_ratio/min": 0.44456789642572403, + "sampling/sampling_logp_difference/max": 0.3246934632460276, + "sampling/sampling_logp_difference/mean": 0.00403062755552431, + "step": 13590, + "step_time": 8.327013289631576, + "student/entropy": 0.13288527689874172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112243185442, + "completions/max_length": 485.6, + "completions/max_terminated_length": 440.56666666666666, + "completions/mean_length": 187.0461212158203, + "completions/mean_terminated_length": 180.1074244181315, + "completions/min_length": 53.06666666666667, + "completions/min_terminated_length": 53.06666666666667, + "entropy": 0.12755054589360953, + "epoch": 0.5172217369840124, + "eval/gt_acc_batch": 0.7963889082272847, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2355193793773651, + "kd/policy_loss": 0.008632331564634417, + "kd/rkl_advantage_mean": 0.7591956538458665, + "kd/rkl_advantage_p95": 5.05312252442042, + "kd/rkl_advantage_std": 2.2291520784298577, + "kd/ssd_loss": 0.0, + "learning_rate": 4.828162381802301e-07, + "loss": 0.03452932834625244, + "num_tokens": 445768754.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888843854269, + "rewards/gt_logging_reward/std": 0.39736758867899574, + "sampling/importance_sampling_ratio/max": 1.8615408062934875, + "sampling/importance_sampling_ratio/mean": 0.9970512727896372, + "sampling/importance_sampling_ratio/min": 0.4248957335948944, + "sampling/sampling_logp_difference/max": 0.3646218279997508, + "sampling/sampling_logp_difference/mean": 0.003981468989513815, + "step": 13620, + "step_time": 8.052310594936232, + "student/entropy": 0.12755054589360953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778673917055, + "completions/max_length": 494.43333333333334, + "completions/max_terminated_length": 461.2, + "completions/mean_length": 186.54195454915364, + "completions/mean_terminated_length": 180.6953145345052, + "completions/min_length": 51.6, + "completions/min_terminated_length": 51.6, + "entropy": 0.1309233820065856, + "epoch": 0.51836099191129, + "eval/gt_acc_batch": 0.795000026623408, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26961806416511536, + "kd/policy_loss": 0.008533356070984155, + "kd/rkl_advantage_mean": 0.837415462732315, + "kd/rkl_advantage_p95": 5.189280221859614, + "kd/rkl_advantage_std": 2.290069223443667, + "kd/ssd_loss": 0.0, + "learning_rate": 4.816769832529525e-07, + "loss": 0.03413343032201131, + "num_tokens": 446737489.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7950000047683716, + "rewards/gt_logging_reward/std": 0.39816976090272266, + "sampling/importance_sampling_ratio/max": 1.9403555075327554, + "sampling/importance_sampling_ratio/mean": 0.9946727136770884, + "sampling/importance_sampling_ratio/min": 0.4119761834541957, + "sampling/sampling_logp_difference/max": 0.35496972799301146, + "sampling/sampling_logp_difference/mean": 0.004071328478554885, + "step": 13650, + "step_time": 7.865297824503311, + "student/entropy": 0.1309233820065856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028888890178253253, + "completions/max_length": 493.1333333333333, + "completions/max_terminated_length": 454.8, + "completions/mean_length": 192.88139851888022, + "completions/mean_terminated_length": 183.4839848836263, + "completions/min_length": 52.13333333333333, + "completions/min_terminated_length": 52.13333333333333, + "entropy": 0.13060363959521054, + "epoch": 0.5195002468385675, + "eval/gt_acc_batch": 0.8033333400885264, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.35230961441993713, + "kd/policy_loss": 0.008649666890657196, + "kd/rkl_advantage_mean": 0.7920275257279475, + "kd/rkl_advantage_p95": 5.1072557429472605, + "kd/rkl_advantage_std": 2.236140798528989, + "kd/ssd_loss": 0.0, + "learning_rate": 4.80537728325675e-07, + "loss": 0.03459866841634115, + "num_tokens": 447734630.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.803333334128062, + "rewards/gt_logging_reward/std": 0.3909426788489024, + "sampling/importance_sampling_ratio/max": 2.00842391649882, + "sampling/importance_sampling_ratio/mean": 0.9978987554709117, + "sampling/importance_sampling_ratio/min": 0.4267040540774663, + "sampling/sampling_logp_difference/max": 0.33734674056371056, + "sampling/sampling_logp_difference/mean": 0.00404376497802635, + "step": 13680, + "step_time": 7.976594755735035, + "student/entropy": 0.13060363959521054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02722222376614809, + "completions/max_length": 475.23333333333335, + "completions/max_terminated_length": 437.9, + "completions/mean_length": 182.46251118977864, + "completions/mean_terminated_length": 173.3874038696289, + "completions/min_length": 54.2, + "completions/min_terminated_length": 54.2, + "entropy": 0.125187465796868, + "epoch": 0.5206395017658452, + "eval/gt_acc_batch": 0.8127777973810831, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30207359790802, + "kd/policy_loss": 0.00835984144359827, + "kd/rkl_advantage_mean": 0.7719736117869616, + "kd/rkl_advantage_p95": 4.9903674066066745, + "kd/rkl_advantage_std": 2.213218104839325, + "kd/ssd_loss": 0.0, + "learning_rate": 4.793984733983975e-07, + "loss": 0.03343936602274577, + "num_tokens": 448685207.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8127777735392253, + "rewards/gt_logging_reward/std": 0.38102691968282065, + "sampling/importance_sampling_ratio/max": 1.8996910373369853, + "sampling/importance_sampling_ratio/mean": 1.0034263590971628, + "sampling/importance_sampling_ratio/min": 0.45933054983615873, + "sampling/sampling_logp_difference/max": 0.34427998860677084, + "sampling/sampling_logp_difference/mean": 0.003889993011641006, + "step": 13710, + "step_time": 7.7680629976346, + "student/entropy": 0.125187465796868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779251337052, + "completions/max_length": 488.7, + "completions/max_terminated_length": 441.1, + "completions/mean_length": 190.2041763305664, + "completions/mean_terminated_length": 181.9239466349284, + "completions/min_length": 51.233333333333334, + "completions/min_terminated_length": 51.233333333333334, + "entropy": 0.12680780297766128, + "epoch": 0.5217787566931227, + "eval/gt_acc_batch": 0.8019444664319356, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32120800018310547, + "kd/policy_loss": 0.00859815781780829, + "kd/rkl_advantage_mean": 0.7344619212361674, + "kd/rkl_advantage_p95": 4.955116772651673, + "kd/rkl_advantage_std": 2.1920243273178737, + "kd/ssd_loss": 0.0, + "learning_rate": 4.782592184711198e-07, + "loss": 0.03439263502756754, + "num_tokens": 449686046.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444406032562, + "rewards/gt_logging_reward/std": 0.39405519564946495, + "sampling/importance_sampling_ratio/max": 1.9913106640179952, + "sampling/importance_sampling_ratio/mean": 0.9999716798464457, + "sampling/importance_sampling_ratio/min": 0.48339502811431884, + "sampling/sampling_logp_difference/max": 0.340974493821462, + "sampling/sampling_logp_difference/mean": 0.003958528100823363, + "step": 13740, + "step_time": 8.095022997328973, + "student/entropy": 0.12680780297766128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02694444600492716, + "completions/max_length": 489.9, + "completions/max_terminated_length": 457.23333333333335, + "completions/mean_length": 185.00639851888022, + "completions/mean_terminated_length": 176.04859619140626, + "completions/min_length": 54.733333333333334, + "completions/min_terminated_length": 54.733333333333334, + "entropy": 0.12894826233386994, + "epoch": 0.5229180116204003, + "eval/gt_acc_batch": 0.8038889030615489, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23327742516994476, + "kd/policy_loss": 0.008628433052217587, + "kd/rkl_advantage_mean": 0.8345113061368465, + "kd/rkl_advantage_p95": 5.236396948496501, + "kd/rkl_advantage_std": 2.315540524323781, + "kd/ssd_loss": 0.0, + "learning_rate": 4.771199635438423e-07, + "loss": 0.03451374371846517, + "num_tokens": 450652437.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8038888931274414, + "rewards/gt_logging_reward/std": 0.3905547688404719, + "sampling/importance_sampling_ratio/max": 2.0025027354558307, + "sampling/importance_sampling_ratio/mean": 0.9950513343016306, + "sampling/importance_sampling_ratio/min": 0.4557744642098745, + "sampling/sampling_logp_difference/max": 0.30751564502716067, + "sampling/sampling_logp_difference/mean": 0.0040189300430938605, + "step": 13770, + "step_time": 7.855921553161655, + "student/entropy": 0.12894826233386994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02888889042660594, + "completions/max_length": 480.3333333333333, + "completions/max_terminated_length": 453.6333333333333, + "completions/mean_length": 191.45362141927083, + "completions/mean_terminated_length": 182.2514912923177, + "completions/min_length": 60.166666666666664, + "completions/min_terminated_length": 60.166666666666664, + "entropy": 0.12216889367749294, + "epoch": 0.5240572665476778, + "eval/gt_acc_batch": 0.7891666809717814, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25446024537086487, + "kd/policy_loss": 0.008162470910853396, + "kd/rkl_advantage_mean": 0.7785483535379172, + "kd/rkl_advantage_p95": 5.051048876841863, + "kd/rkl_advantage_std": 2.222222508986791, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7598070861656474e-07, + "loss": 0.03264988859494527, + "num_tokens": 451653094.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.789166663090388, + "rewards/gt_logging_reward/std": 0.39659317086140317, + "sampling/importance_sampling_ratio/max": 1.896891725063324, + "sampling/importance_sampling_ratio/mean": 1.0005358974138896, + "sampling/importance_sampling_ratio/min": 0.4740508397420247, + "sampling/sampling_logp_difference/max": 0.3425781925519307, + "sampling/sampling_logp_difference/mean": 0.0038158934796229003, + "step": 13800, + "step_time": 7.96781962163465, + "student/entropy": 0.12216889367749294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02750000146528085, + "completions/max_length": 498.3666666666667, + "completions/max_terminated_length": 458.03333333333336, + "completions/mean_length": 195.31862080891926, + "completions/mean_terminated_length": 186.46502176920572, + "completions/min_length": 61.166666666666664, + "completions/min_terminated_length": 61.166666666666664, + "entropy": 0.1278667830551664, + "epoch": 0.5251965214749553, + "eval/gt_acc_batch": 0.7825000186761221, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2778443396091461, + "kd/policy_loss": 0.008574891627843802, + "kd/rkl_advantage_mean": 0.8628701891750097, + "kd/rkl_advantage_p95": 5.163269293308258, + "kd/rkl_advantage_std": 2.2511018107334775, + "kd/ssd_loss": 0.0, + "learning_rate": 4.748414536892872e-07, + "loss": 0.034299568335215254, + "num_tokens": 452661297.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7824999988079071, + "rewards/gt_logging_reward/std": 0.4052983740965525, + "sampling/importance_sampling_ratio/max": 1.8894463141759237, + "sampling/importance_sampling_ratio/mean": 0.9956264754136404, + "sampling/importance_sampling_ratio/min": 0.44286731580893196, + "sampling/sampling_logp_difference/max": 0.34516538778940836, + "sampling/sampling_logp_difference/mean": 0.003961822646670043, + "step": 13830, + "step_time": 7.97011508703678, + "student/entropy": 0.1278667830551664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02861111288269361, + "completions/max_length": 494.73333333333335, + "completions/max_terminated_length": 450.23333333333335, + "completions/mean_length": 189.62362060546874, + "completions/mean_terminated_length": 180.26937713623047, + "completions/min_length": 48.7, + "completions/min_terminated_length": 48.7, + "entropy": 0.12593134908626477, + "epoch": 0.526335776402233, + "eval/gt_acc_batch": 0.8044444680213928, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26076647639274597, + "kd/policy_loss": 0.008303049253299832, + "kd/rkl_advantage_mean": 0.8527796275913715, + "kd/rkl_advantage_p95": 5.199810290336609, + "kd/rkl_advantage_std": 2.2878666977087656, + "kd/ssd_loss": 0.0, + "learning_rate": 4.737021987620096e-07, + "loss": 0.033212198813756304, + "num_tokens": 453644862.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8044444441795349, + "rewards/gt_logging_reward/std": 0.39146906435489653, + "sampling/importance_sampling_ratio/max": 1.8344443321228028, + "sampling/importance_sampling_ratio/mean": 1.0011464973290762, + "sampling/importance_sampling_ratio/min": 0.456776296099027, + "sampling/sampling_logp_difference/max": 0.36933360596497855, + "sampling/sampling_logp_difference/mean": 0.003948131886621316, + "step": 13860, + "step_time": 7.9857836399368045, + "student/entropy": 0.12593134908626477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667573153974, + "completions/max_length": 489.73333333333335, + "completions/max_terminated_length": 465.6333333333333, + "completions/mean_length": 193.71112111409505, + "completions/mean_terminated_length": 187.5854288736979, + "completions/min_length": 48.333333333333336, + "completions/min_terminated_length": 48.333333333333336, + "entropy": 0.13274628445506095, + "epoch": 0.5274750313295105, + "eval/gt_acc_batch": 0.785833348830541, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28369665145874023, + "kd/policy_loss": 0.008904634231779103, + "kd/rkl_advantage_mean": 0.7567404488179211, + "kd/rkl_advantage_p95": 5.0734045525391895, + "kd/rkl_advantage_std": 2.256464385986328, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7256294383473204e-07, + "loss": 0.035618539651234946, + "num_tokens": 454650062.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.785833332935969, + "rewards/gt_logging_reward/std": 0.3997292493780454, + "sampling/importance_sampling_ratio/max": 2.008753768603007, + "sampling/importance_sampling_ratio/mean": 0.9997224688529969, + "sampling/importance_sampling_ratio/min": 0.4006120502948761, + "sampling/sampling_logp_difference/max": 0.32309786081314085, + "sampling/sampling_logp_difference/mean": 0.004097367667903503, + "step": 13890, + "step_time": 8.019089313566898, + "student/entropy": 0.13274628445506095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03166666819403569, + "completions/max_length": 495.1666666666667, + "completions/max_terminated_length": 451.3333333333333, + "completions/mean_length": 196.13917744954426, + "completions/mean_terminated_length": 186.0402374267578, + "completions/min_length": 52.2, + "completions/min_terminated_length": 52.2, + "entropy": 0.1353298483416438, + "epoch": 0.5286142862567881, + "eval/gt_acc_batch": 0.7677778005599976, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27357378602027893, + "kd/policy_loss": 0.009191409504273907, + "kd/rkl_advantage_mean": 0.8681299385925134, + "kd/rkl_advantage_p95": 5.179585138956706, + "kd/rkl_advantage_std": 2.250325361887614, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7142368890745446e-07, + "loss": 0.03676563501358032, + "num_tokens": 455673451.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7677777747313181, + "rewards/gt_logging_reward/std": 0.41145606885353725, + "sampling/importance_sampling_ratio/max": 1.9369396249453226, + "sampling/importance_sampling_ratio/mean": 0.999887736638387, + "sampling/importance_sampling_ratio/min": 0.42798667003711066, + "sampling/sampling_logp_difference/max": 0.3296239177385966, + "sampling/sampling_logp_difference/mean": 0.0041307649187122784, + "step": 13920, + "step_time": 8.22406125130025, + "student/entropy": 0.1353298483416438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.012500000558793545, + "completions/max_length": 461.23333333333335, + "completions/max_terminated_length": 437.93333333333334, + "completions/mean_length": 180.70612030029298, + "completions/mean_terminated_length": 176.6969446818034, + "completions/min_length": 64.03333333333333, + "completions/min_terminated_length": 64.03333333333333, + "entropy": 0.11860144517074028, + "epoch": 0.5297535411840656, + "eval/gt_acc_batch": 0.832500014702479, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2789967358112335, + "kd/policy_loss": 0.0076336984037576865, + "kd/rkl_advantage_mean": 0.7729518757512172, + "kd/rkl_advantage_p95": 4.973008503516515, + "kd/rkl_advantage_std": 2.1891183803478875, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7028443398017693e-07, + "loss": 0.030534797906875612, + "num_tokens": 456616801.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8324999948342641, + "rewards/gt_logging_reward/std": 0.3626602053642273, + "sampling/importance_sampling_ratio/max": 1.908104125658671, + "sampling/importance_sampling_ratio/mean": 1.0020939966042837, + "sampling/importance_sampling_ratio/min": 0.48717716832955676, + "sampling/sampling_logp_difference/max": 0.3304917881886164, + "sampling/sampling_logp_difference/mean": 0.003745162549118201, + "step": 13950, + "step_time": 7.520095770598952, + "student/entropy": 0.11860144517074028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02500000149011612, + "completions/max_length": 489.9, + "completions/max_terminated_length": 448.6666666666667, + "completions/mean_length": 186.6958435058594, + "completions/mean_terminated_length": 178.47743428548176, + "completions/min_length": 50.333333333333336, + "completions/min_terminated_length": 50.333333333333336, + "entropy": 0.13131221774965524, + "epoch": 0.5308927961113432, + "eval/gt_acc_batch": 0.7936111370722453, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34088826179504395, + "kd/policy_loss": 0.009021756675792858, + "kd/rkl_advantage_mean": 0.7565242799910872, + "kd/rkl_advantage_p95": 5.030451864004135, + "kd/rkl_advantage_std": 2.212914439042409, + "kd/ssd_loss": 0.0, + "learning_rate": 4.691451790528994e-07, + "loss": 0.03608702023824056, + "num_tokens": 457597810.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111132303874, + "rewards/gt_logging_reward/std": 0.3957334746917089, + "sampling/importance_sampling_ratio/max": 1.8170052886009216, + "sampling/importance_sampling_ratio/mean": 1.0024266183376311, + "sampling/importance_sampling_ratio/min": 0.47827940781911216, + "sampling/sampling_logp_difference/max": 0.34440837303797406, + "sampling/sampling_logp_difference/mean": 0.004038713787061473, + "step": 13980, + "step_time": 8.058347323072667, + "student/entropy": 0.13131221774965524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555662761132, + "completions/max_length": 479.03333333333336, + "completions/max_terminated_length": 457.03333333333336, + "completions/mean_length": 186.70167694091796, + "completions/mean_terminated_length": 179.91555786132812, + "completions/min_length": 57.4, + "completions/min_terminated_length": 57.4, + "entropy": 0.1322163434078296, + "epoch": 0.5320320510386207, + "eval/gt_acc_batch": 0.7925000170866648, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22778098285198212, + "kd/policy_loss": 0.008738538060182084, + "kd/rkl_advantage_mean": 0.8136056170798838, + "kd/rkl_advantage_p95": 5.186686251560847, + "kd/rkl_advantage_std": 2.271095664302508, + "kd/ssd_loss": 0.0, + "learning_rate": 4.680059241256218e-07, + "loss": 0.03495415449142456, + "num_tokens": 458570368.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7925000031789143, + "rewards/gt_logging_reward/std": 0.3981808731953303, + "sampling/importance_sampling_ratio/max": 1.7830043355623881, + "sampling/importance_sampling_ratio/mean": 0.9996215383211772, + "sampling/importance_sampling_ratio/min": 0.4313208987315496, + "sampling/sampling_logp_difference/max": 0.32099093198776246, + "sampling/sampling_logp_difference/mean": 0.004096685280092061, + "step": 14010, + "step_time": 7.837290285233272, + "student/entropy": 0.1322163434078296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02750000162050128, + "completions/max_length": 501.06666666666666, + "completions/max_terminated_length": 467.56666666666666, + "completions/mean_length": 190.10723215738932, + "completions/mean_terminated_length": 181.12381388346355, + "completions/min_length": 55.733333333333334, + "completions/min_terminated_length": 55.733333333333334, + "entropy": 0.13167274494965872, + "epoch": 0.5331713059658983, + "eval/gt_acc_batch": 0.7886111418406169, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2916111350059509, + "kd/policy_loss": 0.009141125190459813, + "kd/rkl_advantage_mean": 0.8818442208362588, + "kd/rkl_advantage_p95": 5.244532811641693, + "kd/rkl_advantage_std": 2.2945354243119556, + "kd/ssd_loss": 0.0, + "learning_rate": 4.668666691983443e-07, + "loss": 0.03656450112660726, + "num_tokens": 459559530.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.788611114025116, + "rewards/gt_logging_reward/std": 0.40001068214575447, + "sampling/importance_sampling_ratio/max": 2.031305734316508, + "sampling/importance_sampling_ratio/mean": 1.0055910964806876, + "sampling/importance_sampling_ratio/min": 0.45751590728759767, + "sampling/sampling_logp_difference/max": 0.3302497982978821, + "sampling/sampling_logp_difference/mean": 0.0040569905579711, + "step": 14040, + "step_time": 8.065026737634616, + "student/entropy": 0.13167274494965872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001365939776, + "completions/max_length": 500.56666666666666, + "completions/max_terminated_length": 449.96666666666664, + "completions/mean_length": 186.97139892578124, + "completions/mean_terminated_length": 178.72216186523437, + "completions/min_length": 48.733333333333334, + "completions/min_terminated_length": 48.733333333333334, + "entropy": 0.1330014217024048, + "epoch": 0.5343105608931759, + "eval/gt_acc_batch": 0.8100000182787578, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.263800710439682, + "kd/policy_loss": 0.008649396368612845, + "kd/rkl_advantage_mean": 0.7830578133463859, + "kd/rkl_advantage_p95": 5.160121321678162, + "kd/rkl_advantage_std": 2.275480250517527, + "kd/ssd_loss": 0.0, + "learning_rate": 4.657274142710667e-07, + "loss": 0.034597587585449216, + "num_tokens": 460532555.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8100000023841858, + "rewards/gt_logging_reward/std": 0.3837002421418826, + "sampling/importance_sampling_ratio/max": 1.9510464270909627, + "sampling/importance_sampling_ratio/mean": 0.9920869052410126, + "sampling/importance_sampling_ratio/min": 0.4665325323740641, + "sampling/sampling_logp_difference/max": 0.35459613998730977, + "sampling/sampling_logp_difference/mean": 0.004093968775123358, + "step": 14070, + "step_time": 7.958294593698034, + "student/entropy": 0.1330014217024048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334798614182, + "completions/max_length": 482.4, + "completions/max_terminated_length": 452.3666666666667, + "completions/mean_length": 189.1511210123698, + "completions/mean_terminated_length": 181.48666178385417, + "completions/min_length": 59.8, + "completions/min_terminated_length": 59.8, + "entropy": 0.13095820005983114, + "epoch": 0.5354498158204535, + "eval/gt_acc_batch": 0.7755555748939514, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2728184163570404, + "kd/policy_loss": 0.008942694230548416, + "kd/rkl_advantage_mean": 0.7644301899398367, + "kd/rkl_advantage_p95": 5.072128107150395, + "kd/rkl_advantage_std": 2.260962744553884, + "kd/ssd_loss": 0.0, + "learning_rate": 4.645881593437891e-07, + "loss": 0.03577077388763428, + "num_tokens": 461515259.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.775555553038915, + "rewards/gt_logging_reward/std": 0.4097857803106308, + "sampling/importance_sampling_ratio/max": 1.908998962243398, + "sampling/importance_sampling_ratio/mean": 1.0035688936710359, + "sampling/importance_sampling_ratio/min": 0.4268093725045522, + "sampling/sampling_logp_difference/max": 0.30399640202522277, + "sampling/sampling_logp_difference/mean": 0.00404395826626569, + "step": 14100, + "step_time": 7.9358113301360085, + "student/entropy": 0.13095820005983114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0188888899050653, + "completions/max_length": 479.43333333333334, + "completions/max_terminated_length": 434.6666666666667, + "completions/mean_length": 185.80362091064453, + "completions/mean_terminated_length": 179.755717976888, + "completions/min_length": 52.4, + "completions/min_terminated_length": 52.4, + "entropy": 0.1281090594207247, + "epoch": 0.536589070747731, + "eval/gt_acc_batch": 0.7955555816491445, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3171338438987732, + "kd/policy_loss": 0.00862113045101675, + "kd/rkl_advantage_mean": 0.7178408570277194, + "kd/rkl_advantage_p95": 4.922809422016144, + "kd/rkl_advantage_std": 2.1948791911204655, + "kd/ssd_loss": 0.0, + "learning_rate": 4.634489044165116e-07, + "loss": 0.034484525521596275, + "num_tokens": 462496240.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555538336436, + "rewards/gt_logging_reward/std": 0.3949325650930405, + "sampling/importance_sampling_ratio/max": 1.82848379611969, + "sampling/importance_sampling_ratio/mean": 0.9972746471563975, + "sampling/importance_sampling_ratio/min": 0.4678614497184753, + "sampling/sampling_logp_difference/max": 0.32258217136065165, + "sampling/sampling_logp_difference/mean": 0.003982665009486178, + "step": 14130, + "step_time": 7.837802286339381, + "student/entropy": 0.1281090594207247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030000001937150956, + "completions/max_length": 498.1333333333333, + "completions/max_terminated_length": 463.06666666666666, + "completions/mean_length": 188.0269551595052, + "completions/mean_terminated_length": 178.10547434488933, + "completions/min_length": 49.4, + "completions/min_terminated_length": 49.4, + "entropy": 0.13191390068580708, + "epoch": 0.5377283256750085, + "eval/gt_acc_batch": 0.7911111195882161, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27249985933303833, + "kd/policy_loss": 0.008935203154881796, + "kd/rkl_advantage_mean": 0.7406061695267757, + "kd/rkl_advantage_p95": 5.090488608678182, + "kd/rkl_advantage_std": 2.2853286306063336, + "kd/ssd_loss": 0.0, + "learning_rate": 4.62309649489234e-07, + "loss": 0.03574081261952718, + "num_tokens": 463465217.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7911111136277517, + "rewards/gt_logging_reward/std": 0.40392165780067446, + "sampling/importance_sampling_ratio/max": 2.023034652074178, + "sampling/importance_sampling_ratio/mean": 1.0014307816823325, + "sampling/importance_sampling_ratio/min": 0.4638982613881429, + "sampling/sampling_logp_difference/max": 0.32270373702049254, + "sampling/sampling_logp_difference/mean": 0.004053582243310909, + "step": 14160, + "step_time": 7.916466237401377, + "student/entropy": 0.13191390068580708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556720743575, + "completions/max_length": 487.73333333333335, + "completions/max_terminated_length": 451.8333333333333, + "completions/mean_length": 189.17306518554688, + "completions/mean_terminated_length": 182.4264673868815, + "completions/min_length": 52.3, + "completions/min_terminated_length": 52.3, + "entropy": 0.1277829830845197, + "epoch": 0.5388675806022861, + "eval/gt_acc_batch": 0.7933333476384481, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2972525358200073, + "kd/policy_loss": 0.008542922117824977, + "kd/rkl_advantage_mean": 0.7868398564557234, + "kd/rkl_advantage_p95": 5.109304853280386, + "kd/rkl_advantage_std": 2.2591213792562486, + "kd/ssd_loss": 0.0, + "learning_rate": 4.6117039456195643e-07, + "loss": 0.03417169253031413, + "num_tokens": 464456416.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7933333337306976, + "rewards/gt_logging_reward/std": 0.39858792622884115, + "sampling/importance_sampling_ratio/max": 1.9485211769739788, + "sampling/importance_sampling_ratio/mean": 1.0034557362397512, + "sampling/importance_sampling_ratio/min": 0.47325532138347626, + "sampling/sampling_logp_difference/max": 0.33223408460617065, + "sampling/sampling_logp_difference/mean": 0.004001117854689558, + "step": 14190, + "step_time": 7.890491550864923, + "student/entropy": 0.1277829830845197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556907008093, + "completions/max_length": 487.3333333333333, + "completions/max_terminated_length": 438.93333333333334, + "completions/mean_length": 185.93195393880208, + "completions/mean_terminated_length": 179.18723856608074, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.12861260082572698, + "epoch": 0.5400068355295636, + "eval/gt_acc_batch": 0.8047222455342611, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.437302827835083, + "kd/policy_loss": 0.008472359465667979, + "kd/rkl_advantage_mean": 0.7876690263239046, + "kd/rkl_advantage_p95": 5.097296951214473, + "kd/rkl_advantage_std": 2.276056478420893, + "kd/ssd_loss": 0.0, + "learning_rate": 4.600311396346789e-07, + "loss": 0.03388944466908773, + "num_tokens": 465435395.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222236792246, + "rewards/gt_logging_reward/std": 0.3868939866622289, + "sampling/importance_sampling_ratio/max": 1.9082477251688639, + "sampling/importance_sampling_ratio/mean": 0.9980798959732056, + "sampling/importance_sampling_ratio/min": 0.4419197502235572, + "sampling/sampling_logp_difference/max": 0.34326926271120706, + "sampling/sampling_logp_difference/mean": 0.003999324445612728, + "step": 14220, + "step_time": 8.090203091269844, + "student/entropy": 0.12861260082572698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778953313828, + "completions/max_length": 484.93333333333334, + "completions/max_terminated_length": 460.1666666666667, + "completions/mean_length": 188.6252868652344, + "completions/mean_terminated_length": 182.89507446289062, + "completions/min_length": 50.766666666666666, + "completions/min_terminated_length": 50.766666666666666, + "entropy": 0.12615254260599612, + "epoch": 0.5411460904568413, + "eval/gt_acc_batch": 0.8033333460489909, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26796314120292664, + "kd/policy_loss": 0.008588151833585774, + "kd/rkl_advantage_mean": 0.824182416126132, + "kd/rkl_advantage_p95": 5.134262442588806, + "kd/rkl_advantage_std": 2.2486952185630797, + "kd/ssd_loss": 0.0, + "learning_rate": 4.588918847074013e-07, + "loss": 0.034352608521779376, + "num_tokens": 466412734.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8033333420753479, + "rewards/gt_logging_reward/std": 0.3912434091170629, + "sampling/importance_sampling_ratio/max": 1.8951197425524393, + "sampling/importance_sampling_ratio/mean": 1.004019421339035, + "sampling/importance_sampling_ratio/min": 0.42433348794778186, + "sampling/sampling_logp_difference/max": 0.3242478271325429, + "sampling/sampling_logp_difference/mean": 0.003908710445587834, + "step": 14250, + "step_time": 7.881029926898676, + "student/entropy": 0.12615254260599612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111265917619, + "completions/max_length": 473.6333333333333, + "completions/max_terminated_length": 432.26666666666665, + "completions/mean_length": 188.21778767903646, + "completions/mean_terminated_length": 179.76077575683593, + "completions/min_length": 44.733333333333334, + "completions/min_terminated_length": 44.733333333333334, + "entropy": 0.1246906536941727, + "epoch": 0.5422853453841188, + "eval/gt_acc_batch": 0.8105555931727092, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3046474754810333, + "kd/policy_loss": 0.008073354056493069, + "kd/rkl_advantage_mean": 0.7492210269284745, + "kd/rkl_advantage_p95": 4.988565138975779, + "kd/rkl_advantage_std": 2.2147785166899365, + "kd/ssd_loss": 0.0, + "learning_rate": 4.577526297801238e-07, + "loss": 0.03229341109593709, + "num_tokens": 467392974.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8105555534362793, + "rewards/gt_logging_reward/std": 0.38291081885496775, + "sampling/importance_sampling_ratio/max": 1.8150936563809712, + "sampling/importance_sampling_ratio/mean": 0.9977585395177205, + "sampling/importance_sampling_ratio/min": 0.4736273547013601, + "sampling/sampling_logp_difference/max": 0.30562562942504884, + "sampling/sampling_logp_difference/mean": 0.003924941543179254, + "step": 14280, + "step_time": 7.8026003944673, + "student/entropy": 0.1246906536941727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03000000168879827, + "completions/max_length": 501.5, + "completions/max_terminated_length": 458.73333333333335, + "completions/mean_length": 195.0900105794271, + "completions/mean_terminated_length": 185.44005737304687, + "completions/min_length": 57.06666666666667, + "completions/min_terminated_length": 57.06666666666667, + "entropy": 0.13724552188068628, + "epoch": 0.5434246003113964, + "eval/gt_acc_batch": 0.7741666952768962, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3184824287891388, + "kd/policy_loss": 0.009162128332536667, + "kd/rkl_advantage_mean": 0.8390109991033872, + "kd/rkl_advantage_p95": 5.1096123615900675, + "kd/rkl_advantage_std": 2.2231254210074742, + "kd/ssd_loss": 0.0, + "learning_rate": 4.566133748528462e-07, + "loss": 0.03664851983388265, + "num_tokens": 468398898.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7741666674613953, + "rewards/gt_logging_reward/std": 0.4134669174750646, + "sampling/importance_sampling_ratio/max": 2.0095699350039165, + "sampling/importance_sampling_ratio/mean": 0.9991507867972056, + "sampling/importance_sampling_ratio/min": 0.4148773342370987, + "sampling/sampling_logp_difference/max": 0.3441101531187693, + "sampling/sampling_logp_difference/mean": 0.004218136728741229, + "step": 14310, + "step_time": 8.046545662163407, + "student/entropy": 0.13724552188068628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556658655404, + "completions/max_length": 476.8, + "completions/max_terminated_length": 433.8, + "completions/mean_length": 182.37056477864584, + "completions/mean_terminated_length": 175.6757598876953, + "completions/min_length": 60.766666666666666, + "completions/min_terminated_length": 60.766666666666666, + "entropy": 0.1278589957083265, + "epoch": 0.5445638552386739, + "eval/gt_acc_batch": 0.8122222363948822, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2768915891647339, + "kd/policy_loss": 0.008434132154798135, + "kd/rkl_advantage_mean": 0.7375584746400515, + "kd/rkl_advantage_p95": 4.934700310230255, + "kd/rkl_advantage_std": 2.19104775985082, + "kd/ssd_loss": 0.0, + "learning_rate": 4.554741199255687e-07, + "loss": 0.03373652696609497, + "num_tokens": 469353400.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8122222205003102, + "rewards/gt_logging_reward/std": 0.3801257664958636, + "sampling/importance_sampling_ratio/max": 1.919525682926178, + "sampling/importance_sampling_ratio/mean": 1.0021635254224142, + "sampling/importance_sampling_ratio/min": 0.49672264953454337, + "sampling/sampling_logp_difference/max": 0.3533597449461619, + "sampling/sampling_logp_difference/mean": 0.003967809017437199, + "step": 14340, + "step_time": 7.827388152068791, + "student/entropy": 0.1278589957083265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001198301712, + "completions/max_length": 494.3666666666667, + "completions/max_terminated_length": 458.6666666666667, + "completions/mean_length": 185.60473124186197, + "completions/mean_terminated_length": 178.9428741455078, + "completions/min_length": 53.43333333333333, + "completions/min_terminated_length": 53.43333333333333, + "entropy": 0.13205894784380992, + "epoch": 0.5457031101659515, + "eval/gt_acc_batch": 0.7963889102141063, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2869984209537506, + "kd/policy_loss": 0.008894563704961911, + "kd/rkl_advantage_mean": 0.779936836163203, + "kd/rkl_advantage_p95": 5.062445110082626, + "kd/rkl_advantage_std": 2.2575415521860123, + "kd/ssd_loss": 0.0, + "learning_rate": 4.543348649982911e-07, + "loss": 0.035578258832295734, + "num_tokens": 470319465.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888883590698, + "rewards/gt_logging_reward/std": 0.39498642285664876, + "sampling/importance_sampling_ratio/max": 1.8591102759043376, + "sampling/importance_sampling_ratio/mean": 1.0012077689170837, + "sampling/importance_sampling_ratio/min": 0.4524426430463791, + "sampling/sampling_logp_difference/max": 0.3474019070466359, + "sampling/sampling_logp_difference/mean": 0.004075704608112574, + "step": 14370, + "step_time": 7.960193502201097, + "student/entropy": 0.13205894784380992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779375513395, + "completions/max_length": 481.56666666666666, + "completions/max_terminated_length": 438.3, + "completions/mean_length": 191.40167795817058, + "completions/mean_terminated_length": 183.3941914876302, + "completions/min_length": 61.233333333333334, + "completions/min_terminated_length": 61.233333333333334, + "entropy": 0.12285610934098562, + "epoch": 0.546842365093229, + "eval/gt_acc_batch": 0.7858333508173625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3043651878833771, + "kd/policy_loss": 0.00804751859783816, + "kd/rkl_advantage_mean": 0.7266869453092416, + "kd/rkl_advantage_p95": 4.945399794975916, + "kd/rkl_advantage_std": 2.226421195268631, + "kd/ssd_loss": 0.0, + "learning_rate": 4.531956100710135e-07, + "loss": 0.03219006856282552, + "num_tokens": 471312687.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7858333349227905, + "rewards/gt_logging_reward/std": 0.3974597712357839, + "sampling/importance_sampling_ratio/max": 1.9460566480954489, + "sampling/importance_sampling_ratio/mean": 0.9925326625506083, + "sampling/importance_sampling_ratio/min": 0.4048585613568624, + "sampling/sampling_logp_difference/max": 0.3424772421518962, + "sampling/sampling_logp_difference/mean": 0.003805816119226317, + "step": 14400, + "step_time": 7.831805578096828, + "student/entropy": 0.12285610934098562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112721264363, + "completions/max_length": 492.1666666666667, + "completions/max_terminated_length": 446.1333333333333, + "completions/mean_length": 195.12417653401693, + "completions/mean_terminated_length": 186.7240234375, + "completions/min_length": 55.8, + "completions/min_terminated_length": 55.8, + "entropy": 0.12950461332996685, + "epoch": 0.5479816200205065, + "eval/gt_acc_batch": 0.7947222371896108, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26688554883003235, + "kd/policy_loss": 0.008391196542652324, + "kd/rkl_advantage_mean": 0.7083617351017892, + "kd/rkl_advantage_p95": 4.934193418423335, + "kd/rkl_advantage_std": 2.20128775537014, + "kd/ssd_loss": 0.0, + "learning_rate": 4.52056355143736e-07, + "loss": 0.03356478611628214, + "num_tokens": 472323566.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222212950389, + "rewards/gt_logging_reward/std": 0.40067109763622283, + "sampling/importance_sampling_ratio/max": 1.9461580753326415, + "sampling/importance_sampling_ratio/mean": 1.000873827934265, + "sampling/importance_sampling_ratio/min": 0.46605021754900616, + "sampling/sampling_logp_difference/max": 0.3560246706008911, + "sampling/sampling_logp_difference/mean": 0.003946274744036297, + "step": 14430, + "step_time": 8.065704048264402, + "student/entropy": 0.12950461332996685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778934687375, + "completions/max_length": 475.8, + "completions/max_terminated_length": 437.56666666666666, + "completions/mean_length": 183.84278869628906, + "completions/mean_terminated_length": 176.41904907226564, + "completions/min_length": 50.1, + "completions/min_terminated_length": 50.1, + "entropy": 0.12721151920656362, + "epoch": 0.5491208749477842, + "eval/gt_acc_batch": 0.7847222487131754, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31195974349975586, + "kd/policy_loss": 0.008410358768499767, + "kd/rkl_advantage_mean": 0.7828478263070185, + "kd/rkl_advantage_p95": 5.053556867440542, + "kd/rkl_advantage_std": 2.2377468019723894, + "kd/ssd_loss": 0.0, + "learning_rate": 4.509171002164584e-07, + "loss": 0.03364144166310628, + "num_tokens": 473290664.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7847222268581391, + "rewards/gt_logging_reward/std": 0.40486281116803485, + "sampling/importance_sampling_ratio/max": 1.8699603756268819, + "sampling/importance_sampling_ratio/mean": 0.9962499896685283, + "sampling/importance_sampling_ratio/min": 0.44797235280275344, + "sampling/sampling_logp_difference/max": 0.35732428630193075, + "sampling/sampling_logp_difference/mean": 0.003956443350762129, + "step": 14460, + "step_time": 7.799616241499219, + "student/entropy": 0.12721151920656362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223865489165, + "completions/max_length": 493.93333333333334, + "completions/max_terminated_length": 447.06666666666666, + "completions/mean_length": 193.12278747558594, + "completions/mean_terminated_length": 183.2884979248047, + "completions/min_length": 48.333333333333336, + "completions/min_terminated_length": 48.333333333333336, + "entropy": 0.13098448396970827, + "epoch": 0.5502601298750617, + "eval/gt_acc_batch": 0.796111128727595, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2744516432285309, + "kd/policy_loss": 0.008811791877572734, + "kd/rkl_advantage_mean": 0.7832028613736232, + "kd/rkl_advantage_p95": 5.127536849180857, + "kd/rkl_advantage_std": 2.2801956623792647, + "kd/ssd_loss": 0.0, + "learning_rate": 4.497778452891808e-07, + "loss": 0.0352471669514974, + "num_tokens": 474298570.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111108462016, + "rewards/gt_logging_reward/std": 0.39008025676012037, + "sampling/importance_sampling_ratio/max": 1.995458964506785, + "sampling/importance_sampling_ratio/mean": 1.004594471057256, + "sampling/importance_sampling_ratio/min": 0.4439615180095037, + "sampling/sampling_logp_difference/max": 0.3124322195847829, + "sampling/sampling_logp_difference/mean": 0.004005067663577696, + "step": 14490, + "step_time": 7.999133306264412, + "student/entropy": 0.13098448396970827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029444446259488662, + "completions/max_length": 495.23333333333335, + "completions/max_terminated_length": 465.8, + "completions/mean_length": 192.22445526123047, + "completions/mean_terminated_length": 182.7338633219401, + "completions/min_length": 50.3, + "completions/min_terminated_length": 50.3, + "entropy": 0.13106168750673533, + "epoch": 0.5513993848023393, + "eval/gt_acc_batch": 0.7808333575725556, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3008697032928467, + "kd/policy_loss": 0.008835554384859279, + "kd/rkl_advantage_mean": 0.852673387931039, + "kd/rkl_advantage_p95": 5.156245501836141, + "kd/rkl_advantage_std": 2.2747553139925003, + "kd/ssd_loss": 0.0, + "learning_rate": 4.486385903619033e-07, + "loss": 0.03534222443898519, + "num_tokens": 475294482.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7808333297570547, + "rewards/gt_logging_reward/std": 0.4049136092265447, + "sampling/importance_sampling_ratio/max": 1.979413139820099, + "sampling/importance_sampling_ratio/mean": 1.0095176657040914, + "sampling/importance_sampling_ratio/min": 0.45791137566169104, + "sampling/sampling_logp_difference/max": 0.3763920903205872, + "sampling/sampling_logp_difference/mean": 0.004075577156618238, + "step": 14520, + "step_time": 7.999782047532305, + "student/entropy": 0.13106168750673533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111253499985, + "completions/max_length": 474.9, + "completions/max_terminated_length": 444.56666666666666, + "completions/mean_length": 186.72278747558593, + "completions/mean_terminated_length": 178.2408915201823, + "completions/min_length": 54.96666666666667, + "completions/min_terminated_length": 54.96666666666667, + "entropy": 0.12860817952702444, + "epoch": 0.5525386397296168, + "eval/gt_acc_batch": 0.7933333655198415, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25711169838905334, + "kd/policy_loss": 0.008543997213322049, + "kd/rkl_advantage_mean": 0.7369885335365931, + "kd/rkl_advantage_p95": 4.976098597049713, + "kd/rkl_advantage_std": 2.230137065052986, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4749933543462576e-07, + "loss": 0.034175992012023926, + "num_tokens": 476267316.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7933333277702331, + "rewards/gt_logging_reward/std": 0.3977882832288742, + "sampling/importance_sampling_ratio/max": 1.8473928332328797, + "sampling/importance_sampling_ratio/mean": 0.997831251223882, + "sampling/importance_sampling_ratio/min": 0.45278252263863883, + "sampling/sampling_logp_difference/max": 0.32120487292607625, + "sampling/sampling_logp_difference/mean": 0.003953506665614744, + "step": 14550, + "step_time": 7.86307699316676, + "student/entropy": 0.12860817952702444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334662020206, + "completions/max_length": 489.73333333333335, + "completions/max_terminated_length": 452.8, + "completions/mean_length": 186.9569544474284, + "completions/mean_terminated_length": 181.07603352864584, + "completions/min_length": 58.1, + "completions/min_terminated_length": 58.1, + "entropy": 0.12478051135937372, + "epoch": 0.5536778946568944, + "eval/gt_acc_batch": 0.8247222403685252, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1835320144891739, + "kd/policy_loss": 0.007987252545232575, + "kd/rkl_advantage_mean": 0.7970935344696045, + "kd/rkl_advantage_p95": 5.052758999665579, + "kd/rkl_advantage_std": 2.2401954541603724, + "kd/ssd_loss": 0.0, + "learning_rate": 4.463600805073482e-07, + "loss": 0.031949015458424886, + "num_tokens": 477251561.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8247222224871318, + "rewards/gt_logging_reward/std": 0.37059103747208916, + "sampling/importance_sampling_ratio/max": 1.9625274380048117, + "sampling/importance_sampling_ratio/mean": 0.9889938294887543, + "sampling/importance_sampling_ratio/min": 0.4346028923988342, + "sampling/sampling_logp_difference/max": 0.31701832413673403, + "sampling/sampling_logp_difference/mean": 0.0038976720068603756, + "step": 14580, + "step_time": 7.909220107533232, + "student/entropy": 0.12478051135937372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.036666668858379124, + "completions/max_length": 497.6, + "completions/max_terminated_length": 465.23333333333335, + "completions/mean_length": 196.66278737386068, + "completions/mean_terminated_length": 184.81593119303386, + "completions/min_length": 50.96666666666667, + "completions/min_terminated_length": 50.96666666666667, + "entropy": 0.13124432787299156, + "epoch": 0.5548171495841719, + "eval/gt_acc_batch": 0.7700000206629435, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27865561842918396, + "kd/policy_loss": 0.00925178121542558, + "kd/rkl_advantage_mean": 0.8589087894807259, + "kd/rkl_advantage_p95": 5.242187841733297, + "kd/rkl_advantage_std": 2.2911203374465305, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4522082558007065e-07, + "loss": 0.03700713316599528, + "num_tokens": 478288491.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7700000047683716, + "rewards/gt_logging_reward/std": 0.41718982458114623, + "sampling/importance_sampling_ratio/max": 1.9514320850372315, + "sampling/importance_sampling_ratio/mean": 0.9915979226430257, + "sampling/importance_sampling_ratio/min": 0.41511506338914234, + "sampling/sampling_logp_difference/max": 0.33625168800354005, + "sampling/sampling_logp_difference/mean": 0.0040969335163633025, + "step": 14610, + "step_time": 8.259859662701395, + "student/entropy": 0.13124432787299156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02388889032105605, + "completions/max_length": 495.8, + "completions/max_terminated_length": 442.2, + "completions/mean_length": 186.1805648803711, + "completions/mean_terminated_length": 178.3234405517578, + "completions/min_length": 55.333333333333336, + "completions/min_terminated_length": 55.333333333333336, + "entropy": 0.12903570501754683, + "epoch": 0.5559564045114496, + "eval/gt_acc_batch": 0.8072222491105397, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2579575479030609, + "kd/policy_loss": 0.008518146038598691, + "kd/rkl_advantage_mean": 0.8103035018468897, + "kd/rkl_advantage_p95": 5.015791630744934, + "kd/rkl_advantage_std": 2.224917325377464, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4408157065279307e-07, + "loss": 0.034072589874267575, + "num_tokens": 479263469.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222193082174, + "rewards/gt_logging_reward/std": 0.38288371711969377, + "sampling/importance_sampling_ratio/max": 1.8768938938776651, + "sampling/importance_sampling_ratio/mean": 0.9962189773718516, + "sampling/importance_sampling_ratio/min": 0.4684581900636355, + "sampling/sampling_logp_difference/max": 0.328729252020518, + "sampling/sampling_logp_difference/mean": 0.003971867356449365, + "step": 14640, + "step_time": 7.954169170030218, + "student/entropy": 0.12903570501754683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016666667784253755, + "completions/max_length": 477.53333333333336, + "completions/max_terminated_length": 443.23333333333335, + "completions/mean_length": 175.52084197998047, + "completions/mean_terminated_length": 169.95115559895834, + "completions/min_length": 49.56666666666667, + "completions/min_terminated_length": 49.56666666666667, + "entropy": 0.12994326800107955, + "epoch": 0.5570956594387271, + "eval/gt_acc_batch": 0.8216666917006175, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.285945326089859, + "kd/policy_loss": 0.008404760444924856, + "kd/rkl_advantage_mean": 0.84225302208215, + "kd/rkl_advantage_p95": 5.143693939844767, + "kd/rkl_advantage_std": 2.2596986373265584, + "kd/ssd_loss": 0.0, + "learning_rate": 4.429423157255155e-07, + "loss": 0.033619046211242676, + "num_tokens": 480182752.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.821666665871938, + "rewards/gt_logging_reward/std": 0.37485228578249613, + "sampling/importance_sampling_ratio/max": 1.8609692295392355, + "sampling/importance_sampling_ratio/mean": 0.9977297206719716, + "sampling/importance_sampling_ratio/min": 0.4561636020739873, + "sampling/sampling_logp_difference/max": 0.3249599575996399, + "sampling/sampling_logp_difference/mean": 0.004067621318002542, + "step": 14670, + "step_time": 7.684766521669129, + "student/entropy": 0.12994326800107955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030833335034549235, + "completions/max_length": 482.4, + "completions/max_terminated_length": 448.53333333333336, + "completions/mean_length": 188.05167592366536, + "completions/mean_terminated_length": 177.9848492940267, + "completions/min_length": 54.86666666666667, + "completions/min_terminated_length": 54.86666666666667, + "entropy": 0.12949613040934008, + "epoch": 0.5582349143660046, + "eval/gt_acc_batch": 0.7963889141877493, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2759934663772583, + "kd/policy_loss": 0.008721781603526324, + "kd/rkl_advantage_mean": 0.8565758790820837, + "kd/rkl_advantage_p95": 5.206394044558207, + "kd/rkl_advantage_std": 2.277173594633738, + "kd/ssd_loss": 0.0, + "learning_rate": 4.418030607982379e-07, + "loss": 0.03488712310791016, + "num_tokens": 481170562.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888923327128, + "rewards/gt_logging_reward/std": 0.3919052427013715, + "sampling/importance_sampling_ratio/max": 1.9613359093666076, + "sampling/importance_sampling_ratio/mean": 0.9974311470985413, + "sampling/importance_sampling_ratio/min": 0.4282126009464264, + "sampling/sampling_logp_difference/max": 0.36439983248710633, + "sampling/sampling_logp_difference/mean": 0.00400510155595839, + "step": 14700, + "step_time": 7.857621572098287, + "student/entropy": 0.12949613040934008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556969096264, + "completions/max_length": 501.73333333333335, + "completions/max_terminated_length": 462.6666666666667, + "completions/mean_length": 183.54473215738932, + "completions/mean_terminated_length": 176.74147338867186, + "completions/min_length": 47.7, + "completions/min_terminated_length": 47.7, + "entropy": 0.13114777375012637, + "epoch": 0.5593741692932822, + "eval/gt_acc_batch": 0.8041666825612386, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3531908690929413, + "kd/policy_loss": 0.008747491700341925, + "kd/rkl_advantage_mean": 0.8482032224303111, + "kd/rkl_advantage_p95": 5.114320764938991, + "kd/rkl_advantage_std": 2.2371450712283454, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4066380587096037e-07, + "loss": 0.034989968935648603, + "num_tokens": 482130275.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666646798452, + "rewards/gt_logging_reward/std": 0.3930734773476919, + "sampling/importance_sampling_ratio/max": 1.9159248232841493, + "sampling/importance_sampling_ratio/mean": 1.0023906370004019, + "sampling/importance_sampling_ratio/min": 0.4692353387673696, + "sampling/sampling_logp_difference/max": 0.34857372045516966, + "sampling/sampling_logp_difference/mean": 0.004072307166643441, + "step": 14730, + "step_time": 7.975029643499875, + "student/entropy": 0.13114777375012637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018055556683490674, + "completions/max_length": 479.3333333333333, + "completions/max_terminated_length": 449.56666666666666, + "completions/mean_length": 186.2350102742513, + "completions/mean_terminated_length": 180.2590555826823, + "completions/min_length": 57.6, + "completions/min_terminated_length": 57.6, + "entropy": 0.11867553809036811, + "epoch": 0.5605134242205597, + "eval/gt_acc_batch": 0.8286111334959666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34131333231925964, + "kd/policy_loss": 0.007528761878105191, + "kd/rkl_advantage_mean": 0.7645862091332674, + "kd/rkl_advantage_p95": 4.997155328591664, + "kd/rkl_advantage_std": 2.2215456237395603, + "kd/ssd_loss": 0.0, + "learning_rate": 4.395245509436828e-07, + "loss": 0.030115048090616863, + "num_tokens": 483104113.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8286111176013946, + "rewards/gt_logging_reward/std": 0.37212048371632894, + "sampling/importance_sampling_ratio/max": 1.7891842603683472, + "sampling/importance_sampling_ratio/mean": 0.993642270565033, + "sampling/importance_sampling_ratio/min": 0.4611045718193054, + "sampling/sampling_logp_difference/max": 0.36978780627250674, + "sampling/sampling_logp_difference/mean": 0.003695269433471064, + "step": 14760, + "step_time": 7.818187504065766, + "student/entropy": 0.11867553809036811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02555555726091067, + "completions/max_length": 479.8, + "completions/max_terminated_length": 447.8666666666667, + "completions/mean_length": 188.87806599934896, + "completions/mean_terminated_length": 180.53165588378906, + "completions/min_length": 59.333333333333336, + "completions/min_terminated_length": 59.333333333333336, + "entropy": 0.1261564191430807, + "epoch": 0.5616526791478373, + "eval/gt_acc_batch": 0.8158333559830984, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25424909591674805, + "kd/policy_loss": 0.008392402001967033, + "kd/rkl_advantage_mean": 0.7963059127486001, + "kd/rkl_advantage_p95": 5.059913158416748, + "kd/rkl_advantage_std": 2.2320326904455823, + "kd/ssd_loss": 0.0, + "learning_rate": 4.383852960164052e-07, + "loss": 0.03356961409250895, + "num_tokens": 484087458.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8158333321412404, + "rewards/gt_logging_reward/std": 0.37888095527887344, + "sampling/importance_sampling_ratio/max": 1.8959337989489238, + "sampling/importance_sampling_ratio/mean": 0.9944297870000204, + "sampling/importance_sampling_ratio/min": 0.46082624047994614, + "sampling/sampling_logp_difference/max": 0.315626589457194, + "sampling/sampling_logp_difference/mean": 0.003889120346866548, + "step": 14790, + "step_time": 7.841765105200951, + "student/entropy": 0.1261564191430807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445998718343, + "completions/max_length": 489.3, + "completions/max_terminated_length": 458.7, + "completions/mean_length": 194.67389831542968, + "completions/mean_terminated_length": 186.84115397135417, + "completions/min_length": 58.233333333333334, + "completions/min_terminated_length": 58.233333333333334, + "entropy": 0.1269659071539839, + "epoch": 0.5627919340751149, + "eval/gt_acc_batch": 0.7727778057257334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27688223123550415, + "kd/policy_loss": 0.008319795640030254, + "kd/rkl_advantage_mean": 0.8748870355387529, + "kd/rkl_advantage_p95": 5.257646427551905, + "kd/rkl_advantage_std": 2.2771026412645976, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3724604108912773e-07, + "loss": 0.03327918450037638, + "num_tokens": 485093100.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.772777779897054, + "rewards/gt_logging_reward/std": 0.4126951058705648, + "sampling/importance_sampling_ratio/max": 1.892851503690084, + "sampling/importance_sampling_ratio/mean": 0.9920689602692921, + "sampling/importance_sampling_ratio/min": 0.42179983158906303, + "sampling/sampling_logp_difference/max": 0.37629685203234353, + "sampling/sampling_logp_difference/mean": 0.003959286616494259, + "step": 14820, + "step_time": 8.051334394432585, + "student/entropy": 0.1269659071539839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333469927311, + "completions/max_length": 486.3333333333333, + "completions/max_terminated_length": 452.06666666666666, + "completions/mean_length": 184.4652872721354, + "completions/mean_terminated_length": 177.5471649169922, + "completions/min_length": 55.06666666666667, + "completions/min_terminated_length": 55.06666666666667, + "entropy": 0.1252980480591456, + "epoch": 0.5639311890023925, + "eval/gt_acc_batch": 0.800555576880773, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2786824405193329, + "kd/policy_loss": 0.008398722735000775, + "kd/rkl_advantage_mean": 0.7946568898235759, + "kd/rkl_advantage_p95": 5.08853451013565, + "kd/rkl_advantage_std": 2.2454396029313406, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3610678616185015e-07, + "loss": 0.03359489440917969, + "num_tokens": 486068359.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8005555550257365, + "rewards/gt_logging_reward/std": 0.39454602003097533, + "sampling/importance_sampling_ratio/max": 1.9525071382522583, + "sampling/importance_sampling_ratio/mean": 1.0039041618506113, + "sampling/importance_sampling_ratio/min": 0.45630282660325366, + "sampling/sampling_logp_difference/max": 0.33644049962361655, + "sampling/sampling_logp_difference/mean": 0.003926339807609717, + "step": 14850, + "step_time": 8.106788741063792, + "student/entropy": 0.1252980480591456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016111112075547378, + "completions/max_length": 467.9, + "completions/max_terminated_length": 433.23333333333335, + "completions/mean_length": 181.05889841715495, + "completions/mean_terminated_length": 175.7219263712565, + "completions/min_length": 56.766666666666666, + "completions/min_terminated_length": 56.766666666666666, + "entropy": 0.1295476840188106, + "epoch": 0.56507044392967, + "eval/gt_acc_batch": 0.7936111271381379, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34005385637283325, + "kd/policy_loss": 0.008815631210260714, + "kd/rkl_advantage_mean": 0.7409318139272121, + "kd/rkl_advantage_p95": 4.880761867761612, + "kd/rkl_advantage_std": 2.172141740719477, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3496753123457257e-07, + "loss": 0.035262529055277506, + "num_tokens": 487019483.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111152172088, + "rewards/gt_logging_reward/std": 0.39561836073795953, + "sampling/importance_sampling_ratio/max": 1.8938074707984924, + "sampling/importance_sampling_ratio/mean": 0.9934985737005869, + "sampling/importance_sampling_ratio/min": 0.4813077827294668, + "sampling/sampling_logp_difference/max": 0.3207790434360504, + "sampling/sampling_logp_difference/mean": 0.003998855012468994, + "step": 14880, + "step_time": 7.7092128826329525, + "student/entropy": 0.1295476840188106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016111111858238776, + "completions/max_length": 474.06666666666666, + "completions/max_terminated_length": 428.9, + "completions/mean_length": 185.1150095621745, + "completions/mean_terminated_length": 179.83917439778645, + "completions/min_length": 53.266666666666666, + "completions/min_terminated_length": 53.266666666666666, + "entropy": 0.12999116114030282, + "epoch": 0.5662096988569476, + "eval/gt_acc_batch": 0.8072222471237183, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.348001092672348, + "kd/policy_loss": 0.008627593538646276, + "kd/rkl_advantage_mean": 0.7754723818351825, + "kd/rkl_advantage_p95": 4.995095139741897, + "kd/rkl_advantage_std": 2.2205897559722265, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3382827630729504e-07, + "loss": 0.03451037804285685, + "num_tokens": 487982817.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222173213959, + "rewards/gt_logging_reward/std": 0.3874447226524353, + "sampling/importance_sampling_ratio/max": 1.9067360440889993, + "sampling/importance_sampling_ratio/mean": 1.0009732445081074, + "sampling/importance_sampling_ratio/min": 0.45793266172210373, + "sampling/sampling_logp_difference/max": 0.3747342546780904, + "sampling/sampling_logp_difference/mean": 0.003987411200068891, + "step": 14910, + "step_time": 7.796049405565524, + "student/entropy": 0.12999116114030282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112367361785, + "completions/max_length": 492.2, + "completions/max_terminated_length": 447.53333333333336, + "completions/mean_length": 188.02945404052736, + "completions/mean_terminated_length": 181.1200937906901, + "completions/min_length": 50.7, + "completions/min_terminated_length": 50.7, + "entropy": 0.12577752713114024, + "epoch": 0.5673489537842251, + "eval/gt_acc_batch": 0.8113889118035634, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2444296032190323, + "kd/policy_loss": 0.008076839686448995, + "kd/rkl_advantage_mean": 0.8560708520313104, + "kd/rkl_advantage_p95": 5.201135377089183, + "kd/rkl_advantage_std": 2.245233385761579, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3268902138001745e-07, + "loss": 0.032307360569636026, + "num_tokens": 488969923.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8113888959089915, + "rewards/gt_logging_reward/std": 0.38616430858771006, + "sampling/importance_sampling_ratio/max": 1.8590363343556722, + "sampling/importance_sampling_ratio/mean": 0.9978529055913289, + "sampling/importance_sampling_ratio/min": 0.45586527734994886, + "sampling/sampling_logp_difference/max": 0.3754005750020345, + "sampling/sampling_logp_difference/mean": 0.0038810459276040397, + "step": 14940, + "step_time": 8.214343106532276, + "student/entropy": 0.12577752713114024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015833334314326444, + "completions/max_length": 486.1, + "completions/max_terminated_length": 443.9, + "completions/mean_length": 183.20112050374348, + "completions/mean_terminated_length": 177.9965021769206, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.1367311540370186, + "epoch": 0.5684882087115026, + "eval/gt_acc_batch": 0.7950000246365865, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34911081194877625, + "kd/policy_loss": 0.009001181197042267, + "kd/rkl_advantage_mean": 0.8047993111113707, + "kd/rkl_advantage_p95": 5.087614411115647, + "kd/rkl_advantage_std": 2.252368946870168, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3154976645273987e-07, + "loss": 0.03600472609202067, + "num_tokens": 489929951.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7950000027815501, + "rewards/gt_logging_reward/std": 0.395185653368632, + "sampling/importance_sampling_ratio/max": 1.8461347222328186, + "sampling/importance_sampling_ratio/mean": 0.9919510165850322, + "sampling/importance_sampling_ratio/min": 0.44747891326745354, + "sampling/sampling_logp_difference/max": 0.30000892877578733, + "sampling/sampling_logp_difference/mean": 0.0041516999481245875, + "step": 14970, + "step_time": 7.974283711735916, + "student/entropy": 0.1367311540370186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334326744078, + "completions/max_length": 494.96666666666664, + "completions/max_terminated_length": 459.2, + "completions/mean_length": 190.02973124186198, + "completions/mean_terminated_length": 183.3494140625, + "completions/min_length": 53.2, + "completions/min_terminated_length": 53.2, + "entropy": 0.13136992050955693, + "epoch": 0.5696274636387803, + "eval/gt_acc_batch": 0.7938889006773631, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23492155969142914, + "kd/policy_loss": 0.008779669877064104, + "kd/rkl_advantage_mean": 0.8364458842823903, + "kd/rkl_advantage_p95": 5.101889499028524, + "kd/rkl_advantage_std": 2.2145146906375883, + "kd/ssd_loss": 0.0, + "learning_rate": 4.304105115254623e-07, + "loss": 0.03511868715286255, + "num_tokens": 490923466.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7938888887564342, + "rewards/gt_logging_reward/std": 0.39249543224771816, + "sampling/importance_sampling_ratio/max": 1.9053008516629537, + "sampling/importance_sampling_ratio/mean": 1.0008631984392802, + "sampling/importance_sampling_ratio/min": 0.4551946818828583, + "sampling/sampling_logp_difference/max": 0.32081048091252645, + "sampling/sampling_logp_difference/mean": 0.003950778356132408, + "step": 15000, + "step_time": 8.089994428228238, + "student/entropy": 0.13136992050955693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01666666759798924, + "completions/max_length": 481.96666666666664, + "completions/max_terminated_length": 440.8333333333333, + "completions/mean_length": 180.27362263997395, + "completions/mean_terminated_length": 174.74376424153647, + "completions/min_length": 56.4, + "completions/min_terminated_length": 56.4, + "entropy": 0.12867441729952891, + "epoch": 0.5707667185660578, + "eval/gt_acc_batch": 0.8041666845480601, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2652240991592407, + "kd/policy_loss": 0.008298163465224207, + "kd/rkl_advantage_mean": 0.7733783020327488, + "kd/rkl_advantage_p95": 4.938387550910314, + "kd/rkl_advantage_std": 2.206475105881691, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2927125659818476e-07, + "loss": 0.03319265643755595, + "num_tokens": 491877467.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666646798452, + "rewards/gt_logging_reward/std": 0.3914968465765317, + "sampling/importance_sampling_ratio/max": 1.875240687529246, + "sampling/importance_sampling_ratio/mean": 0.9921226640542348, + "sampling/importance_sampling_ratio/min": 0.47616629501183827, + "sampling/sampling_logp_difference/max": 0.32034258047739667, + "sampling/sampling_logp_difference/mean": 0.003998813304739694, + "step": 15030, + "step_time": 7.998703920095674, + "student/entropy": 0.12867441729952891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779381722212, + "completions/max_length": 477.43333333333334, + "completions/max_terminated_length": 446.6, + "completions/mean_length": 191.46778717041016, + "completions/mean_terminated_length": 182.48332875569662, + "completions/min_length": 54.56666666666667, + "completions/min_terminated_length": 54.56666666666667, + "entropy": 0.13434038845201332, + "epoch": 0.5719059734933354, + "eval/gt_acc_batch": 0.7869444688161215, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2313460409641266, + "kd/policy_loss": 0.009210808367546027, + "kd/rkl_advantage_mean": 0.8026962720789015, + "kd/rkl_advantage_p95": 5.114113074541092, + "kd/rkl_advantage_std": 2.2509561757246654, + "kd/ssd_loss": 0.0, + "learning_rate": 4.281320016709072e-07, + "loss": 0.03684323231379191, + "num_tokens": 492871039.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444410006206, + "rewards/gt_logging_reward/std": 0.3994880899786949, + "sampling/importance_sampling_ratio/max": 1.912812642256419, + "sampling/importance_sampling_ratio/mean": 1.0034723699092865, + "sampling/importance_sampling_ratio/min": 0.4437927484512329, + "sampling/sampling_logp_difference/max": 0.32970458070437114, + "sampling/sampling_logp_difference/mean": 0.0040983060064415135, + "step": 15060, + "step_time": 7.941795165268316, + "student/entropy": 0.13434038845201332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112336317697, + "completions/max_length": 478.46666666666664, + "completions/max_terminated_length": 443.7, + "completions/mean_length": 179.22528737386068, + "completions/mean_terminated_length": 172.2721201578776, + "completions/min_length": 59.03333333333333, + "completions/min_terminated_length": 59.03333333333333, + "entropy": 0.13144735265523194, + "epoch": 0.5730452284206129, + "eval/gt_acc_batch": 0.8050000170866648, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24645313620567322, + "kd/policy_loss": 0.008662938146153465, + "kd/rkl_advantage_mean": 0.8035478123774131, + "kd/rkl_advantage_p95": 5.112564738591512, + "kd/rkl_advantage_std": 2.277611975868543, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2699274674362965e-07, + "loss": 0.034651756286621094, + "num_tokens": 493822002.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8049999952316285, + "rewards/gt_logging_reward/std": 0.38890370925267537, + "sampling/importance_sampling_ratio/max": 1.9249722957611084, + "sampling/importance_sampling_ratio/mean": 0.9933929920196534, + "sampling/importance_sampling_ratio/min": 0.4324158291021983, + "sampling/sampling_logp_difference/max": 0.36164778073628745, + "sampling/sampling_logp_difference/mean": 0.004098411952145398, + "step": 15090, + "step_time": 8.00916341660001, + "student/entropy": 0.13144735265523194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01972222321977218, + "completions/max_length": 485.0, + "completions/max_terminated_length": 447.46666666666664, + "completions/mean_length": 185.01112111409506, + "completions/mean_terminated_length": 178.6758270263672, + "completions/min_length": 51.233333333333334, + "completions/min_terminated_length": 51.233333333333334, + "entropy": 0.12549574362734953, + "epoch": 0.5741844833478905, + "eval/gt_acc_batch": 0.8094444652398427, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28451910614967346, + "kd/policy_loss": 0.008159764171189939, + "kd/rkl_advantage_mean": 0.7780690651386977, + "kd/rkl_advantage_p95": 5.0248590012391405, + "kd/rkl_advantage_std": 2.2340725253025693, + "kd/ssd_loss": 0.0, + "learning_rate": 4.258534918163521e-07, + "loss": 0.03263906240463257, + "num_tokens": 494786922.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8094444374243418, + "rewards/gt_logging_reward/std": 0.3869723528623581, + "sampling/importance_sampling_ratio/max": 1.90249977906545, + "sampling/importance_sampling_ratio/mean": 0.9972780883312226, + "sampling/importance_sampling_ratio/min": 0.449739079674085, + "sampling/sampling_logp_difference/max": 0.3390773852666219, + "sampling/sampling_logp_difference/mean": 0.003907989881311854, + "step": 15120, + "step_time": 7.885476231901945, + "student/entropy": 0.12549574362734953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445812453825, + "completions/max_length": 488.7, + "completions/max_terminated_length": 444.23333333333335, + "completions/mean_length": 185.9105651855469, + "completions/mean_terminated_length": 177.93930002848307, + "completions/min_length": 49.666666666666664, + "completions/min_terminated_length": 49.666666666666664, + "entropy": 0.12933529913425446, + "epoch": 0.575323738275168, + "eval/gt_acc_batch": 0.8033333460489909, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30074068903923035, + "kd/policy_loss": 0.008446535023783024, + "kd/rkl_advantage_mean": 0.7386902132381995, + "kd/rkl_advantage_p95": 4.996989820400874, + "kd/rkl_advantage_std": 2.2234066396951677, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2471423688907454e-07, + "loss": 0.03378613789876302, + "num_tokens": 495760600.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.803333334128062, + "rewards/gt_logging_reward/std": 0.3894980102777481, + "sampling/importance_sampling_ratio/max": 1.9617504874865215, + "sampling/importance_sampling_ratio/mean": 0.9992002646128336, + "sampling/importance_sampling_ratio/min": 0.480793189505736, + "sampling/sampling_logp_difference/max": 0.2998013784488042, + "sampling/sampling_logp_difference/mean": 0.003962013210790853, + "step": 15150, + "step_time": 8.034040543599986, + "student/entropy": 0.12933529913425446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030833334910372892, + "completions/max_length": 488.96666666666664, + "completions/max_terminated_length": 448.9, + "completions/mean_length": 193.179731241862, + "completions/mean_terminated_length": 183.13263346354168, + "completions/min_length": 56.233333333333334, + "completions/min_terminated_length": 56.233333333333334, + "entropy": 0.12690923418849706, + "epoch": 0.5764629932024457, + "eval/gt_acc_batch": 0.7891666829586029, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30078554153442383, + "kd/policy_loss": 0.008580896196266015, + "kd/rkl_advantage_mean": 0.8313922794846197, + "kd/rkl_advantage_p95": 5.104433965682984, + "kd/rkl_advantage_std": 2.2482741316159567, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2357498196179695e-07, + "loss": 0.03432358105977376, + "num_tokens": 496761615.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7891666670640309, + "rewards/gt_logging_reward/std": 0.4009433850646019, + "sampling/importance_sampling_ratio/max": 1.950852676232656, + "sampling/importance_sampling_ratio/mean": 1.0018661220868428, + "sampling/importance_sampling_ratio/min": 0.4379060178995132, + "sampling/sampling_logp_difference/max": 0.3031622886657715, + "sampling/sampling_logp_difference/mean": 0.003907645051367581, + "step": 15180, + "step_time": 8.030876906170063, + "student/entropy": 0.12690923418849706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223648180564, + "completions/max_length": 484.23333333333335, + "completions/max_terminated_length": 462.06666666666666, + "completions/mean_length": 193.99834340413412, + "completions/mean_terminated_length": 184.57589569091797, + "completions/min_length": 58.46666666666667, + "completions/min_terminated_length": 58.46666666666667, + "entropy": 0.12933206483721732, + "epoch": 0.5776022481297232, + "eval/gt_acc_batch": 0.7861111183961232, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30495402216911316, + "kd/policy_loss": 0.008787593555947144, + "kd/rkl_advantage_mean": 0.7826602914681037, + "kd/rkl_advantage_p95": 5.024501981337865, + "kd/rkl_advantage_std": 2.2233079512914022, + "kd/ssd_loss": 0.0, + "learning_rate": 4.224357270345194e-07, + "loss": 0.035150369008382164, + "num_tokens": 497764161.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111104488373, + "rewards/gt_logging_reward/std": 0.3969475731253624, + "sampling/importance_sampling_ratio/max": 1.985982648531596, + "sampling/importance_sampling_ratio/mean": 1.001385404666265, + "sampling/importance_sampling_ratio/min": 0.4640386804938316, + "sampling/sampling_logp_difference/max": 0.35490436255931856, + "sampling/sampling_logp_difference/mean": 0.00395402725165089, + "step": 15210, + "step_time": 8.057391296929078, + "student/entropy": 0.12933206483721732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334575096767, + "completions/max_length": 465.3666666666667, + "completions/max_terminated_length": 442.5, + "completions/mean_length": 182.86639760335285, + "completions/mean_terminated_length": 175.98800710042318, + "completions/min_length": 61.1, + "completions/min_terminated_length": 61.1, + "entropy": 0.12440199851989746, + "epoch": 0.5787415030570007, + "eval/gt_acc_batch": 0.8213889102141062, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2680269479751587, + "kd/policy_loss": 0.008077687015368913, + "kd/rkl_advantage_mean": 0.8091907025625308, + "kd/rkl_advantage_p95": 5.126834505796433, + "kd/rkl_advantage_std": 2.2444133937358854, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2129647210724184e-07, + "loss": 0.03231074611345927, + "num_tokens": 498731456.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8213888903458914, + "rewards/gt_logging_reward/std": 0.3671550065279007, + "sampling/importance_sampling_ratio/max": 1.8369230906168619, + "sampling/importance_sampling_ratio/mean": 0.9957556307315827, + "sampling/importance_sampling_ratio/min": 0.438019202152888, + "sampling/sampling_logp_difference/max": 0.3281177262465159, + "sampling/sampling_logp_difference/mean": 0.003868820828696092, + "step": 15240, + "step_time": 8.044955246132547, + "student/entropy": 0.12440199851989746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02000000129143397, + "completions/max_length": 478.93333333333334, + "completions/max_terminated_length": 434.4, + "completions/mean_length": 179.7725092569987, + "completions/mean_terminated_length": 172.89996236165365, + "completions/min_length": 56.43333333333333, + "completions/min_terminated_length": 56.43333333333333, + "entropy": 0.12350963912904263, + "epoch": 0.5798807579842783, + "eval/gt_acc_batch": 0.8244444668293, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2871145009994507, + "kd/policy_loss": 0.007912115469419707, + "kd/rkl_advantage_mean": 0.7846104147533576, + "kd/rkl_advantage_p95": 5.068783607085546, + "kd/rkl_advantage_std": 2.2352878113587695, + "kd/ssd_loss": 0.0, + "learning_rate": 4.2015721717996426e-07, + "loss": 0.03164846102396647, + "num_tokens": 499672925.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.824444442987442, + "rewards/gt_logging_reward/std": 0.3752590358257294, + "sampling/importance_sampling_ratio/max": 1.8502008438110351, + "sampling/importance_sampling_ratio/mean": 1.0034859577814739, + "sampling/importance_sampling_ratio/min": 0.49271436780691147, + "sampling/sampling_logp_difference/max": 0.3523974299430847, + "sampling/sampling_logp_difference/mean": 0.003831335878930986, + "step": 15270, + "step_time": 7.825866373730241, + "student/entropy": 0.12350963912904263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02472222354263067, + "completions/max_length": 489.3666666666667, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 190.0438995361328, + "completions/mean_terminated_length": 181.97462717692056, + "completions/min_length": 47.766666666666666, + "completions/min_terminated_length": 47.766666666666666, + "entropy": 0.12655193104098242, + "epoch": 0.5810200129115558, + "eval/gt_acc_batch": 0.808611132701238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2244393527507782, + "kd/policy_loss": 0.00828826660872437, + "kd/rkl_advantage_mean": 0.7538203207155069, + "kd/rkl_advantage_p95": 5.002336754401525, + "kd/rkl_advantage_std": 2.243658263484637, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1901796225268673e-07, + "loss": 0.03315306504567464, + "num_tokens": 500664635.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8086111068725585, + "rewards/gt_logging_reward/std": 0.3880385140577952, + "sampling/importance_sampling_ratio/max": 1.8010735829671225, + "sampling/importance_sampling_ratio/mean": 0.9997518122196197, + "sampling/importance_sampling_ratio/min": 0.46314718425273893, + "sampling/sampling_logp_difference/max": 0.3276507019996643, + "sampling/sampling_logp_difference/mean": 0.0038854270707815884, + "step": 15300, + "step_time": 8.062427929797074, + "student/entropy": 0.12655193104098242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334643393756, + "completions/max_length": 476.4, + "completions/max_terminated_length": 446.53333333333336, + "completions/mean_length": 192.21389821370443, + "completions/mean_terminated_length": 184.6425989786784, + "completions/min_length": 57.56666666666667, + "completions/min_terminated_length": 57.56666666666667, + "entropy": 0.13125848242392144, + "epoch": 0.5821592678388334, + "eval/gt_acc_batch": 0.7833333591620127, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3745999038219452, + "kd/policy_loss": 0.008662977618708585, + "kd/rkl_advantage_mean": 0.7418439670155446, + "kd/rkl_advantage_p95": 4.9431480666001635, + "kd/rkl_advantage_std": 2.208835549155871, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1787870732540915e-07, + "loss": 0.03465191125869751, + "num_tokens": 501654293.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7833333353201548, + "rewards/gt_logging_reward/std": 0.4051589399576187, + "sampling/importance_sampling_ratio/max": 1.931609364350637, + "sampling/importance_sampling_ratio/mean": 0.996396827697754, + "sampling/importance_sampling_ratio/min": 0.3903307015697161, + "sampling/sampling_logp_difference/max": 0.36801263292630515, + "sampling/sampling_logp_difference/mean": 0.004023080590801934, + "step": 15330, + "step_time": 8.047686284702893, + "student/entropy": 0.13125848242392144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666785875956, + "completions/max_length": 466.93333333333334, + "completions/max_terminated_length": 429.6333333333333, + "completions/mean_length": 188.75973358154297, + "completions/mean_terminated_length": 181.6042037963867, + "completions/min_length": 52.86666666666667, + "completions/min_terminated_length": 52.86666666666667, + "entropy": 0.12459179020176331, + "epoch": 0.5832985227661109, + "eval/gt_acc_batch": 0.8211111227671305, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.253097802400589, + "kd/policy_loss": 0.007671497717577343, + "kd/rkl_advantage_mean": 0.6565991579865416, + "kd/rkl_advantage_p95": 4.736712998151779, + "kd/rkl_advantage_std": 2.145535032947858, + "kd/ssd_loss": 0.0, + "learning_rate": 4.167394523981316e-07, + "loss": 0.03068599502245585, + "num_tokens": 502636588.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8211111128330231, + "rewards/gt_logging_reward/std": 0.367040874560674, + "sampling/importance_sampling_ratio/max": 1.8182965159416198, + "sampling/importance_sampling_ratio/mean": 0.9915614604949952, + "sampling/importance_sampling_ratio/min": 0.44235172470410666, + "sampling/sampling_logp_difference/max": 0.3187384198109309, + "sampling/sampling_logp_difference/mean": 0.003881465387530625, + "step": 15360, + "step_time": 7.955239400363643, + "student/entropy": 0.12459179020176331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333463718494, + "completions/max_length": 491.03333333333336, + "completions/max_terminated_length": 460.03333333333336, + "completions/mean_length": 191.59417724609375, + "completions/mean_terminated_length": 184.81543579101563, + "completions/min_length": 52.833333333333336, + "completions/min_terminated_length": 52.833333333333336, + "entropy": 0.12691999059170483, + "epoch": 0.5844377776933886, + "eval/gt_acc_batch": 0.8077778001626332, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2624358534812927, + "kd/policy_loss": 0.008400994296728944, + "kd/rkl_advantage_mean": 0.7483067738202711, + "kd/rkl_advantage_p95": 4.938881788651148, + "kd/rkl_advantage_std": 2.18638753592968, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1560019747085404e-07, + "loss": 0.033603978157043454, + "num_tokens": 503640695.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8077777783075969, + "rewards/gt_logging_reward/std": 0.38772207001845044, + "sampling/importance_sampling_ratio/max": 1.944317336877187, + "sampling/importance_sampling_ratio/mean": 1.0001565019289653, + "sampling/importance_sampling_ratio/min": 0.46641191144784294, + "sampling/sampling_logp_difference/max": 0.3111032327016195, + "sampling/sampling_logp_difference/mean": 0.0039324759893740215, + "step": 15390, + "step_time": 8.398528395097431, + "student/entropy": 0.12691999059170483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001167257628, + "completions/max_length": 480.6666666666667, + "completions/max_terminated_length": 446.96666666666664, + "completions/mean_length": 191.0069549560547, + "completions/mean_terminated_length": 184.68834228515624, + "completions/min_length": 58.166666666666664, + "completions/min_terminated_length": 58.166666666666664, + "entropy": 0.12796044796705247, + "epoch": 0.5855770326206661, + "eval/gt_acc_batch": 0.7891666928927104, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3009231984615326, + "kd/policy_loss": 0.008347317232983186, + "kd/rkl_advantage_mean": 0.7612043930683285, + "kd/rkl_advantage_p95": 4.9628836691379545, + "kd/rkl_advantage_std": 2.187847074866295, + "kd/ssd_loss": 0.0, + "learning_rate": 4.144609425435765e-07, + "loss": 0.03338926235834758, + "num_tokens": 504635168.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7891666670640309, + "rewards/gt_logging_reward/std": 0.3944397394855817, + "sampling/importance_sampling_ratio/max": 1.9309091250101724, + "sampling/importance_sampling_ratio/mean": 0.9986812750498454, + "sampling/importance_sampling_ratio/min": 0.4757702867190043, + "sampling/sampling_logp_difference/max": 0.35686868826548257, + "sampling/sampling_logp_difference/mean": 0.003986748921064039, + "step": 15420, + "step_time": 8.03136003853336, + "student/entropy": 0.12796044796705247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027500001372148593, + "completions/max_length": 489.3333333333333, + "completions/max_terminated_length": 456.3333333333333, + "completions/mean_length": 191.18417714436848, + "completions/mean_terminated_length": 182.25890401204427, + "completions/min_length": 55.9, + "completions/min_terminated_length": 55.9, + "entropy": 0.12386733833700418, + "epoch": 0.5867162875479437, + "eval/gt_acc_batch": 0.8175000170866649, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26517990231513977, + "kd/policy_loss": 0.008066700918910404, + "kd/rkl_advantage_mean": 0.796012317823867, + "kd/rkl_advantage_p95": 5.085466355085373, + "kd/rkl_advantage_std": 2.2516742517550785, + "kd/ssd_loss": 0.0, + "learning_rate": 4.133216876162989e-07, + "loss": 0.03226680358250936, + "num_tokens": 505621887.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8174999992052714, + "rewards/gt_logging_reward/std": 0.3766180311640104, + "sampling/importance_sampling_ratio/max": 1.930800437927246, + "sampling/importance_sampling_ratio/mean": 1.0057152430216472, + "sampling/importance_sampling_ratio/min": 0.4831873516241709, + "sampling/sampling_logp_difference/max": 0.3149143815040588, + "sampling/sampling_logp_difference/mean": 0.003839653815763692, + "step": 15450, + "step_time": 7.91720277016672, + "student/entropy": 0.12386733833700418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112559835116, + "completions/max_length": 472.3, + "completions/max_terminated_length": 440.23333333333335, + "completions/mean_length": 190.92889862060548, + "completions/mean_terminated_length": 183.36648457845052, + "completions/min_length": 54.766666666666666, + "completions/min_terminated_length": 54.766666666666666, + "entropy": 0.12998690145711103, + "epoch": 0.5878555424752212, + "eval/gt_acc_batch": 0.8069444576899211, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24584195017814636, + "kd/policy_loss": 0.008579699695110321, + "kd/rkl_advantage_mean": 0.729842017699654, + "kd/rkl_advantage_p95": 5.007106119394303, + "kd/rkl_advantage_std": 2.2357634802659354, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1218243268902134e-07, + "loss": 0.034318796793619794, + "num_tokens": 506610655.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8069444477558136, + "rewards/gt_logging_reward/std": 0.38022476732730864, + "sampling/importance_sampling_ratio/max": 1.953475566705068, + "sampling/importance_sampling_ratio/mean": 0.9904627660910289, + "sampling/importance_sampling_ratio/min": 0.42225536455710727, + "sampling/sampling_logp_difference/max": 0.3137604753176371, + "sampling/sampling_logp_difference/mean": 0.004027486002693574, + "step": 15480, + "step_time": 7.956252208000903, + "student/entropy": 0.12998690145711103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334513008596, + "completions/max_length": 469.96666666666664, + "completions/max_terminated_length": 433.73333333333335, + "completions/mean_length": 185.16695454915364, + "completions/mean_terminated_length": 178.34729614257813, + "completions/min_length": 53.03333333333333, + "completions/min_terminated_length": 53.03333333333333, + "entropy": 0.12055580237259468, + "epoch": 0.5889947974024987, + "eval/gt_acc_batch": 0.8341666897137959, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24239039421081543, + "kd/policy_loss": 0.007397564236695567, + "kd/rkl_advantage_mean": 0.7565778630475203, + "kd/rkl_advantage_p95": 5.046579605340957, + "kd/rkl_advantage_std": 2.2380097011725106, + "kd/ssd_loss": 0.0, + "learning_rate": 4.110431777617438e-07, + "loss": 0.029590253035227457, + "num_tokens": 507588888.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8341666658719381, + "rewards/gt_logging_reward/std": 0.3620256811380386, + "sampling/importance_sampling_ratio/max": 1.795862909158071, + "sampling/importance_sampling_ratio/mean": 0.9910203615824381, + "sampling/importance_sampling_ratio/min": 0.45204316278298695, + "sampling/sampling_logp_difference/max": 0.3272618462642034, + "sampling/sampling_logp_difference/mean": 0.0037779625893260044, + "step": 15510, + "step_time": 7.996220321964938, + "student/entropy": 0.12055580237259468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01944444567585985, + "completions/max_length": 499.3, + "completions/max_terminated_length": 460.43333333333334, + "completions/mean_length": 189.09778798421223, + "completions/mean_terminated_length": 182.6711908976237, + "completions/min_length": 60.1, + "completions/min_terminated_length": 60.1, + "entropy": 0.12537546828389168, + "epoch": 0.5901340523297763, + "eval/gt_acc_batch": 0.8022222340106964, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.36262214183807373, + "kd/policy_loss": 0.008325079942975815, + "kd/rkl_advantage_mean": 0.74627833639582, + "kd/rkl_advantage_p95": 5.019305223226548, + "kd/rkl_advantage_std": 2.2426316420237224, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0990392283446623e-07, + "loss": 0.03330031633377075, + "num_tokens": 508576016.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8022222240765889, + "rewards/gt_logging_reward/std": 0.3892242779334386, + "sampling/importance_sampling_ratio/max": 1.85850590467453, + "sampling/importance_sampling_ratio/mean": 1.003237009048462, + "sampling/importance_sampling_ratio/min": 0.44135389626026156, + "sampling/sampling_logp_difference/max": 0.3497146129608154, + "sampling/sampling_logp_difference/mean": 0.0038993463230629763, + "step": 15540, + "step_time": 8.130781616765793, + "student/entropy": 0.12537546828389168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03138889071221153, + "completions/max_length": 495.1, + "completions/max_terminated_length": 452.6666666666667, + "completions/mean_length": 190.94417724609374, + "completions/mean_terminated_length": 180.7211471557617, + "completions/min_length": 54.666666666666664, + "completions/min_terminated_length": 54.666666666666664, + "entropy": 0.1305789766833186, + "epoch": 0.5912733072570538, + "eval/gt_acc_batch": 0.8094444652398427, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29261520504951477, + "kd/policy_loss": 0.008603603423883518, + "kd/rkl_advantage_mean": 0.8360791131854057, + "kd/rkl_advantage_p95": 5.110711552699407, + "kd/rkl_advantage_std": 2.2246329774459204, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0876466790718865e-07, + "loss": 0.03441441456476847, + "num_tokens": 509566591.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8094444493452708, + "rewards/gt_logging_reward/std": 0.38296848237514497, + "sampling/importance_sampling_ratio/max": 1.888165549437205, + "sampling/importance_sampling_ratio/mean": 0.9983424703280132, + "sampling/importance_sampling_ratio/min": 0.44676081587870914, + "sampling/sampling_logp_difference/max": 0.33730498552322385, + "sampling/sampling_logp_difference/mean": 0.004016338824294507, + "step": 15570, + "step_time": 8.13543878403434, + "student/entropy": 0.1305789766833186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029166668187826872, + "completions/max_length": 494.6666666666667, + "completions/max_terminated_length": 455.8666666666667, + "completions/mean_length": 194.7055658976237, + "completions/mean_terminated_length": 185.2229777018229, + "completions/min_length": 48.9, + "completions/min_terminated_length": 48.9, + "entropy": 0.12321169320493937, + "epoch": 0.5924125621843315, + "eval/gt_acc_batch": 0.805277802546819, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.20882534980773926, + "kd/policy_loss": 0.007715173898031935, + "kd/rkl_advantage_mean": 0.7663390563024829, + "kd/rkl_advantage_p95": 5.0452071845531465, + "kd/rkl_advantage_std": 2.225790709257126, + "kd/ssd_loss": 0.0, + "learning_rate": 4.076254129799111e-07, + "loss": 0.030860700209935508, + "num_tokens": 510572163.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8052777767181396, + "rewards/gt_logging_reward/std": 0.3879568725824356, + "sampling/importance_sampling_ratio/max": 1.8268786350886026, + "sampling/importance_sampling_ratio/mean": 0.9874536097049713, + "sampling/importance_sampling_ratio/min": 0.43498461643854774, + "sampling/sampling_logp_difference/max": 0.3354682286580404, + "sampling/sampling_logp_difference/mean": 0.0038333692975963157, + "step": 15600, + "step_time": 8.105524016135314, + "student/entropy": 0.12321169320493937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01805555655931433, + "completions/max_length": 467.43333333333334, + "completions/max_terminated_length": 439.53333333333336, + "completions/mean_length": 191.411398824056, + "completions/mean_terminated_length": 185.55286153157553, + "completions/min_length": 51.03333333333333, + "completions/min_terminated_length": 51.03333333333333, + "entropy": 0.127490694137911, + "epoch": 0.593551817111609, + "eval/gt_acc_batch": 0.7911111354827881, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2831970453262329, + "kd/policy_loss": 0.008454525129248698, + "kd/rkl_advantage_mean": 0.6735275062693593, + "kd/rkl_advantage_p95": 4.846984126170477, + "kd/rkl_advantage_std": 2.181845055023829, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0648615805263354e-07, + "loss": 0.033818097909291585, + "num_tokens": 511572860.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7911111116409302, + "rewards/gt_logging_reward/std": 0.39693643947442375, + "sampling/importance_sampling_ratio/max": 1.9487972696622213, + "sampling/importance_sampling_ratio/mean": 0.998158582051595, + "sampling/importance_sampling_ratio/min": 0.4370227644840876, + "sampling/sampling_logp_difference/max": 0.3262038350105286, + "sampling/sampling_logp_difference/mean": 0.003971382509917021, + "step": 15630, + "step_time": 8.241968348264345, + "student/entropy": 0.127490694137911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02277777884155512, + "completions/max_length": 492.4, + "completions/max_terminated_length": 467.1333333333333, + "completions/mean_length": 186.03417561848957, + "completions/mean_terminated_length": 178.43619842529296, + "completions/min_length": 54.7, + "completions/min_terminated_length": 54.7, + "entropy": 0.12770710003872712, + "epoch": 0.5946910720388866, + "eval/gt_acc_batch": 0.8100000242392222, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26338621973991394, + "kd/policy_loss": 0.008393448534964895, + "kd/rkl_advantage_mean": 0.7845314490298431, + "kd/rkl_advantage_p95": 5.015384034315745, + "kd/rkl_advantage_std": 2.2345471580823264, + "kd/ssd_loss": 0.0, + "learning_rate": 4.05346903125356e-07, + "loss": 0.03357380231221517, + "num_tokens": 512543567.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8099999964237213, + "rewards/gt_logging_reward/std": 0.38509528438250223, + "sampling/importance_sampling_ratio/max": 1.941155727704366, + "sampling/importance_sampling_ratio/mean": 0.9965835154056549, + "sampling/importance_sampling_ratio/min": 0.47325592637062075, + "sampling/sampling_logp_difference/max": 0.37603965600331624, + "sampling/sampling_logp_difference/mean": 0.004009203240275383, + "step": 15660, + "step_time": 8.081632180299009, + "student/entropy": 0.12770710003872712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0269444459117949, + "completions/max_length": 485.1333333333333, + "completions/max_terminated_length": 450.6333333333333, + "completions/mean_length": 185.64778747558594, + "completions/mean_terminated_length": 176.81735331217448, + "completions/min_length": 55.333333333333336, + "completions/min_terminated_length": 55.333333333333336, + "entropy": 0.1290205366909504, + "epoch": 0.5958303269661641, + "eval/gt_acc_batch": 0.7994444568951925, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3199511766433716, + "kd/policy_loss": 0.008379186360010256, + "kd/rkl_advantage_mean": 0.7713562530775865, + "kd/rkl_advantage_p95": 4.987062883377075, + "kd/rkl_advantage_std": 2.2103486488262813, + "kd/ssd_loss": 0.0, + "learning_rate": 4.042076481980785e-07, + "loss": 0.03351674874623616, + "num_tokens": 513517003.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.799444442987442, + "rewards/gt_logging_reward/std": 0.39264953633149463, + "sampling/importance_sampling_ratio/max": 1.951776643594106, + "sampling/importance_sampling_ratio/mean": 0.9988098621368409, + "sampling/importance_sampling_ratio/min": 0.44740179081757864, + "sampling/sampling_logp_difference/max": 0.31686970094839734, + "sampling/sampling_logp_difference/mean": 0.0039893674508978925, + "step": 15690, + "step_time": 8.166299549134177, + "student/entropy": 0.1290205366909504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334513008596, + "completions/max_length": 479.73333333333335, + "completions/max_terminated_length": 446.53333333333336, + "completions/mean_length": 187.210009765625, + "completions/mean_terminated_length": 180.52513682047527, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.12592077398051818, + "epoch": 0.5969695818934417, + "eval/gt_acc_batch": 0.8113888998826345, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26746585965156555, + "kd/policy_loss": 0.008152361256846537, + "kd/rkl_advantage_mean": 0.6917195445547502, + "kd/rkl_advantage_p95": 4.850823251406352, + "kd/rkl_advantage_std": 2.171912948290507, + "kd/ssd_loss": 0.0, + "learning_rate": 4.030683932708009e-07, + "loss": 0.032609446843465166, + "num_tokens": 514492239.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.811388889948527, + "rewards/gt_logging_reward/std": 0.3829143851995468, + "sampling/importance_sampling_ratio/max": 1.796720310052236, + "sampling/importance_sampling_ratio/mean": 0.996383798122406, + "sampling/importance_sampling_ratio/min": 0.46840039392312366, + "sampling/sampling_logp_difference/max": 0.31974584857622784, + "sampling/sampling_logp_difference/mean": 0.003891581227071583, + "step": 15720, + "step_time": 8.05172739102806, + "student/entropy": 0.12592077398051818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333454405268, + "completions/max_length": 481.5, + "completions/max_terminated_length": 460.76666666666665, + "completions/mean_length": 192.13250834147135, + "completions/mean_terminated_length": 185.6835723876953, + "completions/min_length": 48.766666666666666, + "completions/min_terminated_length": 48.766666666666666, + "entropy": 0.12436006100227436, + "epoch": 0.5981088368207192, + "eval/gt_acc_batch": 0.8022222379843394, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3053729236125946, + "kd/policy_loss": 0.008275221484169985, + "kd/rkl_advantage_mean": 0.6893599538365379, + "kd/rkl_advantage_p95": 4.863330550988516, + "kd/rkl_advantage_std": 2.172999294598897, + "kd/ssd_loss": 0.0, + "learning_rate": 4.019291383435233e-07, + "loss": 0.03310088713963826, + "num_tokens": 515492092.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8022222240765889, + "rewards/gt_logging_reward/std": 0.3899690881371498, + "sampling/importance_sampling_ratio/max": 2.0232393662134807, + "sampling/importance_sampling_ratio/mean": 0.9972156465053559, + "sampling/importance_sampling_ratio/min": 0.4345337028304736, + "sampling/sampling_logp_difference/max": 0.374733978509903, + "sampling/sampling_logp_difference/mean": 0.003858750166061024, + "step": 15750, + "step_time": 8.085787655336025, + "student/entropy": 0.12436006100227436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02472222385307153, + "completions/max_length": 482.43333333333334, + "completions/max_terminated_length": 457.5, + "completions/mean_length": 189.0444534301758, + "completions/mean_terminated_length": 181.01937967936198, + "completions/min_length": 58.2, + "completions/min_terminated_length": 58.2, + "entropy": 0.12686703130602836, + "epoch": 0.5992480917479968, + "eval/gt_acc_batch": 0.8077778041362762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.35016950964927673, + "kd/policy_loss": 0.008368013927247375, + "kd/rkl_advantage_mean": 0.7663965344429016, + "kd/rkl_advantage_p95": 5.037122746308644, + "kd/rkl_advantage_std": 2.2239587903022766, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0078988341624573e-07, + "loss": 0.03347205718358358, + "num_tokens": 516471764.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8077777783075969, + "rewards/gt_logging_reward/std": 0.3825808276732763, + "sampling/importance_sampling_ratio/max": 1.8313717563947043, + "sampling/importance_sampling_ratio/mean": 1.0008034884929657, + "sampling/importance_sampling_ratio/min": 0.4530256375670433, + "sampling/sampling_logp_difference/max": 0.3226268231868744, + "sampling/sampling_logp_difference/mean": 0.003932269406504929, + "step": 15780, + "step_time": 7.904400261035092, + "student/entropy": 0.12686703130602836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017222223151475192, + "completions/max_length": 468.1666666666667, + "completions/max_terminated_length": 422.96666666666664, + "completions/mean_length": 182.78250935872396, + "completions/mean_terminated_length": 177.13031209309895, + "completions/min_length": 57.93333333333333, + "completions/min_terminated_length": 57.93333333333333, + "entropy": 0.12802215646952392, + "epoch": 0.6003873466752744, + "eval/gt_acc_batch": 0.8119444529215495, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31542521715164185, + "kd/policy_loss": 0.00829076028506582, + "kd/rkl_advantage_mean": 0.7512514657651385, + "kd/rkl_advantage_p95": 4.8951058705647785, + "kd/rkl_advantage_std": 2.1905432174603146, + "kd/ssd_loss": 0.0, + "learning_rate": 3.996506284889682e-07, + "loss": 0.033163044850031534, + "num_tokens": 517437629.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8119444429874421, + "rewards/gt_logging_reward/std": 0.3818974624077479, + "sampling/importance_sampling_ratio/max": 2.0094470143318177, + "sampling/importance_sampling_ratio/mean": 1.0012769639492034, + "sampling/importance_sampling_ratio/min": 0.49053241809209186, + "sampling/sampling_logp_difference/max": 0.2979117691516876, + "sampling/sampling_logp_difference/mean": 0.003966797220831116, + "step": 15810, + "step_time": 8.063624005065746, + "student/entropy": 0.12802215646952392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112652967375, + "completions/max_length": 474.73333333333335, + "completions/max_terminated_length": 436.1, + "completions/mean_length": 184.00000915527343, + "completions/mean_terminated_length": 176.1875030517578, + "completions/min_length": 58.36666666666667, + "completions/min_terminated_length": 58.36666666666667, + "entropy": 0.12783814184367656, + "epoch": 0.6015266016025519, + "eval/gt_acc_batch": 0.7919444640477499, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30936697125434875, + "kd/policy_loss": 0.008336231739182647, + "kd/rkl_advantage_mean": 0.7829390617087484, + "kd/rkl_advantage_p95": 4.987817718585332, + "kd/rkl_advantage_std": 2.1884882022937138, + "kd/ssd_loss": 0.0, + "learning_rate": 3.985113735616906e-07, + "loss": 0.033344924449920654, + "num_tokens": 518395061.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7919444481531779, + "rewards/gt_logging_reward/std": 0.3997142573197683, + "sampling/importance_sampling_ratio/max": 1.8273363908131917, + "sampling/importance_sampling_ratio/mean": 0.9983710944652557, + "sampling/importance_sampling_ratio/min": 0.481461164355278, + "sampling/sampling_logp_difference/max": 0.30986491044362385, + "sampling/sampling_logp_difference/mean": 0.0039452652679756285, + "step": 15840, + "step_time": 7.885465655365261, + "student/entropy": 0.12783814184367656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112429449956, + "completions/max_length": 481.56666666666666, + "completions/max_terminated_length": 451.06666666666666, + "completions/mean_length": 185.53750966389973, + "completions/mean_terminated_length": 178.69170989990235, + "completions/min_length": 50.56666666666667, + "completions/min_terminated_length": 50.56666666666667, + "entropy": 0.13005870059132577, + "epoch": 0.6026658565298295, + "eval/gt_acc_batch": 0.7866666873296102, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30338582396507263, + "kd/policy_loss": 0.008863912901142612, + "kd/rkl_advantage_mean": 0.7497934848070145, + "kd/rkl_advantage_p95": 4.919593036174774, + "kd/rkl_advantage_std": 2.184302517771721, + "kd/ssd_loss": 0.0, + "learning_rate": 3.9737211863441304e-07, + "loss": 0.035455652077992755, + "num_tokens": 519364340.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7866666674613952, + "rewards/gt_logging_reward/std": 0.4025598704814911, + "sampling/importance_sampling_ratio/max": 1.9306650360425313, + "sampling/importance_sampling_ratio/mean": 0.9984888354937236, + "sampling/importance_sampling_ratio/min": 0.4354955484469732, + "sampling/sampling_logp_difference/max": 0.389822518825531, + "sampling/sampling_logp_difference/mean": 0.0040501301952948175, + "step": 15870, + "step_time": 7.955905679598315, + "student/entropy": 0.13005870059132577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333463718494, + "completions/max_length": 485.46666666666664, + "completions/max_terminated_length": 457.3333333333333, + "completions/mean_length": 185.29973297119142, + "completions/mean_terminated_length": 178.4042236328125, + "completions/min_length": 59.43333333333333, + "completions/min_terminated_length": 59.43333333333333, + "entropy": 0.12818814075241486, + "epoch": 0.603805111457107, + "eval/gt_acc_batch": 0.8102777977784474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3346809446811676, + "kd/policy_loss": 0.008494082741284123, + "kd/rkl_advantage_mean": 0.7604979100326698, + "kd/rkl_advantage_p95": 4.951605884234111, + "kd/rkl_advantage_std": 2.2038119047880174, + "kd/ssd_loss": 0.0, + "learning_rate": 3.962328637071355e-07, + "loss": 0.03397633234659831, + "num_tokens": 520343227.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8102777779102326, + "rewards/gt_logging_reward/std": 0.386552095413208, + "sampling/importance_sampling_ratio/max": 1.963563652833303, + "sampling/importance_sampling_ratio/mean": 1.0053002417087555, + "sampling/importance_sampling_ratio/min": 0.48579760591189064, + "sampling/sampling_logp_difference/max": 0.30724371671676637, + "sampling/sampling_logp_difference/mean": 0.0039039899439861376, + "step": 15900, + "step_time": 8.173177616965647, + "student/entropy": 0.12818814075241486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01833333447575569, + "completions/max_length": 483.93333333333334, + "completions/max_terminated_length": 447.96666666666664, + "completions/mean_length": 187.15750885009766, + "completions/mean_terminated_length": 181.23476867675782, + "completions/min_length": 59.833333333333336, + "completions/min_terminated_length": 59.833333333333336, + "entropy": 0.12395424172282218, + "epoch": 0.6049443663843846, + "eval/gt_acc_batch": 0.8111111362775166, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2872943878173828, + "kd/policy_loss": 0.00802390878670849, + "kd/rkl_advantage_mean": 0.7609133440225074, + "kd/rkl_advantage_p95": 4.943732400735219, + "kd/rkl_advantage_std": 2.190583861867587, + "kd/ssd_loss": 0.0, + "learning_rate": 3.95093608779858e-07, + "loss": 0.03209563891092936, + "num_tokens": 521325554.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8111111144224803, + "rewards/gt_logging_reward/std": 0.3843908444046974, + "sampling/importance_sampling_ratio/max": 1.9063910365104675, + "sampling/importance_sampling_ratio/mean": 1.0015631159146627, + "sampling/importance_sampling_ratio/min": 0.43512921035289764, + "sampling/sampling_logp_difference/max": 0.320579868555069, + "sampling/sampling_logp_difference/mean": 0.0038413260597735645, + "step": 15930, + "step_time": 8.073906587867532, + "student/entropy": 0.12395424172282218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001365939776, + "completions/max_length": 491.4, + "completions/max_terminated_length": 456.8333333333333, + "completions/mean_length": 196.49417521158855, + "completions/mean_terminated_length": 188.44939066569012, + "completions/min_length": 59.5, + "completions/min_terminated_length": 59.5, + "entropy": 0.12766238159189622, + "epoch": 0.6060836213116622, + "eval/gt_acc_batch": 0.7902777949968974, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.256554514169693, + "kd/policy_loss": 0.008613540757990753, + "kd/rkl_advantage_mean": 0.7952773775905371, + "kd/rkl_advantage_p95": 4.9991108993689215, + "kd/rkl_advantage_std": 2.1992622951666516, + "kd/ssd_loss": 0.0, + "learning_rate": 3.939543538525804e-07, + "loss": 0.03445416291554769, + "num_tokens": 522338325.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.790277777115504, + "rewards/gt_logging_reward/std": 0.40132998923460644, + "sampling/importance_sampling_ratio/max": 1.815331264336904, + "sampling/importance_sampling_ratio/mean": 0.996914803981781, + "sampling/importance_sampling_ratio/min": 0.4390860259532928, + "sampling/sampling_logp_difference/max": 0.3148394902547201, + "sampling/sampling_logp_difference/mean": 0.003923622728325427, + "step": 15960, + "step_time": 8.230860503365694, + "student/entropy": 0.12766238159189622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668175409238, + "completions/max_length": 502.1, + "completions/max_terminated_length": 465.06666666666666, + "completions/mean_length": 187.34223327636718, + "completions/mean_terminated_length": 179.3977081298828, + "completions/min_length": 42.833333333333336, + "completions/min_terminated_length": 42.833333333333336, + "entropy": 0.1288131458684802, + "epoch": 0.6072228762389398, + "eval/gt_acc_batch": 0.7850000103314717, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23197075724601746, + "kd/policy_loss": 0.008798420120729133, + "kd/rkl_advantage_mean": 0.8196119910726944, + "kd/rkl_advantage_p95": 5.166665718952815, + "kd/rkl_advantage_std": 2.2843427121639253, + "kd/ssd_loss": 0.0, + "learning_rate": 3.9281509892530287e-07, + "loss": 0.03519367774327596, + "num_tokens": 523324885.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7849999984105428, + "rewards/gt_logging_reward/std": 0.400286465883255, + "sampling/importance_sampling_ratio/max": 1.8532754103342692, + "sampling/importance_sampling_ratio/mean": 1.000154552857081, + "sampling/importance_sampling_ratio/min": 0.4585703447461128, + "sampling/sampling_logp_difference/max": 0.30954309900601706, + "sampling/sampling_logp_difference/mean": 0.003955919916431109, + "step": 15990, + "step_time": 8.165640828472776, + "student/entropy": 0.1288131458684802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223865489165, + "completions/max_length": 483.06666666666666, + "completions/max_terminated_length": 453.6, + "completions/mean_length": 193.79167633056642, + "completions/mean_terminated_length": 184.14105224609375, + "completions/min_length": 60.266666666666666, + "completions/min_terminated_length": 60.266666666666666, + "entropy": 0.13027630367626747, + "epoch": 0.6083621311662173, + "eval/gt_acc_batch": 0.7891666889190674, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3508525788784027, + "kd/policy_loss": 0.008583303424529731, + "kd/rkl_advantage_mean": 0.7743471215168635, + "kd/rkl_advantage_p95": 5.116559010744095, + "kd/rkl_advantage_std": 2.2726360807816186, + "kd/ssd_loss": 0.0, + "learning_rate": 3.916758439980253e-07, + "loss": 0.03433321317036946, + "num_tokens": 524323783.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7891666690508524, + "rewards/gt_logging_reward/std": 0.39616707861423495, + "sampling/importance_sampling_ratio/max": 1.9412779728571574, + "sampling/importance_sampling_ratio/mean": 0.9972905298074086, + "sampling/importance_sampling_ratio/min": 0.4679200957218806, + "sampling/sampling_logp_difference/max": 0.38267941276232403, + "sampling/sampling_logp_difference/mean": 0.0039945051229248445, + "step": 16020, + "step_time": 8.05287980180389, + "student/entropy": 0.13027630367626747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556944260994, + "completions/max_length": 503.43333333333334, + "completions/max_terminated_length": 456.43333333333334, + "completions/mean_length": 188.30806528727214, + "completions/mean_terminated_length": 180.6663553873698, + "completions/min_length": 57.1, + "completions/min_terminated_length": 57.1, + "entropy": 0.1338476326316595, + "epoch": 0.6095013860934948, + "eval/gt_acc_batch": 0.7930555840333303, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4220879375934601, + "kd/policy_loss": 0.008635141882890215, + "kd/rkl_advantage_mean": 0.8012379263838132, + "kd/rkl_advantage_p95": 5.078845445315043, + "kd/rkl_advantage_std": 2.254502280553182, + "kd/ssd_loss": 0.0, + "learning_rate": 3.905365890707477e-07, + "loss": 0.03454056978225708, + "num_tokens": 525302716.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555601914724, + "rewards/gt_logging_reward/std": 0.40146684149901074, + "sampling/importance_sampling_ratio/max": 1.9790518363316854, + "sampling/importance_sampling_ratio/mean": 0.9993346989154815, + "sampling/importance_sampling_ratio/min": 0.4315601160128911, + "sampling/sampling_logp_difference/max": 0.3418779889742533, + "sampling/sampling_logp_difference/mean": 0.004098230266633133, + "step": 16050, + "step_time": 8.140646205830853, + "student/entropy": 0.1338476326316595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015000000751266876, + "completions/max_length": 478.1333333333333, + "completions/max_terminated_length": 446.1666666666667, + "completions/mean_length": 184.49778696695964, + "completions/mean_terminated_length": 179.67063547770184, + "completions/min_length": 58.56666666666667, + "completions/min_terminated_length": 58.56666666666667, + "entropy": 0.12882435272137324, + "epoch": 0.6106406410207724, + "eval/gt_acc_batch": 0.8152777969837188, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2989327609539032, + "kd/policy_loss": 0.008497587560365597, + "kd/rkl_advantage_mean": 0.791849190245072, + "kd/rkl_advantage_p95": 4.986265883843104, + "kd/rkl_advantage_std": 2.1854081749916077, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8939733414347017e-07, + "loss": 0.033990347385406496, + "num_tokens": 526264268.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.815277777115504, + "rewards/gt_logging_reward/std": 0.3724860509236654, + "sampling/importance_sampling_ratio/max": 1.8757411281267802, + "sampling/importance_sampling_ratio/mean": 1.000356262922287, + "sampling/importance_sampling_ratio/min": 0.4390860805908839, + "sampling/sampling_logp_difference/max": 0.3035271147886912, + "sampling/sampling_logp_difference/mean": 0.0039834463813652595, + "step": 16080, + "step_time": 7.943859009534935, + "student/entropy": 0.12882435272137324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445874541997, + "completions/max_length": 491.6, + "completions/max_terminated_length": 449.2, + "completions/mean_length": 190.73528798421225, + "completions/mean_terminated_length": 182.7814488728841, + "completions/min_length": 56.13333333333333, + "completions/min_terminated_length": 56.13333333333333, + "entropy": 0.12868641093373298, + "epoch": 0.6117798959480499, + "eval/gt_acc_batch": 0.8080555737018585, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27515748143196106, + "kd/policy_loss": 0.00831142831981803, + "kd/rkl_advantage_mean": 0.7069383263587952, + "kd/rkl_advantage_p95": 4.8676646769046785, + "kd/rkl_advantage_std": 2.165761078397433, + "kd/ssd_loss": 0.0, + "learning_rate": 3.882580792161926e-07, + "loss": 0.03324571847915649, + "num_tokens": 527250147.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8080555538336436, + "rewards/gt_logging_reward/std": 0.38587050437927245, + "sampling/importance_sampling_ratio/max": 1.8219180742899577, + "sampling/importance_sampling_ratio/mean": 1.0002155482769013, + "sampling/importance_sampling_ratio/min": 0.4363971268137296, + "sampling/sampling_logp_difference/max": 0.36372308135032655, + "sampling/sampling_logp_difference/mean": 0.003929648982981841, + "step": 16110, + "step_time": 8.068288964136931, + "student/entropy": 0.12868641093373298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112938572965, + "completions/max_length": 478.06666666666666, + "completions/max_terminated_length": 441.43333333333334, + "completions/mean_length": 184.91612091064454, + "completions/mean_terminated_length": 176.3590342203776, + "completions/min_length": 47.96666666666667, + "completions/min_terminated_length": 47.96666666666667, + "entropy": 0.12874820275853077, + "epoch": 0.6129191508753276, + "eval/gt_acc_batch": 0.8080555816491445, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3159691095352173, + "kd/policy_loss": 0.008389032234360154, + "kd/rkl_advantage_mean": 0.7229133251744012, + "kd/rkl_advantage_p95": 4.935847183068593, + "kd/rkl_advantage_std": 2.2351902465025586, + "kd/ssd_loss": 0.0, + "learning_rate": 3.87118824288915e-07, + "loss": 0.033556127548217775, + "num_tokens": 528220125.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.808055559794108, + "rewards/gt_logging_reward/std": 0.38251296877861024, + "sampling/importance_sampling_ratio/max": 1.9146992802619933, + "sampling/importance_sampling_ratio/mean": 1.001764863729477, + "sampling/importance_sampling_ratio/min": 0.4380338748296102, + "sampling/sampling_logp_difference/max": 0.329975155989329, + "sampling/sampling_logp_difference/mean": 0.003968514820250372, + "step": 16140, + "step_time": 7.990069626832459, + "student/entropy": 0.12874820275853077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019444445644815764, + "completions/max_length": 489.46666666666664, + "completions/max_terminated_length": 449.3, + "completions/mean_length": 187.7047317504883, + "completions/mean_terminated_length": 181.44164326985677, + "completions/min_length": 53.7, + "completions/min_terminated_length": 53.7, + "entropy": 0.13009910012284914, + "epoch": 0.6140584058026051, + "eval/gt_acc_batch": 0.8019444704055786, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.259034663438797, + "kd/policy_loss": 0.008422326315970471, + "kd/rkl_advantage_mean": 0.7582660774389903, + "kd/rkl_advantage_p95": 5.090832056601842, + "kd/rkl_advantage_std": 2.267538712422053, + "kd/ssd_loss": 0.0, + "learning_rate": 3.859795693616374e-07, + "loss": 0.03368930419286092, + "num_tokens": 529189038.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8019444425900777, + "rewards/gt_logging_reward/std": 0.3891359200080236, + "sampling/importance_sampling_ratio/max": 1.8321316560109457, + "sampling/importance_sampling_ratio/mean": 0.9985219260056813, + "sampling/importance_sampling_ratio/min": 0.47985398570696514, + "sampling/sampling_logp_difference/max": 0.3291045784950256, + "sampling/sampling_logp_difference/mean": 0.003990324990202983, + "step": 16170, + "step_time": 7.981387337631896, + "student/entropy": 0.13009910012284914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223927577337, + "completions/max_length": 501.1, + "completions/max_terminated_length": 457.93333333333334, + "completions/mean_length": 190.8505650838216, + "completions/mean_terminated_length": 181.13305104573567, + "completions/min_length": 45.56666666666667, + "completions/min_terminated_length": 45.56666666666667, + "entropy": 0.12872015809019408, + "epoch": 0.6151976607298827, + "eval/gt_acc_batch": 0.7908333440621694, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.317183256149292, + "kd/policy_loss": 0.008498742411999653, + "kd/rkl_advantage_mean": 0.8482374099393686, + "kd/rkl_advantage_p95": 5.175614484151205, + "kd/rkl_advantage_std": 2.278575149178505, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8484031443435995e-07, + "loss": 0.033994972705841064, + "num_tokens": 530177612.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7908333361148834, + "rewards/gt_logging_reward/std": 0.3989820212125778, + "sampling/importance_sampling_ratio/max": 1.953744900226593, + "sampling/importance_sampling_ratio/mean": 1.0024462521076203, + "sampling/importance_sampling_ratio/min": 0.43393003443876904, + "sampling/sampling_logp_difference/max": 0.3179595470428467, + "sampling/sampling_logp_difference/mean": 0.003945093105236689, + "step": 16200, + "step_time": 8.175325140032024, + "student/entropy": 0.12872015809019408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016944445483386517, + "completions/max_length": 474.46666666666664, + "completions/max_terminated_length": 442.5, + "completions/mean_length": 178.73556416829427, + "completions/mean_terminated_length": 173.11146901448566, + "completions/min_length": 53.1, + "completions/min_terminated_length": 53.1, + "entropy": 0.12931654298057157, + "epoch": 0.6163369156571602, + "eval/gt_acc_batch": 0.8147222439448039, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3079073131084442, + "kd/policy_loss": 0.008387955877697095, + "kd/rkl_advantage_mean": 0.7893757448221247, + "kd/rkl_advantage_p95": 5.00601985057195, + "kd/rkl_advantage_std": 2.203719703356425, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8370105950708237e-07, + "loss": 0.03355183204015096, + "num_tokens": 531124068.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8147222201029459, + "rewards/gt_logging_reward/std": 0.3830281287431717, + "sampling/importance_sampling_ratio/max": 1.9008373061815897, + "sampling/importance_sampling_ratio/mean": 0.9966725329558055, + "sampling/importance_sampling_ratio/min": 0.4917365392049154, + "sampling/sampling_logp_difference/max": 0.3297815084457397, + "sampling/sampling_logp_difference/mean": 0.0039880018138016265, + "step": 16230, + "step_time": 8.041853319767203, + "student/entropy": 0.12931654298057157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556733161212, + "completions/max_length": 492.4, + "completions/max_terminated_length": 463.96666666666664, + "completions/mean_length": 192.27362263997395, + "completions/mean_terminated_length": 184.07227020263673, + "completions/min_length": 56.7, + "completions/min_terminated_length": 56.7, + "entropy": 0.13239434839536746, + "epoch": 0.6174761705844378, + "eval/gt_acc_batch": 0.7844444632530212, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4053783118724823, + "kd/policy_loss": 0.00906479285331443, + "kd/rkl_advantage_mean": 0.7650435421693449, + "kd/rkl_advantage_p95": 4.982504747311274, + "kd/rkl_advantage_std": 2.234773101409276, + "kd/ssd_loss": 0.0, + "learning_rate": 3.825618045798048e-07, + "loss": 0.036259170373280844, + "num_tokens": 532106565.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7844444473584493, + "rewards/gt_logging_reward/std": 0.4066020756959915, + "sampling/importance_sampling_ratio/max": 1.9832794864972432, + "sampling/importance_sampling_ratio/mean": 1.0032292147477468, + "sampling/importance_sampling_ratio/min": 0.4702116395036379, + "sampling/sampling_logp_difference/max": 0.3445621053377787, + "sampling/sampling_logp_difference/mean": 0.004073410946875811, + "step": 16260, + "step_time": 8.044667978666258, + "student/entropy": 0.13239434839536746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.032500001788139346, + "completions/max_length": 499.6333333333333, + "completions/max_terminated_length": 463.7, + "completions/mean_length": 195.9894551595052, + "completions/mean_terminated_length": 185.38331247965496, + "completions/min_length": 50.13333333333333, + "completions/min_terminated_length": 50.13333333333333, + "entropy": 0.1281928534929951, + "epoch": 0.6186154255117153, + "eval/gt_acc_batch": 0.7594444692134857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29615482687950134, + "kd/policy_loss": 0.008919656649231911, + "kd/rkl_advantage_mean": 0.7089311379939318, + "kd/rkl_advantage_p95": 4.92143691778183, + "kd/rkl_advantage_std": 2.1982248316208524, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8142254965252725e-07, + "loss": 0.03567862510681152, + "num_tokens": 533121103.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7594444394111634, + "rewards/gt_logging_reward/std": 0.42100370724995934, + "sampling/importance_sampling_ratio/max": 1.9626025517781576, + "sampling/importance_sampling_ratio/mean": 0.9974181075890859, + "sampling/importance_sampling_ratio/min": 0.40109802583853404, + "sampling/sampling_logp_difference/max": 0.3809069236119588, + "sampling/sampling_logp_difference/mean": 0.003946674017546077, + "step": 16290, + "step_time": 8.21119038723021, + "student/entropy": 0.1281928534929951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777903402845, + "completions/max_length": 496.03333333333336, + "completions/max_terminated_length": 453.5, + "completions/mean_length": 189.37278747558594, + "completions/mean_terminated_length": 181.0870559692383, + "completions/min_length": 54.93333333333333, + "completions/min_terminated_length": 54.93333333333333, + "entropy": 0.13218690666059654, + "epoch": 0.6197546804389928, + "eval/gt_acc_batch": 0.8041666785875956, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23816561698913574, + "kd/policy_loss": 0.008901640528347344, + "kd/rkl_advantage_mean": 0.7862682494024436, + "kd/rkl_advantage_p95": 5.0194881121317545, + "kd/rkl_advantage_std": 2.2352590103944143, + "kd/ssd_loss": 0.0, + "learning_rate": 3.8028329472524967e-07, + "loss": 0.03560655911763509, + "num_tokens": 534108205.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666626930237, + "rewards/gt_logging_reward/std": 0.3898299276828766, + "sampling/importance_sampling_ratio/max": 2.0544403274854024, + "sampling/importance_sampling_ratio/mean": 0.9976440032323202, + "sampling/importance_sampling_ratio/min": 0.45273423691590625, + "sampling/sampling_logp_difference/max": 0.3422394076983134, + "sampling/sampling_logp_difference/mean": 0.004077131821153064, + "step": 16320, + "step_time": 8.01834190809944, + "student/entropy": 0.13218690666059654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000094374021, + "completions/max_length": 490.8333333333333, + "completions/max_terminated_length": 454.1, + "completions/mean_length": 188.10389912923176, + "completions/mean_terminated_length": 182.32041676839194, + "completions/min_length": 57.766666666666666, + "completions/min_terminated_length": 57.766666666666666, + "entropy": 0.13077647797763348, + "epoch": 0.6208939353662705, + "eval/gt_acc_batch": 0.796111132701238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3429872691631317, + "kd/policy_loss": 0.00862401796427245, + "kd/rkl_advantage_mean": 0.7836837870379288, + "kd/rkl_advantage_p95": 4.987880390882492, + "kd/rkl_advantage_std": 2.2046846876541775, + "kd/ssd_loss": 0.0, + "learning_rate": 3.791440397979721e-07, + "loss": 0.03449607690175374, + "num_tokens": 535088643.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.796111112833023, + "rewards/gt_logging_reward/std": 0.3942525893449783, + "sampling/importance_sampling_ratio/max": 1.9104328632354737, + "sampling/importance_sampling_ratio/mean": 0.9968473315238953, + "sampling/importance_sampling_ratio/min": 0.45274848540623985, + "sampling/sampling_logp_difference/max": 0.33758111397425333, + "sampling/sampling_logp_difference/mean": 0.003958714737867315, + "step": 16350, + "step_time": 8.29930556039326, + "student/entropy": 0.13077647797763348 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018611112361152968, + "completions/max_length": 469.06666666666666, + "completions/max_terminated_length": 434.46666666666664, + "completions/mean_length": 185.74639892578125, + "completions/mean_terminated_length": 179.76029154459636, + "completions/min_length": 53.86666666666667, + "completions/min_terminated_length": 53.86666666666667, + "entropy": 0.1272642644122243, + "epoch": 0.622033190293548, + "eval/gt_acc_batch": 0.796111136674881, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.40915447473526, + "kd/policy_loss": 0.008324152879261722, + "kd/rkl_advantage_mean": 0.7545023831228416, + "kd/rkl_advantage_p95": 4.930125202735265, + "kd/rkl_advantage_std": 2.189967851837476, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7800478487069456e-07, + "loss": 0.0332966148853302, + "num_tokens": 536062570.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111187934875, + "rewards/gt_logging_reward/std": 0.3958402246236801, + "sampling/importance_sampling_ratio/max": 1.903787942727407, + "sampling/importance_sampling_ratio/mean": 0.9968758702278138, + "sampling/importance_sampling_ratio/min": 0.45758094390233356, + "sampling/sampling_logp_difference/max": 0.327685292561849, + "sampling/sampling_logp_difference/mean": 0.003954170496823887, + "step": 16380, + "step_time": 8.029695574897536, + "student/entropy": 0.1272642644122243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666810711225, + "completions/max_length": 487.9, + "completions/max_terminated_length": 452.76666666666665, + "completions/mean_length": 186.55528818766277, + "completions/mean_terminated_length": 179.54840901692708, + "completions/min_length": 51.3, + "completions/min_terminated_length": 51.3, + "entropy": 0.12495134522517522, + "epoch": 0.6231724452208256, + "eval/gt_acc_batch": 0.809166685740153, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29333174228668213, + "kd/policy_loss": 0.008216502511641011, + "kd/rkl_advantage_mean": 0.7364287225607161, + "kd/rkl_advantage_p95": 4.960745553175609, + "kd/rkl_advantage_std": 2.2038942406574886, + "kd/ssd_loss": 0.0, + "learning_rate": 3.76865529943417e-07, + "loss": 0.03286601305007934, + "num_tokens": 537037009.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8091666618982951, + "rewards/gt_logging_reward/std": 0.380447319149971, + "sampling/importance_sampling_ratio/max": 1.89793594678243, + "sampling/importance_sampling_ratio/mean": 1.0024449427922566, + "sampling/importance_sampling_ratio/min": 0.4571714719136556, + "sampling/sampling_logp_difference/max": 0.3684006631374359, + "sampling/sampling_logp_difference/mean": 0.0038417460784936947, + "step": 16410, + "step_time": 8.15918818139762, + "student/entropy": 0.12495134522517522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556553105513, + "completions/max_length": 480.23333333333335, + "completions/max_terminated_length": 446.43333333333334, + "completions/mean_length": 186.05139872233073, + "completions/mean_terminated_length": 180.98852081298827, + "completions/min_length": 57.266666666666666, + "completions/min_terminated_length": 57.266666666666666, + "entropy": 0.12538273800164462, + "epoch": 0.6243117001481031, + "eval/gt_acc_batch": 0.7972222367922465, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3025725781917572, + "kd/policy_loss": 0.008372696888788294, + "kd/rkl_advantage_mean": 0.7380722592895229, + "kd/rkl_advantage_p95": 4.966238047679266, + "kd/rkl_advantage_std": 2.222761232654254, + "kd/ssd_loss": 0.0, + "learning_rate": 3.757262750161394e-07, + "loss": 0.03349079291025798, + "num_tokens": 538007818.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7972222248713176, + "rewards/gt_logging_reward/std": 0.3965550710757573, + "sampling/importance_sampling_ratio/max": 2.075756084918976, + "sampling/importance_sampling_ratio/mean": 1.008975859483083, + "sampling/importance_sampling_ratio/min": 0.49477258920669553, + "sampling/sampling_logp_difference/max": 0.31146361827850344, + "sampling/sampling_logp_difference/mean": 0.0038684601817900936, + "step": 16440, + "step_time": 7.9924219875683775, + "student/entropy": 0.12538273800164462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444562618931, + "completions/max_length": 492.23333333333335, + "completions/max_terminated_length": 450.56666666666666, + "completions/mean_length": 185.8975092569987, + "completions/mean_terminated_length": 177.97116394042968, + "completions/min_length": 46.13333333333333, + "completions/min_terminated_length": 46.13333333333333, + "entropy": 0.1298248674099644, + "epoch": 0.6254509550753807, + "eval/gt_acc_batch": 0.8111111223697662, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30794087052345276, + "kd/policy_loss": 0.008177121874177828, + "kd/rkl_advantage_mean": 0.7349569372701807, + "kd/rkl_advantage_p95": 4.883764714002609, + "kd/rkl_advantage_std": 2.1962475409110387, + "kd/ssd_loss": 0.0, + "learning_rate": 3.745870200888619e-07, + "loss": 0.032708489894866945, + "num_tokens": 538983249.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8111111104488373, + "rewards/gt_logging_reward/std": 0.3826622838775317, + "sampling/importance_sampling_ratio/max": 1.9064668615659077, + "sampling/importance_sampling_ratio/mean": 0.9988359530766805, + "sampling/importance_sampling_ratio/min": 0.4255235016345978, + "sampling/sampling_logp_difference/max": 0.31299150983492535, + "sampling/sampling_logp_difference/mean": 0.003951947235812743, + "step": 16470, + "step_time": 8.098057282835361, + "student/entropy": 0.1298248674099644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500001130004723, + "completions/max_length": 489.26666666666665, + "completions/max_terminated_length": 452.1333333333333, + "completions/mean_length": 182.30472920735676, + "completions/mean_terminated_length": 176.4876256306966, + "completions/min_length": 57.266666666666666, + "completions/min_terminated_length": 57.266666666666666, + "entropy": 0.12459181528538465, + "epoch": 0.6265902100026582, + "eval/gt_acc_batch": 0.822222230831782, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.37383216619491577, + "kd/policy_loss": 0.007679929916048422, + "kd/rkl_advantage_mean": 0.8093613032872479, + "kd/rkl_advantage_p95": 5.0979839861392975, + "kd/rkl_advantage_std": 2.264000352223714, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7344776516158434e-07, + "loss": 0.03071972131729126, + "num_tokens": 539945738.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8222222248713176, + "rewards/gt_logging_reward/std": 0.37302082280317944, + "sampling/importance_sampling_ratio/max": 1.8568974177042643, + "sampling/importance_sampling_ratio/mean": 0.9992794672648112, + "sampling/importance_sampling_ratio/min": 0.4493908127148946, + "sampling/sampling_logp_difference/max": 0.3342802921930949, + "sampling/sampling_logp_difference/mean": 0.003856910113245249, + "step": 16500, + "step_time": 8.131358830500782, + "student/entropy": 0.12459181528538465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014444445197780927, + "completions/max_length": 476.43333333333334, + "completions/max_terminated_length": 436.23333333333335, + "completions/mean_length": 184.18334350585937, + "completions/mean_terminated_length": 179.54222971598307, + "completions/min_length": 55.3, + "completions/min_terminated_length": 55.3, + "entropy": 0.12352447416633368, + "epoch": 0.6277294649299359, + "eval/gt_acc_batch": 0.8158333619435628, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28680166602134705, + "kd/policy_loss": 0.007762846554396674, + "kd/rkl_advantage_mean": 0.7396268751161794, + "kd/rkl_advantage_p95": 4.9695456226666765, + "kd/rkl_advantage_std": 2.1886898557345074, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7230851023430675e-07, + "loss": 0.031051385402679443, + "num_tokens": 540908510.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8158333281675975, + "rewards/gt_logging_reward/std": 0.38190851906935375, + "sampling/importance_sampling_ratio/max": 1.9092254757881164, + "sampling/importance_sampling_ratio/mean": 0.9973410189151763, + "sampling/importance_sampling_ratio/min": 0.4904051899909973, + "sampling/sampling_logp_difference/max": 0.30880643328030905, + "sampling/sampling_logp_difference/mean": 0.0038363789596284428, + "step": 16530, + "step_time": 7.830856922963479, + "student/entropy": 0.12352447416633368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666779667139, + "completions/max_length": 471.76666666666665, + "completions/max_terminated_length": 444.8333333333333, + "completions/mean_length": 187.8586201985677, + "completions/mean_terminated_length": 180.84404754638672, + "completions/min_length": 55.666666666666664, + "completions/min_terminated_length": 55.666666666666664, + "entropy": 0.12806627259900172, + "epoch": 0.6288687198572134, + "eval/gt_acc_batch": 0.8008333524068196, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32657936215400696, + "kd/policy_loss": 0.008411666882845262, + "kd/rkl_advantage_mean": 0.758517024666071, + "kd/rkl_advantage_p95": 5.004856856664022, + "kd/rkl_advantage_std": 2.2214075615008673, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7116925530702917e-07, + "loss": 0.03364667495091756, + "num_tokens": 541887969.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333265781402, + "rewards/gt_logging_reward/std": 0.3943203012148539, + "sampling/importance_sampling_ratio/max": 1.9125505487124126, + "sampling/importance_sampling_ratio/mean": 1.0027405003706613, + "sampling/importance_sampling_ratio/min": 0.47687788903713224, + "sampling/sampling_logp_difference/max": 0.3559423327445984, + "sampling/sampling_logp_difference/mean": 0.0039725710405036805, + "step": 16560, + "step_time": 7.865853979802341, + "student/entropy": 0.12806627259900172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334227403006, + "completions/max_length": 464.2, + "completions/max_terminated_length": 433.2, + "completions/mean_length": 181.95028686523438, + "completions/mean_terminated_length": 175.93243611653645, + "completions/min_length": 57.8, + "completions/min_terminated_length": 57.8, + "entropy": 0.1254055700575312, + "epoch": 0.630007974784491, + "eval/gt_acc_batch": 0.8194444636503856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2698708772659302, + "kd/policy_loss": 0.007787124531265969, + "kd/rkl_advantage_mean": 0.6661436108251413, + "kd/rkl_advantage_p95": 4.808536885182063, + "kd/rkl_advantage_std": 2.1458729803562164, + "kd/ssd_loss": 0.0, + "learning_rate": 3.7003000037975164e-07, + "loss": 0.031148497263590494, + "num_tokens": 542842086.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8194444417953491, + "rewards/gt_logging_reward/std": 0.3757772545019786, + "sampling/importance_sampling_ratio/max": 1.8904400865236919, + "sampling/importance_sampling_ratio/mean": 0.9985626379648844, + "sampling/importance_sampling_ratio/min": 0.462022398908933, + "sampling/sampling_logp_difference/max": 0.3346476912498474, + "sampling/sampling_logp_difference/mean": 0.0038626741152256727, + "step": 16590, + "step_time": 7.899620647964184, + "student/entropy": 0.1254055700575312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028888890768090883, + "completions/max_length": 500.8666666666667, + "completions/max_terminated_length": 450.6333333333333, + "completions/mean_length": 190.79889729817708, + "completions/mean_terminated_length": 181.4381851196289, + "completions/min_length": 57.86666666666667, + "completions/min_terminated_length": 57.86666666666667, + "entropy": 0.12932433895766735, + "epoch": 0.6311472297117685, + "eval/gt_acc_batch": 0.771666685740153, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3497261703014374, + "kd/policy_loss": 0.008726177737116814, + "kd/rkl_advantage_mean": 0.7588584430176222, + "kd/rkl_advantage_p95": 4.994052294890086, + "kd/rkl_advantage_std": 2.223782985409101, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6889074545247406e-07, + "loss": 0.03490471045176188, + "num_tokens": 543845170.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7716666678587596, + "rewards/gt_logging_reward/std": 0.4156635344028473, + "sampling/importance_sampling_ratio/max": 1.8684284408887228, + "sampling/importance_sampling_ratio/mean": 0.9956352293491364, + "sampling/importance_sampling_ratio/min": 0.40860153834025065, + "sampling/sampling_logp_difference/max": 0.3387289623419444, + "sampling/sampling_logp_difference/mean": 0.003987524417849878, + "step": 16620, + "step_time": 8.402167396065973, + "student/entropy": 0.12932433895766735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888889780888953, + "completions/max_length": 471.93333333333334, + "completions/max_terminated_length": 439.2, + "completions/mean_length": 184.36778666178387, + "completions/mean_terminated_length": 178.2680196126302, + "completions/min_length": 55.4, + "completions/min_terminated_length": 55.4, + "entropy": 0.12329084128141403, + "epoch": 0.632286484639046, + "eval/gt_acc_batch": 0.8200000246365865, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28595054149627686, + "kd/policy_loss": 0.00826877610719142, + "kd/rkl_advantage_mean": 0.7214710847785075, + "kd/rkl_advantage_p95": 4.951477764050166, + "kd/rkl_advantage_std": 2.1896701008081436, + "kd/ssd_loss": 0.0, + "learning_rate": 3.677514905251965e-07, + "loss": 0.033075108130772905, + "num_tokens": 544809774.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.82000000278155, + "rewards/gt_logging_reward/std": 0.3703742951154709, + "sampling/importance_sampling_ratio/max": 1.902244253953298, + "sampling/importance_sampling_ratio/mean": 1.004546622435252, + "sampling/importance_sampling_ratio/min": 0.48086852033933003, + "sampling/sampling_logp_difference/max": 0.3271129488945007, + "sampling/sampling_logp_difference/mean": 0.003816502292950948, + "step": 16650, + "step_time": 7.985863657931137, + "student/entropy": 0.12329084128141403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026666668336838485, + "completions/max_length": 494.56666666666666, + "completions/max_terminated_length": 458.96666666666664, + "completions/mean_length": 194.1894541422526, + "completions/mean_terminated_length": 185.63214365641275, + "completions/min_length": 54.06666666666667, + "completions/min_terminated_length": 54.06666666666667, + "entropy": 0.12744026804963748, + "epoch": 0.6334257395663236, + "eval/gt_acc_batch": 0.7958333532015482, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24376645684242249, + "kd/policy_loss": 0.008584142930340022, + "kd/rkl_advantage_mean": 0.8068246103823185, + "kd/rkl_advantage_p95": 5.054715953270594, + "kd/rkl_advantage_std": 2.216339596112569, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6661223559791895e-07, + "loss": 0.03433656692504883, + "num_tokens": 545812568.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7958333333333333, + "rewards/gt_logging_reward/std": 0.3990170627832413, + "sampling/importance_sampling_ratio/max": 1.88309938510259, + "sampling/importance_sampling_ratio/mean": 0.9974883834520976, + "sampling/importance_sampling_ratio/min": 0.4514936794837316, + "sampling/sampling_logp_difference/max": 0.33097117642561596, + "sampling/sampling_logp_difference/mean": 0.003914167871698737, + "step": 16680, + "step_time": 8.12900812367055, + "student/entropy": 0.12744026804963748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112783352534, + "completions/max_length": 471.5, + "completions/max_terminated_length": 440.03333333333336, + "completions/mean_length": 181.84556427001954, + "completions/mean_terminated_length": 173.0122843424479, + "completions/min_length": 53.333333333333336, + "completions/min_terminated_length": 53.333333333333336, + "entropy": 0.1310638548185428, + "epoch": 0.6345649944936012, + "eval/gt_acc_batch": 0.8158333579699198, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2601781189441681, + "kd/policy_loss": 0.008611928365038086, + "kd/rkl_advantage_mean": 0.7426661500086387, + "kd/rkl_advantage_p95": 4.938623595237732, + "kd/rkl_advantage_std": 2.2072240600983304, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6547298067064137e-07, + "loss": 0.03444771766662598, + "num_tokens": 546775068.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8158333341280619, + "rewards/gt_logging_reward/std": 0.3750457356373469, + "sampling/importance_sampling_ratio/max": 1.9897431572278341, + "sampling/importance_sampling_ratio/mean": 0.994774212439855, + "sampling/importance_sampling_ratio/min": 0.4466517319281896, + "sampling/sampling_logp_difference/max": 0.3173393766085307, + "sampling/sampling_logp_difference/mean": 0.004001408590314289, + "step": 16710, + "step_time": 7.8749277216091285, + "student/entropy": 0.1310638548185428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445806245008, + "completions/max_length": 466.1, + "completions/max_terminated_length": 438.4, + "completions/mean_length": 186.42001037597657, + "completions/mean_terminated_length": 179.26867014567057, + "completions/min_length": 48.03333333333333, + "completions/min_terminated_length": 48.03333333333333, + "entropy": 0.13055832019696634, + "epoch": 0.6357042494208788, + "eval/gt_acc_batch": 0.8147222459316253, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4370981752872467, + "kd/policy_loss": 0.008759688444357986, + "kd/rkl_advantage_mean": 0.7667011284036562, + "kd/rkl_advantage_p95": 4.991180789470673, + "kd/rkl_advantage_std": 2.200049901008606, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6433372574336384e-07, + "loss": 0.03503875335057576, + "num_tokens": 547743812.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8147222260634105, + "rewards/gt_logging_reward/std": 0.3804353604714076, + "sampling/importance_sampling_ratio/max": 1.9546062390009562, + "sampling/importance_sampling_ratio/mean": 0.9984741330146789, + "sampling/importance_sampling_ratio/min": 0.44320075313250223, + "sampling/sampling_logp_difference/max": 0.30802469650904335, + "sampling/sampling_logp_difference/mean": 0.004014152040084203, + "step": 16740, + "step_time": 7.8832811877975475, + "student/entropy": 0.13055832019696634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033888890966773035, + "completions/max_length": 492.7, + "completions/max_terminated_length": 457.3666666666667, + "completions/mean_length": 195.19250996907553, + "completions/mean_terminated_length": 184.2764119466146, + "completions/min_length": 61.2, + "completions/min_terminated_length": 61.2, + "entropy": 0.12950787084798018, + "epoch": 0.6368435043481563, + "eval/gt_acc_batch": 0.7961111227671306, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2133023887872696, + "kd/policy_loss": 0.008775795251131058, + "kd/rkl_advantage_mean": 0.7557705676803986, + "kd/rkl_advantage_p95": 4.955019611120224, + "kd/rkl_advantage_std": 2.217440726359685, + "kd/ssd_loss": 0.0, + "learning_rate": 3.631944708160863e-07, + "loss": 0.03510317802429199, + "num_tokens": 548760817.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111108462016, + "rewards/gt_logging_reward/std": 0.39317729075749713, + "sampling/importance_sampling_ratio/max": 1.9034319082895914, + "sampling/importance_sampling_ratio/mean": 1.0023000856240591, + "sampling/importance_sampling_ratio/min": 0.45405703485012056, + "sampling/sampling_logp_difference/max": 0.31260815262794495, + "sampling/sampling_logp_difference/mean": 0.003969692423318823, + "step": 16770, + "step_time": 8.309821190134, + "student/entropy": 0.12950787084798018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890761882066, + "completions/max_length": 496.56666666666666, + "completions/max_terminated_length": 462.6666666666667, + "completions/mean_length": 196.08389892578126, + "completions/mean_terminated_length": 187.7781962076823, + "completions/min_length": 58.43333333333333, + "completions/min_terminated_length": 58.43333333333333, + "entropy": 0.13090988372763, + "epoch": 0.6379827592754339, + "eval/gt_acc_batch": 0.7644444704055786, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28201431035995483, + "kd/policy_loss": 0.009022985579213128, + "kd/rkl_advantage_mean": 0.8293279712398847, + "kd/rkl_advantage_p95": 5.189117453495661, + "kd/rkl_advantage_std": 2.2660762131214143, + "kd/ssd_loss": 0.0, + "learning_rate": 3.620552158888087e-07, + "loss": 0.036091947555541994, + "num_tokens": 549787239.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7644444465637207, + "rewards/gt_logging_reward/std": 0.4131416355570157, + "sampling/importance_sampling_ratio/max": 1.9011201500892638, + "sampling/importance_sampling_ratio/mean": 1.0022559384504954, + "sampling/importance_sampling_ratio/min": 0.46953366200129193, + "sampling/sampling_logp_difference/max": 0.3316683411598206, + "sampling/sampling_logp_difference/mean": 0.004019130649976432, + "step": 16800, + "step_time": 8.357869617029792, + "student/entropy": 0.13090988372763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112410823503, + "completions/max_length": 484.43333333333334, + "completions/max_terminated_length": 432.2, + "completions/mean_length": 189.45001068115235, + "completions/mean_terminated_length": 180.9223642985026, + "completions/min_length": 54.96666666666667, + "completions/min_terminated_length": 54.96666666666667, + "entropy": 0.1302058808505535, + "epoch": 0.6391220142027114, + "eval/gt_acc_batch": 0.7902778029441834, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2863351106643677, + "kd/policy_loss": 0.008717595421088239, + "kd/rkl_advantage_mean": 0.7980396802226702, + "kd/rkl_advantage_p95": 5.033965384960174, + "kd/rkl_advantage_std": 2.2028055797020594, + "kd/ssd_loss": 0.0, + "learning_rate": 3.6091596096153114e-07, + "loss": 0.03487038215001424, + "num_tokens": 550770395.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.790277777115504, + "rewards/gt_logging_reward/std": 0.4044573217630386, + "sampling/importance_sampling_ratio/max": 1.9070892373720805, + "sampling/importance_sampling_ratio/mean": 0.9986464281876882, + "sampling/importance_sampling_ratio/min": 0.4457494924465815, + "sampling/sampling_logp_difference/max": 0.3311859846115112, + "sampling/sampling_logp_difference/mean": 0.004034633461075524, + "step": 16830, + "step_time": 8.132317678266554, + "student/entropy": 0.1302058808505535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333482965827, + "completions/max_length": 479.03333333333336, + "completions/max_terminated_length": 449.23333333333335, + "completions/mean_length": 190.50528818766276, + "completions/mean_terminated_length": 182.91724853515626, + "completions/min_length": 57.36666666666667, + "completions/min_terminated_length": 57.36666666666667, + "entropy": 0.12861852670709292, + "epoch": 0.640261269129989, + "eval/gt_acc_batch": 0.8044444660345713, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26407188177108765, + "kd/policy_loss": 0.008428666713492324, + "kd/rkl_advantage_mean": 0.8468839629242818, + "kd/rkl_advantage_p95": 5.182201846440633, + "kd/rkl_advantage_std": 2.256565195322037, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5977670603425356e-07, + "loss": 0.033714667956034346, + "num_tokens": 551755390.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8044444441795349, + "rewards/gt_logging_reward/std": 0.3888797665635745, + "sampling/importance_sampling_ratio/max": 1.9703713377316794, + "sampling/importance_sampling_ratio/mean": 0.9970546662807465, + "sampling/importance_sampling_ratio/min": 0.46315187017122905, + "sampling/sampling_logp_difference/max": 0.321373571952184, + "sampling/sampling_logp_difference/mean": 0.003977269252451758, + "step": 16860, + "step_time": 8.021342025401342, + "student/entropy": 0.12861852670709292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888889936109383, + "completions/max_length": 475.26666666666665, + "completions/max_terminated_length": 452.46666666666664, + "completions/mean_length": 186.71417592366535, + "completions/mean_terminated_length": 180.74634602864583, + "completions/min_length": 58.56666666666667, + "completions/min_terminated_length": 58.56666666666667, + "entropy": 0.132790748650829, + "epoch": 0.6414005240572666, + "eval/gt_acc_batch": 0.8013889114061992, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33089545369148254, + "kd/policy_loss": 0.00864062579542709, + "kd/rkl_advantage_mean": 0.7926197704548638, + "kd/rkl_advantage_p95": 4.967120695114136, + "kd/rkl_advantage_std": 2.199816984931628, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5863745110697603e-07, + "loss": 0.03456250031789144, + "num_tokens": 552724953.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888915379842, + "rewards/gt_logging_reward/std": 0.39046584367752074, + "sampling/importance_sampling_ratio/max": 1.8505292097727457, + "sampling/importance_sampling_ratio/mean": 0.9931596696376801, + "sampling/importance_sampling_ratio/min": 0.4621580223242442, + "sampling/sampling_logp_difference/max": 0.313336169719696, + "sampling/sampling_logp_difference/mean": 0.004056859753715495, + "step": 16890, + "step_time": 7.953456612932496, + "student/entropy": 0.132790748650829 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014722223052134116, + "completions/max_length": 473.5, + "completions/max_terminated_length": 442.9, + "completions/mean_length": 181.39667510986328, + "completions/mean_terminated_length": 176.60907135009765, + "completions/min_length": 51.666666666666664, + "completions/min_terminated_length": 51.666666666666664, + "entropy": 0.12692044793317717, + "epoch": 0.6425397789845441, + "eval/gt_acc_batch": 0.8225000202655792, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29109060764312744, + "kd/policy_loss": 0.008347628182188297, + "kd/rkl_advantage_mean": 0.7645384796584646, + "kd/rkl_advantage_p95": 4.921144419908524, + "kd/rkl_advantage_std": 2.1904951632022858, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5749819617969845e-07, + "loss": 0.03339051405588786, + "num_tokens": 553684949.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8225000003973643, + "rewards/gt_logging_reward/std": 0.37071799784898757, + "sampling/importance_sampling_ratio/max": 1.8854875445365906, + "sampling/importance_sampling_ratio/mean": 0.9971880853176117, + "sampling/importance_sampling_ratio/min": 0.4574228525161743, + "sampling/sampling_logp_difference/max": 0.35574683745702107, + "sampling/sampling_logp_difference/mean": 0.00394845602568239, + "step": 16920, + "step_time": 8.03289802436775, + "student/entropy": 0.12692044793317717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03194444617256522, + "completions/max_length": 500.2, + "completions/max_terminated_length": 456.56666666666666, + "completions/mean_length": 200.96084238688152, + "completions/mean_terminated_length": 190.8514139811198, + "completions/min_length": 62.93333333333333, + "completions/min_terminated_length": 62.93333333333333, + "entropy": 0.12581306000550588, + "epoch": 0.6436790339118217, + "eval/gt_acc_batch": 0.7830555697282156, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.18658971786499023, + "kd/policy_loss": 0.008529637957690284, + "kd/rkl_advantage_mean": 0.7302560514460007, + "kd/rkl_advantage_p95": 4.95717288851738, + "kd/rkl_advantage_std": 2.184102706114451, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5635894125242087e-07, + "loss": 0.03411855300267538, + "num_tokens": 554712192.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7830555617809296, + "rewards/gt_logging_reward/std": 0.40173674523830416, + "sampling/importance_sampling_ratio/max": 1.9459283788998922, + "sampling/importance_sampling_ratio/mean": 1.0045204480489096, + "sampling/importance_sampling_ratio/min": 0.4331026092171669, + "sampling/sampling_logp_difference/max": 0.3185398558775584, + "sampling/sampling_logp_difference/mean": 0.0038957555157442887, + "step": 16950, + "step_time": 8.106574442635369, + "student/entropy": 0.12581306000550588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890103747448, + "completions/max_length": 486.93333333333334, + "completions/max_terminated_length": 450.8333333333333, + "completions/mean_length": 189.81973215738932, + "completions/mean_terminated_length": 182.23128204345704, + "completions/min_length": 55.93333333333333, + "completions/min_terminated_length": 55.93333333333333, + "entropy": 0.12517688553780318, + "epoch": 0.6448182888390992, + "eval/gt_acc_batch": 0.7980555713176727, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23848868906497955, + "kd/policy_loss": 0.00832369676985157, + "kd/rkl_advantage_mean": 0.6480079291388392, + "kd/rkl_advantage_p95": 4.901359939575196, + "kd/rkl_advantage_std": 2.232854505379995, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5521968632514334e-07, + "loss": 0.03329478700955709, + "num_tokens": 555697663.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7980555534362793, + "rewards/gt_logging_reward/std": 0.3926996052265167, + "sampling/importance_sampling_ratio/max": 1.9332472840944925, + "sampling/importance_sampling_ratio/mean": 1.0043156842390697, + "sampling/importance_sampling_ratio/min": 0.4467013200124105, + "sampling/sampling_logp_difference/max": 0.3129841009775797, + "sampling/sampling_logp_difference/mean": 0.0038931862683966754, + "step": 16980, + "step_time": 7.9257947397592945, + "student/entropy": 0.12517688553780318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223865489165, + "completions/max_length": 493.3666666666667, + "completions/max_terminated_length": 442.1, + "completions/mean_length": 189.5900100708008, + "completions/mean_terminated_length": 179.7471923828125, + "completions/min_length": 48.96666666666667, + "completions/min_terminated_length": 48.96666666666667, + "entropy": 0.12464943962792556, + "epoch": 0.6459575437663768, + "eval/gt_acc_batch": 0.8025000135103861, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27047643065452576, + "kd/policy_loss": 0.008202873372162382, + "kd/rkl_advantage_mean": 0.7720872076422286, + "kd/rkl_advantage_p95": 5.037077375253042, + "kd/rkl_advantage_std": 2.2325597325960795, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5408043139786575e-07, + "loss": 0.03281149864196777, + "num_tokens": 556686859.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8025000035762787, + "rewards/gt_logging_reward/std": 0.3918683489163717, + "sampling/importance_sampling_ratio/max": 1.8840449412663778, + "sampling/importance_sampling_ratio/mean": 0.9984977940718333, + "sampling/importance_sampling_ratio/min": 0.42580653751889863, + "sampling/sampling_logp_difference/max": 0.3353710254033407, + "sampling/sampling_logp_difference/mean": 0.003849335752117137, + "step": 17010, + "step_time": 7.98488733356741, + "student/entropy": 0.12464943962792556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890265176695, + "completions/max_length": 494.23333333333335, + "completions/max_terminated_length": 454.7, + "completions/mean_length": 189.18251088460286, + "completions/mean_terminated_length": 180.70476888020832, + "completions/min_length": 50.46666666666667, + "completions/min_terminated_length": 50.46666666666667, + "entropy": 0.13043951236953338, + "epoch": 0.6470967986936543, + "eval/gt_acc_batch": 0.7861111323038737, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2660060226917267, + "kd/policy_loss": 0.008643310008725773, + "kd/rkl_advantage_mean": 0.7926251156100382, + "kd/rkl_advantage_p95": 5.034605377912522, + "kd/rkl_advantage_std": 2.2338148544232053, + "kd/ssd_loss": 0.0, + "learning_rate": 3.529411764705882e-07, + "loss": 0.03457323710123698, + "num_tokens": 557664068.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111124356588, + "rewards/gt_logging_reward/std": 0.40044122536977134, + "sampling/importance_sampling_ratio/max": 1.8600624283154805, + "sampling/importance_sampling_ratio/mean": 1.0003699362277985, + "sampling/importance_sampling_ratio/min": 0.4648956427971522, + "sampling/sampling_logp_difference/max": 0.31121462980906167, + "sampling/sampling_logp_difference/mean": 0.003996648173779249, + "step": 17040, + "step_time": 8.048448121971644, + "student/entropy": 0.13043951236953338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02750000189989805, + "completions/max_length": 481.03333333333336, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 190.1408437093099, + "completions/mean_terminated_length": 181.21231231689453, + "completions/min_length": 55.7, + "completions/min_terminated_length": 55.7, + "entropy": 0.12673679540554683, + "epoch": 0.648236053620932, + "eval/gt_acc_batch": 0.8138889094193776, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3182868957519531, + "kd/policy_loss": 0.008340878057060764, + "kd/rkl_advantage_mean": 0.752524463708202, + "kd/rkl_advantage_p95": 5.011719028155009, + "kd/rkl_advantage_std": 2.199068667491277, + "kd/ssd_loss": 0.0, + "learning_rate": 3.518019215433107e-07, + "loss": 0.03336351712544759, + "num_tokens": 558667535.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8138888895511627, + "rewards/gt_logging_reward/std": 0.3823909064133962, + "sampling/importance_sampling_ratio/max": 1.8473361055056254, + "sampling/importance_sampling_ratio/mean": 0.9927049557367961, + "sampling/importance_sampling_ratio/min": 0.45785174518823624, + "sampling/sampling_logp_difference/max": 0.4298693756262461, + "sampling/sampling_logp_difference/mean": 0.00387502727098763, + "step": 17070, + "step_time": 8.16017706253139, + "student/entropy": 0.12673679540554683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02777777925754587, + "completions/max_length": 484.2, + "completions/max_terminated_length": 451.6666666666667, + "completions/mean_length": 182.57278747558593, + "completions/mean_terminated_length": 173.11978403727213, + "completions/min_length": 51.53333333333333, + "completions/min_terminated_length": 51.53333333333333, + "entropy": 0.129990264835457, + "epoch": 0.6493753085482095, + "eval/gt_acc_batch": 0.7991666853427887, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25680431723594666, + "kd/policy_loss": 0.00845349082762065, + "kd/rkl_advantage_mean": 0.7881077564321458, + "kd/rkl_advantage_p95": 4.997448068857193, + "kd/rkl_advantage_std": 2.2257494290669757, + "kd/ssd_loss": 0.0, + "learning_rate": 3.506626666160331e-07, + "loss": 0.033813969294230146, + "num_tokens": 559626861.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7991666615009307, + "rewards/gt_logging_reward/std": 0.39131142099698385, + "sampling/importance_sampling_ratio/max": 1.8216785033543905, + "sampling/importance_sampling_ratio/mean": 0.9983015437920888, + "sampling/importance_sampling_ratio/min": 0.4365095059076945, + "sampling/sampling_logp_difference/max": 0.3363833675781886, + "sampling/sampling_logp_difference/mean": 0.004028104168052475, + "step": 17100, + "step_time": 8.071341763609476, + "student/entropy": 0.129990264835457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112621923287, + "completions/max_length": 490.5, + "completions/max_terminated_length": 458.93333333333334, + "completions/mean_length": 185.7497309366862, + "completions/mean_terminated_length": 178.0465296427409, + "completions/min_length": 56.56666666666667, + "completions/min_terminated_length": 56.56666666666667, + "entropy": 0.12880011362334093, + "epoch": 0.6505145634754871, + "eval/gt_acc_batch": 0.8050000190734863, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2536056339740753, + "kd/policy_loss": 0.0084313633677084, + "kd/rkl_advantage_mean": 0.8496054710820318, + "kd/rkl_advantage_p95": 5.23048862417539, + "kd/rkl_advantage_std": 2.3166631162166595, + "kd/ssd_loss": 0.0, + "learning_rate": 3.4952341168875553e-07, + "loss": 0.033725460370381675, + "num_tokens": 560597712.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8050000031789144, + "rewards/gt_logging_reward/std": 0.38492599030335745, + "sampling/importance_sampling_ratio/max": 1.8876159270604451, + "sampling/importance_sampling_ratio/mean": 0.9998113413651785, + "sampling/importance_sampling_ratio/min": 0.45339525640010836, + "sampling/sampling_logp_difference/max": 0.31308398246765134, + "sampling/sampling_logp_difference/mean": 0.004005709810492893, + "step": 17130, + "step_time": 8.032414309601881, + "student/entropy": 0.12880011362334093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500001067916552, + "completions/max_length": 486.1, + "completions/max_terminated_length": 454.6333333333333, + "completions/mean_length": 188.8425094604492, + "completions/mean_terminated_length": 183.32730458577473, + "completions/min_length": 55.166666666666664, + "completions/min_terminated_length": 55.166666666666664, + "entropy": 0.12512787009278933, + "epoch": 0.6516538184027646, + "eval/gt_acc_batch": 0.8175000270207723, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24812979996204376, + "kd/policy_loss": 0.008056775713339449, + "kd/rkl_advantage_mean": 0.709441336352999, + "kd/rkl_advantage_p95": 4.9326574881871545, + "kd/rkl_advantage_std": 2.214345129330953, + "kd/ssd_loss": 0.0, + "learning_rate": 3.48384156761478e-07, + "loss": 0.03222710291544596, + "num_tokens": 561584121.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8175000051657358, + "rewards/gt_logging_reward/std": 0.3823355803887049, + "sampling/importance_sampling_ratio/max": 1.8817971110343934, + "sampling/importance_sampling_ratio/mean": 1.0015168885389965, + "sampling/importance_sampling_ratio/min": 0.44526663223902385, + "sampling/sampling_logp_difference/max": 0.375621239344279, + "sampling/sampling_logp_difference/mean": 0.0038643741281703115, + "step": 17160, + "step_time": 7.939830409031129, + "student/entropy": 0.12512787009278933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444565102458, + "completions/max_length": 499.6, + "completions/max_terminated_length": 469.46666666666664, + "completions/mean_length": 192.77334238688152, + "completions/mean_terminated_length": 185.7891591389974, + "completions/min_length": 53.7, + "completions/min_terminated_length": 53.7, + "entropy": 0.12967084149519603, + "epoch": 0.6527930733300421, + "eval/gt_acc_batch": 0.7975000301996867, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3100634813308716, + "kd/policy_loss": 0.008209549899523457, + "kd/rkl_advantage_mean": 0.8336216099560261, + "kd/rkl_advantage_p95": 5.1515682776769, + "kd/rkl_advantage_std": 2.2405625651280086, + "kd/ssd_loss": 0.0, + "learning_rate": 3.472449018342004e-07, + "loss": 0.03283819357554118, + "num_tokens": 562584993.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7974999964237213, + "rewards/gt_logging_reward/std": 0.3893975590666135, + "sampling/importance_sampling_ratio/max": 1.8936812241872152, + "sampling/importance_sampling_ratio/mean": 0.9946183045705159, + "sampling/importance_sampling_ratio/min": 0.4260357399781545, + "sampling/sampling_logp_difference/max": 0.3180759867032369, + "sampling/sampling_logp_difference/mean": 0.003975346319687863, + "step": 17190, + "step_time": 8.159894287334948, + "student/entropy": 0.12967084149519603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02527777897194028, + "completions/max_length": 490.5, + "completions/max_terminated_length": 450.43333333333334, + "completions/mean_length": 184.88695475260417, + "completions/mean_terminated_length": 176.6697738647461, + "completions/min_length": 47.9, + "completions/min_terminated_length": 47.9, + "entropy": 0.12816196090231338, + "epoch": 0.6539323282573197, + "eval/gt_acc_batch": 0.8030555725097657, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.39181390404701233, + "kd/policy_loss": 0.008119095965715436, + "kd/rkl_advantage_mean": 0.8203365982820591, + "kd/rkl_advantage_p95": 5.122635807593664, + "kd/rkl_advantage_std": 2.2617140690485638, + "kd/ssd_loss": 0.0, + "learning_rate": 3.4610564690692284e-07, + "loss": 0.0324763834476471, + "num_tokens": 563557042.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555546283722, + "rewards/gt_logging_reward/std": 0.38667806386947634, + "sampling/importance_sampling_ratio/max": 1.7836159467697144, + "sampling/importance_sampling_ratio/mean": 0.9991901059945424, + "sampling/importance_sampling_ratio/min": 0.4507124811410904, + "sampling/sampling_logp_difference/max": 0.34797885616620383, + "sampling/sampling_logp_difference/mean": 0.003964576971096297, + "step": 17220, + "step_time": 8.003491304906978, + "student/entropy": 0.12816196090231338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02138888988023003, + "completions/max_length": 482.9, + "completions/max_terminated_length": 453.46666666666664, + "completions/mean_length": 187.54806620279948, + "completions/mean_terminated_length": 180.58749084472657, + "completions/min_length": 46.9, + "completions/min_terminated_length": 46.9, + "entropy": 0.12660679935167232, + "epoch": 0.6550715831845972, + "eval/gt_acc_batch": 0.8147222479184468, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2492290884256363, + "kd/policy_loss": 0.008193143360161532, + "kd/rkl_advantage_mean": 0.6728134886361659, + "kd/rkl_advantage_p95": 4.818406353394191, + "kd/rkl_advantage_std": 2.173099362850189, + "kd/ssd_loss": 0.0, + "learning_rate": 3.4496639197964525e-07, + "loss": 0.03277257283528646, + "num_tokens": 564528575.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8147222181161244, + "rewards/gt_logging_reward/std": 0.3841874122619629, + "sampling/importance_sampling_ratio/max": 1.8976428707440693, + "sampling/importance_sampling_ratio/mean": 1.0029410700003305, + "sampling/importance_sampling_ratio/min": 0.4632793088754018, + "sampling/sampling_logp_difference/max": 0.337332155307134, + "sampling/sampling_logp_difference/mean": 0.003923756489530206, + "step": 17250, + "step_time": 7.93261963952779, + "student/entropy": 0.12660679935167232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112156262, + "completions/max_length": 485.6666666666667, + "completions/max_terminated_length": 452.6, + "completions/mean_length": 190.3930658976237, + "completions/mean_terminated_length": 182.70277404785156, + "completions/min_length": 54.733333333333334, + "completions/min_terminated_length": 54.733333333333334, + "entropy": 0.12307022170474131, + "epoch": 0.6562108381118749, + "eval/gt_acc_batch": 0.8147222439448039, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3426455855369568, + "kd/policy_loss": 0.008079214851992826, + "kd/rkl_advantage_mean": 0.7779317287107309, + "kd/rkl_advantage_p95": 5.13288760582606, + "kd/rkl_advantage_std": 2.256614649295807, + "kd/ssd_loss": 0.0, + "learning_rate": 3.438271370523677e-07, + "loss": 0.03231686353683472, + "num_tokens": 565513966.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8147222240765889, + "rewards/gt_logging_reward/std": 0.38011477192242943, + "sampling/importance_sampling_ratio/max": 1.8073400855064392, + "sampling/importance_sampling_ratio/mean": 1.0021860977013906, + "sampling/importance_sampling_ratio/min": 0.4736773798863093, + "sampling/sampling_logp_difference/max": 0.3347885966300964, + "sampling/sampling_logp_difference/mean": 0.0038397989235818388, + "step": 17280, + "step_time": 7.932388707695646, + "student/entropy": 0.12307022170474131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055557515472174, + "completions/max_length": 498.26666666666665, + "completions/max_terminated_length": 457.3, + "completions/mean_length": 186.28389790852864, + "completions/mean_terminated_length": 176.88760172526042, + "completions/min_length": 59.3, + "completions/min_terminated_length": 59.3, + "entropy": 0.12739157664279144, + "epoch": 0.6573500930391524, + "eval/gt_acc_batch": 0.8041666845480601, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26202431321144104, + "kd/policy_loss": 0.008401907450752333, + "kd/rkl_advantage_mean": 0.8452710379225512, + "kd/rkl_advantage_p95": 5.186262981096903, + "kd/rkl_advantage_std": 2.261862533291181, + "kd/ssd_loss": 0.0, + "learning_rate": 3.426878821250902e-07, + "loss": 0.033607629934946696, + "num_tokens": 566478956.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666646798452, + "rewards/gt_logging_reward/std": 0.39313697020212807, + "sampling/importance_sampling_ratio/max": 1.8522125442822774, + "sampling/importance_sampling_ratio/mean": 0.9998597145080567, + "sampling/importance_sampling_ratio/min": 0.41462819228569664, + "sampling/sampling_logp_difference/max": 0.3409214456876119, + "sampling/sampling_logp_difference/mean": 0.003935561732699474, + "step": 17310, + "step_time": 7.937692051599152, + "student/entropy": 0.12739157664279144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01805555649722616, + "completions/max_length": 463.06666666666666, + "completions/max_terminated_length": 433.93333333333334, + "completions/mean_length": 184.32056427001953, + "completions/mean_terminated_length": 178.4252182006836, + "completions/min_length": 56.7, + "completions/min_terminated_length": 56.7, + "entropy": 0.12021523397415876, + "epoch": 0.65848934796643, + "eval/gt_acc_batch": 0.8380555748939514, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2985924184322357, + "kd/policy_loss": 0.007507918745977804, + "kd/rkl_advantage_mean": 0.6578180702907654, + "kd/rkl_advantage_p95": 4.787430010239283, + "kd/rkl_advantage_std": 2.144655391573906, + "kd/ssd_loss": 0.0, + "learning_rate": 3.415486271978126e-07, + "loss": 0.03003167708714803, + "num_tokens": 567441918.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8380555609862009, + "rewards/gt_logging_reward/std": 0.3616993298133214, + "sampling/importance_sampling_ratio/max": 1.8850119471549989, + "sampling/importance_sampling_ratio/mean": 0.9961894830067952, + "sampling/importance_sampling_ratio/min": 0.4024794687827428, + "sampling/sampling_logp_difference/max": 0.3355924646059672, + "sampling/sampling_logp_difference/mean": 0.003739680373109877, + "step": 17340, + "step_time": 7.6842571621595805, + "student/entropy": 0.12021523397415876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02694444628432393, + "completions/max_length": 493.8666666666667, + "completions/max_terminated_length": 458.46666666666664, + "completions/mean_length": 190.57862091064453, + "completions/mean_terminated_length": 181.90733388264974, + "completions/min_length": 58.03333333333333, + "completions/min_terminated_length": 58.03333333333333, + "entropy": 0.12291164596875509, + "epoch": 0.6596286028937075, + "eval/gt_acc_batch": 0.8022222459316254, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1959981918334961, + "kd/policy_loss": 0.008044641414502014, + "kd/rkl_advantage_mean": 0.7724793929606676, + "kd/rkl_advantage_p95": 5.11864776412646, + "kd/rkl_advantage_std": 2.2524529536565145, + "kd/ssd_loss": 0.0, + "learning_rate": 3.404093722705351e-07, + "loss": 0.03217856486638387, + "num_tokens": 568429041.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8022222220897675, + "rewards/gt_logging_reward/std": 0.39161916077136993, + "sampling/importance_sampling_ratio/max": 2.0102009495099384, + "sampling/importance_sampling_ratio/mean": 1.002364852031072, + "sampling/importance_sampling_ratio/min": 0.4630582511425018, + "sampling/sampling_logp_difference/max": 0.3216120441754659, + "sampling/sampling_logp_difference/mean": 0.003817741402114431, + "step": 17370, + "step_time": 7.940266089568225, + "student/entropy": 0.12291164596875509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02277777884155512, + "completions/max_length": 482.56666666666666, + "completions/max_terminated_length": 444.03333333333336, + "completions/mean_length": 184.4930669148763, + "completions/mean_terminated_length": 176.88331502278646, + "completions/min_length": 54.53333333333333, + "completions/min_terminated_length": 54.53333333333333, + "entropy": 0.1272055149078369, + "epoch": 0.6607678578209851, + "eval/gt_acc_batch": 0.8136111279328664, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2287134826183319, + "kd/policy_loss": 0.008458635624265298, + "kd/rkl_advantage_mean": 0.7212472068145871, + "kd/rkl_advantage_p95": 4.909848507245382, + "kd/rkl_advantage_std": 2.2212389171123506, + "kd/ssd_loss": 0.0, + "learning_rate": 3.392701173432575e-07, + "loss": 0.033834540843963624, + "num_tokens": 569398376.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813611110051473, + "rewards/gt_logging_reward/std": 0.38213937332232795, + "sampling/importance_sampling_ratio/max": 1.9362521370251973, + "sampling/importance_sampling_ratio/mean": 1.0004419406255087, + "sampling/importance_sampling_ratio/min": 0.46223601549863813, + "sampling/sampling_logp_difference/max": 0.37812764644622804, + "sampling/sampling_logp_difference/mean": 0.003940871551943322, + "step": 17400, + "step_time": 7.8952496529256075, + "student/entropy": 0.1272055149078369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000109896064, + "completions/max_length": 467.43333333333334, + "completions/max_terminated_length": 439.3, + "completions/mean_length": 184.19195353190105, + "completions/mean_terminated_length": 178.53575083414714, + "completions/min_length": 54.7, + "completions/min_terminated_length": 54.7, + "entropy": 0.12391647100448608, + "epoch": 0.6619071127482626, + "eval/gt_acc_batch": 0.8213889271020889, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29260334372520447, + "kd/policy_loss": 0.007686402378991867, + "kd/rkl_advantage_mean": 0.6604364017024636, + "kd/rkl_advantage_p95": 4.822237334648768, + "kd/rkl_advantage_std": 2.1723181307315826, + "kd/ssd_loss": 0.0, + "learning_rate": 3.381308624159799e-07, + "loss": 0.03074561357498169, + "num_tokens": 570366139.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8213888833920161, + "rewards/gt_logging_reward/std": 0.3662651225924492, + "sampling/importance_sampling_ratio/max": 1.8376686334609986, + "sampling/importance_sampling_ratio/mean": 0.9948789397875468, + "sampling/importance_sampling_ratio/min": 0.4434926142295202, + "sampling/sampling_logp_difference/max": 0.3453893423080444, + "sampling/sampling_logp_difference/mean": 0.003806181865123411, + "step": 17430, + "step_time": 7.843234439462928, + "student/entropy": 0.12391647100448608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890283803145, + "completions/max_length": 475.53333333333336, + "completions/max_terminated_length": 449.1666666666667, + "completions/mean_length": 183.92556610107422, + "completions/mean_terminated_length": 177.04452412923177, + "completions/min_length": 58.03333333333333, + "completions/min_terminated_length": 58.03333333333333, + "entropy": 0.12861754813541967, + "epoch": 0.6630463676755401, + "eval/gt_acc_batch": 0.804722249507904, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26947760581970215, + "kd/policy_loss": 0.008294671089000378, + "kd/rkl_advantage_mean": 0.7002710719282429, + "kd/rkl_advantage_p95": 4.949443423748017, + "kd/rkl_advantage_std": 2.189201246698697, + "kd/ssd_loss": 0.0, + "learning_rate": 3.369916074887024e-07, + "loss": 0.0331786851088206, + "num_tokens": 571330775.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222256660461, + "rewards/gt_logging_reward/std": 0.38997937937577565, + "sampling/importance_sampling_ratio/max": 1.8600077470143637, + "sampling/importance_sampling_ratio/mean": 1.000118871529897, + "sampling/importance_sampling_ratio/min": 0.48650990625222523, + "sampling/sampling_logp_difference/max": 0.31823251247406004, + "sampling/sampling_logp_difference/mean": 0.003964298738477131, + "step": 17460, + "step_time": 7.937490180737223, + "student/entropy": 0.12861754813541967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001334895692, + "completions/max_length": 494.8, + "completions/max_terminated_length": 454.4, + "completions/mean_length": 188.11528778076172, + "completions/mean_terminated_length": 179.8384256998698, + "completions/min_length": 52.9, + "completions/min_terminated_length": 52.9, + "entropy": 0.13024234250187874, + "epoch": 0.6641856226028178, + "eval/gt_acc_batch": 0.8066666821638743, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2412887066602707, + "kd/policy_loss": 0.008754815525996188, + "kd/rkl_advantage_mean": 0.7652453483392795, + "kd/rkl_advantage_p95": 4.981199435393015, + "kd/rkl_advantage_std": 2.1962982128063837, + "kd/ssd_loss": 0.0, + "learning_rate": 3.358523525614248e-07, + "loss": 0.0350192666053772, + "num_tokens": 572319078.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8066666702429454, + "rewards/gt_logging_reward/std": 0.38653584371010463, + "sampling/importance_sampling_ratio/max": 1.9506943027178447, + "sampling/importance_sampling_ratio/mean": 1.0027129828929902, + "sampling/importance_sampling_ratio/min": 0.4954705576101939, + "sampling/sampling_logp_difference/max": 0.3265507996082306, + "sampling/sampling_logp_difference/mean": 0.004002032754942775, + "step": 17490, + "step_time": 8.146531678866207, + "student/entropy": 0.13024234250187874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777779027819633, + "completions/max_length": 492.3333333333333, + "completions/max_terminated_length": 441.9, + "completions/mean_length": 181.60000864664713, + "completions/mean_terminated_length": 174.14453786214193, + "completions/min_length": 53.1, + "completions/min_terminated_length": 53.1, + "entropy": 0.12845517043024302, + "epoch": 0.6653248775300953, + "eval/gt_acc_batch": 0.8261111319065094, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4585038721561432, + "kd/policy_loss": 0.008344640322805692, + "kd/rkl_advantage_mean": 0.7679364129726309, + "kd/rkl_advantage_p95": 5.108111164967219, + "kd/rkl_advantage_std": 2.278198626637459, + "kd/ssd_loss": 0.0, + "learning_rate": 3.347130976341472e-07, + "loss": 0.033378565311431886, + "num_tokens": 573273638.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.826111110051473, + "rewards/gt_logging_reward/std": 0.36905793895324074, + "sampling/importance_sampling_ratio/max": 2.042654585838318, + "sampling/importance_sampling_ratio/mean": 1.0005733370780945, + "sampling/importance_sampling_ratio/min": 0.46291246712207795, + "sampling/sampling_logp_difference/max": 0.3283746679623922, + "sampling/sampling_logp_difference/mean": 0.003936907237706085, + "step": 17520, + "step_time": 7.962564455492733, + "student/entropy": 0.12845517043024302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026666668181618055, + "completions/max_length": 500.8, + "completions/max_terminated_length": 447.8, + "completions/mean_length": 190.62862243652344, + "completions/mean_terminated_length": 181.86478678385416, + "completions/min_length": 51.166666666666664, + "completions/min_terminated_length": 51.166666666666664, + "entropy": 0.13295833431184292, + "epoch": 0.6664641324573729, + "eval/gt_acc_batch": 0.7916666766007742, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3519384264945984, + "kd/policy_loss": 0.008860076228544737, + "kd/rkl_advantage_mean": 0.800262009104093, + "kd/rkl_advantage_p95": 5.066384834051132, + "kd/rkl_advantage_std": 2.2498834441105524, + "kd/ssd_loss": 0.0, + "learning_rate": 3.3357384270686964e-07, + "loss": 0.03544031381607056, + "num_tokens": 574262829.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7916666706403096, + "rewards/gt_logging_reward/std": 0.40027037958304085, + "sampling/importance_sampling_ratio/max": 1.8876842856407166, + "sampling/importance_sampling_ratio/mean": 0.990189915895462, + "sampling/importance_sampling_ratio/min": 0.39682947595914203, + "sampling/sampling_logp_difference/max": 0.3215497016906738, + "sampling/sampling_logp_difference/mean": 0.004083842597901821, + "step": 17550, + "step_time": 8.019523767268401, + "student/entropy": 0.13295833431184292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889051352938, + "completions/max_length": 490.0, + "completions/max_terminated_length": 451.6333333333333, + "completions/mean_length": 190.0511210123698, + "completions/mean_terminated_length": 181.46961873372396, + "completions/min_length": 53.96666666666667, + "completions/min_terminated_length": 53.96666666666667, + "entropy": 0.13090494039158027, + "epoch": 0.6676033873846504, + "eval/gt_acc_batch": 0.79277779062589, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24259047210216522, + "kd/policy_loss": 0.008506533354132746, + "kd/rkl_advantage_mean": 0.8372690501933296, + "kd/rkl_advantage_p95": 5.053820435206095, + "kd/rkl_advantage_std": 2.231183784206708, + "kd/ssd_loss": 0.0, + "learning_rate": 3.3243458777959217e-07, + "loss": 0.034026137987772626, + "num_tokens": 575247981.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7927777806917826, + "rewards/gt_logging_reward/std": 0.398642701903979, + "sampling/importance_sampling_ratio/max": 1.808309535185496, + "sampling/importance_sampling_ratio/mean": 0.9883547385533651, + "sampling/importance_sampling_ratio/min": 0.4439965049425761, + "sampling/sampling_logp_difference/max": 0.31943644881248473, + "sampling/sampling_logp_difference/mean": 0.0040115293969089786, + "step": 17580, + "step_time": 8.142879592701986, + "student/entropy": 0.13090494039158027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890221714973, + "completions/max_length": 492.46666666666664, + "completions/max_terminated_length": 451.9, + "completions/mean_length": 187.41862080891926, + "completions/mean_terminated_length": 180.32379760742188, + "completions/min_length": 57.733333333333334, + "completions/min_terminated_length": 57.733333333333334, + "entropy": 0.127741383202374, + "epoch": 0.668742642311928, + "eval/gt_acc_batch": 0.8130555768807729, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23289693892002106, + "kd/policy_loss": 0.008100119281637792, + "kd/rkl_advantage_mean": 0.7520952455699443, + "kd/rkl_advantage_p95": 4.958979135751724, + "kd/rkl_advantage_std": 2.2088383555412294, + "kd/ssd_loss": 0.0, + "learning_rate": 3.312953328523146e-07, + "loss": 0.03240047891934713, + "num_tokens": 576230120.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8130555550257365, + "rewards/gt_logging_reward/std": 0.38249687751134237, + "sampling/importance_sampling_ratio/max": 1.8718886097272238, + "sampling/importance_sampling_ratio/mean": 0.9942612111568451, + "sampling/importance_sampling_ratio/min": 0.406381560365359, + "sampling/sampling_logp_difference/max": 0.3465886195500692, + "sampling/sampling_logp_difference/mean": 0.003957128711044788, + "step": 17610, + "step_time": 8.086330650762344, + "student/entropy": 0.127741383202374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000100582838, + "completions/max_length": 487.06666666666666, + "completions/max_terminated_length": 452.3, + "completions/mean_length": 179.71973215738933, + "completions/mean_terminated_length": 173.82500508626302, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.1260816395903627, + "epoch": 0.6698818972392055, + "eval/gt_acc_batch": 0.8327778021494547, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3257884085178375, + "kd/policy_loss": 0.00811724223700973, + "kd/rkl_advantage_mean": 0.8224388501296441, + "kd/rkl_advantage_p95": 5.134667722384135, + "kd/rkl_advantage_std": 2.265880990028381, + "kd/ssd_loss": 0.0, + "learning_rate": 3.30156077925037e-07, + "loss": 0.032468966643015545, + "num_tokens": 577173455.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8327777763207753, + "rewards/gt_logging_reward/std": 0.3665173803766569, + "sampling/importance_sampling_ratio/max": 1.8417413552602133, + "sampling/importance_sampling_ratio/mean": 0.9995017925898234, + "sampling/importance_sampling_ratio/min": 0.4501650477449099, + "sampling/sampling_logp_difference/max": 0.31654035250345863, + "sampling/sampling_logp_difference/mean": 0.0038304586972420416, + "step": 17640, + "step_time": 7.8305025284372585, + "student/entropy": 0.1260816395903627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028888890612870456, + "completions/max_length": 495.06666666666666, + "completions/max_terminated_length": 445.1666666666667, + "completions/mean_length": 181.57084197998046, + "completions/mean_terminated_length": 171.85578002929688, + "completions/min_length": 49.333333333333336, + "completions/min_terminated_length": 49.333333333333336, + "entropy": 0.1256029965976874, + "epoch": 0.6710211521664832, + "eval/gt_acc_batch": 0.8119444608688354, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.301008015871048, + "kd/policy_loss": 0.008078069045829276, + "kd/rkl_advantage_mean": 0.8618729834275048, + "kd/rkl_advantage_p95": 5.155529946088791, + "kd/rkl_advantage_std": 2.2501130123933155, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2901682299775947e-07, + "loss": 0.032312277952829996, + "num_tokens": 578119862.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.811944446961085, + "rewards/gt_logging_reward/std": 0.37728563447793323, + "sampling/importance_sampling_ratio/max": 1.9106628537178039, + "sampling/importance_sampling_ratio/mean": 0.9954687217871349, + "sampling/importance_sampling_ratio/min": 0.41723570475975674, + "sampling/sampling_logp_difference/max": 0.3137870629628499, + "sampling/sampling_logp_difference/mean": 0.00384935907398661, + "step": 17670, + "step_time": 8.023740875794708, + "student/entropy": 0.1256029965976874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02944444598009189, + "completions/max_length": 495.6, + "completions/max_terminated_length": 463.93333333333334, + "completions/mean_length": 188.01751149495442, + "completions/mean_terminated_length": 178.5489298502604, + "completions/min_length": 59.46666666666667, + "completions/min_terminated_length": 59.46666666666667, + "entropy": 0.13582610028485456, + "epoch": 0.6721604070937607, + "eval/gt_acc_batch": 0.7941666881243388, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3141253888607025, + "kd/policy_loss": 0.008911592460935935, + "kd/rkl_advantage_mean": 0.7694632877906163, + "kd/rkl_advantage_p95": 4.980625418821971, + "kd/rkl_advantage_std": 2.2073015958070754, + "kd/ssd_loss": 0.0, + "learning_rate": 3.278775680704819e-07, + "loss": 0.03564637104670207, + "num_tokens": 579090677.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7941666722297669, + "rewards/gt_logging_reward/std": 0.3942030747731527, + "sampling/importance_sampling_ratio/max": 1.9174003680547078, + "sampling/importance_sampling_ratio/mean": 1.0027616699536641, + "sampling/importance_sampling_ratio/min": 0.4200414627790451, + "sampling/sampling_logp_difference/max": 0.3492885649204254, + "sampling/sampling_logp_difference/mean": 0.004133965959772468, + "step": 17700, + "step_time": 7.995640997001707, + "student/entropy": 0.13582610028485456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667604198058, + "completions/max_length": 481.03333333333336, + "completions/max_terminated_length": 441.3333333333333, + "completions/mean_length": 195.67639923095703, + "completions/mean_terminated_length": 189.54854685465494, + "completions/min_length": 60.43333333333333, + "completions/min_terminated_length": 60.43333333333333, + "entropy": 0.1291210301220417, + "epoch": 0.6732996620210382, + "eval/gt_acc_batch": 0.8036111334959666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24315160512924194, + "kd/policy_loss": 0.008430358967355762, + "kd/rkl_advantage_mean": 0.7433569273600976, + "kd/rkl_advantage_p95": 5.022708815336228, + "kd/rkl_advantage_std": 2.2195707082748415, + "kd/ssd_loss": 0.0, + "learning_rate": 3.267383131432043e-07, + "loss": 0.033721431096394854, + "num_tokens": 580094472.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111076672872, + "rewards/gt_logging_reward/std": 0.39150675336519875, + "sampling/importance_sampling_ratio/max": 1.8727708498636881, + "sampling/importance_sampling_ratio/mean": 0.9999435901641845, + "sampling/importance_sampling_ratio/min": 0.474872417251269, + "sampling/sampling_logp_difference/max": 0.3092583288749059, + "sampling/sampling_logp_difference/mean": 0.0039841874114548165, + "step": 17730, + "step_time": 7.943550622064504, + "student/entropy": 0.1291210301220417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888890029241642, + "completions/max_length": 480.0, + "completions/max_terminated_length": 453.1, + "completions/mean_length": 187.85084279378256, + "completions/mean_terminated_length": 181.81392720540364, + "completions/min_length": 55.7, + "completions/min_terminated_length": 55.7, + "entropy": 0.12886380193134148, + "epoch": 0.6744389169483158, + "eval/gt_acc_batch": 0.8036111235618592, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27165961265563965, + "kd/policy_loss": 0.00849630232163084, + "kd/rkl_advantage_mean": 0.7642403537174687, + "kd/rkl_advantage_p95": 5.006518101692199, + "kd/rkl_advantage_std": 2.2279286036888757, + "kd/ssd_loss": 0.0, + "learning_rate": 3.255990582159268e-07, + "loss": 0.03398520946502685, + "num_tokens": 581075639.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111136277516, + "rewards/gt_logging_reward/std": 0.38833606044451396, + "sampling/importance_sampling_ratio/max": 1.8867033839225769, + "sampling/importance_sampling_ratio/mean": 0.9968628287315369, + "sampling/importance_sampling_ratio/min": 0.4064067636926969, + "sampling/sampling_logp_difference/max": 0.327189177274704, + "sampling/sampling_logp_difference/mean": 0.003958731726743281, + "step": 17760, + "step_time": 7.851146203500685, + "student/entropy": 0.12886380193134148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02361111268401146, + "completions/max_length": 490.46666666666664, + "completions/max_terminated_length": 441.7, + "completions/mean_length": 189.4336201985677, + "completions/mean_terminated_length": 181.74365641276043, + "completions/min_length": 54.56666666666667, + "completions/min_terminated_length": 54.56666666666667, + "entropy": 0.12563642027477423, + "epoch": 0.6755781718755933, + "eval/gt_acc_batch": 0.8052777846654257, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24795067310333252, + "kd/policy_loss": 0.008076871250523254, + "kd/rkl_advantage_mean": 0.847195222399508, + "kd/rkl_advantage_p95": 5.172544811169306, + "kd/rkl_advantage_std": 2.2653516153494517, + "kd/ssd_loss": 0.0, + "learning_rate": 3.244598032886492e-07, + "loss": 0.03230748176574707, + "num_tokens": 582063736.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8052777767181396, + "rewards/gt_logging_reward/std": 0.3890812615553538, + "sampling/importance_sampling_ratio/max": 1.8630433797836303, + "sampling/importance_sampling_ratio/mean": 0.9973360737164815, + "sampling/importance_sampling_ratio/min": 0.43950029611587527, + "sampling/sampling_logp_difference/max": 0.31749341686566673, + "sampling/sampling_logp_difference/mean": 0.0038571629983683428, + "step": 17790, + "step_time": 7.946709738534022, + "student/entropy": 0.12563642027477423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028333335183560848, + "completions/max_length": 484.4, + "completions/max_terminated_length": 459.4, + "completions/mean_length": 188.76167755126954, + "completions/mean_terminated_length": 179.54772542317707, + "completions/min_length": 45.06666666666667, + "completions/min_terminated_length": 45.06666666666667, + "entropy": 0.1328347146511078, + "epoch": 0.676717426802871, + "eval/gt_acc_batch": 0.7980555673440297, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26817095279693604, + "kd/policy_loss": 0.008812753208136808, + "kd/rkl_advantage_mean": 0.698044736062487, + "kd/rkl_advantage_p95": 4.858969604969024, + "kd/rkl_advantage_std": 2.2173392554124196, + "kd/ssd_loss": 0.0, + "learning_rate": 3.233205483613716e-07, + "loss": 0.0352510134379069, + "num_tokens": 583049494.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7980555554231008, + "rewards/gt_logging_reward/std": 0.3944853434960047, + "sampling/importance_sampling_ratio/max": 1.8414246519406636, + "sampling/importance_sampling_ratio/mean": 0.997216937939326, + "sampling/importance_sampling_ratio/min": 0.4437927315632502, + "sampling/sampling_logp_difference/max": 0.33489974538485207, + "sampling/sampling_logp_difference/mean": 0.004074512957595288, + "step": 17820, + "step_time": 8.03971927149687, + "student/entropy": 0.1328347146511078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02555555688838164, + "completions/max_length": 479.26666666666665, + "completions/max_terminated_length": 441.1666666666667, + "completions/mean_length": 187.09917755126952, + "completions/mean_terminated_length": 178.61787923177084, + "completions/min_length": 48.333333333333336, + "completions/min_terminated_length": 48.333333333333336, + "entropy": 0.1273341982314984, + "epoch": 0.6778566817301485, + "eval/gt_acc_batch": 0.7958333512147268, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3095136880874634, + "kd/policy_loss": 0.008327373468394702, + "kd/rkl_advantage_mean": 0.7624353275634348, + "kd/rkl_advantage_p95": 5.011302584409714, + "kd/rkl_advantage_std": 2.215672713518143, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2218129343409414e-07, + "loss": 0.03330949743588765, + "num_tokens": 584026539.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7958333313465118, + "rewards/gt_logging_reward/std": 0.3955939928690592, + "sampling/importance_sampling_ratio/max": 1.9509803851445515, + "sampling/importance_sampling_ratio/mean": 0.9963717103004456, + "sampling/importance_sampling_ratio/min": 0.43224813615282376, + "sampling/sampling_logp_difference/max": 0.31671832005182904, + "sampling/sampling_logp_difference/mean": 0.003914232303698858, + "step": 17850, + "step_time": 7.873290835100731, + "student/entropy": 0.1273341982314984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01694444523503383, + "completions/max_length": 479.4, + "completions/max_terminated_length": 442.9, + "completions/mean_length": 184.20000966389975, + "completions/mean_terminated_length": 178.5193364461263, + "completions/min_length": 59.43333333333333, + "completions/min_terminated_length": 59.43333333333333, + "entropy": 0.1276683614899715, + "epoch": 0.6789959366574261, + "eval/gt_acc_batch": 0.8002777914206187, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31665438413619995, + "kd/policy_loss": 0.00846880291743825, + "kd/rkl_advantage_mean": 0.7790825539578994, + "kd/rkl_advantage_p95": 5.007539200782776, + "kd/rkl_advantage_std": 2.211745300889015, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2104203850681655e-07, + "loss": 0.03387520710627238, + "num_tokens": 584985595.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8002777735392252, + "rewards/gt_logging_reward/std": 0.38736618409554163, + "sampling/importance_sampling_ratio/max": 1.995789150396983, + "sampling/importance_sampling_ratio/mean": 1.007623302936554, + "sampling/importance_sampling_ratio/min": 0.43800482352574666, + "sampling/sampling_logp_difference/max": 0.3270734906196594, + "sampling/sampling_logp_difference/mean": 0.0039473214186728, + "step": 17880, + "step_time": 7.911463196604745, + "student/entropy": 0.1276683614899715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02472222357367476, + "completions/max_length": 488.73333333333335, + "completions/max_terminated_length": 446.96666666666664, + "completions/mean_length": 187.4747319539388, + "completions/mean_terminated_length": 179.40337320963542, + "completions/min_length": 58.2, + "completions/min_terminated_length": 58.2, + "entropy": 0.12990355882793664, + "epoch": 0.6801351915847036, + "eval/gt_acc_batch": 0.8163889070351918, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.45883074402809143, + "kd/policy_loss": 0.008337611479995151, + "kd/rkl_advantage_mean": 0.812320146523416, + "kd/rkl_advantage_p95": 5.132954478263855, + "kd/rkl_advantage_std": 2.2567496170600254, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1990278357953897e-07, + "loss": 0.03335044781366984, + "num_tokens": 585963952.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8163888891537984, + "rewards/gt_logging_reward/std": 0.38127917846043907, + "sampling/importance_sampling_ratio/max": 1.9115824182828267, + "sampling/importance_sampling_ratio/mean": 0.9972885489463806, + "sampling/importance_sampling_ratio/min": 0.43443761169910433, + "sampling/sampling_logp_difference/max": 0.3232491175333659, + "sampling/sampling_logp_difference/mean": 0.00402447241358459, + "step": 17910, + "step_time": 7.961873410265738, + "student/entropy": 0.12990355882793664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556584149598, + "completions/max_length": 472.26666666666665, + "completions/max_terminated_length": 437.4, + "completions/mean_length": 186.0219528198242, + "completions/mean_terminated_length": 180.986528523763, + "completions/min_length": 50.733333333333334, + "completions/min_terminated_length": 50.733333333333334, + "entropy": 0.12586873341351748, + "epoch": 0.6812744465119812, + "eval/gt_acc_batch": 0.7975000301996867, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22059522569179535, + "kd/policy_loss": 0.008376707745871197, + "kd/rkl_advantage_mean": 0.6987958053592592, + "kd/rkl_advantage_p95": 4.8451431731383, + "kd/rkl_advantage_std": 2.1673806617657343, + "kd/ssd_loss": 0.0, + "learning_rate": 3.187635286522614e-07, + "loss": 0.03350683450698853, + "num_tokens": 586943623.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7975000003973644, + "rewards/gt_logging_reward/std": 0.393998991449674, + "sampling/importance_sampling_ratio/max": 1.8826738317807516, + "sampling/importance_sampling_ratio/mean": 0.9981965343157451, + "sampling/importance_sampling_ratio/min": 0.4577892005443573, + "sampling/sampling_logp_difference/max": 0.2787577311197917, + "sampling/sampling_logp_difference/mean": 0.0038746320564920704, + "step": 17940, + "step_time": 7.868905318831094, + "student/entropy": 0.12586873341351748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02000000107412537, + "completions/max_length": 483.96666666666664, + "completions/max_terminated_length": 431.0, + "completions/mean_length": 186.6716761271159, + "completions/mean_terminated_length": 180.1531992594401, + "completions/min_length": 51.43333333333333, + "completions/min_terminated_length": 51.43333333333333, + "entropy": 0.12898040575285752, + "epoch": 0.6824137014392587, + "eval/gt_acc_batch": 0.8141666968663533, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25929930806159973, + "kd/policy_loss": 0.008538192180761446, + "kd/rkl_advantage_mean": 0.7483343536655108, + "kd/rkl_advantage_p95": 5.078054487705231, + "kd/rkl_advantage_std": 2.2675636768341065, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1762427372498386e-07, + "loss": 0.034152770042419435, + "num_tokens": 587915505.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.814166667064031, + "rewards/gt_logging_reward/std": 0.3828497757514318, + "sampling/importance_sampling_ratio/max": 1.8483790596326193, + "sampling/importance_sampling_ratio/mean": 1.002558410167694, + "sampling/importance_sampling_ratio/min": 0.4654486685991287, + "sampling/sampling_logp_difference/max": 0.30356703599294027, + "sampling/sampling_logp_difference/mean": 0.003995450166985393, + "step": 17970, + "step_time": 7.946609368337279, + "student/entropy": 0.12898040575285752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02138889009753863, + "completions/max_length": 495.93333333333334, + "completions/max_terminated_length": 459.73333333333335, + "completions/mean_length": 187.5788996378581, + "completions/mean_terminated_length": 180.5467519124349, + "completions/min_length": 48.266666666666666, + "completions/min_terminated_length": 48.266666666666666, + "entropy": 0.13041285661359628, + "epoch": 0.6835529563665362, + "eval/gt_acc_batch": 0.807500014702479, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25505396723747253, + "kd/policy_loss": 0.008540194052814816, + "kd/rkl_advantage_mean": 0.8163311689471205, + "kd/rkl_advantage_p95": 5.127550554275513, + "kd/rkl_advantage_std": 2.2722776492436725, + "kd/ssd_loss": 0.0, + "learning_rate": 3.164850187977063e-07, + "loss": 0.03416076898574829, + "num_tokens": 588902909.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8075000007947286, + "rewards/gt_logging_reward/std": 0.3871312712629636, + "sampling/importance_sampling_ratio/max": 1.916427528858185, + "sampling/importance_sampling_ratio/mean": 0.9972895920276642, + "sampling/importance_sampling_ratio/min": 0.45244673391183216, + "sampling/sampling_logp_difference/max": 0.3035268763701121, + "sampling/sampling_logp_difference/mean": 0.003981121652759612, + "step": 18000, + "step_time": 8.257267534701775, + "student/entropy": 0.13041285661359628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779220292965, + "completions/max_length": 492.2, + "completions/max_terminated_length": 449.3666666666667, + "completions/mean_length": 185.27278645833334, + "completions/mean_terminated_length": 177.00454813639323, + "completions/min_length": 54.4, + "completions/min_terminated_length": 54.4, + "entropy": 0.13612093056241673, + "epoch": 0.6846922112938139, + "eval/gt_acc_batch": 0.7966666877269745, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3508550226688385, + "kd/policy_loss": 0.009112033044220879, + "kd/rkl_advantage_mean": 0.8423595079531272, + "kd/rkl_advantage_p95": 5.182273242870966, + "kd/rkl_advantage_std": 2.26228499909242, + "kd/ssd_loss": 0.0, + "learning_rate": 3.153457638704287e-07, + "loss": 0.036448129018147785, + "num_tokens": 589863291.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7966666698455811, + "rewards/gt_logging_reward/std": 0.3900836298863093, + "sampling/importance_sampling_ratio/max": 1.9809180736541747, + "sampling/importance_sampling_ratio/mean": 1.0028761307398477, + "sampling/importance_sampling_ratio/min": 0.4263595978418986, + "sampling/sampling_logp_difference/max": 0.31735443870226543, + "sampling/sampling_logp_difference/mean": 0.004101817108069857, + "step": 18030, + "step_time": 7.969392490602331, + "student/entropy": 0.13612093056241673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444571932157, + "completions/max_length": 482.3, + "completions/max_terminated_length": 457.5, + "completions/mean_length": 191.6047332763672, + "completions/mean_terminated_length": 183.73922729492188, + "completions/min_length": 51.43333333333333, + "completions/min_terminated_length": 51.43333333333333, + "entropy": 0.1245240855962038, + "epoch": 0.6858314662210914, + "eval/gt_acc_batch": 0.7952778081099192, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2773171663284302, + "kd/policy_loss": 0.00806598345710275, + "kd/rkl_advantage_mean": 0.769089749113967, + "kd/rkl_advantage_p95": 4.931274106105168, + "kd/rkl_advantage_std": 2.1733868777751923, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1420650894315117e-07, + "loss": 0.032263930638631186, + "num_tokens": 590860468.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777723471324, + "rewards/gt_logging_reward/std": 0.39235001703103384, + "sampling/importance_sampling_ratio/max": 1.9071621457735697, + "sampling/importance_sampling_ratio/mean": 0.9987006743748983, + "sampling/importance_sampling_ratio/min": 0.45650686224301656, + "sampling/sampling_logp_difference/max": 0.3229703962802887, + "sampling/sampling_logp_difference/mean": 0.00384234170584629, + "step": 18060, + "step_time": 8.15610914560578, + "student/entropy": 0.1245240855962038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388889787097774, + "completions/max_length": 489.4, + "completions/max_terminated_length": 447.1333333333333, + "completions/mean_length": 189.81778767903646, + "completions/mean_terminated_length": 182.89275410970052, + "completions/min_length": 50.833333333333336, + "completions/min_terminated_length": 50.833333333333336, + "entropy": 0.126194712271293, + "epoch": 0.686970721148369, + "eval/gt_acc_batch": 0.7936111350854238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3101235628128052, + "kd/policy_loss": 0.008460897182036813, + "kd/rkl_advantage_mean": 0.7577066428959369, + "kd/rkl_advantage_p95": 4.963706485430399, + "kd/rkl_advantage_std": 2.208876476685206, + "kd/ssd_loss": 0.0, + "learning_rate": 3.130672540158736e-07, + "loss": 0.033843588829040525, + "num_tokens": 591858756.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111132303874, + "rewards/gt_logging_reward/std": 0.39424096842606865, + "sampling/importance_sampling_ratio/max": 1.953086825211843, + "sampling/importance_sampling_ratio/mean": 0.9994511743386586, + "sampling/importance_sampling_ratio/min": 0.4649672736724218, + "sampling/sampling_logp_difference/max": 0.3292308966318766, + "sampling/sampling_logp_difference/mean": 0.0038776816024134555, + "step": 18090, + "step_time": 8.18072950176235, + "student/entropy": 0.126194712271293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0100000006146729, + "completions/max_length": 468.56666666666666, + "completions/max_terminated_length": 443.1, + "completions/mean_length": 179.47917785644532, + "completions/mean_terminated_length": 176.17530619303386, + "completions/min_length": 62.36666666666667, + "completions/min_terminated_length": 62.36666666666667, + "entropy": 0.13196246325969696, + "epoch": 0.6881099760756465, + "eval/gt_acc_batch": 0.8144444723924001, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23561455309391022, + "kd/policy_loss": 0.008443762305735921, + "kd/rkl_advantage_mean": 0.7640486222381393, + "kd/rkl_advantage_p95": 5.045073366165161, + "kd/rkl_advantage_std": 2.2376035432020824, + "kd/ssd_loss": 0.0, + "learning_rate": 3.11927999088596e-07, + "loss": 0.0337750514348348, + "num_tokens": 592803241.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8144444465637207, + "rewards/gt_logging_reward/std": 0.38380595048268634, + "sampling/importance_sampling_ratio/max": 1.866903058687846, + "sampling/importance_sampling_ratio/mean": 0.9974591275056203, + "sampling/importance_sampling_ratio/min": 0.4490860124429067, + "sampling/sampling_logp_difference/max": 0.3206980268160502, + "sampling/sampling_logp_difference/mean": 0.004080821884175142, + "step": 18120, + "step_time": 7.840722847765816, + "student/entropy": 0.13196246325969696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890283803145, + "completions/max_length": 482.8, + "completions/max_terminated_length": 446.23333333333335, + "completions/mean_length": 189.53834431966146, + "completions/mean_terminated_length": 182.65403696695964, + "completions/min_length": 51.266666666666666, + "completions/min_terminated_length": 51.266666666666666, + "entropy": 0.12737260311841964, + "epoch": 0.6892492310029241, + "eval/gt_acc_batch": 0.7991666853427887, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3646838963031769, + "kd/policy_loss": 0.008553746433851, + "kd/rkl_advantage_mean": 0.7762618708851127, + "kd/rkl_advantage_p95": 5.111948704719543, + "kd/rkl_advantage_std": 2.248295947909355, + "kd/ssd_loss": 0.0, + "learning_rate": 3.107887441613185e-07, + "loss": 0.034214981396993, + "num_tokens": 593786811.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7991666694482168, + "rewards/gt_logging_reward/std": 0.3915697505076726, + "sampling/importance_sampling_ratio/max": 1.948603904247284, + "sampling/importance_sampling_ratio/mean": 0.9995978891849517, + "sampling/importance_sampling_ratio/min": 0.46480485796928406, + "sampling/sampling_logp_difference/max": 0.35054067373275755, + "sampling/sampling_logp_difference/mean": 0.003951631983121236, + "step": 18150, + "step_time": 7.887799388097482, + "student/entropy": 0.12737260311841964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016944445483386517, + "completions/max_length": 482.1, + "completions/max_terminated_length": 462.93333333333334, + "completions/mean_length": 191.54806772867838, + "completions/mean_terminated_length": 186.09014841715495, + "completions/min_length": 48.833333333333336, + "completions/min_terminated_length": 48.833333333333336, + "entropy": 0.13261126037687063, + "epoch": 0.6903884859302016, + "eval/gt_acc_batch": 0.7861111263434092, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3227193057537079, + "kd/policy_loss": 0.009115590118259813, + "kd/rkl_advantage_mean": 0.6986501305053632, + "kd/rkl_advantage_p95": 4.9062960088253025, + "kd/rkl_advantage_std": 2.1996230045954386, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0964948923404094e-07, + "loss": 0.036462366580963135, + "num_tokens": 594785592.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111144224803, + "rewards/gt_logging_reward/std": 0.40417528450489043, + "sampling/importance_sampling_ratio/max": 1.9665724754333496, + "sampling/importance_sampling_ratio/mean": 1.005012478431066, + "sampling/importance_sampling_ratio/min": 0.4638899366060893, + "sampling/sampling_logp_difference/max": 0.3497488498687744, + "sampling/sampling_logp_difference/mean": 0.00411854285436372, + "step": 18180, + "step_time": 7.984861002365748, + "student/entropy": 0.13261126037687063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944445539265872, + "completions/max_length": 471.1, + "completions/max_terminated_length": 446.5, + "completions/mean_length": 192.70528818766277, + "completions/mean_terminated_length": 183.98812052408854, + "completions/min_length": 57.266666666666666, + "completions/min_terminated_length": 57.266666666666666, + "entropy": 0.1292762207488219, + "epoch": 0.6915277408574793, + "eval/gt_acc_batch": 0.7858333508173625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22560624778270721, + "kd/policy_loss": 0.008570722960090886, + "kd/rkl_advantage_mean": 0.7268507499868672, + "kd/rkl_advantage_p95": 4.936966741085053, + "kd/rkl_advantage_std": 2.1818471868832905, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0851023430676336e-07, + "loss": 0.034282898902893065, + "num_tokens": 595774907.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7858333309491475, + "rewards/gt_logging_reward/std": 0.4002948120236397, + "sampling/importance_sampling_ratio/max": 1.87258491118749, + "sampling/importance_sampling_ratio/mean": 0.9989691098531087, + "sampling/importance_sampling_ratio/min": 0.44489890038967134, + "sampling/sampling_logp_difference/max": 0.3068312029043833, + "sampling/sampling_logp_difference/mean": 0.003979794300782184, + "step": 18210, + "step_time": 7.911307927898209, + "student/entropy": 0.1292762207488219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02305555638546745, + "completions/max_length": 490.53333333333336, + "completions/max_terminated_length": 446.4, + "completions/mean_length": 187.82417704264324, + "completions/mean_terminated_length": 180.14046478271484, + "completions/min_length": 61.96666666666667, + "completions/min_terminated_length": 61.96666666666667, + "entropy": 0.1282666942725579, + "epoch": 0.6926669957847568, + "eval/gt_acc_batch": 0.8011111299196879, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26365894079208374, + "kd/policy_loss": 0.008515885472297669, + "kd/rkl_advantage_mean": 0.7651928345983227, + "kd/rkl_advantage_p95": 5.047714086373647, + "kd/rkl_advantage_std": 2.237474959095319, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0737097937948583e-07, + "loss": 0.034063541889190675, + "num_tokens": 596756042.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8011111120382944, + "rewards/gt_logging_reward/std": 0.3891572400927544, + "sampling/importance_sampling_ratio/max": 1.9272836089134215, + "sampling/importance_sampling_ratio/mean": 0.9976880351702372, + "sampling/importance_sampling_ratio/min": 0.45717196762561796, + "sampling/sampling_logp_difference/max": 0.3149083495140076, + "sampling/sampling_logp_difference/mean": 0.003947656229138374, + "step": 18240, + "step_time": 8.085744847095338, + "student/entropy": 0.1282666942725579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0269444459117949, + "completions/max_length": 495.3, + "completions/max_terminated_length": 448.8, + "completions/mean_length": 189.26639862060546, + "completions/mean_terminated_length": 180.30506591796876, + "completions/min_length": 50.6, + "completions/min_terminated_length": 50.6, + "entropy": 0.12943279383083184, + "epoch": 0.6938062507120343, + "eval/gt_acc_batch": 0.8138888875643412, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.409175306558609, + "kd/policy_loss": 0.008587579252586389, + "kd/rkl_advantage_mean": 0.8263406481749068, + "kd/rkl_advantage_p95": 5.0353517214457195, + "kd/rkl_advantage_std": 2.2363624374071756, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0623172445220825e-07, + "loss": 0.03435031970342, + "num_tokens": 597737969.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8138888895511627, + "rewards/gt_logging_reward/std": 0.3789021243651708, + "sampling/importance_sampling_ratio/max": 1.9318847378094992, + "sampling/importance_sampling_ratio/mean": 1.000699096918106, + "sampling/importance_sampling_ratio/min": 0.4502990633249283, + "sampling/sampling_logp_difference/max": 0.3312809189160665, + "sampling/sampling_logp_difference/mean": 0.003978108917362988, + "step": 18270, + "step_time": 8.075375041169657, + "student/entropy": 0.12943279383083184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223387410244, + "completions/max_length": 474.7, + "completions/max_terminated_length": 434.76666666666665, + "completions/mean_length": 185.9627888997396, + "completions/mean_terminated_length": 177.73510131835937, + "completions/min_length": 57.36666666666667, + "completions/min_terminated_length": 57.36666666666667, + "entropy": 0.1315958693002661, + "epoch": 0.6949455056393119, + "eval/gt_acc_batch": 0.7936111231644948, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2774946987628937, + "kd/policy_loss": 0.008677724758551145, + "kd/rkl_advantage_mean": 0.6989152099005878, + "kd/rkl_advantage_p95": 4.906930540005366, + "kd/rkl_advantage_std": 2.2174929777781167, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0509246952493067e-07, + "loss": 0.034710899988810225, + "num_tokens": 598717803.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111132303874, + "rewards/gt_logging_reward/std": 0.39885537773370744, + "sampling/importance_sampling_ratio/max": 1.8834928035736085, + "sampling/importance_sampling_ratio/mean": 0.9972959180672963, + "sampling/importance_sampling_ratio/min": 0.4221644302209218, + "sampling/sampling_logp_difference/max": 0.34836281538009645, + "sampling/sampling_logp_difference/mean": 0.004001338826492428, + "step": 18300, + "step_time": 8.091076636928483, + "student/entropy": 0.1315958693002661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01916666782150666, + "completions/max_length": 466.43333333333334, + "completions/max_terminated_length": 427.3666666666667, + "completions/mean_length": 184.68111928304037, + "completions/mean_terminated_length": 178.45380859375, + "completions/min_length": 53.333333333333336, + "completions/min_terminated_length": 53.333333333333336, + "entropy": 0.12554976145426433, + "epoch": 0.6960847605665894, + "eval/gt_acc_batch": 0.8102778017520904, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2703593969345093, + "kd/policy_loss": 0.008044734443925942, + "kd/rkl_advantage_mean": 0.7189149659126997, + "kd/rkl_advantage_p95": 4.897382819652558, + "kd/rkl_advantage_std": 2.1929639448722202, + "kd/ssd_loss": 0.0, + "learning_rate": 3.039532145976531e-07, + "loss": 0.03217894236246745, + "num_tokens": 599694919.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8102777798970541, + "rewards/gt_logging_reward/std": 0.38523093461990354, + "sampling/importance_sampling_ratio/max": 1.8721401969591775, + "sampling/importance_sampling_ratio/mean": 0.994475652774175, + "sampling/importance_sampling_ratio/min": 0.46803710410992305, + "sampling/sampling_logp_difference/max": 0.3130121290683746, + "sampling/sampling_logp_difference/mean": 0.003861436798858146, + "step": 18330, + "step_time": 8.048193770062062, + "student/entropy": 0.12554976145426433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112150053183, + "completions/max_length": 491.53333333333336, + "completions/max_terminated_length": 455.6333333333333, + "completions/mean_length": 193.2144551595052, + "completions/mean_terminated_length": 186.54857635498047, + "completions/min_length": 68.3, + "completions/min_terminated_length": 68.3, + "entropy": 0.12933761949340503, + "epoch": 0.697224015493867, + "eval/gt_acc_batch": 0.7947222411632537, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.5516051650047302, + "kd/policy_loss": 0.008925543913695341, + "kd/rkl_advantage_mean": 0.7979816662768523, + "kd/rkl_advantage_p95": 5.079685970147451, + "kd/rkl_advantage_std": 2.2260403553644816, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0281395967037556e-07, + "loss": 0.03570217688878377, + "num_tokens": 600690475.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222193082174, + "rewards/gt_logging_reward/std": 0.3942178308963776, + "sampling/importance_sampling_ratio/max": 1.936139424641927, + "sampling/importance_sampling_ratio/mean": 0.9973698814709981, + "sampling/importance_sampling_ratio/min": 0.4176129331191381, + "sampling/sampling_logp_difference/max": 0.3905489524205526, + "sampling/sampling_logp_difference/mean": 0.0040051479591056705, + "step": 18360, + "step_time": 8.016814453700015, + "student/entropy": 0.12933761949340503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779065072538, + "completions/max_length": 477.53333333333336, + "completions/max_terminated_length": 448.56666666666666, + "completions/mean_length": 186.31389770507812, + "completions/mean_terminated_length": 178.13377431233724, + "completions/min_length": 58.233333333333334, + "completions/min_terminated_length": 58.233333333333334, + "entropy": 0.12646811020871004, + "epoch": 0.6983632704211445, + "eval/gt_acc_batch": 0.8013889054457347, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.35066571831703186, + "kd/policy_loss": 0.008118312124861405, + "kd/rkl_advantage_mean": 0.8019596740603447, + "kd/rkl_advantage_p95": 5.090736709038416, + "kd/rkl_advantage_std": 2.251822625597318, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0167470474309797e-07, + "loss": 0.032473246256510414, + "num_tokens": 601664925.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888895511627, + "rewards/gt_logging_reward/std": 0.3846681584914525, + "sampling/importance_sampling_ratio/max": 1.8872914075851441, + "sampling/importance_sampling_ratio/mean": 1.000750050942103, + "sampling/importance_sampling_ratio/min": 0.4506725122531255, + "sampling/sampling_logp_difference/max": 0.3208147803942362, + "sampling/sampling_logp_difference/mean": 0.003907567247127494, + "step": 18390, + "step_time": 7.933137268692372, + "student/entropy": 0.12646811020871004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02472222372889519, + "completions/max_length": 498.8333333333333, + "completions/max_terminated_length": 447.03333333333336, + "completions/mean_length": 192.03473154703775, + "completions/mean_terminated_length": 183.96815643310546, + "completions/min_length": 54.86666666666667, + "completions/min_terminated_length": 54.86666666666667, + "entropy": 0.13556651708980402, + "epoch": 0.6995025253484222, + "eval/gt_acc_batch": 0.7788889169692993, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24275439977645874, + "kd/policy_loss": 0.009344894105258088, + "kd/rkl_advantage_mean": 0.9469573999444644, + "kd/rkl_advantage_p95": 5.400123810768127, + "kd/rkl_advantage_std": 2.326257519920667, + "kd/ssd_loss": 0.0, + "learning_rate": 3.0053544981582044e-07, + "loss": 0.0373795747756958, + "num_tokens": 602674386.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7788888871669769, + "rewards/gt_logging_reward/std": 0.4094826857248942, + "sampling/importance_sampling_ratio/max": 1.9177453756332397, + "sampling/importance_sampling_ratio/mean": 1.003491848707199, + "sampling/importance_sampling_ratio/min": 0.47103525400161744, + "sampling/sampling_logp_difference/max": 0.35701558788617455, + "sampling/sampling_logp_difference/mean": 0.004133848768348495, + "step": 18420, + "step_time": 8.422095310839358, + "student/entropy": 0.13556651708980402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01833333441366752, + "completions/max_length": 492.0, + "completions/max_terminated_length": 449.1666666666667, + "completions/mean_length": 184.79445241292316, + "completions/mean_terminated_length": 178.81987965901692, + "completions/min_length": 55.333333333333336, + "completions/min_terminated_length": 55.333333333333336, + "entropy": 0.12976668663322927, + "epoch": 0.7006417802756997, + "eval/gt_acc_batch": 0.8116666833559673, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.19064480066299438, + "kd/policy_loss": 0.00818750433002909, + "kd/rkl_advantage_mean": 0.8144569670160612, + "kd/rkl_advantage_p95": 5.0740978936354315, + "kd/rkl_advantage_std": 2.2395573715368906, + "kd/ssd_loss": 0.0, + "learning_rate": 2.993961948885429e-07, + "loss": 0.032750014464060465, + "num_tokens": 603643062.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8116666674613953, + "rewards/gt_logging_reward/std": 0.3838582346836726, + "sampling/importance_sampling_ratio/max": 1.8573846260706584, + "sampling/importance_sampling_ratio/mean": 1.0007722099622092, + "sampling/importance_sampling_ratio/min": 0.4489586422840754, + "sampling/sampling_logp_difference/max": 0.32277405659357705, + "sampling/sampling_logp_difference/mean": 0.0039819115307182075, + "step": 18450, + "step_time": 8.155393820236592, + "student/entropy": 0.12976668663322927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03166666865969698, + "completions/max_length": 500.46666666666664, + "completions/max_terminated_length": 455.8, + "completions/mean_length": 189.86362050374348, + "completions/mean_terminated_length": 179.49260711669922, + "completions/min_length": 52.333333333333336, + "completions/min_terminated_length": 52.333333333333336, + "entropy": 0.12812492387990157, + "epoch": 0.7017810352029773, + "eval/gt_acc_batch": 0.7822222292423249, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2484414130449295, + "kd/policy_loss": 0.00853080946059587, + "kd/rkl_advantage_mean": 0.8934882594272494, + "kd/rkl_advantage_p95": 5.1840221385161085, + "kd/rkl_advantage_std": 2.2489487419525784, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9825693996126533e-07, + "loss": 0.03412323792775472, + "num_tokens": 604632027.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7822222193082173, + "rewards/gt_logging_reward/std": 0.40344332754611967, + "sampling/importance_sampling_ratio/max": 1.9275755167007447, + "sampling/importance_sampling_ratio/mean": 0.9969559113184611, + "sampling/importance_sampling_ratio/min": 0.43207543790340425, + "sampling/sampling_logp_difference/max": 0.32221940557161965, + "sampling/sampling_logp_difference/mean": 0.0038992984918877482, + "step": 18480, + "step_time": 8.170352719763953, + "student/entropy": 0.12812492387990157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223468124866, + "completions/max_length": 493.2, + "completions/max_terminated_length": 450.2, + "completions/mean_length": 180.8238972981771, + "completions/mean_terminated_length": 174.27778065999348, + "completions/min_length": 53.333333333333336, + "completions/min_terminated_length": 53.333333333333336, + "entropy": 0.12932220920920373, + "epoch": 0.7029202901302548, + "eval/gt_acc_batch": 0.8027777949968974, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3711436986923218, + "kd/policy_loss": 0.00817838727380149, + "kd/rkl_advantage_mean": 0.7520150609935323, + "kd/rkl_advantage_p95": 4.985353815555572, + "kd/rkl_advantage_std": 2.212083871165911, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9711768503398775e-07, + "loss": 0.03271355231602987, + "num_tokens": 605592721.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8027777791023254, + "rewards/gt_logging_reward/std": 0.38588288525740305, + "sampling/importance_sampling_ratio/max": 1.7852956732114156, + "sampling/importance_sampling_ratio/mean": 0.9960488061110179, + "sampling/importance_sampling_ratio/min": 0.4664416174093882, + "sampling/sampling_logp_difference/max": 0.31902802785237633, + "sampling/sampling_logp_difference/mean": 0.003971459033588569, + "step": 18510, + "step_time": 8.14868825719847, + "student/entropy": 0.12932220920920373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055557298163572, + "completions/max_length": 492.23333333333335, + "completions/max_terminated_length": 451.46666666666664, + "completions/mean_length": 191.56251068115233, + "completions/mean_terminated_length": 182.4316899617513, + "completions/min_length": 54.86666666666667, + "completions/min_terminated_length": 54.86666666666667, + "entropy": 0.13414881179730098, + "epoch": 0.7040595450575323, + "eval/gt_acc_batch": 0.7952777922153473, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34319522976875305, + "kd/policy_loss": 0.008842326467856765, + "kd/rkl_advantage_mean": 0.8037486221641302, + "kd/rkl_advantage_p95": 5.101621939738592, + "kd/rkl_advantage_std": 2.2420857548713684, + "kd/ssd_loss": 0.0, + "learning_rate": 2.959784301067102e-07, + "loss": 0.03536931276321411, + "num_tokens": 606588770.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777743339539, + "rewards/gt_logging_reward/std": 0.3961900889873505, + "sampling/importance_sampling_ratio/max": 1.8794790426890056, + "sampling/importance_sampling_ratio/mean": 0.9971237679322561, + "sampling/importance_sampling_ratio/min": 0.45948495815197626, + "sampling/sampling_logp_difference/max": 0.3388438562552134, + "sampling/sampling_logp_difference/mean": 0.004106915928423405, + "step": 18540, + "step_time": 8.14916591863827, + "student/entropy": 0.13414881179730098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556875964005, + "completions/max_length": 483.7, + "completions/max_terminated_length": 450.5, + "completions/mean_length": 182.25167643229167, + "completions/mean_terminated_length": 175.31168263753256, + "completions/min_length": 43.266666666666666, + "completions/min_terminated_length": 43.266666666666666, + "entropy": 0.12910201642662286, + "epoch": 0.7051987999848099, + "eval/gt_acc_batch": 0.7994444727897644, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3604404628276825, + "kd/policy_loss": 0.008293058729032054, + "kd/rkl_advantage_mean": 0.8595464533815781, + "kd/rkl_advantage_p95": 5.2220585187276205, + "kd/rkl_advantage_std": 2.274797679980596, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9483917517943264e-07, + "loss": 0.03317224184672038, + "num_tokens": 607555940.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7994444449742635, + "rewards/gt_logging_reward/std": 0.3926816691954931, + "sampling/importance_sampling_ratio/max": 1.836533002058665, + "sampling/importance_sampling_ratio/mean": 1.0025392770767212, + "sampling/importance_sampling_ratio/min": 0.408458536863327, + "sampling/sampling_logp_difference/max": 0.32733068664868675, + "sampling/sampling_logp_difference/mean": 0.003943121107295155, + "step": 18570, + "step_time": 8.205209675896914, + "student/entropy": 0.12910201642662286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02305555666486422, + "completions/max_length": 490.1333333333333, + "completions/max_terminated_length": 452.1333333333333, + "completions/mean_length": 184.77862091064452, + "completions/mean_terminated_length": 176.99105682373047, + "completions/min_length": 57.4, + "completions/min_terminated_length": 57.4, + "entropy": 0.1270523158212503, + "epoch": 0.7063380549120875, + "eval/gt_acc_batch": 0.8108333508173625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.38210341334342957, + "kd/policy_loss": 0.008407066892444467, + "kd/rkl_advantage_mean": 0.792829422528545, + "kd/rkl_advantage_p95": 5.054631618658702, + "kd/rkl_advantage_std": 2.22312508225441, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9369992025215506e-07, + "loss": 0.03362826903661092, + "num_tokens": 608522495.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8108333289623261, + "rewards/gt_logging_reward/std": 0.3892966320117315, + "sampling/importance_sampling_ratio/max": 1.9868064363797506, + "sampling/importance_sampling_ratio/mean": 1.001477736234665, + "sampling/importance_sampling_ratio/min": 0.4532947967449824, + "sampling/sampling_logp_difference/max": 0.31704949140548705, + "sampling/sampling_logp_difference/mean": 0.003910982678644359, + "step": 18600, + "step_time": 8.06083397672628, + "student/entropy": 0.1270523158212503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03111111276472608, + "completions/max_length": 492.1, + "completions/max_terminated_length": 433.73333333333335, + "completions/mean_length": 185.83334401448568, + "completions/mean_terminated_length": 175.3925013224284, + "completions/min_length": 56.333333333333336, + "completions/min_terminated_length": 56.333333333333336, + "entropy": 0.12957003855456908, + "epoch": 0.7074773098393651, + "eval/gt_acc_batch": 0.8138889054457347, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30980634689331055, + "kd/policy_loss": 0.008473196000947307, + "kd/rkl_advantage_mean": 0.8185330686469873, + "kd/rkl_advantage_p95": 5.123369429508845, + "kd/rkl_advantage_std": 2.2605281730492908, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9256066532487747e-07, + "loss": 0.03389278650283813, + "num_tokens": 609487983.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8138888935248058, + "rewards/gt_logging_reward/std": 0.3765319590767225, + "sampling/importance_sampling_ratio/max": 1.911739722887675, + "sampling/importance_sampling_ratio/mean": 0.9965421557426453, + "sampling/importance_sampling_ratio/min": 0.4455436944961548, + "sampling/sampling_logp_difference/max": 0.3370644966761271, + "sampling/sampling_logp_difference/mean": 0.003933078943130871, + "step": 18630, + "step_time": 8.13633809129921, + "student/entropy": 0.12957003855456908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014722223207354546, + "completions/max_length": 480.93333333333334, + "completions/max_terminated_length": 439.1666666666667, + "completions/mean_length": 187.73000946044922, + "completions/mean_terminated_length": 182.9732630411784, + "completions/min_length": 54.8, + "completions/min_terminated_length": 54.8, + "entropy": 0.13046239241957663, + "epoch": 0.7086165647666426, + "eval/gt_acc_batch": 0.7822222371896108, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33178022503852844, + "kd/policy_loss": 0.008701390627538785, + "kd/rkl_advantage_mean": 0.8307648420333862, + "kd/rkl_advantage_p95": 5.046725553274155, + "kd/rkl_advantage_std": 2.2208530684312184, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9142141039759994e-07, + "loss": 0.03480556011199951, + "num_tokens": 610465251.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7822222193082173, + "rewards/gt_logging_reward/std": 0.4049077779054642, + "sampling/importance_sampling_ratio/max": 2.023377617200216, + "sampling/importance_sampling_ratio/mean": 1.003094643354416, + "sampling/importance_sampling_ratio/min": 0.47097368637720743, + "sampling/sampling_logp_difference/max": 0.34797490934530895, + "sampling/sampling_logp_difference/mean": 0.003945659101009369, + "step": 18660, + "step_time": 8.05546142287009, + "student/entropy": 0.13046239241957663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112305273613, + "completions/max_length": 493.53333333333336, + "completions/max_terminated_length": 433.5, + "completions/mean_length": 177.9100102742513, + "completions/mean_terminated_length": 170.84434509277344, + "completions/min_length": 59.43333333333333, + "completions/min_terminated_length": 59.43333333333333, + "entropy": 0.12462754286825657, + "epoch": 0.7097558196939202, + "eval/gt_acc_batch": 0.8141666849454244, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2696957588195801, + "kd/policy_loss": 0.008103168156230823, + "kd/rkl_advantage_mean": 0.790746673506995, + "kd/rkl_advantage_p95": 5.004683154821396, + "kd/rkl_advantage_std": 2.228380317489306, + "kd/ssd_loss": 0.0, + "learning_rate": 2.902821554703224e-07, + "loss": 0.03241267204284668, + "num_tokens": 611399623.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8141666650772095, + "rewards/gt_logging_reward/std": 0.3812244971593221, + "sampling/importance_sampling_ratio/max": 1.9128605365753173, + "sampling/importance_sampling_ratio/mean": 1.0031359374523163, + "sampling/importance_sampling_ratio/min": 0.4329400186737378, + "sampling/sampling_logp_difference/max": 0.31673192779223125, + "sampling/sampling_logp_difference/mean": 0.0038473621166000764, + "step": 18690, + "step_time": 7.918726723168705, + "student/entropy": 0.12462754286825657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01333333421498537, + "completions/max_length": 469.8, + "completions/max_terminated_length": 445.73333333333335, + "completions/mean_length": 183.21084238688152, + "completions/mean_terminated_length": 178.8848424275716, + "completions/min_length": 62.6, + "completions/min_terminated_length": 62.6, + "entropy": 0.12514458366980155, + "epoch": 0.7108950746211977, + "eval/gt_acc_batch": 0.7986111283302307, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1981181800365448, + "kd/policy_loss": 0.008164219639729709, + "kd/rkl_advantage_mean": 0.6318269823988278, + "kd/rkl_advantage_p95": 4.650505977869034, + "kd/rkl_advantage_std": 2.136773830652237, + "kd/ssd_loss": 0.0, + "learning_rate": 2.8914290054304483e-07, + "loss": 0.032656880219777425, + "num_tokens": 612357110.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111164093017, + "rewards/gt_logging_reward/std": 0.39082243541876477, + "sampling/importance_sampling_ratio/max": 1.8958666284879049, + "sampling/importance_sampling_ratio/mean": 0.997365798552831, + "sampling/importance_sampling_ratio/min": 0.4414369732141495, + "sampling/sampling_logp_difference/max": 0.3165182371934255, + "sampling/sampling_logp_difference/mean": 0.0038914022501558066, + "step": 18720, + "step_time": 7.783124997171884, + "student/entropy": 0.12514458366980155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02305555666486422, + "completions/max_length": 492.26666666666665, + "completions/max_terminated_length": 454.76666666666665, + "completions/mean_length": 180.53695271809895, + "completions/mean_terminated_length": 172.75641276041668, + "completions/min_length": 45.1, + "completions/min_terminated_length": 45.1, + "entropy": 0.13477735910564662, + "epoch": 0.7120343295484753, + "eval/gt_acc_batch": 0.8066666901111603, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3227863609790802, + "kd/policy_loss": 0.008505434593341002, + "kd/rkl_advantage_mean": 0.8152099175999562, + "kd/rkl_advantage_p95": 5.114200355609258, + "kd/rkl_advantage_std": 2.284795719385147, + "kd/ssd_loss": 0.0, + "learning_rate": 2.880036456157673e-07, + "loss": 0.03402174313863118, + "num_tokens": 613318547.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8066666702429454, + "rewards/gt_logging_reward/std": 0.39032358129819233, + "sampling/importance_sampling_ratio/max": 1.8816378394762674, + "sampling/importance_sampling_ratio/mean": 0.9994307418664297, + "sampling/importance_sampling_ratio/min": 0.45114853531122207, + "sampling/sampling_logp_difference/max": 0.292666886250178, + "sampling/sampling_logp_difference/mean": 0.0038967344366634886, + "step": 18750, + "step_time": 8.062442385536269, + "student/entropy": 0.13477735910564662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333454405268, + "completions/max_length": 472.9, + "completions/max_terminated_length": 448.1333333333333, + "completions/mean_length": 188.7452865600586, + "completions/mean_terminated_length": 182.1093729654948, + "completions/min_length": 56.2, + "completions/min_terminated_length": 56.2, + "entropy": 0.1276249962548415, + "epoch": 0.7131735844757529, + "eval/gt_acc_batch": 0.7980555812517802, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27223095297813416, + "kd/policy_loss": 0.00844787698976385, + "kd/rkl_advantage_mean": 0.7390564364691575, + "kd/rkl_advantage_p95": 4.908010822534561, + "kd/rkl_advantage_std": 2.1594148268302282, + "kd/ssd_loss": 0.0, + "learning_rate": 2.868643906884897e-07, + "loss": 0.033791510264078777, + "num_tokens": 614296246.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7980555514494578, + "rewards/gt_logging_reward/std": 0.3897443945209185, + "sampling/importance_sampling_ratio/max": 1.9309635996818542, + "sampling/importance_sampling_ratio/mean": 1.003533265988032, + "sampling/importance_sampling_ratio/min": 0.4657454987366994, + "sampling/sampling_logp_difference/max": 0.3028684695561727, + "sampling/sampling_logp_difference/mean": 0.003919799307671686, + "step": 18780, + "step_time": 7.997021963363902, + "student/entropy": 0.1276249962548415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02694444606701533, + "completions/max_length": 485.7, + "completions/max_terminated_length": 456.1333333333333, + "completions/mean_length": 190.79973297119142, + "completions/mean_terminated_length": 182.16696370442708, + "completions/min_length": 47.833333333333336, + "completions/min_terminated_length": 47.833333333333336, + "entropy": 0.13260359621296328, + "epoch": 0.7143128394030304, + "eval/gt_acc_batch": 0.7833333532015483, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2590438425540924, + "kd/policy_loss": 0.008870318803625802, + "kd/rkl_advantage_mean": 0.7966099987427394, + "kd/rkl_advantage_p95": 5.080254073937734, + "kd/rkl_advantage_std": 2.2536922067403795, + "kd/ssd_loss": 0.0, + "learning_rate": 2.8572513576121214e-07, + "loss": 0.03548127810160319, + "num_tokens": 615299197.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7833333293596904, + "rewards/gt_logging_reward/std": 0.4032714645067851, + "sampling/importance_sampling_ratio/max": 1.98001473347346, + "sampling/importance_sampling_ratio/mean": 1.003070205450058, + "sampling/importance_sampling_ratio/min": 0.462103167672952, + "sampling/sampling_logp_difference/max": 0.34302109479904175, + "sampling/sampling_logp_difference/mean": 0.004037820198573172, + "step": 18810, + "step_time": 8.259865591799219, + "student/entropy": 0.13260359621296328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03583333566784859, + "completions/max_length": 505.96666666666664, + "completions/max_terminated_length": 458.8666666666667, + "completions/mean_length": 192.21556599934897, + "completions/mean_terminated_length": 180.47741139729817, + "completions/min_length": 55.3, + "completions/min_terminated_length": 55.3, + "entropy": 0.12838153758396706, + "epoch": 0.715452094330308, + "eval/gt_acc_batch": 0.799444454908371, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3584807813167572, + "kd/policy_loss": 0.008528082284222667, + "kd/rkl_advantage_mean": 0.7936316055556139, + "kd/rkl_advantage_p95": 5.129116360346476, + "kd/rkl_advantage_std": 2.2658176998297375, + "kd/ssd_loss": 0.0, + "learning_rate": 2.845858808339346e-07, + "loss": 0.03411232630411784, + "num_tokens": 616289661.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.799444442987442, + "rewards/gt_logging_reward/std": 0.3923456440369288, + "sampling/importance_sampling_ratio/max": 1.9391244411468507, + "sampling/importance_sampling_ratio/mean": 0.9995732545852661, + "sampling/importance_sampling_ratio/min": 0.43518434166908265, + "sampling/sampling_logp_difference/max": 0.325477659702301, + "sampling/sampling_logp_difference/mean": 0.003898715685742597, + "step": 18840, + "step_time": 8.2121027410952, + "student/entropy": 0.12838153758396706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02388889022792379, + "completions/max_length": 489.43333333333334, + "completions/max_terminated_length": 438.0, + "completions/mean_length": 189.53973185221355, + "completions/mean_terminated_length": 181.6225336710612, + "completions/min_length": 56.1, + "completions/min_terminated_length": 56.1, + "entropy": 0.12492978939165672, + "epoch": 0.7165913492575855, + "eval/gt_acc_batch": 0.8058333535989125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3185071349143982, + "kd/policy_loss": 0.007842431634586925, + "kd/rkl_advantage_mean": 0.8254496946930885, + "kd/rkl_advantage_p95": 5.119173242648443, + "kd/rkl_advantage_std": 2.2329300304253894, + "kd/ssd_loss": 0.0, + "learning_rate": 2.83446625906657e-07, + "loss": 0.03136972188949585, + "num_tokens": 617275940.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8058333377043406, + "rewards/gt_logging_reward/std": 0.3841863547762235, + "sampling/importance_sampling_ratio/max": 1.9304044922192891, + "sampling/importance_sampling_ratio/mean": 1.0006747742493947, + "sampling/importance_sampling_ratio/min": 0.4503905822833379, + "sampling/sampling_logp_difference/max": 0.3059377352396647, + "sampling/sampling_logp_difference/mean": 0.003828979500879844, + "step": 18870, + "step_time": 8.13756590153207, + "student/entropy": 0.12492978939165672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777779307216407, + "completions/max_length": 487.73333333333335, + "completions/max_terminated_length": 453.53333333333336, + "completions/mean_length": 186.90306498209637, + "completions/mean_terminated_length": 179.29770100911458, + "completions/min_length": 55.733333333333334, + "completions/min_terminated_length": 55.733333333333334, + "entropy": 0.12325595089544852, + "epoch": 0.7177306041848631, + "eval/gt_acc_batch": 0.8061111450195313, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25789549946784973, + "kd/policy_loss": 0.008228989958297461, + "kd/rkl_advantage_mean": 0.7814322414807975, + "kd/rkl_advantage_p95": 5.042327606678009, + "kd/rkl_advantage_std": 2.2370506087938944, + "kd/ssd_loss": 0.0, + "learning_rate": 2.8230737097937944e-07, + "loss": 0.03291596372922261, + "num_tokens": 618255327.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8061111092567443, + "rewards/gt_logging_reward/std": 0.3886197497447332, + "sampling/importance_sampling_ratio/max": 1.8504010756810507, + "sampling/importance_sampling_ratio/mean": 0.9934372584025065, + "sampling/importance_sampling_ratio/min": 0.43379037578900653, + "sampling/sampling_logp_difference/max": 0.33207580049832663, + "sampling/sampling_logp_difference/mean": 0.00377567324321717, + "step": 18900, + "step_time": 8.166459698903298, + "student/entropy": 0.12325595089544852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026666668492058912, + "completions/max_length": 486.6, + "completions/max_terminated_length": 453.2, + "completions/mean_length": 189.51862080891928, + "completions/mean_terminated_length": 180.69070536295573, + "completions/min_length": 48.233333333333334, + "completions/min_terminated_length": 48.233333333333334, + "entropy": 0.13149923322101434, + "epoch": 0.7188698591121406, + "eval/gt_acc_batch": 0.7886111438274384, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31630635261535645, + "kd/policy_loss": 0.00903317701498357, + "kd/rkl_advantage_mean": 0.7808940712362528, + "kd/rkl_advantage_p95": 5.026230070988337, + "kd/rkl_advantage_std": 2.2258971919616064, + "kd/ssd_loss": 0.0, + "learning_rate": 2.811681160521019e-07, + "loss": 0.03613270918528239, + "num_tokens": 619246842.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7886111080646515, + "rewards/gt_logging_reward/std": 0.39939035177230836, + "sampling/importance_sampling_ratio/max": 1.9848928292592367, + "sampling/importance_sampling_ratio/mean": 1.004356865088145, + "sampling/importance_sampling_ratio/min": 0.4334350248177846, + "sampling/sampling_logp_difference/max": 0.32934614419937136, + "sampling/sampling_logp_difference/mean": 0.003987930668517947, + "step": 18930, + "step_time": 8.205049830965192, + "student/entropy": 0.13149923322101434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334835867087, + "completions/max_length": 492.4, + "completions/max_terminated_length": 451.03333333333336, + "completions/mean_length": 189.304177347819, + "completions/mean_terminated_length": 180.7540730794271, + "completions/min_length": 57.833333333333336, + "completions/min_terminated_length": 57.833333333333336, + "entropy": 0.12802238712708155, + "epoch": 0.7200091140394183, + "eval/gt_acc_batch": 0.8077777942021688, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2956252694129944, + "kd/policy_loss": 0.00838618869893253, + "kd/rkl_advantage_mean": 0.825469506594042, + "kd/rkl_advantage_p95": 5.062785450617472, + "kd/rkl_advantage_std": 2.2340629398822784, + "kd/ssd_loss": 0.0, + "learning_rate": 2.800288611248244e-07, + "loss": 0.03354475498199463, + "num_tokens": 620224449.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8077777743339538, + "rewards/gt_logging_reward/std": 0.3875200092792511, + "sampling/importance_sampling_ratio/max": 1.9556353489557903, + "sampling/importance_sampling_ratio/mean": 1.0024316608905792, + "sampling/importance_sampling_ratio/min": 0.45567113558451333, + "sampling/sampling_logp_difference/max": 0.3246176858743032, + "sampling/sampling_logp_difference/mean": 0.0039634844986721875, + "step": 18960, + "step_time": 7.991382958001728, + "student/entropy": 0.12802238712708155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778680125872, + "completions/max_length": 485.6333333333333, + "completions/max_terminated_length": 438.96666666666664, + "completions/mean_length": 184.6269536336263, + "completions/mean_terminated_length": 177.86489969889323, + "completions/min_length": 54.36666666666667, + "completions/min_terminated_length": 54.36666666666667, + "entropy": 0.12631419083724418, + "epoch": 0.7211483689666958, + "eval/gt_acc_batch": 0.8161111255486806, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26834091544151306, + "kd/policy_loss": 0.0079883647384122, + "kd/rkl_advantage_mean": 0.7527104582637548, + "kd/rkl_advantage_p95": 4.93022270600001, + "kd/rkl_advantage_std": 2.200076950589816, + "kd/ssd_loss": 0.0, + "learning_rate": 2.788896061975468e-07, + "loss": 0.031953457991282144, + "num_tokens": 621186274.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8161111156145732, + "rewards/gt_logging_reward/std": 0.38258205552895863, + "sampling/importance_sampling_ratio/max": 1.8114596803983052, + "sampling/importance_sampling_ratio/mean": 0.9981013258298238, + "sampling/importance_sampling_ratio/min": 0.47251108090082805, + "sampling/sampling_logp_difference/max": 0.30775660077730815, + "sampling/sampling_logp_difference/mean": 0.0038826532584304613, + "step": 18990, + "step_time": 8.040193271801884, + "student/entropy": 0.12631419083724418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112497746944, + "completions/max_length": 498.96666666666664, + "completions/max_terminated_length": 461.8333333333333, + "completions/mean_length": 190.69167734781902, + "completions/mean_terminated_length": 182.9242935180664, + "completions/min_length": 56.96666666666667, + "completions/min_terminated_length": 56.96666666666667, + "entropy": 0.12708753732343514, + "epoch": 0.7222876238939734, + "eval/gt_acc_batch": 0.797222246726354, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24790042638778687, + "kd/policy_loss": 0.00854038818506524, + "kd/rkl_advantage_mean": 0.7310247719443093, + "kd/rkl_advantage_p95": 4.959832914670309, + "kd/rkl_advantage_std": 2.200832423567772, + "kd/ssd_loss": 0.0, + "learning_rate": 2.777503512702692e-07, + "loss": 0.03416155576705933, + "num_tokens": 622180892.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7972222228844961, + "rewards/gt_logging_reward/std": 0.39433205425739287, + "sampling/importance_sampling_ratio/max": 1.904612406094869, + "sampling/importance_sampling_ratio/mean": 1.004942035675049, + "sampling/importance_sampling_ratio/min": 0.4705708851416906, + "sampling/sampling_logp_difference/max": 0.3227923889954885, + "sampling/sampling_logp_difference/mean": 0.0038692069705575705, + "step": 19020, + "step_time": 8.128762692765061, + "student/entropy": 0.12708753732343514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667641450962, + "completions/max_length": 487.96666666666664, + "completions/max_terminated_length": 454.6, + "completions/mean_length": 191.6055658976237, + "completions/mean_terminated_length": 184.59213460286458, + "completions/min_length": 46.7, + "completions/min_terminated_length": 46.7, + "entropy": 0.13156115698317686, + "epoch": 0.7234268788212509, + "eval/gt_acc_batch": 0.783611128727595, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27806273102760315, + "kd/policy_loss": 0.008740294222176696, + "kd/rkl_advantage_mean": 0.8223446052211026, + "kd/rkl_advantage_p95": 5.094746828079224, + "kd/rkl_advantage_std": 2.2412268936634065, + "kd/ssd_loss": 0.0, + "learning_rate": 2.766110963429917e-07, + "loss": 0.0349611759185791, + "num_tokens": 623187664.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7836111148198446, + "rewards/gt_logging_reward/std": 0.40551029741764066, + "sampling/importance_sampling_ratio/max": 1.8870457649230956, + "sampling/importance_sampling_ratio/mean": 1.0015461405118307, + "sampling/importance_sampling_ratio/min": 0.43552455902099607, + "sampling/sampling_logp_difference/max": 0.3065626154343287, + "sampling/sampling_logp_difference/mean": 0.00402178760462751, + "step": 19050, + "step_time": 8.26893453420295, + "student/entropy": 0.13156115698317686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778959522645, + "completions/max_length": 490.3, + "completions/max_terminated_length": 455.3666666666667, + "completions/mean_length": 186.85306498209636, + "completions/mean_terminated_length": 180.2141881306966, + "completions/min_length": 53.333333333333336, + "completions/min_terminated_length": 53.333333333333336, + "entropy": 0.13149737535665432, + "epoch": 0.7245661337485284, + "eval/gt_acc_batch": 0.8100000202655793, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.36305731534957886, + "kd/policy_loss": 0.00878937779343687, + "kd/rkl_advantage_mean": 0.7868192375792812, + "kd/rkl_advantage_p95": 4.974683066209157, + "kd/rkl_advantage_std": 2.2175570358832677, + "kd/ssd_loss": 0.0, + "learning_rate": 2.754718414157141e-07, + "loss": 0.035157513618469236, + "num_tokens": 624165951.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8100000023841858, + "rewards/gt_logging_reward/std": 0.385286819934845, + "sampling/importance_sampling_ratio/max": 2.0065584341684977, + "sampling/importance_sampling_ratio/mean": 1.0079496184984842, + "sampling/importance_sampling_ratio/min": 0.477982164422671, + "sampling/sampling_logp_difference/max": 0.3064212123552958, + "sampling/sampling_logp_difference/mean": 0.004017945557522277, + "step": 19080, + "step_time": 8.072730235867978, + "student/entropy": 0.13149737535665432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666816920042, + "completions/max_length": 479.8666666666667, + "completions/max_terminated_length": 428.3, + "completions/mean_length": 184.008620707194, + "completions/mean_terminated_length": 176.95087381998698, + "completions/min_length": 58.233333333333334, + "completions/min_terminated_length": 58.233333333333334, + "entropy": 0.13473638215412695, + "epoch": 0.725705388675806, + "eval/gt_acc_batch": 0.7941666801770528, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2604370415210724, + "kd/policy_loss": 0.008903717984988664, + "kd/rkl_advantage_mean": 0.7919267082897326, + "kd/rkl_advantage_p95": 5.022930471102397, + "kd/rkl_advantage_std": 2.1952030271291734, + "kd/ssd_loss": 0.0, + "learning_rate": 2.743325864884365e-07, + "loss": 0.03561487197875977, + "num_tokens": 625126926.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7941666682561238, + "rewards/gt_logging_reward/std": 0.3958113143841426, + "sampling/importance_sampling_ratio/max": 1.9125473300615947, + "sampling/importance_sampling_ratio/mean": 1.0062893827756245, + "sampling/importance_sampling_ratio/min": 0.4518770967920621, + "sampling/sampling_logp_difference/max": 0.31133819818496705, + "sampling/sampling_logp_difference/mean": 0.004126173739011089, + "step": 19110, + "step_time": 7.9330710192007245, + "student/entropy": 0.13473638215412695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223474333683, + "completions/max_length": 486.3666666666667, + "completions/max_terminated_length": 447.03333333333336, + "completions/mean_length": 187.34500986735026, + "completions/mean_terminated_length": 180.0403055826823, + "completions/min_length": 52.63333333333333, + "completions/min_terminated_length": 52.63333333333333, + "entropy": 0.12583524299164614, + "epoch": 0.7268446436030835, + "eval/gt_acc_batch": 0.8100000242392222, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.277184396982193, + "kd/policy_loss": 0.0081218947423622, + "kd/rkl_advantage_mean": 0.7302921103934447, + "kd/rkl_advantage_p95": 4.979685960213343, + "kd/rkl_advantage_std": 2.2320472339789075, + "kd/ssd_loss": 0.0, + "learning_rate": 2.73193331561159e-07, + "loss": 0.03248758316040039, + "num_tokens": 626109848.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8100000023841858, + "rewards/gt_logging_reward/std": 0.38400597472985587, + "sampling/importance_sampling_ratio/max": 1.8010416547457377, + "sampling/importance_sampling_ratio/mean": 1.0017314811547597, + "sampling/importance_sampling_ratio/min": 0.4824485868215561, + "sampling/sampling_logp_difference/max": 0.33777461846669515, + "sampling/sampling_logp_difference/mean": 0.003825637154902021, + "step": 19140, + "step_time": 8.055420617297447, + "student/entropy": 0.12583524299164614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667573153974, + "completions/max_length": 473.06666666666666, + "completions/max_terminated_length": 434.6, + "completions/mean_length": 180.18889872233072, + "completions/mean_terminated_length": 173.82629801432293, + "completions/min_length": 52.266666666666666, + "completions/min_terminated_length": 52.266666666666666, + "entropy": 0.12961737333486478, + "epoch": 0.7279838985303612, + "eval/gt_acc_batch": 0.8111111323038737, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3378889560699463, + "kd/policy_loss": 0.008563372966212532, + "kd/rkl_advantage_mean": 0.7370686170955499, + "kd/rkl_advantage_p95": 4.972315973043441, + "kd/rkl_advantage_std": 2.218448344866435, + "kd/ssd_loss": 0.0, + "learning_rate": 2.720540766338814e-07, + "loss": 0.03425349394480388, + "num_tokens": 627058544.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8111111124356588, + "rewards/gt_logging_reward/std": 0.38053431262572607, + "sampling/importance_sampling_ratio/max": 1.8748007933298747, + "sampling/importance_sampling_ratio/mean": 1.0007248123486836, + "sampling/importance_sampling_ratio/min": 0.43037160535653435, + "sampling/sampling_logp_difference/max": 0.32828500668207805, + "sampling/sampling_logp_difference/mean": 0.003980298472257952, + "step": 19170, + "step_time": 7.904598762301612, + "student/entropy": 0.12961737333486478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017500000881652038, + "completions/max_length": 487.1666666666667, + "completions/max_terminated_length": 442.6333333333333, + "completions/mean_length": 189.279177347819, + "completions/mean_terminated_length": 183.5934844970703, + "completions/min_length": 55.5, + "completions/min_terminated_length": 55.5, + "entropy": 0.12582531521717708, + "epoch": 0.7291231534576387, + "eval/gt_acc_batch": 0.8188888986905416, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30019691586494446, + "kd/policy_loss": 0.008307304477784782, + "kd/rkl_advantage_mean": 0.7365399579207103, + "kd/rkl_advantage_p95": 4.931020029385885, + "kd/rkl_advantage_std": 2.18638577858607, + "kd/ssd_loss": 0.0, + "learning_rate": 2.7091482170660383e-07, + "loss": 0.033229213953018186, + "num_tokens": 628033277.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8188888907432557, + "rewards/gt_logging_reward/std": 0.3786193976799647, + "sampling/importance_sampling_ratio/max": 1.8907293796539306, + "sampling/importance_sampling_ratio/mean": 1.0033932030200958, + "sampling/importance_sampling_ratio/min": 0.44517294863859813, + "sampling/sampling_logp_difference/max": 0.33348788420359293, + "sampling/sampling_logp_difference/mean": 0.0038342515705153345, + "step": 19200, + "step_time": 7.915141133273331, + "student/entropy": 0.12582531521717708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02183908179145435, + "completions/max_length": 467.0, + "completions/max_terminated_length": 423.51724137931035, + "completions/mean_length": 183.6396642224542, + "completions/mean_terminated_length": 176.42342297784214, + "completions/min_length": 56.55172413793103, + "completions/min_terminated_length": 56.55172413793103, + "entropy": 0.13032252219473495, + "epoch": 0.7302624083849163, + "eval/gt_acc_batch": 0.8000000221975918, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34196704626083374, + "kd/policy_loss": 0.008231781555564496, + "kd/rkl_advantage_mean": 0.806012436747551, + "kd/rkl_advantage_p95": 4.985342181962112, + "kd/rkl_advantage_std": 2.1841452573907785, + "kd/ssd_loss": 0.0, + "learning_rate": 2.6977556677932636e-07, + "loss": 0.031829559803009035, + "num_tokens": 628959751.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7999999995889335, + "rewards/gt_logging_reward/std": 0.3913719690051572, + "sampling/importance_sampling_ratio/max": 1.979108530899574, + "sampling/importance_sampling_ratio/mean": 1.0015781316263923, + "sampling/importance_sampling_ratio/min": 0.4514777095153414, + "sampling/sampling_logp_difference/max": 0.3490932193295709, + "sampling/sampling_logp_difference/mean": 0.004045210329109225, + "step": 19230, + "step_time": 7.560036902467255, + "student/entropy": 0.13032252219473495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030555557242284218, + "completions/max_length": 497.1, + "completions/max_terminated_length": 450.76666666666665, + "completions/mean_length": 188.30167592366536, + "completions/mean_terminated_length": 178.19794413248698, + "completions/min_length": 54.7, + "completions/min_terminated_length": 54.7, + "entropy": 0.13210447101543346, + "epoch": 0.7314016633121938, + "eval/gt_acc_batch": 0.8083333472410837, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.317132830619812, + "kd/policy_loss": 0.008624310583885138, + "kd/rkl_advantage_mean": 0.8453960252304872, + "kd/rkl_advantage_p95": 5.1813263416290285, + "kd/rkl_advantage_std": 2.283019922176997, + "kd/ssd_loss": 0.0, + "learning_rate": 2.6863631185204877e-07, + "loss": 0.03449724515279134, + "num_tokens": 629943533.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8083333392937978, + "rewards/gt_logging_reward/std": 0.38801578680674237, + "sampling/importance_sampling_ratio/max": 1.8775590419769288, + "sampling/importance_sampling_ratio/mean": 1.0010281205177307, + "sampling/importance_sampling_ratio/min": 0.45583374400933585, + "sampling/sampling_logp_difference/max": 0.304938202102979, + "sampling/sampling_logp_difference/mean": 0.003977814138246079, + "step": 19260, + "step_time": 8.143341417360352, + "student/entropy": 0.13210447101543346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02250000120451053, + "completions/max_length": 494.56666666666666, + "completions/max_terminated_length": 450.73333333333335, + "completions/mean_length": 189.23750864664714, + "completions/mean_terminated_length": 182.0042526245117, + "completions/min_length": 58.1, + "completions/min_terminated_length": 58.1, + "entropy": 0.12999957849582036, + "epoch": 0.7325409182394714, + "eval/gt_acc_batch": 0.8072222451368968, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2692919373512268, + "kd/policy_loss": 0.008633186602188895, + "kd/rkl_advantage_mean": 0.7971085583791137, + "kd/rkl_advantage_p95": 5.064895957708359, + "kd/rkl_advantage_std": 2.224267645676931, + "kd/ssd_loss": 0.0, + "learning_rate": 2.674970569247712e-07, + "loss": 0.03453274965286255, + "num_tokens": 630926788.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222272555033, + "rewards/gt_logging_reward/std": 0.3865293741226196, + "sampling/importance_sampling_ratio/max": 1.9388410170873007, + "sampling/importance_sampling_ratio/mean": 0.999427487452825, + "sampling/importance_sampling_ratio/min": 0.43871418833732606, + "sampling/sampling_logp_difference/max": 0.3171274503072103, + "sampling/sampling_logp_difference/mean": 0.003936924366280436, + "step": 19290, + "step_time": 8.110810611000245, + "student/entropy": 0.12999957849582036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030833334724108378, + "completions/max_length": 480.3666666666667, + "completions/max_terminated_length": 451.9, + "completions/mean_length": 191.11278737386067, + "completions/mean_terminated_length": 181.02879842122397, + "completions/min_length": 56.46666666666667, + "completions/min_terminated_length": 56.46666666666667, + "entropy": 0.13327580944945414, + "epoch": 0.7336801731667489, + "eval/gt_acc_batch": 0.7813889046510061, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.45897042751312256, + "kd/policy_loss": 0.00898876841335247, + "kd/rkl_advantage_mean": 0.7612913464506467, + "kd/rkl_advantage_p95": 4.937490085760753, + "kd/rkl_advantage_std": 2.1859763006369275, + "kd/ssd_loss": 0.0, + "learning_rate": 2.6635780199749366e-07, + "loss": 0.03595507542292277, + "num_tokens": 631923970.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7813888867696126, + "rewards/gt_logging_reward/std": 0.40441614935795467, + "sampling/importance_sampling_ratio/max": 1.8986808816591898, + "sampling/importance_sampling_ratio/mean": 0.9979060252507528, + "sampling/importance_sampling_ratio/min": 0.3856195519367854, + "sampling/sampling_logp_difference/max": 0.3505401690800985, + "sampling/sampling_logp_difference/mean": 0.0040578966417039435, + "step": 19320, + "step_time": 8.161738468461165, + "student/entropy": 0.13327580944945414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334860702357, + "completions/max_length": 486.8, + "completions/max_terminated_length": 464.4, + "completions/mean_length": 185.22834370930988, + "completions/mean_terminated_length": 177.51339060465494, + "completions/min_length": 50.733333333333334, + "completions/min_terminated_length": 50.733333333333334, + "entropy": 0.13296963156511385, + "epoch": 0.7348194280940266, + "eval/gt_acc_batch": 0.8008333504199981, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3559264838695526, + "kd/policy_loss": 0.008843824835882213, + "kd/rkl_advantage_mean": 0.8328322950129707, + "kd/rkl_advantage_p95": 5.180996797482172, + "kd/rkl_advantage_std": 2.2722189724445343, + "kd/ssd_loss": 0.0, + "learning_rate": 2.652185470702161e-07, + "loss": 0.035375301043192545, + "num_tokens": 632909160.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333345254263, + "rewards/gt_logging_reward/std": 0.39144307176272075, + "sampling/importance_sampling_ratio/max": 1.9314284245173137, + "sampling/importance_sampling_ratio/mean": 0.9984660903612773, + "sampling/importance_sampling_ratio/min": 0.4762565518418948, + "sampling/sampling_logp_difference/max": 0.3511220246553421, + "sampling/sampling_logp_difference/mean": 0.004058385244570672, + "step": 19350, + "step_time": 8.137571561970011, + "student/entropy": 0.13296963156511385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01472222302109003, + "completions/max_length": 474.96666666666664, + "completions/max_terminated_length": 433.5, + "completions/mean_length": 179.2697311401367, + "completions/mean_terminated_length": 174.45407358805338, + "completions/min_length": 60.53333333333333, + "completions/min_terminated_length": 60.53333333333333, + "entropy": 0.12542189123729866, + "epoch": 0.7359586830213041, + "eval/gt_acc_batch": 0.821666689713796, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24359279870986938, + "kd/policy_loss": 0.00789370912243612, + "kd/rkl_advantage_mean": 0.7711478068027645, + "kd/rkl_advantage_p95": 5.076553642749786, + "kd/rkl_advantage_std": 2.228021421035131, + "kd/ssd_loss": 0.0, + "learning_rate": 2.640792921429385e-07, + "loss": 0.03157483537991842, + "num_tokens": 633853499.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.821666665871938, + "rewards/gt_logging_reward/std": 0.3759480704863866, + "sampling/importance_sampling_ratio/max": 1.8693518439928691, + "sampling/importance_sampling_ratio/mean": 1.0034370879332224, + "sampling/importance_sampling_ratio/min": 0.48747678299744923, + "sampling/sampling_logp_difference/max": 0.31028189659118655, + "sampling/sampling_logp_difference/mean": 0.003851521434262395, + "step": 19380, + "step_time": 7.751471898734841, + "student/entropy": 0.12542189123729866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555662761132, + "completions/max_length": 488.23333333333335, + "completions/max_terminated_length": 435.8333333333333, + "completions/mean_length": 184.27556559244792, + "completions/mean_terminated_length": 177.53091227213542, + "completions/min_length": 52.9, + "completions/min_terminated_length": 52.9, + "entropy": 0.13226532358676196, + "epoch": 0.7370979379485816, + "eval/gt_acc_batch": 0.7975000282128651, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2965153753757477, + "kd/policy_loss": 0.008964819001266732, + "kd/rkl_advantage_mean": 0.7819308566550414, + "kd/rkl_advantage_p95": 5.041588701804479, + "kd/rkl_advantage_std": 2.2478778918584186, + "kd/ssd_loss": 0.0, + "learning_rate": 2.629400372156609e-07, + "loss": 0.035859278837839764, + "num_tokens": 634810579.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7975000003973644, + "rewards/gt_logging_reward/std": 0.39203281899293263, + "sampling/importance_sampling_ratio/max": 1.943582812945048, + "sampling/importance_sampling_ratio/mean": 0.9999742945035298, + "sampling/importance_sampling_ratio/min": 0.4360542764266332, + "sampling/sampling_logp_difference/max": 0.2960124413172404, + "sampling/sampling_logp_difference/mean": 0.004066380734244982, + "step": 19410, + "step_time": 7.89203226553218, + "student/entropy": 0.13226532358676196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556789040567, + "completions/max_length": 490.1, + "completions/max_terminated_length": 438.8666666666667, + "completions/mean_length": 188.46862131754557, + "completions/mean_terminated_length": 180.8783218383789, + "completions/min_length": 57.36666666666667, + "completions/min_terminated_length": 57.36666666666667, + "entropy": 0.12879397366195916, + "epoch": 0.7382371928758592, + "eval/gt_acc_batch": 0.7966666877269745, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27390459179878235, + "kd/policy_loss": 0.008536899815468738, + "kd/rkl_advantage_mean": 0.7296721260373791, + "kd/rkl_advantage_p95": 4.964423048496246, + "kd/rkl_advantage_std": 2.232636331518491, + "kd/ssd_loss": 0.0, + "learning_rate": 2.618007822883834e-07, + "loss": 0.034147604306538897, + "num_tokens": 635793250.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7966666698455811, + "rewards/gt_logging_reward/std": 0.39449888120094934, + "sampling/importance_sampling_ratio/max": 1.904247530301412, + "sampling/importance_sampling_ratio/mean": 1.0002993623415628, + "sampling/importance_sampling_ratio/min": 0.41962619324525197, + "sampling/sampling_logp_difference/max": 0.37840380469957985, + "sampling/sampling_logp_difference/mean": 0.003960203550135096, + "step": 19440, + "step_time": 8.012527602303695, + "student/entropy": 0.12879397366195916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445899377267, + "completions/max_length": 482.1333333333333, + "completions/max_terminated_length": 437.8, + "completions/mean_length": 186.4313990275065, + "completions/mean_terminated_length": 179.18441212972004, + "completions/min_length": 55.7, + "completions/min_terminated_length": 55.7, + "entropy": 0.12804486316939195, + "epoch": 0.7393764478031367, + "eval/gt_acc_batch": 0.801111121972402, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30575495958328247, + "kd/policy_loss": 0.008544389662953714, + "kd/rkl_advantage_mean": 0.7090903775456051, + "kd/rkl_advantage_p95": 4.897886943817139, + "kd/rkl_advantage_std": 2.1874799112478893, + "kd/ssd_loss": 0.0, + "learning_rate": 2.606615273611058e-07, + "loss": 0.03417756160100301, + "num_tokens": 636768787.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.801111110051473, + "rewards/gt_logging_reward/std": 0.39319525261720023, + "sampling/importance_sampling_ratio/max": 1.8935348550478617, + "sampling/importance_sampling_ratio/mean": 0.9975704828898112, + "sampling/importance_sampling_ratio/min": 0.4313364734252294, + "sampling/sampling_logp_difference/max": 0.32644379933675133, + "sampling/sampling_logp_difference/mean": 0.003922679756457607, + "step": 19470, + "step_time": 8.154283256965574, + "student/entropy": 0.12804486316939195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02555555695046981, + "completions/max_length": 495.0, + "completions/max_terminated_length": 458.3333333333333, + "completions/mean_length": 193.44140014648437, + "completions/mean_terminated_length": 185.09754333496093, + "completions/min_length": 46.2, + "completions/min_terminated_length": 46.2, + "entropy": 0.12722696823378404, + "epoch": 0.7405157027304143, + "eval/gt_acc_batch": 0.7925000150998434, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29781484603881836, + "kd/policy_loss": 0.008491856288552905, + "kd/rkl_advantage_mean": 0.7590056039082508, + "kd/rkl_advantage_p95": 5.078929046789805, + "kd/rkl_advantage_std": 2.2637422968943914, + "kd/ssd_loss": 0.0, + "learning_rate": 2.595222724338282e-07, + "loss": 0.03396742343902588, + "num_tokens": 637774840.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7924999952316284, + "rewards/gt_logging_reward/std": 0.39845245629549025, + "sampling/importance_sampling_ratio/max": 1.9071011543273926, + "sampling/importance_sampling_ratio/mean": 1.0040442963441214, + "sampling/importance_sampling_ratio/min": 0.4842746168375015, + "sampling/sampling_logp_difference/max": 0.3291155139605204, + "sampling/sampling_logp_difference/mean": 0.0038875183168177803, + "step": 19500, + "step_time": 8.320591368034366, + "student/entropy": 0.12722696823378404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.010833334022512039, + "completions/max_length": 471.1666666666667, + "completions/max_terminated_length": 444.03333333333336, + "completions/mean_length": 181.7894546508789, + "completions/mean_terminated_length": 178.21392720540365, + "completions/min_length": 48.766666666666666, + "completions/min_terminated_length": 48.766666666666666, + "entropy": 0.12836513525495927, + "epoch": 0.7416549576576918, + "eval/gt_acc_batch": 0.8163889070351918, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.7235389947891235, + "kd/policy_loss": 0.00831828751640084, + "kd/rkl_advantage_mean": 0.7598971753536413, + "kd/rkl_advantage_p95": 4.980448603630066, + "kd/rkl_advantage_std": 2.2115337401628494, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5838301750655074e-07, + "loss": 0.033273152510325116, + "num_tokens": 638721074.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8163888891537984, + "rewards/gt_logging_reward/std": 0.38240728676319125, + "sampling/importance_sampling_ratio/max": 1.865463423728943, + "sampling/importance_sampling_ratio/mean": 1.0002119521299997, + "sampling/importance_sampling_ratio/min": 0.4542728583017985, + "sampling/sampling_logp_difference/max": 0.29543550809224445, + "sampling/sampling_logp_difference/mean": 0.003938382646689813, + "step": 19530, + "step_time": 8.033308284730689, + "student/entropy": 0.12836513525495927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556813875834, + "completions/max_length": 478.9, + "completions/max_terminated_length": 455.3666666666667, + "completions/mean_length": 185.1644551595052, + "completions/mean_terminated_length": 178.39063618977863, + "completions/min_length": 55.733333333333334, + "completions/min_terminated_length": 55.733333333333334, + "entropy": 0.12620271053165197, + "epoch": 0.7427942125849695, + "eval/gt_acc_batch": 0.8083333472410837, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28973111510276794, + "kd/policy_loss": 0.008260869867323587, + "kd/rkl_advantage_mean": 0.7570522269544502, + "kd/rkl_advantage_p95": 5.011263100306193, + "kd/rkl_advantage_std": 2.23166080613931, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5724376257927316e-07, + "loss": 0.03304347991943359, + "num_tokens": 639686698.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8083333293596904, + "rewards/gt_logging_reward/std": 0.3868274211883545, + "sampling/importance_sampling_ratio/max": 1.9244820396105449, + "sampling/importance_sampling_ratio/mean": 0.9966843008995057, + "sampling/importance_sampling_ratio/min": 0.45407118648290634, + "sampling/sampling_logp_difference/max": 0.3307483692963918, + "sampling/sampling_logp_difference/mean": 0.0038931434197972217, + "step": 19560, + "step_time": 8.120924717635111, + "student/entropy": 0.12620271053165197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778897434474, + "completions/max_length": 469.43333333333334, + "completions/max_terminated_length": 445.6333333333333, + "completions/mean_length": 186.55889892578125, + "completions/mean_terminated_length": 179.91060740152994, + "completions/min_length": 53.266666666666666, + "completions/min_terminated_length": 53.266666666666666, + "entropy": 0.12711496371775866, + "epoch": 0.743933467512247, + "eval/gt_acc_batch": 0.8002777973810832, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2639774680137634, + "kd/policy_loss": 0.007949693681439385, + "kd/rkl_advantage_mean": 0.7916605478773514, + "kd/rkl_advantage_p95": 5.156017396847407, + "kd/rkl_advantage_std": 2.257316647966703, + "kd/ssd_loss": 0.0, + "learning_rate": 2.561045076519956e-07, + "loss": 0.031798774003982545, + "num_tokens": 640671270.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8002777735392252, + "rewards/gt_logging_reward/std": 0.3884796549876531, + "sampling/importance_sampling_ratio/max": 1.8561038851737977, + "sampling/importance_sampling_ratio/mean": 0.9944007635116577, + "sampling/importance_sampling_ratio/min": 0.4689375470081965, + "sampling/sampling_logp_difference/max": 0.3065548559029897, + "sampling/sampling_logp_difference/mean": 0.003914768473866085, + "step": 19590, + "step_time": 8.207603816769552, + "student/entropy": 0.12711496371775866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889042039712, + "completions/max_length": 487.56666666666666, + "completions/max_terminated_length": 455.6333333333333, + "completions/mean_length": 190.75223236083986, + "completions/mean_terminated_length": 181.9724858601888, + "completions/min_length": 61.266666666666666, + "completions/min_terminated_length": 61.266666666666666, + "entropy": 0.1298315931111574, + "epoch": 0.7450727224395246, + "eval/gt_acc_batch": 0.7841666837533315, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34257563948631287, + "kd/policy_loss": 0.008503481411995988, + "kd/rkl_advantage_mean": 0.7953653243059914, + "kd/rkl_advantage_p95": 5.075889070828755, + "kd/rkl_advantage_std": 2.244518346587817, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5496525272471805e-07, + "loss": 0.03401393095652262, + "num_tokens": 641665874.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7841666678587595, + "rewards/gt_logging_reward/std": 0.40789507031440736, + "sampling/importance_sampling_ratio/max": 1.9015865643819174, + "sampling/importance_sampling_ratio/mean": 0.997674963871638, + "sampling/importance_sampling_ratio/min": 0.4555261512597402, + "sampling/sampling_logp_difference/max": 0.32917863726615904, + "sampling/sampling_logp_difference/mean": 0.003975241111281017, + "step": 19620, + "step_time": 8.2731588326608, + "student/entropy": 0.1298315931111574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112559835116, + "completions/max_length": 496.06666666666666, + "completions/max_terminated_length": 449.3333333333333, + "completions/mean_length": 183.2702875773112, + "completions/mean_terminated_length": 175.51051432291666, + "completions/min_length": 57.3, + "completions/min_terminated_length": 57.3, + "entropy": 0.12850976524253685, + "epoch": 0.7462119773668021, + "eval/gt_acc_batch": 0.7930555740992228, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.310396671295166, + "kd/policy_loss": 0.00876221347716637, + "kd/rkl_advantage_mean": 0.7948073625875016, + "kd/rkl_advantage_p95": 5.062232597668966, + "kd/rkl_advantage_std": 2.254484365383784, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5382599779744047e-07, + "loss": 0.035048862298329674, + "num_tokens": 642623279.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7930555542310079, + "rewards/gt_logging_reward/std": 0.3992587695519129, + "sampling/importance_sampling_ratio/max": 1.9491121967633565, + "sampling/importance_sampling_ratio/mean": 1.00113951365153, + "sampling/importance_sampling_ratio/min": 0.4520309418439865, + "sampling/sampling_logp_difference/max": 0.3498302161693573, + "sampling/sampling_logp_difference/mean": 0.003959835188773771, + "step": 19650, + "step_time": 8.238327447032983, + "student/entropy": 0.12850976524253685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333469927311, + "completions/max_length": 480.96666666666664, + "completions/max_terminated_length": 443.8333333333333, + "completions/mean_length": 187.39334411621093, + "completions/mean_terminated_length": 180.5657964070638, + "completions/min_length": 52.666666666666664, + "completions/min_terminated_length": 52.666666666666664, + "entropy": 0.12679010362674792, + "epoch": 0.7473512322940796, + "eval/gt_acc_batch": 0.8216666936874389, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3078823685646057, + "kd/policy_loss": 0.007990691098772611, + "kd/rkl_advantage_mean": 0.6734390700856845, + "kd/rkl_advantage_p95": 4.862349017461141, + "kd/rkl_advantage_std": 2.1925534665584565, + "kd/ssd_loss": 0.0, + "learning_rate": 2.526867428701629e-07, + "loss": 0.031962766249974565, + "num_tokens": 643605887.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8216666718324025, + "rewards/gt_logging_reward/std": 0.379154505332311, + "sampling/importance_sampling_ratio/max": 1.7827107350031535, + "sampling/importance_sampling_ratio/mean": 0.9989740530649821, + "sampling/importance_sampling_ratio/min": 0.4922326495250066, + "sampling/sampling_logp_difference/max": 0.32603750824928285, + "sampling/sampling_logp_difference/mean": 0.0038839524999881784, + "step": 19680, + "step_time": 8.076102761201522, + "student/entropy": 0.12679010362674792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778959522645, + "completions/max_length": 495.2, + "completions/max_terminated_length": 464.03333333333336, + "completions/mean_length": 186.2997319539388, + "completions/mean_terminated_length": 179.6750966389974, + "completions/min_length": 59.53333333333333, + "completions/min_terminated_length": 59.53333333333333, + "entropy": 0.12673180686930816, + "epoch": 0.7484904872213572, + "eval/gt_acc_batch": 0.8013889094193777, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3209072947502136, + "kd/policy_loss": 0.008358414876662815, + "kd/rkl_advantage_mean": 0.7763866887117425, + "kd/rkl_advantage_p95": 5.094975358247757, + "kd/rkl_advantage_std": 2.26398828625679, + "kd/ssd_loss": 0.0, + "learning_rate": 2.515474879428853e-07, + "loss": 0.03343366781870524, + "num_tokens": 644582014.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888915379842, + "rewards/gt_logging_reward/std": 0.3899892215927442, + "sampling/importance_sampling_ratio/max": 1.8391324917475382, + "sampling/importance_sampling_ratio/mean": 0.993171199162801, + "sampling/importance_sampling_ratio/min": 0.4423011526465416, + "sampling/sampling_logp_difference/max": 0.3218663235505422, + "sampling/sampling_logp_difference/mean": 0.003897423429104189, + "step": 19710, + "step_time": 8.218176582028779, + "student/entropy": 0.12673180686930816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01500000072022279, + "completions/max_length": 481.6333333333333, + "completions/max_terminated_length": 431.03333333333336, + "completions/mean_length": 182.3047302246094, + "completions/mean_terminated_length": 177.2401555379232, + "completions/min_length": 55.733333333333334, + "completions/min_terminated_length": 55.733333333333334, + "entropy": 0.13139221432308357, + "epoch": 0.7496297421486348, + "eval/gt_acc_batch": 0.817222245534261, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26999449729919434, + "kd/policy_loss": 0.008433338138274848, + "kd/rkl_advantage_mean": 0.8774571143711607, + "kd/rkl_advantage_p95": 5.226573189099629, + "kd/rkl_advantage_std": 2.2509266316890715, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5040823301560777e-07, + "loss": 0.03373335202534993, + "num_tokens": 645526575.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8172222236792247, + "rewards/gt_logging_reward/std": 0.37583180169264474, + "sampling/importance_sampling_ratio/max": 1.9272438486417134, + "sampling/importance_sampling_ratio/mean": 1.001880031824112, + "sampling/importance_sampling_ratio/min": 0.446943587064743, + "sampling/sampling_logp_difference/max": 0.3117194930712382, + "sampling/sampling_logp_difference/mean": 0.003981241281144321, + "step": 19740, + "step_time": 7.984754919894233, + "student/entropy": 0.13139221432308357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445526848235, + "completions/max_length": 484.8, + "completions/max_terminated_length": 457.76666666666665, + "completions/mean_length": 188.80750885009766, + "completions/mean_terminated_length": 181.6355412801107, + "completions/min_length": 54.766666666666666, + "completions/min_terminated_length": 54.766666666666666, + "entropy": 0.13366201426833868, + "epoch": 0.7507689970759124, + "eval/gt_acc_batch": 0.7861111303170522, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2261309027671814, + "kd/policy_loss": 0.00899652723261776, + "kd/rkl_advantage_mean": 0.7244734970231851, + "kd/rkl_advantage_p95": 4.98481352130572, + "kd/rkl_advantage_std": 2.2169239113728207, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4926897808833024e-07, + "loss": 0.03598611354827881, + "num_tokens": 646511834.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7861111104488373, + "rewards/gt_logging_reward/std": 0.4022670795520147, + "sampling/importance_sampling_ratio/max": 1.9687163670857748, + "sampling/importance_sampling_ratio/mean": 1.0022607147693634, + "sampling/importance_sampling_ratio/min": 0.4264992376168569, + "sampling/sampling_logp_difference/max": 0.3464935580889384, + "sampling/sampling_logp_difference/mean": 0.004049272874059776, + "step": 19770, + "step_time": 8.177774197265778, + "student/entropy": 0.13366201426833868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112243185442, + "completions/max_length": 481.5, + "completions/max_terminated_length": 447.43333333333334, + "completions/mean_length": 187.53334248860676, + "completions/mean_terminated_length": 180.5811014811198, + "completions/min_length": 57.46666666666667, + "completions/min_terminated_length": 57.46666666666667, + "entropy": 0.1255962581684192, + "epoch": 0.7519082520031899, + "eval/gt_acc_batch": 0.8030555784702301, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28510990738868713, + "kd/policy_loss": 0.007905050068317602, + "kd/rkl_advantage_mean": 0.7714004440504747, + "kd/rkl_advantage_p95": 4.989827267328898, + "kd/rkl_advantage_std": 2.204768345753352, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4812972316105266e-07, + "loss": 0.03162020047505697, + "num_tokens": 647495882.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555526415507, + "rewards/gt_logging_reward/std": 0.3913274695475896, + "sampling/importance_sampling_ratio/max": 1.8493103663126627, + "sampling/importance_sampling_ratio/mean": 0.9994418303171794, + "sampling/importance_sampling_ratio/min": 0.44341179231802624, + "sampling/sampling_logp_difference/max": 0.35790724754333497, + "sampling/sampling_logp_difference/mean": 0.003836966158511738, + "step": 19800, + "step_time": 8.272996518494134, + "student/entropy": 0.1255962581684192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015000000813355048, + "completions/max_length": 477.7, + "completions/max_terminated_length": 450.3666666666667, + "completions/mean_length": 186.86639760335285, + "completions/mean_terminated_length": 182.005815633138, + "completions/min_length": 63.5, + "completions/min_terminated_length": 63.5, + "entropy": 0.1222544901072979, + "epoch": 0.7530475069304675, + "eval/gt_acc_batch": 0.8122222483158111, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3483871519565582, + "kd/policy_loss": 0.00798060125671327, + "kd/rkl_advantage_mean": 0.8047697834049662, + "kd/rkl_advantage_p95": 5.064788546164831, + "kd/rkl_advantage_std": 2.2264504611492155, + "kd/ssd_loss": 0.0, + "learning_rate": 2.469904682337751e-07, + "loss": 0.03192240993181864, + "num_tokens": 648464081.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8122222224871317, + "rewards/gt_logging_reward/std": 0.37681624641021094, + "sampling/importance_sampling_ratio/max": 1.9371894598007202, + "sampling/importance_sampling_ratio/mean": 1.0028741717338563, + "sampling/importance_sampling_ratio/min": 0.45523566504319507, + "sampling/sampling_logp_difference/max": 0.3247182905673981, + "sampling/sampling_logp_difference/mean": 0.0037766827270388605, + "step": 19830, + "step_time": 8.067869361765528, + "student/entropy": 0.1222544901072979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779251337052, + "completions/max_length": 483.6666666666667, + "completions/max_terminated_length": 448.06666666666666, + "completions/mean_length": 188.09195353190105, + "completions/mean_terminated_length": 179.71124064127605, + "completions/min_length": 54.5, + "completions/min_terminated_length": 54.5, + "entropy": 0.12388698402792216, + "epoch": 0.754186761857745, + "eval/gt_acc_batch": 0.7913889149824779, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.388114333152771, + "kd/policy_loss": 0.008181636427373936, + "kd/rkl_advantage_mean": 0.8452639158815145, + "kd/rkl_advantage_p95": 5.1399304111798605, + "kd/rkl_advantage_std": 2.2439719259738924, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4585121330649755e-07, + "loss": 0.032726548115412396, + "num_tokens": 649452996.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7913888911406199, + "rewards/gt_logging_reward/std": 0.3983986794948578, + "sampling/importance_sampling_ratio/max": 1.802250858147939, + "sampling/importance_sampling_ratio/mean": 0.9979208807150522, + "sampling/importance_sampling_ratio/min": 0.45878295799096425, + "sampling/sampling_logp_difference/max": 0.3567464590072632, + "sampling/sampling_logp_difference/mean": 0.003797354370666047, + "step": 19860, + "step_time": 8.333770604800277, + "student/entropy": 0.12388698402792216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02305555691321691, + "completions/max_length": 489.1333333333333, + "completions/max_terminated_length": 459.06666666666666, + "completions/mean_length": 189.98862050374348, + "completions/mean_terminated_length": 182.51639404296876, + "completions/min_length": 55.4, + "completions/min_terminated_length": 55.4, + "entropy": 0.13450771756470203, + "epoch": 0.7553260167850226, + "eval/gt_acc_batch": 0.7919444640477499, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25787797570228577, + "kd/policy_loss": 0.008987945194045702, + "kd/rkl_advantage_mean": 0.7270278632019956, + "kd/rkl_advantage_p95": 4.947769403457642, + "kd/rkl_advantage_std": 2.2166728963454565, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4471195837921997e-07, + "loss": 0.03595178524653117, + "num_tokens": 650448595.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7919444421927134, + "rewards/gt_logging_reward/std": 0.39638554453849795, + "sampling/importance_sampling_ratio/max": 1.9669509530067444, + "sampling/importance_sampling_ratio/mean": 1.0036982774734498, + "sampling/importance_sampling_ratio/min": 0.4349796627958616, + "sampling/sampling_logp_difference/max": 0.2999541759490967, + "sampling/sampling_logp_difference/mean": 0.004082537163048983, + "step": 19890, + "step_time": 8.344489080503505, + "student/entropy": 0.13450771756470203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.031666668504476546, + "completions/max_length": 485.1333333333333, + "completions/max_terminated_length": 451.3666666666667, + "completions/mean_length": 190.0919537862142, + "completions/mean_terminated_length": 179.92456741333007, + "completions/min_length": 53.6, + "completions/min_terminated_length": 53.6, + "entropy": 0.1305525541305542, + "epoch": 0.7564652717123002, + "eval/gt_acc_batch": 0.796111128727595, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.37085169553756714, + "kd/policy_loss": 0.008563092273349563, + "kd/rkl_advantage_mean": 0.7898605638494094, + "kd/rkl_advantage_p95": 4.977546898523967, + "kd/rkl_advantage_std": 2.191403034329414, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4357270345194244e-07, + "loss": 0.03425236543019613, + "num_tokens": 651434334.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7961111187934875, + "rewards/gt_logging_reward/std": 0.3882455547650655, + "sampling/importance_sampling_ratio/max": 1.9246365110079446, + "sampling/importance_sampling_ratio/mean": 1.0015799363454183, + "sampling/importance_sampling_ratio/min": 0.4641634831825892, + "sampling/sampling_logp_difference/max": 0.331625239054362, + "sampling/sampling_logp_difference/mean": 0.003939435662080844, + "step": 19920, + "step_time": 8.291937018596219, + "student/entropy": 0.1305525541305542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277779176831247, + "completions/max_length": 485.1, + "completions/max_terminated_length": 449.3333333333333, + "completions/mean_length": 186.81389973958332, + "completions/mean_terminated_length": 180.28101450602213, + "completions/min_length": 55.46666666666667, + "completions/min_terminated_length": 55.46666666666667, + "entropy": 0.12847357988357544, + "epoch": 0.7576045266395777, + "eval/gt_acc_batch": 0.7977778017520905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28305739164352417, + "kd/policy_loss": 0.008177311635032916, + "kd/rkl_advantage_mean": 0.8303352258478601, + "kd/rkl_advantage_p95": 5.0494480351607, + "kd/rkl_advantage_std": 2.2440278003613154, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4243344852466486e-07, + "loss": 0.0327092448870341, + "num_tokens": 652401904.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.797777779897054, + "rewards/gt_logging_reward/std": 0.3916856919725736, + "sampling/importance_sampling_ratio/max": 1.8594923535982768, + "sampling/importance_sampling_ratio/mean": 0.998709503809611, + "sampling/importance_sampling_ratio/min": 0.45400549868742623, + "sampling/sampling_logp_difference/max": 0.34520491361618044, + "sampling/sampling_logp_difference/mean": 0.0039034358380983275, + "step": 19950, + "step_time": 8.159155991429968, + "student/entropy": 0.12847357988357544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111265917619, + "completions/max_length": 475.8333333333333, + "completions/max_terminated_length": 446.26666666666665, + "completions/mean_length": 191.04556681315105, + "completions/mean_terminated_length": 182.55660909016927, + "completions/min_length": 56.166666666666664, + "completions/min_terminated_length": 56.166666666666664, + "entropy": 0.13232018916557234, + "epoch": 0.7587437815668553, + "eval/gt_acc_batch": 0.7775000174840291, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27149832248687744, + "kd/policy_loss": 0.00895920314748461, + "kd/rkl_advantage_mean": 0.7623641296289861, + "kd/rkl_advantage_p95": 5.0228143016497295, + "kd/rkl_advantage_std": 2.245679821570714, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4129419359738727e-07, + "loss": 0.035836819807688394, + "num_tokens": 653384572.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7775000055631002, + "rewards/gt_logging_reward/std": 0.40682370265324913, + "sampling/importance_sampling_ratio/max": 1.9168914357821147, + "sampling/importance_sampling_ratio/mean": 1.0050819873809815, + "sampling/importance_sampling_ratio/min": 0.47154206931591036, + "sampling/sampling_logp_difference/max": 0.3092679738998413, + "sampling/sampling_logp_difference/mean": 0.0040049545389289655, + "step": 19980, + "step_time": 8.21666705206153, + "student/entropy": 0.13232018916557234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.031111112982034683, + "completions/max_length": 497.8, + "completions/max_terminated_length": 469.6666666666667, + "completions/mean_length": 199.01361999511718, + "completions/mean_terminated_length": 189.04684804280598, + "completions/min_length": 51.4, + "completions/min_terminated_length": 51.4, + "entropy": 0.13351108152419328, + "epoch": 0.7598830364941328, + "eval/gt_acc_batch": 0.7825000186761221, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22841492295265198, + "kd/policy_loss": 0.008406857892987318, + "kd/rkl_advantage_mean": 0.727429825005432, + "kd/rkl_advantage_p95": 4.921079760789871, + "kd/rkl_advantage_std": 2.187630224227905, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4015493867010974e-07, + "loss": 0.033627438545227054, + "num_tokens": 654400485.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7824999948342641, + "rewards/gt_logging_reward/std": 0.4033319503068924, + "sampling/importance_sampling_ratio/max": 1.8490193168322244, + "sampling/importance_sampling_ratio/mean": 0.9912253399689992, + "sampling/importance_sampling_ratio/min": 0.4354811171690623, + "sampling/sampling_logp_difference/max": 0.31484848658243814, + "sampling/sampling_logp_difference/mean": 0.003932747027526299, + "step": 20010, + "step_time": 8.47252599527031, + "student/entropy": 0.13351108152419328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556882172822, + "completions/max_length": 497.3666666666667, + "completions/max_terminated_length": 444.3, + "completions/mean_length": 185.99861958821614, + "completions/mean_terminated_length": 178.42871907552083, + "completions/min_length": 54.93333333333333, + "completions/min_terminated_length": 54.93333333333333, + "entropy": 0.1296739850193262, + "epoch": 0.7610222914214104, + "eval/gt_acc_batch": 0.8061111330986023, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24005495011806488, + "kd/policy_loss": 0.008468670580380906, + "kd/rkl_advantage_mean": 0.8473821502178908, + "kd/rkl_advantage_p95": 5.088946072260539, + "kd/rkl_advantage_std": 2.235593613982201, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3901568374283216e-07, + "loss": 0.03387468655904134, + "num_tokens": 655380480.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8061111092567443, + "rewards/gt_logging_reward/std": 0.38893579741319023, + "sampling/importance_sampling_ratio/max": 1.8896080414454142, + "sampling/importance_sampling_ratio/mean": 0.9988877773284912, + "sampling/importance_sampling_ratio/min": 0.46019707520802816, + "sampling/sampling_logp_difference/max": 0.3136323591073354, + "sampling/sampling_logp_difference/mean": 0.003967275563627482, + "step": 20040, + "step_time": 8.341845629169256, + "student/entropy": 0.1296739850193262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277779344469307, + "completions/max_length": 491.6, + "completions/max_terminated_length": 454.8666666666667, + "completions/mean_length": 193.0166768391927, + "completions/mean_terminated_length": 184.76156158447264, + "completions/min_length": 57.733333333333334, + "completions/min_terminated_length": 57.733333333333334, + "entropy": 0.1395179795101285, + "epoch": 0.7621615463486879, + "eval/gt_acc_batch": 0.7966666877269745, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2531838119029999, + "kd/policy_loss": 0.00913243896405523, + "kd/rkl_advantage_mean": 0.7581034948428472, + "kd/rkl_advantage_p95": 5.002742932240168, + "kd/rkl_advantage_std": 2.2148460666338603, + "kd/ssd_loss": 0.0, + "learning_rate": 2.378764288155546e-07, + "loss": 0.036529759565989174, + "num_tokens": 656380084.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7966666678587596, + "rewards/gt_logging_reward/std": 0.3985517054796219, + "sampling/importance_sampling_ratio/max": 2.022644905249278, + "sampling/importance_sampling_ratio/mean": 1.0091464837392172, + "sampling/importance_sampling_ratio/min": 0.44264207084973656, + "sampling/sampling_logp_difference/max": 0.3095769762992859, + "sampling/sampling_logp_difference/mean": 0.004059667830976347, + "step": 20070, + "step_time": 8.255384985994898, + "student/entropy": 0.1395179795101285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028333335183560848, + "completions/max_length": 489.3333333333333, + "completions/max_terminated_length": 461.46666666666664, + "completions/mean_length": 195.10306599934896, + "completions/mean_terminated_length": 186.05101013183594, + "completions/min_length": 54.96666666666667, + "completions/min_terminated_length": 54.96666666666667, + "entropy": 0.12498614713549613, + "epoch": 0.7633008012759656, + "eval/gt_acc_batch": 0.7902777949968974, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30757397413253784, + "kd/policy_loss": 0.008221563725965098, + "kd/rkl_advantage_mean": 0.7908283287038406, + "kd/rkl_advantage_p95": 5.207660073041916, + "kd/rkl_advantage_std": 2.297394529978434, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3673717388827705e-07, + "loss": 0.03288624882698059, + "num_tokens": 657392151.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.790277773141861, + "rewards/gt_logging_reward/std": 0.39762401282787324, + "sampling/importance_sampling_ratio/max": 1.818430209159851, + "sampling/importance_sampling_ratio/mean": 1.0007449785868328, + "sampling/importance_sampling_ratio/min": 0.43533316552639006, + "sampling/sampling_logp_difference/max": 0.31465014616648357, + "sampling/sampling_logp_difference/mean": 0.0038063563018416366, + "step": 20100, + "step_time": 8.334112043300411, + "student/entropy": 0.12498614713549613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013611111976206302, + "completions/max_length": 463.76666666666665, + "completions/max_terminated_length": 445.73333333333335, + "completions/mean_length": 189.67667541503906, + "completions/mean_terminated_length": 185.3237274169922, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.12397052260736624, + "epoch": 0.7644400562032431, + "eval/gt_acc_batch": 0.8011111279328664, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29247042536735535, + "kd/policy_loss": 0.008016562626774732, + "kd/rkl_advantage_mean": 0.7292116859927773, + "kd/rkl_advantage_p95": 4.931886845827103, + "kd/rkl_advantage_std": 2.1652101506789525, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3559791896099952e-07, + "loss": 0.03206625580787659, + "num_tokens": 658383123.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8011111080646515, + "rewards/gt_logging_reward/std": 0.38807089775800707, + "sampling/importance_sampling_ratio/max": 1.7525275588035583, + "sampling/importance_sampling_ratio/mean": 0.9953193624814352, + "sampling/importance_sampling_ratio/min": 0.482170761624972, + "sampling/sampling_logp_difference/max": 0.3035338441530863, + "sampling/sampling_logp_difference/mean": 0.0037959063891321422, + "step": 20130, + "step_time": 8.211766632875273, + "student/entropy": 0.12397052260736624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02183908182356892, + "completions/max_length": 465.8965517241379, + "completions/max_terminated_length": 443.9310344827586, + "completions/mean_length": 187.06409007105333, + "completions/mean_terminated_length": 179.79784209152749, + "completions/min_length": 55.41379310344828, + "completions/min_terminated_length": 55.41379310344828, + "entropy": 0.12227650230814671, + "epoch": 0.7655793111305207, + "eval/gt_acc_batch": 0.8068965735106632, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23376566171646118, + "kd/policy_loss": 0.008023715923251263, + "kd/rkl_advantage_mean": 0.6686869071764422, + "kd/rkl_advantage_p95": 4.7949009928210025, + "kd/rkl_advantage_std": 2.1588204564719367, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3445866403372194e-07, + "loss": 0.031025036176045736, + "num_tokens": 659322130.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8068965488466723, + "rewards/gt_logging_reward/std": 0.38692529756447364, + "sampling/importance_sampling_ratio/max": 1.9866512561666554, + "sampling/importance_sampling_ratio/mean": 1.0014176964759827, + "sampling/importance_sampling_ratio/min": 0.48028547702164487, + "sampling/sampling_logp_difference/max": 0.32457565028091956, + "sampling/sampling_logp_difference/mean": 0.0037395433314016155, + "step": 20160, + "step_time": 7.794941440600087, + "student/entropy": 0.12227650230814671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026666668461014828, + "completions/max_length": 481.1, + "completions/max_terminated_length": 457.3666666666667, + "completions/mean_length": 182.7791768391927, + "completions/mean_terminated_length": 173.97730509440103, + "completions/min_length": 53.5, + "completions/min_terminated_length": 53.5, + "entropy": 0.12608017306774855, + "epoch": 0.7667185660577982, + "eval/gt_acc_batch": 0.810277795791626, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29796749353408813, + "kd/policy_loss": 0.008385820490851377, + "kd/rkl_advantage_mean": 0.7675553504610434, + "kd/rkl_advantage_p95": 5.103670253356298, + "kd/rkl_advantage_std": 2.2750742544730502, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3331940910644438e-07, + "loss": 0.03354328473409017, + "num_tokens": 660276631.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8102777818838756, + "rewards/gt_logging_reward/std": 0.37860896239678066, + "sampling/importance_sampling_ratio/max": 1.9109833280245463, + "sampling/importance_sampling_ratio/mean": 0.9987269083658854, + "sampling/importance_sampling_ratio/min": 0.46298396537701286, + "sampling/sampling_logp_difference/max": 0.3181571920712789, + "sampling/sampling_logp_difference/mean": 0.0038858320641641814, + "step": 20190, + "step_time": 8.140770838894726, + "student/entropy": 0.12608017306774855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277779021610817, + "completions/max_length": 471.53333333333336, + "completions/max_terminated_length": 446.8333333333333, + "completions/mean_length": 186.3833429972331, + "completions/mean_terminated_length": 179.8139862060547, + "completions/min_length": 50.43333333333333, + "completions/min_terminated_length": 50.43333333333333, + "entropy": 0.12775179861734312, + "epoch": 0.7678578209850757, + "eval/gt_acc_batch": 0.7977778037389119, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2782203257083893, + "kd/policy_loss": 0.008399450307479128, + "kd/rkl_advantage_mean": 0.8026129634430011, + "kd/rkl_advantage_p95": 5.132111656665802, + "kd/rkl_advantage_std": 2.269930386543274, + "kd/ssd_loss": 0.0, + "learning_rate": 2.321801541791668e-07, + "loss": 0.03359781106313069, + "num_tokens": 661247083.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.797777775923411, + "rewards/gt_logging_reward/std": 0.392411212126414, + "sampling/importance_sampling_ratio/max": 1.8501734932263691, + "sampling/importance_sampling_ratio/mean": 1.0059687654177347, + "sampling/importance_sampling_ratio/min": 0.4661195605993271, + "sampling/sampling_logp_difference/max": 0.34161778887112937, + "sampling/sampling_logp_difference/mean": 0.003893120974923174, + "step": 20220, + "step_time": 8.126092381765678, + "student/entropy": 0.12775179861734312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222224045544863, + "completions/max_length": 482.3666666666667, + "completions/max_terminated_length": 446.23333333333335, + "completions/mean_length": 190.9275095621745, + "completions/mean_terminated_length": 182.14341939290364, + "completions/min_length": 52.06666666666667, + "completions/min_terminated_length": 52.06666666666667, + "entropy": 0.12853542870531479, + "epoch": 0.7689970759123533, + "eval/gt_acc_batch": 0.7969444513320922, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2571999430656433, + "kd/policy_loss": 0.008592863984328384, + "kd/rkl_advantage_mean": 0.7894450464596351, + "kd/rkl_advantage_p95": 5.052782917022705, + "kd/rkl_advantage_std": 2.229092053572337, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3104089925188924e-07, + "loss": 0.03437145948410034, + "num_tokens": 662240878.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444453716278, + "rewards/gt_logging_reward/std": 0.39511067668596905, + "sampling/importance_sampling_ratio/max": 1.9363954504330954, + "sampling/importance_sampling_ratio/mean": 0.994359815120697, + "sampling/importance_sampling_ratio/min": 0.4603819489479065, + "sampling/sampling_logp_difference/max": 0.31499665379524233, + "sampling/sampling_logp_difference/mean": 0.0038775968520591655, + "step": 20250, + "step_time": 8.336811973161335, + "student/entropy": 0.12853542870531479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017222223213563363, + "completions/max_length": 481.0, + "completions/max_terminated_length": 436.53333333333336, + "completions/mean_length": 184.62362009684244, + "completions/mean_terminated_length": 178.977587890625, + "completions/min_length": 53.833333333333336, + "completions/min_terminated_length": 53.833333333333336, + "entropy": 0.13090084809809924, + "epoch": 0.7701363308396308, + "eval/gt_acc_batch": 0.7847222447395324, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.39712971448898315, + "kd/policy_loss": 0.008615159635276844, + "kd/rkl_advantage_mean": 0.7762270097931226, + "kd/rkl_advantage_p95": 5.071673744916916, + "kd/rkl_advantage_std": 2.2615598797798158, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2990164432461171e-07, + "loss": 0.03446063995361328, + "num_tokens": 663207931.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7847222288449606, + "rewards/gt_logging_reward/std": 0.40106662213802335, + "sampling/importance_sampling_ratio/max": 1.8396191795667012, + "sampling/importance_sampling_ratio/mean": 0.9960631469885508, + "sampling/importance_sampling_ratio/min": 0.4672684828440348, + "sampling/sampling_logp_difference/max": 0.3500140905380249, + "sampling/sampling_logp_difference/mean": 0.00397274117761602, + "step": 20280, + "step_time": 8.209912867564707, + "student/entropy": 0.13090084809809924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001012037198, + "completions/max_length": 468.46666666666664, + "completions/max_terminated_length": 447.76666666666665, + "completions/mean_length": 189.07000986735025, + "completions/mean_terminated_length": 182.74456837972005, + "completions/min_length": 56.43333333333333, + "completions/min_terminated_length": 56.43333333333333, + "entropy": 0.1293072992314895, + "epoch": 0.7712755857669085, + "eval/gt_acc_batch": 0.7947222491105398, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4622788727283478, + "kd/policy_loss": 0.008500051653633515, + "kd/rkl_advantage_mean": 0.6777686673022496, + "kd/rkl_advantage_p95": 4.867590502897898, + "kd/rkl_advantage_std": 2.205088542898496, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2876238939733413e-07, + "loss": 0.034000213940938315, + "num_tokens": 664196015.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222173213959, + "rewards/gt_logging_reward/std": 0.3935557991266251, + "sampling/importance_sampling_ratio/max": 1.8839170257250468, + "sampling/importance_sampling_ratio/mean": 0.9971115310986837, + "sampling/importance_sampling_ratio/min": 0.43642234603563945, + "sampling/sampling_logp_difference/max": 0.3154336074988047, + "sampling/sampling_logp_difference/mean": 0.003927576045195262, + "step": 20310, + "step_time": 7.975428503796381, + "student/entropy": 0.1293072992314895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029166668467223645, + "completions/max_length": 488.73333333333335, + "completions/max_terminated_length": 448.26666666666665, + "completions/mean_length": 194.82667592366536, + "completions/mean_terminated_length": 185.24785766601562, + "completions/min_length": 57.7, + "completions/min_terminated_length": 57.7, + "entropy": 0.12776682718346516, + "epoch": 0.772414840694186, + "eval/gt_acc_batch": 0.7863889078299204, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2926277816295624, + "kd/policy_loss": 0.00847606971122635, + "kd/rkl_advantage_mean": 0.7514126753279319, + "kd/rkl_advantage_p95": 5.007332567373911, + "kd/rkl_advantage_std": 2.2179048667351404, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2762313447005658e-07, + "loss": 0.033904282251993816, + "num_tokens": 665201407.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7863888899485271, + "rewards/gt_logging_reward/std": 0.4020795678098997, + "sampling/importance_sampling_ratio/max": 1.9262135942777, + "sampling/importance_sampling_ratio/mean": 1.0002436061700186, + "sampling/importance_sampling_ratio/min": 0.43968753814697265, + "sampling/sampling_logp_difference/max": 0.34240060249964394, + "sampling/sampling_logp_difference/mean": 0.0038962303117538494, + "step": 20340, + "step_time": 8.116740273094425, + "student/entropy": 0.12776682718346516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667573153974, + "completions/max_length": 475.3, + "completions/max_terminated_length": 434.46666666666664, + "completions/mean_length": 182.8372314453125, + "completions/mean_terminated_length": 176.41494903564453, + "completions/min_length": 49.733333333333334, + "completions/min_terminated_length": 49.733333333333334, + "entropy": 0.12733068155745667, + "epoch": 0.7735540956214636, + "eval/gt_acc_batch": 0.807500010728836, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30256038904190063, + "kd/policy_loss": 0.0083961200589935, + "kd/rkl_advantage_mean": 0.7129727742324273, + "kd/rkl_advantage_p95": 4.96230438152949, + "kd/rkl_advantage_std": 2.2137958923975627, + "kd/ssd_loss": 0.0, + "learning_rate": 2.26483879542779e-07, + "loss": 0.03358448346455892, + "num_tokens": 666163197.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8075000007947286, + "rewards/gt_logging_reward/std": 0.38715618004401525, + "sampling/importance_sampling_ratio/max": 1.8037002364794412, + "sampling/importance_sampling_ratio/mean": 0.9960067252318064, + "sampling/importance_sampling_ratio/min": 0.4721009537577629, + "sampling/sampling_logp_difference/max": 0.3322459717591604, + "sampling/sampling_logp_difference/mean": 0.003890217173223694, + "step": 20370, + "step_time": 8.01645794492506, + "student/entropy": 0.12733068155745667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112212141354, + "completions/max_length": 486.2, + "completions/max_terminated_length": 455.3, + "completions/mean_length": 184.96889902750652, + "completions/mean_terminated_length": 178.1186945597331, + "completions/min_length": 57.13333333333333, + "completions/min_terminated_length": 57.13333333333333, + "entropy": 0.12820448856800795, + "epoch": 0.7746933505487411, + "eval/gt_acc_batch": 0.8083333551883698, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25930845737457275, + "kd/policy_loss": 0.008388680642625938, + "kd/rkl_advantage_mean": 0.8331831268966198, + "kd/rkl_advantage_p95": 5.090759726365407, + "kd/rkl_advantage_std": 2.2311219443877537, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2534462461550144e-07, + "loss": 0.03355472087860108, + "num_tokens": 667127925.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8083333333333333, + "rewards/gt_logging_reward/std": 0.38204803665479026, + "sampling/importance_sampling_ratio/max": 1.9556318958600363, + "sampling/importance_sampling_ratio/mean": 1.0001160661379496, + "sampling/importance_sampling_ratio/min": 0.47620013455549876, + "sampling/sampling_logp_difference/max": 0.35184895396232607, + "sampling/sampling_logp_difference/mean": 0.003915569256059825, + "step": 20400, + "step_time": 7.957684768066974, + "student/entropy": 0.12820448856800795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0302777794500192, + "completions/max_length": 507.3666666666667, + "completions/max_terminated_length": 474.1333333333333, + "completions/mean_length": 195.3394546508789, + "completions/mean_terminated_length": 185.4649622599284, + "completions/min_length": 52.5, + "completions/min_terminated_length": 52.5, + "entropy": 0.13439681877692541, + "epoch": 0.7758326054760187, + "eval/gt_acc_batch": 0.7800000230471293, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31644415855407715, + "kd/policy_loss": 0.008886761129057655, + "kd/rkl_advantage_mean": 0.7976116010298332, + "kd/rkl_advantage_p95": 5.1088983873526255, + "kd/rkl_advantage_std": 2.2580823649962745, + "kd/ssd_loss": 0.0, + "learning_rate": 2.242053696882239e-07, + "loss": 0.03554704189300537, + "num_tokens": 668134595.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7800000031789144, + "rewards/gt_logging_reward/std": 0.406439150373141, + "sampling/importance_sampling_ratio/max": 1.919628377755483, + "sampling/importance_sampling_ratio/mean": 0.9980144759019216, + "sampling/importance_sampling_ratio/min": 0.43747264544169107, + "sampling/sampling_logp_difference/max": 0.3267417629559835, + "sampling/sampling_logp_difference/mean": 0.004038412061830362, + "step": 20430, + "step_time": 8.264020851098273, + "student/entropy": 0.13439681877692541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556795249384, + "completions/max_length": 497.06666666666666, + "completions/max_terminated_length": 447.1, + "completions/mean_length": 191.98750915527344, + "completions/mean_terminated_length": 183.74403330485026, + "completions/min_length": 55.63333333333333, + "completions/min_terminated_length": 55.63333333333333, + "entropy": 0.13024160744001467, + "epoch": 0.7769718604032962, + "eval/gt_acc_batch": 0.7975000202655792, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29700151085853577, + "kd/policy_loss": 0.008485517602336283, + "kd/rkl_advantage_mean": 0.8106122590601444, + "kd/rkl_advantage_p95": 5.158261066675186, + "kd/rkl_advantage_std": 2.2664575318495435, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2306611476094633e-07, + "loss": 0.03394207159678141, + "num_tokens": 669134918.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7975000043710073, + "rewards/gt_logging_reward/std": 0.39130662952860196, + "sampling/importance_sampling_ratio/max": 1.8729846835136414, + "sampling/importance_sampling_ratio/mean": 0.9984130283196767, + "sampling/importance_sampling_ratio/min": 0.46468472182750703, + "sampling/sampling_logp_difference/max": 0.3190427303314209, + "sampling/sampling_logp_difference/mean": 0.003921035102879008, + "step": 20460, + "step_time": 8.35740302723813, + "student/entropy": 0.13024160744001467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013055556453764439, + "completions/max_length": 469.4, + "completions/max_terminated_length": 447.4, + "completions/mean_length": 182.32528737386068, + "completions/mean_terminated_length": 178.07462615966796, + "completions/min_length": 56.9, + "completions/min_terminated_length": 56.9, + "entropy": 0.12661665181318918, + "epoch": 0.7781111153305738, + "eval/gt_acc_batch": 0.8161111215750376, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3114558756351471, + "kd/policy_loss": 0.0079575753537938, + "kd/rkl_advantage_mean": 0.7409959439188242, + "kd/rkl_advantage_p95": 4.903533246119817, + "kd/rkl_advantage_std": 2.1717200110356014, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2192685983366877e-07, + "loss": 0.031830302874247235, + "num_tokens": 670085777.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8161111096541087, + "rewards/gt_logging_reward/std": 0.37958431939284004, + "sampling/importance_sampling_ratio/max": 1.8112470666567484, + "sampling/importance_sampling_ratio/mean": 0.99882164200147, + "sampling/importance_sampling_ratio/min": 0.477987410624822, + "sampling/sampling_logp_difference/max": 0.32637261152267455, + "sampling/sampling_logp_difference/mean": 0.003871117765083909, + "step": 20490, + "step_time": 7.794535872465349, + "student/entropy": 0.12661665181318918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888889936109383, + "completions/max_length": 479.4, + "completions/max_terminated_length": 443.4, + "completions/mean_length": 188.68028767903647, + "completions/mean_terminated_length": 182.5912094116211, + "completions/min_length": 48.1, + "completions/min_terminated_length": 48.1, + "entropy": 0.12709727038939794, + "epoch": 0.7792503702578514, + "eval/gt_acc_batch": 0.811388905843099, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2774048447608948, + "kd/policy_loss": 0.00833214686329787, + "kd/rkl_advantage_mean": 0.7962485716678203, + "kd/rkl_advantage_p95": 5.080854288736979, + "kd/rkl_advantage_std": 2.2329776336749396, + "kd/ssd_loss": 0.0, + "learning_rate": 2.207876049063912e-07, + "loss": 0.0333285927772522, + "num_tokens": 671072186.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.811388889948527, + "rewards/gt_logging_reward/std": 0.3784083108107249, + "sampling/importance_sampling_ratio/max": 1.937760063012441, + "sampling/importance_sampling_ratio/mean": 1.0034212589263916, + "sampling/importance_sampling_ratio/min": 0.4434123381972313, + "sampling/sampling_logp_difference/max": 0.36574379205703733, + "sampling/sampling_logp_difference/mean": 0.0038859563802058497, + "step": 20520, + "step_time": 8.137478721667625, + "student/entropy": 0.12709727038939794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015277778760840495, + "completions/max_length": 470.4, + "completions/max_terminated_length": 436.7, + "completions/mean_length": 179.57667643229166, + "completions/mean_terminated_length": 174.49473317464194, + "completions/min_length": 58.6, + "completions/min_terminated_length": 58.6, + "entropy": 0.1250147915755709, + "epoch": 0.7803896251851289, + "eval/gt_acc_batch": 0.8272222359975179, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2733175456523895, + "kd/policy_loss": 0.008075692469719797, + "kd/rkl_advantage_mean": 0.7823130888553957, + "kd/rkl_advantage_p95": 5.107070451974868, + "kd/rkl_advantage_std": 2.2532985190550487, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1964834997911366e-07, + "loss": 0.03230277299880981, + "num_tokens": 672022550.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.827222224076589, + "rewards/gt_logging_reward/std": 0.3677007665236791, + "sampling/importance_sampling_ratio/max": 1.8743006904919943, + "sampling/importance_sampling_ratio/mean": 0.9976526101430258, + "sampling/importance_sampling_ratio/min": 0.4566743363936742, + "sampling/sampling_logp_difference/max": 0.32676708896954854, + "sampling/sampling_logp_difference/mean": 0.003806903787578146, + "step": 20550, + "step_time": 7.949342761800896, + "student/entropy": 0.1250147915755709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0280555572360754, + "completions/max_length": 491.6666666666667, + "completions/max_terminated_length": 445.7, + "completions/mean_length": 190.12417551676432, + "completions/mean_terminated_length": 180.85047912597656, + "completions/min_length": 53.63333333333333, + "completions/min_terminated_length": 53.63333333333333, + "entropy": 0.12474353766689698, + "epoch": 0.7815288801124065, + "eval/gt_acc_batch": 0.8022222499052684, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25872039794921875, + "kd/policy_loss": 0.007990100781898945, + "kd/rkl_advantage_mean": 0.7386558180054029, + "kd/rkl_advantage_p95": 4.91678938070933, + "kd/rkl_advantage_std": 2.1653066982825595, + "kd/ssd_loss": 0.0, + "learning_rate": 2.185090950518361e-07, + "loss": 0.03196040789286295, + "num_tokens": 673025021.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8022222240765889, + "rewards/gt_logging_reward/std": 0.3924072136481603, + "sampling/importance_sampling_ratio/max": 1.7982855637868245, + "sampling/importance_sampling_ratio/mean": 0.9973934431870778, + "sampling/importance_sampling_ratio/min": 0.4604299326737722, + "sampling/sampling_logp_difference/max": 0.32739702065785725, + "sampling/sampling_logp_difference/mean": 0.0038345685228705407, + "step": 20580, + "step_time": 8.336836162526742, + "student/entropy": 0.12474353766689698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0247222236978511, + "completions/max_length": 486.93333333333334, + "completions/max_terminated_length": 446.43333333333334, + "completions/mean_length": 189.30612030029297, + "completions/mean_terminated_length": 181.31784311930338, + "completions/min_length": 52.03333333333333, + "completions/min_terminated_length": 52.03333333333333, + "entropy": 0.12915779805431762, + "epoch": 0.782668135039684, + "eval/gt_acc_batch": 0.8061111191908519, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34086868166923523, + "kd/policy_loss": 0.008408397401217372, + "kd/rkl_advantage_mean": 0.8313202444463968, + "kd/rkl_advantage_p95": 5.174305558204651, + "kd/rkl_advantage_std": 2.2662012706200283, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1736984012455852e-07, + "loss": 0.03363358577092489, + "num_tokens": 674018059.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8061111072699229, + "rewards/gt_logging_reward/std": 0.3844926506280899, + "sampling/importance_sampling_ratio/max": 1.9471496105194093, + "sampling/importance_sampling_ratio/mean": 1.001315047343572, + "sampling/importance_sampling_ratio/min": 0.43048441807428994, + "sampling/sampling_logp_difference/max": 0.31256946325302126, + "sampling/sampling_logp_difference/mean": 0.003987694329892596, + "step": 20610, + "step_time": 8.228230388935966, + "student/entropy": 0.12915779805431762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018055556528270244, + "completions/max_length": 495.03333333333336, + "completions/max_terminated_length": 455.53333333333336, + "completions/mean_length": 183.38612009684246, + "completions/mean_terminated_length": 177.21105550130207, + "completions/min_length": 50.03333333333333, + "completions/min_terminated_length": 50.03333333333333, + "entropy": 0.1295291451116403, + "epoch": 0.7838073899669616, + "eval/gt_acc_batch": 0.8027778168519338, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2735559940338135, + "kd/policy_loss": 0.008408554876223207, + "kd/rkl_advantage_mean": 0.7839258414382736, + "kd/rkl_advantage_p95": 5.0856968522071835, + "kd/rkl_advantage_std": 2.2452591558297477, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1623058519728096e-07, + "loss": 0.03363422552744547, + "num_tokens": 674975385.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8027777830759685, + "rewards/gt_logging_reward/std": 0.3929351935784022, + "sampling/importance_sampling_ratio/max": 1.967348035176595, + "sampling/importance_sampling_ratio/mean": 1.0070539156595866, + "sampling/importance_sampling_ratio/min": 0.4514552672704061, + "sampling/sampling_logp_difference/max": 0.3462051272392273, + "sampling/sampling_logp_difference/mean": 0.003984668385237455, + "step": 20640, + "step_time": 8.08317957483911, + "student/entropy": 0.1295291451116403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111250395576, + "completions/max_length": 494.3, + "completions/max_terminated_length": 465.4, + "completions/mean_length": 192.83167622884113, + "completions/mean_terminated_length": 184.36162770589192, + "completions/min_length": 52.46666666666667, + "completions/min_terminated_length": 52.46666666666667, + "entropy": 0.12714999597519636, + "epoch": 0.7849466448942392, + "eval/gt_acc_batch": 0.8063889145851135, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29043424129486084, + "kd/policy_loss": 0.008480954273060585, + "kd/rkl_advantage_mean": 0.8095453088482221, + "kd/rkl_advantage_p95": 5.142375206947326, + "kd/rkl_advantage_std": 2.258004918694496, + "kd/ssd_loss": 0.0, + "learning_rate": 2.150913302700034e-07, + "loss": 0.0339238166809082, + "num_tokens": 675976443.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8063888847827911, + "rewards/gt_logging_reward/std": 0.3893670529127121, + "sampling/importance_sampling_ratio/max": 1.9321671009063721, + "sampling/importance_sampling_ratio/mean": 0.9970778902371724, + "sampling/importance_sampling_ratio/min": 0.4497093717257182, + "sampling/sampling_logp_difference/max": 0.3071626683076223, + "sampling/sampling_logp_difference/mean": 0.0038644756268089017, + "step": 20670, + "step_time": 8.20494957366803, + "student/entropy": 0.12714999597519636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001198301712, + "completions/max_length": 482.76666666666665, + "completions/max_terminated_length": 449.7, + "completions/mean_length": 181.37723185221355, + "completions/mean_terminated_length": 174.77051493326823, + "completions/min_length": 50.5, + "completions/min_terminated_length": 50.5, + "entropy": 0.13238883428275586, + "epoch": 0.7860858998215168, + "eval/gt_acc_batch": 0.7886111338933309, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31483423709869385, + "kd/policy_loss": 0.009092929549903298, + "kd/rkl_advantage_mean": 0.6969040374892453, + "kd/rkl_advantage_p95": 4.834889809290568, + "kd/rkl_advantage_std": 2.177247029542923, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1395207534272585e-07, + "loss": 0.036371715863545734, + "num_tokens": 676940585.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7886111179987589, + "rewards/gt_logging_reward/std": 0.3999864846467972, + "sampling/importance_sampling_ratio/max": 1.8913215080897012, + "sampling/importance_sampling_ratio/mean": 0.991120582818985, + "sampling/importance_sampling_ratio/min": 0.45748386283715564, + "sampling/sampling_logp_difference/max": 0.32482681969801586, + "sampling/sampling_logp_difference/mean": 0.0039768167305737736, + "step": 20700, + "step_time": 8.2495124487672, + "student/entropy": 0.13238883428275586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277779114743075, + "completions/max_length": 480.0, + "completions/max_terminated_length": 431.6666666666667, + "completions/mean_length": 183.4636225382487, + "completions/mean_terminated_length": 176.64421895345052, + "completions/min_length": 52.13333333333333, + "completions/min_terminated_length": 52.13333333333333, + "entropy": 0.12650155425071716, + "epoch": 0.7872251547487943, + "eval/gt_acc_batch": 0.8083333512147267, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30248844623565674, + "kd/policy_loss": 0.007829507708083838, + "kd/rkl_advantage_mean": 0.766255919697384, + "kd/rkl_advantage_p95": 4.961375705401102, + "kd/rkl_advantage_std": 2.2081872185071307, + "kd/ssd_loss": 0.0, + "learning_rate": 2.128128204154483e-07, + "loss": 0.03131803274154663, + "num_tokens": 677900590.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8083333273728689, + "rewards/gt_logging_reward/std": 0.38521882792313894, + "sampling/importance_sampling_ratio/max": 1.9455652077992758, + "sampling/importance_sampling_ratio/mean": 1.0036179065704345, + "sampling/importance_sampling_ratio/min": 0.4743225465218226, + "sampling/sampling_logp_difference/max": 0.336443022886912, + "sampling/sampling_logp_difference/mean": 0.003864831360988319, + "step": 20730, + "step_time": 7.845900852660027, + "student/entropy": 0.12650155425071716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02138889003545046, + "completions/max_length": 501.8333333333333, + "completions/max_terminated_length": 464.26666666666665, + "completions/mean_length": 191.30612131754557, + "completions/mean_terminated_length": 184.2746790568034, + "completions/min_length": 55.93333333333333, + "completions/min_terminated_length": 55.93333333333333, + "entropy": 0.13384631363054117, + "epoch": 0.7883644096760718, + "eval/gt_acc_batch": 0.7922222356001536, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32767125964164734, + "kd/policy_loss": 0.009036393758530417, + "kd/rkl_advantage_mean": 0.7906889160474141, + "kd/rkl_advantage_p95": 5.1145099322001135, + "kd/rkl_advantage_std": 2.268378095825513, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1167356548817071e-07, + "loss": 0.03614558378855388, + "num_tokens": 678891324.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222236792247, + "rewards/gt_logging_reward/std": 0.3980928326646487, + "sampling/importance_sampling_ratio/max": 1.939883021513621, + "sampling/importance_sampling_ratio/mean": 1.003896524508794, + "sampling/importance_sampling_ratio/min": 0.46015554865201314, + "sampling/sampling_logp_difference/max": 0.307533323764801, + "sampling/sampling_logp_difference/mean": 0.004090004476408163, + "step": 20760, + "step_time": 8.102029238430744, + "student/entropy": 0.13384631363054117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166667989144724, + "completions/max_length": 505.8666666666667, + "completions/max_terminated_length": 475.43333333333334, + "completions/mean_length": 191.73639831542968, + "completions/mean_terminated_length": 183.8676737467448, + "completions/min_length": 54.03333333333333, + "completions/min_terminated_length": 54.03333333333333, + "entropy": 0.13296898659318684, + "epoch": 0.7895036646033494, + "eval/gt_acc_batch": 0.7927778005599976, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28788962960243225, + "kd/policy_loss": 0.009044903571096559, + "kd/rkl_advantage_mean": 0.7999830225327362, + "kd/rkl_advantage_p95": 5.118143264452616, + "kd/rkl_advantage_std": 2.284919469555219, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1053431056089316e-07, + "loss": 0.036179614067077634, + "num_tokens": 679898695.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7927777747313182, + "rewards/gt_logging_reward/std": 0.4014110763867696, + "sampling/importance_sampling_ratio/max": 1.9107422947883606, + "sampling/importance_sampling_ratio/mean": 0.999746960401535, + "sampling/importance_sampling_ratio/min": 0.4534277538458506, + "sampling/sampling_logp_difference/max": 0.3153413494427999, + "sampling/sampling_logp_difference/mean": 0.004006055183708668, + "step": 20790, + "step_time": 8.461855065626635, + "student/entropy": 0.13296898659318684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112305273613, + "completions/max_length": 472.6333333333333, + "completions/max_terminated_length": 446.56666666666666, + "completions/mean_length": 192.06973317464193, + "completions/mean_terminated_length": 185.31256256103515, + "completions/min_length": 57.266666666666666, + "completions/min_terminated_length": 57.266666666666666, + "entropy": 0.12618211389829714, + "epoch": 0.7906429195306269, + "eval/gt_acc_batch": 0.7769444624582926, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28828006982803345, + "kd/policy_loss": 0.008607525440553824, + "kd/rkl_advantage_mean": 0.7057499185282116, + "kd/rkl_advantage_p95": 4.88022495508194, + "kd/rkl_advantage_std": 2.193454028169314, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0939505563361563e-07, + "loss": 0.03443010250727336, + "num_tokens": 680896458.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7769444425900777, + "rewards/gt_logging_reward/std": 0.4092552165190379, + "sampling/importance_sampling_ratio/max": 1.9662815928459167, + "sampling/importance_sampling_ratio/mean": 1.0038348774115244, + "sampling/importance_sampling_ratio/min": 0.46886705557505287, + "sampling/sampling_logp_difference/max": 0.33966626624266305, + "sampling/sampling_logp_difference/mean": 0.0038516991461316746, + "step": 20820, + "step_time": 8.054982615999567, + "student/entropy": 0.12618211389829714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01833333441366752, + "completions/max_length": 460.2, + "completions/max_terminated_length": 425.76666666666665, + "completions/mean_length": 179.75167694091797, + "completions/mean_terminated_length": 173.8018834431966, + "completions/min_length": 54.93333333333333, + "completions/min_terminated_length": 54.93333333333333, + "entropy": 0.12987176875273387, + "epoch": 0.7917821744579046, + "eval/gt_acc_batch": 0.7980555733044942, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2645399868488312, + "kd/policy_loss": 0.008225047215819358, + "kd/rkl_advantage_mean": 0.6820656571472985, + "kd/rkl_advantage_p95": 4.794006917874018, + "kd/rkl_advantage_std": 2.186598411202431, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0825580070633805e-07, + "loss": 0.03290019035339355, + "num_tokens": 681843796.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7980555554231008, + "rewards/gt_logging_reward/std": 0.3875097299615542, + "sampling/importance_sampling_ratio/max": 1.973408003648122, + "sampling/importance_sampling_ratio/mean": 0.9969303965568542, + "sampling/importance_sampling_ratio/min": 0.4358108381430308, + "sampling/sampling_logp_difference/max": 0.3313009242216746, + "sampling/sampling_logp_difference/mean": 0.0039783066449066, + "step": 20850, + "step_time": 8.147455522969056, + "student/entropy": 0.12987176875273387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223927577337, + "completions/max_length": 475.76666666666665, + "completions/max_terminated_length": 450.6, + "completions/mean_length": 190.93195343017578, + "completions/mean_terminated_length": 181.2626159667969, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.13212150962402422, + "epoch": 0.7929214293851821, + "eval/gt_acc_batch": 0.77333336075147, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2849154770374298, + "kd/policy_loss": 0.009008160023950041, + "kd/rkl_advantage_mean": 0.7475448032375425, + "kd/rkl_advantage_p95": 4.969313657283783, + "kd/rkl_advantage_std": 2.1991303543249767, + "kd/ssd_loss": 0.0, + "learning_rate": 2.071165457790605e-07, + "loss": 0.036032644907633464, + "num_tokens": 682848863.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7733333309491476, + "rewards/gt_logging_reward/std": 0.4101488202810287, + "sampling/importance_sampling_ratio/max": 1.9111072738965353, + "sampling/importance_sampling_ratio/mean": 0.9972355405489604, + "sampling/importance_sampling_ratio/min": 0.45075533737738926, + "sampling/sampling_logp_difference/max": 0.3197549968957901, + "sampling/sampling_logp_difference/mean": 0.003992000004897515, + "step": 20880, + "step_time": 8.52326264863465, + "student/entropy": 0.13212150962402422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013611111789941788, + "completions/max_length": 486.23333333333335, + "completions/max_terminated_length": 440.5, + "completions/mean_length": 179.97862091064454, + "completions/mean_terminated_length": 175.42652079264323, + "completions/min_length": 59.4, + "completions/min_terminated_length": 59.4, + "entropy": 0.12789772152900697, + "epoch": 0.7940606843124597, + "eval/gt_acc_batch": 0.8172222415606181, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29409629106521606, + "kd/policy_loss": 0.008386804023757577, + "kd/rkl_advantage_mean": 0.7272642849013209, + "kd/rkl_advantage_p95": 4.896261469523112, + "kd/rkl_advantage_std": 2.197756173213323, + "kd/ssd_loss": 0.0, + "learning_rate": 2.059772908517829e-07, + "loss": 0.03354722261428833, + "num_tokens": 683800610.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8172222216924031, + "rewards/gt_logging_reward/std": 0.3801286409298579, + "sampling/importance_sampling_ratio/max": 1.8273634274800619, + "sampling/importance_sampling_ratio/mean": 0.998584904273351, + "sampling/importance_sampling_ratio/min": 0.4629117200771968, + "sampling/sampling_logp_difference/max": 0.3172488133112589, + "sampling/sampling_logp_difference/mean": 0.0038750738138332965, + "step": 20910, + "step_time": 8.344998847233365, + "student/entropy": 0.12789772152900697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030000001844018697, + "completions/max_length": 490.1333333333333, + "completions/max_terminated_length": 459.4, + "completions/mean_length": 194.3577880859375, + "completions/mean_terminated_length": 184.73109486897786, + "completions/min_length": 51.6, + "completions/min_terminated_length": 51.6, + "entropy": 0.1318827496220668, + "epoch": 0.7951999392397372, + "eval/gt_acc_batch": 0.7775000095367431, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2835569977760315, + "kd/policy_loss": 0.008599493947500984, + "kd/rkl_advantage_mean": 0.7423185429225365, + "kd/rkl_advantage_p95": 4.951168139775594, + "kd/rkl_advantage_std": 2.206764267881711, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0483803592450535e-07, + "loss": 0.03439797957738241, + "num_tokens": 684809058.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7775000015894572, + "rewards/gt_logging_reward/std": 0.40780978202819823, + "sampling/importance_sampling_ratio/max": 1.8538680871327717, + "sampling/importance_sampling_ratio/mean": 0.9940573831399282, + "sampling/importance_sampling_ratio/min": 0.42840509017308553, + "sampling/sampling_logp_difference/max": 0.29723299741744996, + "sampling/sampling_logp_difference/mean": 0.0039596220711246135, + "step": 20940, + "step_time": 8.397919805869849, + "student/entropy": 0.1318827496220668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055557068437336, + "completions/max_length": 499.8, + "completions/max_terminated_length": 472.26666666666665, + "completions/mean_length": 191.39917755126953, + "completions/mean_terminated_length": 183.81275024414063, + "completions/min_length": 53.9, + "completions/min_terminated_length": 53.9, + "entropy": 0.130138896095256, + "epoch": 0.7963391941670148, + "eval/gt_acc_batch": 0.781944461663564, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3593582212924957, + "kd/policy_loss": 0.008704493575108548, + "kd/rkl_advantage_mean": 0.8481934975832701, + "kd/rkl_advantage_p95": 5.217655406395594, + "kd/rkl_advantage_std": 2.2757728179295857, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0369878099722782e-07, + "loss": 0.03481797377268473, + "num_tokens": 685803983.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7819444437821707, + "rewards/gt_logging_reward/std": 0.4063135474920273, + "sampling/importance_sampling_ratio/max": 1.846528740723928, + "sampling/importance_sampling_ratio/mean": 0.9944670220216115, + "sampling/importance_sampling_ratio/min": 0.41029917597770693, + "sampling/sampling_logp_difference/max": 0.3373536547025045, + "sampling/sampling_logp_difference/mean": 0.003911465282241503, + "step": 20970, + "step_time": 8.462876100897363, + "student/entropy": 0.130138896095256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334382623433, + "completions/max_length": 490.8666666666667, + "completions/max_terminated_length": 463.3, + "completions/mean_length": 185.86528727213542, + "completions/mean_terminated_length": 179.8620819091797, + "completions/min_length": 58.86666666666667, + "completions/min_terminated_length": 58.86666666666667, + "entropy": 0.12768895191450913, + "epoch": 0.7974784490942923, + "eval/gt_acc_batch": 0.8013888955116272, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29044830799102783, + "kd/policy_loss": 0.008308052979797746, + "kd/rkl_advantage_mean": 0.8175796000039858, + "kd/rkl_advantage_p95": 5.062593924999237, + "kd/rkl_advantage_std": 2.2508513212203978, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0255952606995024e-07, + "loss": 0.03323222398757934, + "num_tokens": 686774394.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888915379842, + "rewards/gt_logging_reward/std": 0.3929796705643336, + "sampling/importance_sampling_ratio/max": 1.8803833603858948, + "sampling/importance_sampling_ratio/mean": 1.0027410606543223, + "sampling/importance_sampling_ratio/min": 0.4828127185503642, + "sampling/sampling_logp_difference/max": 0.3038428246974945, + "sampling/sampling_logp_difference/mean": 0.0038897542515769603, + "step": 21000, + "step_time": 8.314082260394935, + "student/entropy": 0.12768895191450913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111253499985, + "completions/max_length": 489.43333333333334, + "completions/max_terminated_length": 457.06666666666666, + "completions/mean_length": 187.55806427001954, + "completions/mean_terminated_length": 178.97872772216797, + "completions/min_length": 53.266666666666666, + "completions/min_terminated_length": 53.266666666666666, + "entropy": 0.13096216283738613, + "epoch": 0.7986177040215698, + "eval/gt_acc_batch": 0.8055555860201518, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31705352663993835, + "kd/policy_loss": 0.00852491333692645, + "kd/rkl_advantage_mean": 0.7536177281290293, + "kd/rkl_advantage_p95": 5.037333605686824, + "kd/rkl_advantage_std": 2.2187119563420614, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0142027114267268e-07, + "loss": 0.034099654356638594, + "num_tokens": 687753307.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8055555542310079, + "rewards/gt_logging_reward/std": 0.3906245400508245, + "sampling/importance_sampling_ratio/max": 1.9551949659983316, + "sampling/importance_sampling_ratio/mean": 0.9988263050715128, + "sampling/importance_sampling_ratio/min": 0.43537881374359133, + "sampling/sampling_logp_difference/max": 0.29422687689463295, + "sampling/sampling_logp_difference/mean": 0.003993981385913988, + "step": 21030, + "step_time": 8.255806891002187, + "student/entropy": 0.13096216283738613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112528791032, + "completions/max_length": 481.23333333333335, + "completions/max_terminated_length": 440.2, + "completions/mean_length": 191.4452891031901, + "completions/mean_terminated_length": 183.75846354166666, + "completions/min_length": 56.93333333333333, + "completions/min_terminated_length": 56.93333333333333, + "entropy": 0.13012899328023195, + "epoch": 0.7997569589488475, + "eval/gt_acc_batch": 0.800000011920929, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.5345389246940613, + "kd/policy_loss": 0.00909100923454389, + "kd/rkl_advantage_mean": 0.7432518806308508, + "kd/rkl_advantage_p95": 4.99458401799202, + "kd/rkl_advantage_std": 2.2186006546020507, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0028101621539513e-07, + "loss": 0.03636404275894165, + "num_tokens": 688751294.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8, + "rewards/gt_logging_reward/std": 0.39726589719454447, + "sampling/importance_sampling_ratio/max": 1.916915241877238, + "sampling/importance_sampling_ratio/mean": 0.9994440277417501, + "sampling/importance_sampling_ratio/min": 0.45043237805366515, + "sampling/sampling_logp_difference/max": 0.3412542382876078, + "sampling/sampling_logp_difference/mean": 0.003987054449195663, + "step": 21060, + "step_time": 8.216286845761351, + "student/entropy": 0.13012899328023195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02750000162050128, + "completions/max_length": 501.4, + "completions/max_terminated_length": 476.5, + "completions/mean_length": 197.3202885945638, + "completions/mean_terminated_length": 188.42169952392578, + "completions/min_length": 55.2, + "completions/min_terminated_length": 55.2, + "entropy": 0.12855937325706085, + "epoch": 0.800896213876125, + "eval/gt_acc_batch": 0.7777777930100759, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.39833003282546997, + "kd/policy_loss": 0.00861167978340139, + "kd/rkl_advantage_mean": 0.7904579558720192, + "kd/rkl_advantage_p95": 5.086360319455465, + "kd/rkl_advantage_std": 2.248361203074455, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9914176128811755e-07, + "loss": 0.03444672425587972, + "num_tokens": 689762527.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.777777777115504, + "rewards/gt_logging_reward/std": 0.40930961569150287, + "sampling/importance_sampling_ratio/max": 1.9196407755215963, + "sampling/importance_sampling_ratio/mean": 0.9992976903915405, + "sampling/importance_sampling_ratio/min": 0.4622038662433624, + "sampling/sampling_logp_difference/max": 0.31345277627309165, + "sampling/sampling_logp_difference/mean": 0.00391203563194722, + "step": 21090, + "step_time": 8.27229422122958, + "student/entropy": 0.12855937325706085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013750000856816769, + "completions/max_length": 463.2, + "completions/max_terminated_length": 419.85, + "completions/mean_length": 186.58417587280275, + "completions/mean_terminated_length": 182.1340789794922, + "completions/min_length": 58.65, + "completions/min_terminated_length": 58.65, + "entropy": 0.12939047580584884, + "epoch": 0.8020354688034026, + "eval/gt_acc_batch": 0.7875000208616256, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24375629425048828, + "kd/policy_loss": 0.008131009031785652, + "kd/rkl_advantage_mean": 0.6957314797269646, + "kd/rkl_advantage_p95": 4.799916455149651, + "kd/rkl_advantage_std": 2.151377236843109, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9800250636084002e-07, + "loss": 0.03252404034137726, + "num_tokens": 690754535.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7874999940395355, + "rewards/gt_logging_reward/std": 0.40385905653238297, + "sampling/importance_sampling_ratio/max": 1.8248178124427796, + "sampling/importance_sampling_ratio/mean": 0.9947557032108307, + "sampling/importance_sampling_ratio/min": 0.4518837913870811, + "sampling/sampling_logp_difference/max": 0.3027741551399231, + "sampling/sampling_logp_difference/mean": 0.003884607832878828, + "step": 21120, + "step_time": 7.913813526606828, + "student/entropy": 0.12939047580584884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112336317697, + "completions/max_length": 489.43333333333334, + "completions/max_terminated_length": 453.3666666666667, + "completions/mean_length": 192.8969533284505, + "completions/mean_terminated_length": 186.09339345296223, + "completions/min_length": 58.666666666666664, + "completions/min_terminated_length": 58.666666666666664, + "entropy": 0.13123528690387806, + "epoch": 0.8031747237306801, + "eval/gt_acc_batch": 0.7786111295223236, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29372453689575195, + "kd/policy_loss": 0.008425119538636257, + "kd/rkl_advantage_mean": 0.7806499759977062, + "kd/rkl_advantage_p95": 4.984885656833649, + "kd/rkl_advantage_std": 2.219469228386879, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9686325143356243e-07, + "loss": 0.03370047807693481, + "num_tokens": 691742708.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7786111116409302, + "rewards/gt_logging_reward/std": 0.40786136786142985, + "sampling/importance_sampling_ratio/max": 1.9203442653020224, + "sampling/importance_sampling_ratio/mean": 0.9948749522368113, + "sampling/importance_sampling_ratio/min": 0.44870459934075674, + "sampling/sampling_logp_difference/max": 0.315538227558136, + "sampling/sampling_logp_difference/mean": 0.003905812643157939, + "step": 21150, + "step_time": 7.937665155903475, + "student/entropy": 0.13123528690387806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02888889048869411, + "completions/max_length": 493.1333333333333, + "completions/max_terminated_length": 456.1333333333333, + "completions/mean_length": 194.62501017252603, + "completions/mean_terminated_length": 185.605344136556, + "completions/min_length": 53.8, + "completions/min_terminated_length": 53.8, + "entropy": 0.13520498418559632, + "epoch": 0.8043139786579577, + "eval/gt_acc_batch": 0.781388912598292, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24039152264595032, + "kd/policy_loss": 0.009118565280611316, + "kd/rkl_advantage_mean": 0.7751866079866886, + "kd/rkl_advantage_p95": 5.142460225025813, + "kd/rkl_advantage_std": 2.269443773229917, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9572399650628488e-07, + "loss": 0.03647425969441732, + "num_tokens": 692755294.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7813888847827911, + "rewards/gt_logging_reward/std": 0.40438465078671776, + "sampling/importance_sampling_ratio/max": 1.9347715894381206, + "sampling/importance_sampling_ratio/mean": 1.0023068030675253, + "sampling/importance_sampling_ratio/min": 0.4184630736708641, + "sampling/sampling_logp_difference/max": 0.3452309250831604, + "sampling/sampling_logp_difference/mean": 0.004065493059655031, + "step": 21180, + "step_time": 8.157987681033168, + "student/entropy": 0.13520498418559632 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02833333509042859, + "completions/max_length": 498.96666666666664, + "completions/max_terminated_length": 451.2, + "completions/mean_length": 191.89945424397786, + "completions/mean_terminated_length": 182.65874532063802, + "completions/min_length": 47.333333333333336, + "completions/min_terminated_length": 47.333333333333336, + "entropy": 0.13326559762159984, + "epoch": 0.8054532335852352, + "eval/gt_acc_batch": 0.7869444648424785, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2724500894546509, + "kd/policy_loss": 0.008758121812328075, + "kd/rkl_advantage_mean": 0.8229799826939901, + "kd/rkl_advantage_p95": 5.141605557998021, + "kd/rkl_advantage_std": 2.256368734439214, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9458474157900732e-07, + "loss": 0.03503248294194539, + "num_tokens": 693749308.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444489479065, + "rewards/gt_logging_reward/std": 0.3995523348450661, + "sampling/importance_sampling_ratio/max": 1.9108855247497558, + "sampling/importance_sampling_ratio/mean": 0.9965784549713135, + "sampling/importance_sampling_ratio/min": 0.40940639277299246, + "sampling/sampling_logp_difference/max": 0.3992016871770223, + "sampling/sampling_logp_difference/mean": 0.003991637953246633, + "step": 21210, + "step_time": 8.045575109260001, + "student/entropy": 0.13326559762159984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028888890519738197, + "completions/max_length": 482.3, + "completions/max_terminated_length": 448.8333333333333, + "completions/mean_length": 191.83834381103514, + "completions/mean_terminated_length": 182.60110880533855, + "completions/min_length": 54.733333333333334, + "completions/min_terminated_length": 54.733333333333334, + "entropy": 0.13152330443263055, + "epoch": 0.8065924885125129, + "eval/gt_acc_batch": 0.7869444588820139, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24029015004634857, + "kd/policy_loss": 0.008610709855565802, + "kd/rkl_advantage_mean": 0.7209165226047237, + "kd/rkl_advantage_p95": 4.917790903647741, + "kd/rkl_advantage_std": 2.2127308348814645, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9344548665172977e-07, + "loss": 0.034442838033040366, + "num_tokens": 694756566.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7869444449742635, + "rewards/gt_logging_reward/std": 0.40343409578005474, + "sampling/importance_sampling_ratio/max": 1.8957260092099508, + "sampling/importance_sampling_ratio/mean": 0.9964794476826986, + "sampling/importance_sampling_ratio/min": 0.44753661900758746, + "sampling/sampling_logp_difference/max": 0.3381652812163035, + "sampling/sampling_logp_difference/mean": 0.003952939948067069, + "step": 21240, + "step_time": 8.051577260938938, + "student/entropy": 0.13152330443263055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02222222350537777, + "completions/max_length": 499.26666666666665, + "completions/max_terminated_length": 461.96666666666664, + "completions/mean_length": 194.73473154703777, + "completions/mean_terminated_length": 187.49263712565104, + "completions/min_length": 58.2, + "completions/min_terminated_length": 58.2, + "entropy": 0.1273087098573645, + "epoch": 0.8077317434397904, + "eval/gt_acc_batch": 0.7975000202655792, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28558242321014404, + "kd/policy_loss": 0.008055321513287103, + "kd/rkl_advantage_mean": 0.7246089035955569, + "kd/rkl_advantage_p95": 4.897933711608251, + "kd/rkl_advantage_std": 2.1819740047057468, + "kd/ssd_loss": 0.0, + "learning_rate": 1.923062317244522e-07, + "loss": 0.0322212815284729, + "num_tokens": 695762723.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7974999944368998, + "rewards/gt_logging_reward/std": 0.3955924371878306, + "sampling/importance_sampling_ratio/max": 1.965640119711558, + "sampling/importance_sampling_ratio/mean": 0.9942101458708446, + "sampling/importance_sampling_ratio/min": 0.45960119664669036, + "sampling/sampling_logp_difference/max": 0.2868567486604055, + "sampling/sampling_logp_difference/mean": 0.0038575592217966912, + "step": 21270, + "step_time": 7.991447956441941, + "student/entropy": 0.1273087098573645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018055556590358415, + "completions/max_length": 484.73333333333335, + "completions/max_terminated_length": 449.3, + "completions/mean_length": 183.897509765625, + "completions/mean_terminated_length": 177.93607533772786, + "completions/min_length": 48.666666666666664, + "completions/min_terminated_length": 48.666666666666664, + "entropy": 0.1233679989973704, + "epoch": 0.8088709983670679, + "eval/gt_acc_batch": 0.8072222411632538, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33473846316337585, + "kd/policy_loss": 0.007935358013492077, + "kd/rkl_advantage_mean": 0.6881830565631389, + "kd/rkl_advantage_p95": 4.779024289051692, + "kd/rkl_advantage_std": 2.1548807621002197, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9116697679717463e-07, + "loss": 0.03174143234888713, + "num_tokens": 696724722.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222173213959, + "rewards/gt_logging_reward/std": 0.3881600668032964, + "sampling/importance_sampling_ratio/max": 1.8383774201075236, + "sampling/importance_sampling_ratio/mean": 0.9970783611138662, + "sampling/importance_sampling_ratio/min": 0.45794594784577686, + "sampling/sampling_logp_difference/max": 0.3310838023821513, + "sampling/sampling_logp_difference/mean": 0.0037320230699454744, + "step": 21300, + "step_time": 7.909751482368059, + "student/entropy": 0.1233679989973704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778673917055, + "completions/max_length": 490.3, + "completions/max_terminated_length": 456.06666666666666, + "completions/mean_length": 187.6077870686849, + "completions/mean_terminated_length": 181.94668833414713, + "completions/min_length": 59.06666666666667, + "completions/min_terminated_length": 59.06666666666667, + "entropy": 0.13464793724318344, + "epoch": 0.8100102532943455, + "eval/gt_acc_batch": 0.7986111342906952, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4245620667934418, + "kd/policy_loss": 0.008983809013928597, + "kd/rkl_advantage_mean": 0.7822890217726429, + "kd/rkl_advantage_p95": 5.114395962158839, + "kd/rkl_advantage_std": 2.2638967007398607, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9002772186989707e-07, + "loss": 0.03593524297078451, + "num_tokens": 697699550.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111084620158, + "rewards/gt_logging_reward/std": 0.3967046856880188, + "sampling/importance_sampling_ratio/max": 1.8580148855845133, + "sampling/importance_sampling_ratio/mean": 0.9994392613569896, + "sampling/importance_sampling_ratio/min": 0.4498531957467397, + "sampling/sampling_logp_difference/max": 0.362181959549586, + "sampling/sampling_logp_difference/mean": 0.004044527358685931, + "step": 21330, + "step_time": 7.825559357461558, + "student/entropy": 0.13464793724318344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445775200924, + "completions/max_length": 488.3, + "completions/max_terminated_length": 445.3333333333333, + "completions/mean_length": 190.79334411621093, + "completions/mean_terminated_length": 183.86802571614584, + "completions/min_length": 54.3, + "completions/min_terminated_length": 54.3, + "entropy": 0.1274471778422594, + "epoch": 0.811149508221623, + "eval/gt_acc_batch": 0.7816666762034098, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25712937116622925, + "kd/policy_loss": 0.008506203854146103, + "kd/rkl_advantage_mean": 0.7519914568712314, + "kd/rkl_advantage_p95": 5.061613750457764, + "kd/rkl_advantage_std": 2.237518349289894, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8888846694261952e-07, + "loss": 0.03402482271194458, + "num_tokens": 698701814.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7816666662693024, + "rewards/gt_logging_reward/std": 0.4059631168842316, + "sampling/importance_sampling_ratio/max": 1.8578107714653016, + "sampling/importance_sampling_ratio/mean": 0.9978854934374491, + "sampling/importance_sampling_ratio/min": 0.4645260771115621, + "sampling/sampling_logp_difference/max": 0.335895444949468, + "sampling/sampling_logp_difference/mean": 0.003906302984493474, + "step": 21360, + "step_time": 8.11195277600588, + "student/entropy": 0.1274471778422594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112280438343, + "completions/max_length": 501.76666666666665, + "completions/max_terminated_length": 453.0, + "completions/mean_length": 182.49695332845053, + "completions/mean_terminated_length": 174.50060526529947, + "completions/min_length": 52.86666666666667, + "completions/min_terminated_length": 52.86666666666667, + "entropy": 0.1298230215907097, + "epoch": 0.8122887631489006, + "eval/gt_acc_batch": 0.8013889074325562, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.315913587808609, + "kd/policy_loss": 0.008438163456351807, + "kd/rkl_advantage_mean": 0.7535806958253185, + "kd/rkl_advantage_p95": 5.06130144794782, + "kd/rkl_advantage_std": 2.258491728703181, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8774921201534196e-07, + "loss": 0.03375264803568522, + "num_tokens": 699659107.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888875643412, + "rewards/gt_logging_reward/std": 0.39203139742215476, + "sampling/importance_sampling_ratio/max": 1.8814522663752238, + "sampling/importance_sampling_ratio/mean": 1.0021841903527577, + "sampling/importance_sampling_ratio/min": 0.4689969380696615, + "sampling/sampling_logp_difference/max": 0.32055510183175406, + "sampling/sampling_logp_difference/mean": 0.003913690041129788, + "step": 21390, + "step_time": 7.9487288718043905, + "student/entropy": 0.1298230215907097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556789040567, + "completions/max_length": 482.56666666666666, + "completions/max_terminated_length": 456.9, + "completions/mean_length": 190.2841771443685, + "completions/mean_terminated_length": 182.85309397379558, + "completions/min_length": 51.266666666666666, + "completions/min_terminated_length": 51.266666666666666, + "entropy": 0.12893607821315528, + "epoch": 0.8134280180761781, + "eval/gt_acc_batch": 0.8005555808544159, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24541622400283813, + "kd/policy_loss": 0.007977969773734609, + "kd/rkl_advantage_mean": 0.7155021130902848, + "kd/rkl_advantage_p95": 4.905288430054982, + "kd/rkl_advantage_std": 2.205449614922206, + "kd/ssd_loss": 0.0, + "learning_rate": 1.866099570880644e-07, + "loss": 0.031911879777908325, + "num_tokens": 700643418.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8005555510520935, + "rewards/gt_logging_reward/std": 0.39207402169704436, + "sampling/importance_sampling_ratio/max": 1.8734707554181418, + "sampling/importance_sampling_ratio/mean": 0.9976408084233602, + "sampling/importance_sampling_ratio/min": 0.46584392885367076, + "sampling/sampling_logp_difference/max": 0.3085514783859253, + "sampling/sampling_logp_difference/mean": 0.0038716528797522187, + "step": 21420, + "step_time": 7.978377699369836, + "student/entropy": 0.12893607821315528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02138889003545046, + "completions/max_length": 488.76666666666665, + "completions/max_terminated_length": 463.96666666666664, + "completions/mean_length": 195.21862030029297, + "completions/mean_terminated_length": 188.56088358561198, + "completions/min_length": 53.166666666666664, + "completions/min_terminated_length": 53.166666666666664, + "entropy": 0.1308124886204799, + "epoch": 0.8145672730034558, + "eval/gt_acc_batch": 0.784166689713796, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25789692997932434, + "kd/policy_loss": 0.00879050512254859, + "kd/rkl_advantage_mean": 0.7033161686422924, + "kd/rkl_advantage_p95": 4.891208859284719, + "kd/rkl_advantage_std": 2.1766762415568035, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8547070216078682e-07, + "loss": 0.03516202767690023, + "num_tokens": 701649581.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7841666638851166, + "rewards/gt_logging_reward/std": 0.401549357175827, + "sampling/importance_sampling_ratio/max": 1.9654080549875894, + "sampling/importance_sampling_ratio/mean": 0.9998113751411438, + "sampling/importance_sampling_ratio/min": 0.45466448962688444, + "sampling/sampling_logp_difference/max": 0.3273669362068176, + "sampling/sampling_logp_difference/mean": 0.003996564641905328, + "step": 21450, + "step_time": 7.951938066908042, + "student/entropy": 0.1308124886204799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223250816264, + "completions/max_length": 485.9, + "completions/max_terminated_length": 441.43333333333334, + "completions/mean_length": 188.1672337849935, + "completions/mean_terminated_length": 181.79902038574218, + "completions/min_length": 61.4, + "completions/min_terminated_length": 61.4, + "entropy": 0.1277515217040976, + "epoch": 0.8157065279307333, + "eval/gt_acc_batch": 0.8108333647251129, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23814278841018677, + "kd/policy_loss": 0.008341093972558156, + "kd/rkl_advantage_mean": 0.7082819820245884, + "kd/rkl_advantage_p95": 4.880656522512436, + "kd/rkl_advantage_std": 2.175767601529757, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8433144723350927e-07, + "loss": 0.03336437543233236, + "num_tokens": 702627831.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8108333349227905, + "rewards/gt_logging_reward/std": 0.3865358720223109, + "sampling/importance_sampling_ratio/max": 1.9074494043986003, + "sampling/importance_sampling_ratio/mean": 1.0041028002897898, + "sampling/importance_sampling_ratio/min": 0.41827363818883895, + "sampling/sampling_logp_difference/max": 0.31163255174954735, + "sampling/sampling_logp_difference/mean": 0.0039051345627134043, + "step": 21480, + "step_time": 7.952675979033423, + "student/entropy": 0.1277515217040976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666795189182, + "completions/max_length": 488.06666666666666, + "completions/max_terminated_length": 441.1333333333333, + "completions/mean_length": 185.55945485432943, + "completions/mean_terminated_length": 178.31074015299478, + "completions/min_length": 54.2, + "completions/min_terminated_length": 54.2, + "entropy": 0.12891173884272575, + "epoch": 0.8168457828580109, + "eval/gt_acc_batch": 0.8130555709203084, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3037206530570984, + "kd/policy_loss": 0.008567850538141405, + "kd/rkl_advantage_mean": 0.7182860840189581, + "kd/rkl_advantage_p95": 4.831400942802429, + "kd/rkl_advantage_std": 2.179238090912501, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8319219230623174e-07, + "loss": 0.0342713991800944, + "num_tokens": 703598509.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813055549065272, + "rewards/gt_logging_reward/std": 0.3845926821231842, + "sampling/importance_sampling_ratio/max": 1.9487300634384155, + "sampling/importance_sampling_ratio/mean": 1.012421230475108, + "sampling/importance_sampling_ratio/min": 0.4840393697222074, + "sampling/sampling_logp_difference/max": 0.315477458635966, + "sampling/sampling_logp_difference/mean": 0.0039002410136163234, + "step": 21510, + "step_time": 7.865735871833749, + "student/entropy": 0.12891173884272575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112715055546, + "completions/max_length": 496.1, + "completions/max_terminated_length": 459.8, + "completions/mean_length": 189.77889811197917, + "completions/mean_terminated_length": 182.0191212972005, + "completions/min_length": 50.766666666666666, + "completions/min_terminated_length": 50.766666666666666, + "entropy": 0.12962638940662147, + "epoch": 0.8179850377852884, + "eval/gt_acc_batch": 0.785833356777827, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3081289827823639, + "kd/policy_loss": 0.008272406097967177, + "kd/rkl_advantage_mean": 0.802731096620361, + "kd/rkl_advantage_p95": 5.093313046296438, + "kd/rkl_advantage_std": 2.2428653965393703, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8205293737895416e-07, + "loss": 0.03308962186177571, + "num_tokens": 704585529.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.785833332935969, + "rewards/gt_logging_reward/std": 0.40284262001514437, + "sampling/importance_sampling_ratio/max": 1.8613884488741557, + "sampling/importance_sampling_ratio/mean": 0.9970019578933715, + "sampling/importance_sampling_ratio/min": 0.4254110644261042, + "sampling/sampling_logp_difference/max": 0.3769297202428182, + "sampling/sampling_logp_difference/mean": 0.003939149117407699, + "step": 21540, + "step_time": 8.198885394664831, + "student/entropy": 0.12962638940662147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0369444467437764, + "completions/max_length": 496.73333333333335, + "completions/max_terminated_length": 466.1666666666667, + "completions/mean_length": 196.07695414225262, + "completions/mean_terminated_length": 184.15743509928384, + "completions/min_length": 56.56666666666667, + "completions/min_terminated_length": 56.56666666666667, + "entropy": 0.13428463619202374, + "epoch": 0.8191242927125659, + "eval/gt_acc_batch": 0.7850000162919363, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22197501361370087, + "kd/policy_loss": 0.008920211750470723, + "kd/rkl_advantage_mean": 0.7505288790911436, + "kd/rkl_advantage_p95": 5.039597378174464, + "kd/rkl_advantage_std": 2.2287869215011598, + "kd/ssd_loss": 0.0, + "learning_rate": 1.809136824516766e-07, + "loss": 0.035680846373240156, + "num_tokens": 705599998.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7849999984105428, + "rewards/gt_logging_reward/std": 0.4009191979964574, + "sampling/importance_sampling_ratio/max": 1.9622156262397765, + "sampling/importance_sampling_ratio/mean": 1.0072839776674907, + "sampling/importance_sampling_ratio/min": 0.4721508582433065, + "sampling/sampling_logp_difference/max": 0.2947230875492096, + "sampling/sampling_logp_difference/mean": 0.0040404104006787145, + "step": 21570, + "step_time": 8.311679606260926, + "student/entropy": 0.13428463619202374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001390775046, + "completions/max_length": 473.7, + "completions/max_terminated_length": 452.26666666666665, + "completions/mean_length": 192.9594533284505, + "completions/mean_terminated_length": 185.8704396565755, + "completions/min_length": 52.96666666666667, + "completions/min_terminated_length": 52.96666666666667, + "entropy": 0.12730694226920605, + "epoch": 0.8202635476398435, + "eval/gt_acc_batch": 0.7891666889190674, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2751025855541229, + "kd/policy_loss": 0.008675953743901725, + "kd/rkl_advantage_mean": 0.7575737199435632, + "kd/rkl_advantage_p95": 4.971784994999568, + "kd/rkl_advantage_std": 2.1853046278158823, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7977442752439904e-07, + "loss": 0.034703818957010905, + "num_tokens": 706605836.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7891666611035665, + "rewards/gt_logging_reward/std": 0.39875810643037163, + "sampling/importance_sampling_ratio/max": 1.951008645693461, + "sampling/importance_sampling_ratio/mean": 1.0007342358430227, + "sampling/importance_sampling_ratio/min": 0.4613120873769124, + "sampling/sampling_logp_difference/max": 0.297554948925972, + "sampling/sampling_logp_difference/mean": 0.003863970749080181, + "step": 21600, + "step_time": 8.151641402333432, + "student/entropy": 0.12730694226920605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02500000155220429, + "completions/max_length": 474.96666666666664, + "completions/max_terminated_length": 444.8, + "completions/mean_length": 185.7738993326823, + "completions/mean_terminated_length": 177.5375951131185, + "completions/min_length": 47.93333333333333, + "completions/min_terminated_length": 47.93333333333333, + "entropy": 0.13689549683282773, + "epoch": 0.8214028025671211, + "eval/gt_acc_batch": 0.7941666821638743, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.281383216381073, + "kd/policy_loss": 0.0085822031852634, + "kd/rkl_advantage_mean": 0.7488358478372296, + "kd/rkl_advantage_p95": 4.940113818645477, + "kd/rkl_advantage_std": 2.190806539853414, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7863517259712146e-07, + "loss": 0.03432881434758504, + "num_tokens": 707579078.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7941666702429454, + "rewards/gt_logging_reward/std": 0.39418881783882775, + "sampling/importance_sampling_ratio/max": 1.8925624092419941, + "sampling/importance_sampling_ratio/mean": 0.9996952394644419, + "sampling/importance_sampling_ratio/min": 0.4581642975409826, + "sampling/sampling_logp_difference/max": 0.3225720008214315, + "sampling/sampling_logp_difference/mean": 0.004093757535641392, + "step": 21630, + "step_time": 8.003894641640363, + "student/entropy": 0.13689549683282773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334488173325, + "completions/max_length": 498.93333333333334, + "completions/max_terminated_length": 446.93333333333334, + "completions/mean_length": 188.41612091064454, + "completions/mean_terminated_length": 180.75244954427083, + "completions/min_length": 54.56666666666667, + "completions/min_terminated_length": 54.56666666666667, + "entropy": 0.13225641225775084, + "epoch": 0.8225420574943987, + "eval/gt_acc_batch": 0.7808333535989126, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3649830222129822, + "kd/policy_loss": 0.008655792102217674, + "kd/rkl_advantage_mean": 0.7970033611481389, + "kd/rkl_advantage_p95": 5.0687026103337605, + "kd/rkl_advantage_std": 2.246586122115453, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7749591766984393e-07, + "loss": 0.0346231738726298, + "num_tokens": 708555840.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7808333337306976, + "rewards/gt_logging_reward/std": 0.40490318139394127, + "sampling/importance_sampling_ratio/max": 1.9233240127563476, + "sampling/importance_sampling_ratio/mean": 1.0001326898733776, + "sampling/importance_sampling_ratio/min": 0.46862563292185466, + "sampling/sampling_logp_difference/max": 0.30760535796483357, + "sampling/sampling_logp_difference/mean": 0.003980898717418313, + "step": 21660, + "step_time": 8.109931588194256, + "student/entropy": 0.13225641225775084 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016388889960944654, + "completions/max_length": 483.06666666666666, + "completions/max_terminated_length": 444.06666666666666, + "completions/mean_length": 179.68028615315754, + "completions/mean_terminated_length": 174.20833892822264, + "completions/min_length": 56.266666666666666, + "completions/min_terminated_length": 56.266666666666666, + "entropy": 0.1339274414504568, + "epoch": 0.8236813124216762, + "eval/gt_acc_batch": 0.8047222435474396, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2913224697113037, + "kd/policy_loss": 0.008849023721025636, + "kd/rkl_advantage_mean": 0.821362184608976, + "kd/rkl_advantage_p95": 5.158576792478561, + "kd/rkl_advantage_std": 2.2609405746062596, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7635666274256635e-07, + "loss": 0.035396103064219156, + "num_tokens": 709502313.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222216924032, + "rewards/gt_logging_reward/std": 0.38899335960547127, + "sampling/importance_sampling_ratio/max": 1.9313522100448608, + "sampling/importance_sampling_ratio/mean": 0.9980152169863383, + "sampling/importance_sampling_ratio/min": 0.45628765324751536, + "sampling/sampling_logp_difference/max": 0.3049539844195048, + "sampling/sampling_logp_difference/mean": 0.004048929231551786, + "step": 21690, + "step_time": 7.927884894470723, + "student/entropy": 0.1339274414504568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555668969949, + "completions/max_length": 492.76666666666665, + "completions/max_terminated_length": 458.56666666666666, + "completions/mean_length": 194.79250996907552, + "completions/mean_terminated_length": 188.20710042317708, + "completions/min_length": 54.46666666666667, + "completions/min_terminated_length": 54.46666666666667, + "entropy": 0.13108997823049626, + "epoch": 0.8248205673489538, + "eval/gt_acc_batch": 0.7819444755713145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2820308804512024, + "kd/policy_loss": 0.008627937355777248, + "kd/rkl_advantage_mean": 0.7535503803907583, + "kd/rkl_advantage_p95": 4.941602178414663, + "kd/rkl_advantage_std": 2.194266207019488, + "kd/ssd_loss": 0.0, + "learning_rate": 1.752174078152888e-07, + "loss": 0.034511748949686685, + "num_tokens": 710505710.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7819444378217061, + "rewards/gt_logging_reward/std": 0.4088441570599874, + "sampling/importance_sampling_ratio/max": 1.9086131691932677, + "sampling/importance_sampling_ratio/mean": 0.9902771373589834, + "sampling/importance_sampling_ratio/min": 0.45349346896012627, + "sampling/sampling_logp_difference/max": 0.32025028665860494, + "sampling/sampling_logp_difference/mean": 0.003978815209120512, + "step": 21720, + "step_time": 8.124112357834626, + "student/entropy": 0.13108997823049626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025277778909852108, + "completions/max_length": 485.5, + "completions/max_terminated_length": 459.03333333333336, + "completions/mean_length": 192.97945454915364, + "completions/mean_terminated_length": 184.81041514078777, + "completions/min_length": 57.56666666666667, + "completions/min_terminated_length": 57.56666666666667, + "entropy": 0.13384941754241783, + "epoch": 0.8259598222762313, + "eval/gt_acc_batch": 0.7816666762034098, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2847552001476288, + "kd/policy_loss": 0.008700640420041357, + "kd/rkl_advantage_mean": 0.7606034453958272, + "kd/rkl_advantage_p95": 4.917656515041987, + "kd/rkl_advantage_std": 2.1838763962189356, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7407815288801124e-07, + "loss": 0.0348025639851888, + "num_tokens": 711521572.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7816666662693024, + "rewards/gt_logging_reward/std": 0.4054734428723653, + "sampling/importance_sampling_ratio/max": 1.9535720467567443, + "sampling/importance_sampling_ratio/mean": 1.0012876788775127, + "sampling/importance_sampling_ratio/min": 0.45312662521998087, + "sampling/sampling_logp_difference/max": 0.313635923465093, + "sampling/sampling_logp_difference/mean": 0.004057838302105665, + "step": 21750, + "step_time": 8.487316269236423, + "student/entropy": 0.13384941754241783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0188888899050653, + "completions/max_length": 489.96666666666664, + "completions/max_terminated_length": 451.1, + "completions/mean_length": 187.47917683919272, + "completions/mean_terminated_length": 181.3087422688802, + "completions/min_length": 55.266666666666666, + "completions/min_terminated_length": 55.266666666666666, + "entropy": 0.12976673282682896, + "epoch": 0.827099077203509, + "eval/gt_acc_batch": 0.8027777930100759, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2743168771266937, + "kd/policy_loss": 0.008542749425396323, + "kd/rkl_advantage_mean": 0.7472719506671032, + "kd/rkl_advantage_p95": 5.019225203990937, + "kd/rkl_advantage_std": 2.2214262624581655, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7293889796073366e-07, + "loss": 0.034171001116434736, + "num_tokens": 712493337.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8027777810891469, + "rewards/gt_logging_reward/std": 0.38906519263982775, + "sampling/importance_sampling_ratio/max": 1.891133761405945, + "sampling/importance_sampling_ratio/mean": 1.005589618285497, + "sampling/importance_sampling_ratio/min": 0.46531993001699445, + "sampling/sampling_logp_difference/max": 0.3254246711730957, + "sampling/sampling_logp_difference/mean": 0.003973469836637378, + "step": 21780, + "step_time": 8.066514998096197, + "student/entropy": 0.12976673282682896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001396983863, + "completions/max_length": 496.46666666666664, + "completions/max_terminated_length": 463.3, + "completions/mean_length": 188.13945465087892, + "completions/mean_terminated_length": 179.76343790690103, + "completions/min_length": 51.833333333333336, + "completions/min_terminated_length": 51.833333333333336, + "entropy": 0.12750049320360024, + "epoch": 0.8282383321307865, + "eval/gt_acc_batch": 0.796666689713796, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3356053829193115, + "kd/policy_loss": 0.008072864638719087, + "kd/rkl_advantage_mean": 0.8099790142228206, + "kd/rkl_advantage_p95": 5.022656569878261, + "kd/rkl_advantage_std": 2.2168882469336193, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7179964303345613e-07, + "loss": 0.03229145606358846, + "num_tokens": 713468895.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7966666698455811, + "rewards/gt_logging_reward/std": 0.39292255143324534, + "sampling/importance_sampling_ratio/max": 1.7541900078455608, + "sampling/importance_sampling_ratio/mean": 0.9984619041283925, + "sampling/importance_sampling_ratio/min": 0.4451137820879618, + "sampling/sampling_logp_difference/max": 0.32538766860961915, + "sampling/sampling_logp_difference/mean": 0.0039301776637633646, + "step": 21810, + "step_time": 8.180330047561437, + "student/entropy": 0.12750049320360024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026666668492058912, + "completions/max_length": 487.8666666666667, + "completions/max_terminated_length": 456.03333333333336, + "completions/mean_length": 194.3425099690755, + "completions/mean_terminated_length": 185.69802551269532, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.13061127346009016, + "epoch": 0.8293775870580641, + "eval/gt_acc_batch": 0.7963889102141063, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2515995502471924, + "kd/policy_loss": 0.008087613041667888, + "kd/rkl_advantage_mean": 0.7912805884766082, + "kd/rkl_advantage_p95": 5.046743402878444, + "kd/rkl_advantage_std": 2.2295059730609257, + "kd/ssd_loss": 0.0, + "learning_rate": 1.7066038810617854e-07, + "loss": 0.03235045274098714, + "num_tokens": 714466640.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888843854269, + "rewards/gt_logging_reward/std": 0.39457153479258217, + "sampling/importance_sampling_ratio/max": 1.941011361281077, + "sampling/importance_sampling_ratio/mean": 0.9991806228955586, + "sampling/importance_sampling_ratio/min": 0.40545007785161336, + "sampling/sampling_logp_difference/max": 0.309822682539622, + "sampling/sampling_logp_difference/mean": 0.003951842547394335, + "step": 21840, + "step_time": 8.082345298602013, + "student/entropy": 0.13061127346009016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016388889712591965, + "completions/max_length": 464.8, + "completions/max_terminated_length": 434.03333333333336, + "completions/mean_length": 186.94056396484376, + "completions/mean_terminated_length": 181.6878860473633, + "completions/min_length": 53.53333333333333, + "completions/min_terminated_length": 53.53333333333333, + "entropy": 0.1262030846749743, + "epoch": 0.8305168419853416, + "eval/gt_acc_batch": 0.8091666956742605, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.257894903421402, + "kd/policy_loss": 0.00782846058330809, + "kd/rkl_advantage_mean": 0.6854274295891325, + "kd/rkl_advantage_p95": 4.829876321554184, + "kd/rkl_advantage_std": 2.1377787311871845, + "kd/ssd_loss": 0.0, + "learning_rate": 1.69521133178901e-07, + "loss": 0.03131384452184041, + "num_tokens": 715433242.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8091666618982951, + "rewards/gt_logging_reward/std": 0.38261617223421734, + "sampling/importance_sampling_ratio/max": 1.7856500267982482, + "sampling/importance_sampling_ratio/mean": 0.9956918517748515, + "sampling/importance_sampling_ratio/min": 0.4791534016529719, + "sampling/sampling_logp_difference/max": 0.3144776701927185, + "sampling/sampling_logp_difference/mean": 0.0038534905295819046, + "step": 21870, + "step_time": 7.846956185001181, + "student/entropy": 0.1262030846749743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778673917055, + "completions/max_length": 482.3, + "completions/max_terminated_length": 451.0, + "completions/mean_length": 183.95000864664715, + "completions/mean_terminated_length": 178.10933634440104, + "completions/min_length": 48.7, + "completions/min_terminated_length": 48.7, + "entropy": 0.12232558038085699, + "epoch": 0.8316560969126191, + "eval/gt_acc_batch": 0.8183333575725555, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26129063963890076, + "kd/policy_loss": 0.007748832454672084, + "kd/rkl_advantage_mean": 0.7645033324758211, + "kd/rkl_advantage_p95": 5.0221162418524425, + "kd/rkl_advantage_std": 2.235727126399676, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6838187825162343e-07, + "loss": 0.030995333194732667, + "num_tokens": 716392270.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8183333277702332, + "rewards/gt_logging_reward/std": 0.3790676787495613, + "sampling/importance_sampling_ratio/max": 1.8259406447410584, + "sampling/importance_sampling_ratio/mean": 0.9997441112995148, + "sampling/importance_sampling_ratio/min": 0.49157346884409586, + "sampling/sampling_logp_difference/max": 0.3152689814567566, + "sampling/sampling_logp_difference/mean": 0.0037906839822729427, + "step": 21900, + "step_time": 7.896479809162944, + "student/entropy": 0.12232558038085699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223387410244, + "completions/max_length": 477.56666666666666, + "completions/max_terminated_length": 450.53333333333336, + "completions/mean_length": 194.79640045166016, + "completions/mean_terminated_length": 186.9627919514974, + "completions/min_length": 56.666666666666664, + "completions/min_terminated_length": 56.666666666666664, + "entropy": 0.1300292134905855, + "epoch": 0.8327953518398967, + "eval/gt_acc_batch": 0.7891666909058889, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2436823844909668, + "kd/policy_loss": 0.00861390216741711, + "kd/rkl_advantage_mean": 0.7133435395080596, + "kd/rkl_advantage_p95": 4.949803876876831, + "kd/rkl_advantage_std": 2.178648580114047, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6724262332434588e-07, + "loss": 0.034455609321594236, + "num_tokens": 717407377.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7891666690508524, + "rewards/gt_logging_reward/std": 0.39803616801897684, + "sampling/importance_sampling_ratio/max": 1.941292687257131, + "sampling/importance_sampling_ratio/mean": 0.9965042908986409, + "sampling/importance_sampling_ratio/min": 0.42858380675315855, + "sampling/sampling_logp_difference/max": 0.3182550092538198, + "sampling/sampling_logp_difference/mean": 0.003997007485789557, + "step": 21930, + "step_time": 8.166973190870097, + "student/entropy": 0.1300292134905855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277779021610817, + "completions/max_length": 481.53333333333336, + "completions/max_terminated_length": 450.23333333333335, + "completions/mean_length": 188.7033442179362, + "completions/mean_terminated_length": 182.20982767740887, + "completions/min_length": 49.03333333333333, + "completions/min_terminated_length": 49.03333333333333, + "entropy": 0.13779965521146853, + "epoch": 0.8339346067671742, + "eval/gt_acc_batch": 0.7833333472410838, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2729966938495636, + "kd/policy_loss": 0.008726057532476262, + "kd/rkl_advantage_mean": 0.795087803589801, + "kd/rkl_advantage_p95": 4.997982154289882, + "kd/rkl_advantage_std": 2.199200164278348, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6610336839706832e-07, + "loss": 0.034904225667317705, + "num_tokens": 718383077.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7833333373069763, + "rewards/gt_logging_reward/std": 0.4009580830732981, + "sampling/importance_sampling_ratio/max": 1.8972206672032674, + "sampling/importance_sampling_ratio/mean": 0.9964913626511892, + "sampling/importance_sampling_ratio/min": 0.48788846135139463, + "sampling/sampling_logp_difference/max": 0.35476712783177694, + "sampling/sampling_logp_difference/mean": 0.004088336912294229, + "step": 21960, + "step_time": 8.053667612030404, + "student/entropy": 0.13779965521146853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026111112472911677, + "completions/max_length": 477.23333333333335, + "completions/max_terminated_length": 433.93333333333334, + "completions/mean_length": 186.49667612711588, + "completions/mean_terminated_length": 177.95895334879557, + "completions/min_length": 55.53333333333333, + "completions/min_terminated_length": 55.53333333333333, + "entropy": 0.13304974629233282, + "epoch": 0.8350738616944519, + "eval/gt_acc_batch": 0.7994444787502288, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2885095179080963, + "kd/policy_loss": 0.008483153051929548, + "kd/rkl_advantage_mean": 0.7997201787307858, + "kd/rkl_advantage_p95": 5.041120620568593, + "kd/rkl_advantage_std": 2.2363890896240872, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6496411346979074e-07, + "loss": 0.033932614326477054, + "num_tokens": 719365665.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7994444449742635, + "rewards/gt_logging_reward/std": 0.3897714138031006, + "sampling/importance_sampling_ratio/max": 1.7875962177912395, + "sampling/importance_sampling_ratio/mean": 0.9909016092618307, + "sampling/importance_sampling_ratio/min": 0.4791289319594701, + "sampling/sampling_logp_difference/max": 0.33542196949323017, + "sampling/sampling_logp_difference/mean": 0.004028124874457717, + "step": 21990, + "step_time": 8.312347773298582, + "student/entropy": 0.13304974629233282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02305555697530508, + "completions/max_length": 486.46666666666664, + "completions/max_terminated_length": 434.46666666666664, + "completions/mean_length": 185.91667785644532, + "completions/mean_terminated_length": 178.40764414469402, + "completions/min_length": 59.03333333333333, + "completions/min_terminated_length": 59.03333333333333, + "entropy": 0.1312519058585167, + "epoch": 0.8362131166217294, + "eval/gt_acc_batch": 0.7986111303170522, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3779628276824951, + "kd/policy_loss": 0.008799421459358806, + "kd/rkl_advantage_mean": 0.7534091283877691, + "kd/rkl_advantage_p95": 4.972088917096456, + "kd/rkl_advantage_std": 2.2000416457653045, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6382485854251318e-07, + "loss": 0.03519768714904785, + "num_tokens": 720333317.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111104488373, + "rewards/gt_logging_reward/std": 0.39396326839923856, + "sampling/importance_sampling_ratio/max": 1.9584973971048991, + "sampling/importance_sampling_ratio/mean": 1.0005714197953541, + "sampling/importance_sampling_ratio/min": 0.4431596711277962, + "sampling/sampling_logp_difference/max": 0.3609576880931854, + "sampling/sampling_logp_difference/mean": 0.003970329145280023, + "step": 22020, + "step_time": 8.084651628466478, + "student/entropy": 0.1312519058585167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112621923287, + "completions/max_length": 486.73333333333335, + "completions/max_terminated_length": 449.53333333333336, + "completions/mean_length": 185.02500915527344, + "completions/mean_terminated_length": 177.12046966552734, + "completions/min_length": 45.63333333333333, + "completions/min_terminated_length": 45.63333333333333, + "entropy": 0.12716341987252236, + "epoch": 0.837352371549007, + "eval/gt_acc_batch": 0.7969444632530213, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.39055272936820984, + "kd/policy_loss": 0.008252689254004509, + "kd/rkl_advantage_mean": 0.6966283041362961, + "kd/rkl_advantage_p95": 4.919476302464803, + "kd/rkl_advantage_std": 2.2218957523504894, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6268560361523563e-07, + "loss": 0.033010758956273395, + "num_tokens": 721296471.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444493452708, + "rewards/gt_logging_reward/std": 0.39337474505106607, + "sampling/importance_sampling_ratio/max": 1.9045813123385111, + "sampling/importance_sampling_ratio/mean": 1.0026594996452332, + "sampling/importance_sampling_ratio/min": 0.4484321688612302, + "sampling/sampling_logp_difference/max": 0.35066500306129456, + "sampling/sampling_logp_difference/mean": 0.003907778665112952, + "step": 22050, + "step_time": 7.9944261062288815, + "student/entropy": 0.12716341987252236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01805555655931433, + "completions/max_length": 485.8666666666667, + "completions/max_terminated_length": 437.3666666666667, + "completions/mean_length": 187.33556518554687, + "completions/mean_terminated_length": 181.45034891764323, + "completions/min_length": 49.36666666666667, + "completions/min_terminated_length": 49.36666666666667, + "entropy": 0.1274328616137306, + "epoch": 0.8384916264762845, + "eval/gt_acc_batch": 0.8175000190734864, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28067290782928467, + "kd/policy_loss": 0.007994890061672777, + "kd/rkl_advantage_mean": 0.7603579053034385, + "kd/rkl_advantage_p95": 4.991238663593928, + "kd/rkl_advantage_std": 2.2179590155680975, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6154634868795807e-07, + "loss": 0.03197956085205078, + "num_tokens": 722273159.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8175000031789144, + "rewards/gt_logging_reward/std": 0.3745128850142161, + "sampling/importance_sampling_ratio/max": 1.8468910058339436, + "sampling/importance_sampling_ratio/mean": 0.9936999877293905, + "sampling/importance_sampling_ratio/min": 0.47307549317677816, + "sampling/sampling_logp_difference/max": 0.32768351038297017, + "sampling/sampling_logp_difference/mean": 0.003869962653455635, + "step": 22080, + "step_time": 8.107962816328897, + "student/entropy": 0.1274328616137306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778897434474, + "completions/max_length": 484.56666666666666, + "completions/max_terminated_length": 443.2, + "completions/mean_length": 182.75973205566407, + "completions/mean_terminated_length": 176.09385782877604, + "completions/min_length": 54.4, + "completions/min_terminated_length": 54.4, + "entropy": 0.1325146062920491, + "epoch": 0.8396308814035621, + "eval/gt_acc_batch": 0.7986111323038737, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.4273342788219452, + "kd/policy_loss": 0.008730388956610113, + "kd/rkl_advantage_mean": 0.8023306039472421, + "kd/rkl_advantage_p95": 5.143548009792964, + "kd/rkl_advantage_std": 2.2659287889798483, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6040709376068051e-07, + "loss": 0.03492155869801839, + "num_tokens": 723232398.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111104488373, + "rewards/gt_logging_reward/std": 0.3939600576957067, + "sampling/importance_sampling_ratio/max": 1.909198788801829, + "sampling/importance_sampling_ratio/mean": 1.0009625931580861, + "sampling/importance_sampling_ratio/min": 0.418133877714475, + "sampling/sampling_logp_difference/max": 0.33799819548924764, + "sampling/sampling_logp_difference/mean": 0.004049147890570263, + "step": 22110, + "step_time": 8.173245803065948, + "student/entropy": 0.1325146062920491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023333334426085154, + "completions/max_length": 485.1, + "completions/max_terminated_length": 461.5, + "completions/mean_length": 195.76389923095704, + "completions/mean_terminated_length": 188.17799377441406, + "completions/min_length": 52.9, + "completions/min_terminated_length": 52.9, + "entropy": 0.12475171070545912, + "epoch": 0.8407701363308396, + "eval/gt_acc_batch": 0.8030555705229442, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29038041830062866, + "kd/policy_loss": 0.007953320589149371, + "kd/rkl_advantage_mean": 0.7325929601831983, + "kd/rkl_advantage_p95": 4.893853304783503, + "kd/rkl_advantage_std": 2.172626096010208, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5926783883340296e-07, + "loss": 0.03181328574816386, + "num_tokens": 724237924.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555586020152, + "rewards/gt_logging_reward/std": 0.3932329515616099, + "sampling/importance_sampling_ratio/max": 2.0123003840446474, + "sampling/importance_sampling_ratio/mean": 0.999665230512619, + "sampling/importance_sampling_ratio/min": 0.47834560871124265, + "sampling/sampling_logp_difference/max": 0.313453471660614, + "sampling/sampling_logp_difference/mean": 0.0038195600577940543, + "step": 22140, + "step_time": 8.21171269856762, + "student/entropy": 0.12475171070545912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445874541997, + "completions/max_length": 502.2, + "completions/max_terminated_length": 459.8666666666667, + "completions/mean_length": 192.5791763305664, + "completions/mean_terminated_length": 184.63370564778646, + "completions/min_length": 57.5, + "completions/min_terminated_length": 57.5, + "entropy": 0.1325329078361392, + "epoch": 0.8419093912581171, + "eval/gt_acc_batch": 0.7763889054457347, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2585466206073761, + "kd/policy_loss": 0.008656649163458496, + "kd/rkl_advantage_mean": 0.781512791228791, + "kd/rkl_advantage_p95": 4.956156925360362, + "kd/rkl_advantage_std": 2.1873933414618176, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5812858390612538e-07, + "loss": 0.03462659517923991, + "num_tokens": 725229337.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7763888875643412, + "rewards/gt_logging_reward/std": 0.41076376338799797, + "sampling/importance_sampling_ratio/max": 1.8384151816368104, + "sampling/importance_sampling_ratio/mean": 0.9994127770264943, + "sampling/importance_sampling_ratio/min": 0.4386747757593791, + "sampling/sampling_logp_difference/max": 0.32744612693786623, + "sampling/sampling_logp_difference/mean": 0.004014613844143848, + "step": 22170, + "step_time": 8.272946898335553, + "student/entropy": 0.1325329078361392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02777777931963404, + "completions/max_length": 494.8666666666667, + "completions/max_terminated_length": 450.3, + "completions/mean_length": 196.88778788248698, + "completions/mean_terminated_length": 188.0404810587565, + "completions/min_length": 55.13333333333333, + "completions/min_terminated_length": 55.13333333333333, + "entropy": 0.13313586711883546, + "epoch": 0.8430486461853948, + "eval/gt_acc_batch": 0.7691666861375173, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3742421269416809, + "kd/policy_loss": 0.009073434649811437, + "kd/rkl_advantage_mean": 0.7770038907726605, + "kd/rkl_advantage_p95": 5.0364954829216, + "kd/rkl_advantage_std": 2.2213787029186887, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5698932897884782e-07, + "loss": 0.036293745040893555, + "num_tokens": 726245957.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7691666682561239, + "rewards/gt_logging_reward/std": 0.41436497668425243, + "sampling/importance_sampling_ratio/max": 1.9681978265444438, + "sampling/importance_sampling_ratio/mean": 1.0085690359274546, + "sampling/importance_sampling_ratio/min": 0.4166793256998062, + "sampling/sampling_logp_difference/max": 0.35652629534403485, + "sampling/sampling_logp_difference/mean": 0.004000342870131135, + "step": 22200, + "step_time": 8.35064946123748, + "student/entropy": 0.13313586711883546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02416666808227698, + "completions/max_length": 491.4, + "completions/max_terminated_length": 468.96666666666664, + "completions/mean_length": 186.5572311401367, + "completions/mean_terminated_length": 178.61560567220053, + "completions/min_length": 55.63333333333333, + "completions/min_terminated_length": 55.63333333333333, + "entropy": 0.13243679162114858, + "epoch": 0.8441879011126723, + "eval/gt_acc_batch": 0.7930555840333303, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31247201561927795, + "kd/policy_loss": 0.008413689961889759, + "kd/rkl_advantage_mean": 0.7919532576575875, + "kd/rkl_advantage_p95": 5.088038432598114, + "kd/rkl_advantage_std": 2.252047916253408, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5585007405157026e-07, + "loss": 0.03365476926167806, + "num_tokens": 727207995.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.793055550257365, + "rewards/gt_logging_reward/std": 0.3971247146526972, + "sampling/importance_sampling_ratio/max": 1.8080207268397013, + "sampling/importance_sampling_ratio/mean": 1.0013012290000916, + "sampling/importance_sampling_ratio/min": 0.49271173278490704, + "sampling/sampling_logp_difference/max": 0.3150181392828623, + "sampling/sampling_logp_difference/mean": 0.004003947670571506, + "step": 22230, + "step_time": 7.908665397901981, + "student/entropy": 0.13243679162114858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889051352938, + "completions/max_length": 493.06666666666666, + "completions/max_terminated_length": 460.2, + "completions/mean_length": 186.25473175048828, + "completions/mean_terminated_length": 177.5077865600586, + "completions/min_length": 51.8, + "completions/min_terminated_length": 51.8, + "entropy": 0.13526720715065796, + "epoch": 0.8453271560399499, + "eval/gt_acc_batch": 0.8072222292423248, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2858476936817169, + "kd/policy_loss": 0.008623088089128336, + "kd/rkl_advantage_mean": 0.8256231498594085, + "kd/rkl_advantage_p95": 5.08225781917572, + "kd/rkl_advantage_std": 2.243401731053988, + "kd/ssd_loss": 0.0, + "learning_rate": 1.547108191242927e-07, + "loss": 0.034492357571919756, + "num_tokens": 728182680.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222252686818, + "rewards/gt_logging_reward/std": 0.3881797323624293, + "sampling/importance_sampling_ratio/max": 1.8032938877741496, + "sampling/importance_sampling_ratio/mean": 0.9973913292090099, + "sampling/importance_sampling_ratio/min": 0.44764808217684426, + "sampling/sampling_logp_difference/max": 0.3074962337811788, + "sampling/sampling_logp_difference/mean": 0.004107640784544249, + "step": 22260, + "step_time": 8.169953117502155, + "student/entropy": 0.13526720715065796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777779214084148, + "completions/max_length": 479.3, + "completions/max_terminated_length": 441.73333333333335, + "completions/mean_length": 192.5747309366862, + "completions/mean_terminated_length": 185.19730580647786, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.13079874627292157, + "epoch": 0.8464664109672274, + "eval/gt_acc_batch": 0.8036111255486806, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27309751510620117, + "kd/policy_loss": 0.008394794488170494, + "kd/rkl_advantage_mean": 0.7464213964218894, + "kd/rkl_advantage_p95": 4.971682759126027, + "kd/rkl_advantage_std": 2.182406407594681, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5357156419701515e-07, + "loss": 0.033579174677530924, + "num_tokens": 729188253.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111116409301, + "rewards/gt_logging_reward/std": 0.38983963926633197, + "sampling/importance_sampling_ratio/max": 1.8818811535835267, + "sampling/importance_sampling_ratio/mean": 0.9968519548575083, + "sampling/importance_sampling_ratio/min": 0.42139291763305664, + "sampling/sampling_logp_difference/max": 0.31740962465604144, + "sampling/sampling_logp_difference/mean": 0.003916265691320101, + "step": 22290, + "step_time": 8.17872572777366, + "student/entropy": 0.13079874627292157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020833334326744078, + "completions/max_length": 480.9, + "completions/max_terminated_length": 452.6, + "completions/mean_length": 185.180007425944, + "completions/mean_terminated_length": 178.4642593383789, + "completions/min_length": 47.86666666666667, + "completions/min_terminated_length": 47.86666666666667, + "entropy": 0.13869922409454982, + "epoch": 0.847605665894505, + "eval/gt_acc_batch": 0.7888889014720917, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24101538956165314, + "kd/policy_loss": 0.0088563627039548, + "kd/rkl_advantage_mean": 0.7582301416744789, + "kd/rkl_advantage_p95": 4.902921529610952, + "kd/rkl_advantage_std": 2.1760416785875956, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5243230926973757e-07, + "loss": 0.035425448417663576, + "num_tokens": 730153829.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7888888935248057, + "rewards/gt_logging_reward/std": 0.39918265243371326, + "sampling/importance_sampling_ratio/max": 1.8938840230305989, + "sampling/importance_sampling_ratio/mean": 0.9947291672229767, + "sampling/importance_sampling_ratio/min": 0.42905299812555314, + "sampling/sampling_logp_difference/max": 0.3330888032913208, + "sampling/sampling_logp_difference/mean": 0.004204424781103929, + "step": 22320, + "step_time": 8.082094586228292, + "student/entropy": 0.13869922409454982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03388889074946443, + "completions/max_length": 488.23333333333335, + "completions/max_terminated_length": 452.1, + "completions/mean_length": 199.21751098632814, + "completions/mean_terminated_length": 188.52822926839193, + "completions/min_length": 38.8, + "completions/min_terminated_length": 38.8, + "entropy": 0.13278496060520412, + "epoch": 0.8487449208217825, + "eval/gt_acc_batch": 0.7752777894337972, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2958698868751526, + "kd/policy_loss": 0.008597005469103655, + "kd/rkl_advantage_mean": 0.8313729441724718, + "kd/rkl_advantage_p95": 5.160448896884918, + "kd/rkl_advantage_std": 2.247417418162028, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5129305434246004e-07, + "loss": 0.03438801765441894, + "num_tokens": 731174132.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7752777775128682, + "rewards/gt_logging_reward/std": 0.41012552479902903, + "sampling/importance_sampling_ratio/max": 1.9079638242721557, + "sampling/importance_sampling_ratio/mean": 1.000334765513738, + "sampling/importance_sampling_ratio/min": 0.446453199783961, + "sampling/sampling_logp_difference/max": 0.3564078489939372, + "sampling/sampling_logp_difference/mean": 0.004011299430082242, + "step": 22350, + "step_time": 8.183145063959333, + "student/entropy": 0.13278496060520412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02250000142181913, + "completions/max_length": 491.3333333333333, + "completions/max_terminated_length": 449.6666666666667, + "completions/mean_length": 189.1550094604492, + "completions/mean_terminated_length": 181.88352813720704, + "completions/min_length": 54.86666666666667, + "completions/min_terminated_length": 54.86666666666667, + "entropy": 0.13055445284893116, + "epoch": 0.8498841757490602, + "eval/gt_acc_batch": 0.8013889094193777, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3019527196884155, + "kd/policy_loss": 0.008280803228262811, + "kd/rkl_advantage_mean": 0.7814196186761061, + "kd/rkl_advantage_p95": 5.035306944449743, + "kd/rkl_advantage_std": 2.206218820810318, + "kd/ssd_loss": 0.0, + "learning_rate": 1.5015379941518246e-07, + "loss": 0.03312321305274964, + "num_tokens": 732156746.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888915379842, + "rewards/gt_logging_reward/std": 0.38919543027877807, + "sampling/importance_sampling_ratio/max": 1.9520423849423727, + "sampling/importance_sampling_ratio/mean": 0.9978225847085317, + "sampling/importance_sampling_ratio/min": 0.4314105858405431, + "sampling/sampling_logp_difference/max": 0.354723991950353, + "sampling/sampling_logp_difference/mean": 0.0039725443503508965, + "step": 22380, + "step_time": 8.111582698162723, + "student/entropy": 0.13055445284893116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223722686372, + "completions/max_length": 486.1666666666667, + "completions/max_terminated_length": 449.73333333333335, + "completions/mean_length": 190.33389943440756, + "completions/mean_terminated_length": 183.07573699951172, + "completions/min_length": 52.1, + "completions/min_terminated_length": 52.1, + "entropy": 0.12960996987919013, + "epoch": 0.8510234306763377, + "eval/gt_acc_batch": 0.80694446961085, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32027289271354675, + "kd/policy_loss": 0.008315072030139465, + "kd/rkl_advantage_mean": 0.6679072124883533, + "kd/rkl_advantage_p95": 4.822523794571558, + "kd/rkl_advantage_std": 2.181551867723465, + "kd/ssd_loss": 0.0, + "learning_rate": 1.490145444879049e-07, + "loss": 0.033260289827982584, + "num_tokens": 733144276.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8069444437821706, + "rewards/gt_logging_reward/std": 0.38922295371691384, + "sampling/importance_sampling_ratio/max": 1.888064738114675, + "sampling/importance_sampling_ratio/mean": 1.0018999576568604, + "sampling/importance_sampling_ratio/min": 0.4401904304822286, + "sampling/sampling_logp_difference/max": 0.30855729182561237, + "sampling/sampling_logp_difference/mean": 0.003941412440811595, + "step": 22410, + "step_time": 8.065161557405371, + "student/entropy": 0.12960996987919013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02111111208796501, + "completions/max_length": 484.06666666666666, + "completions/max_terminated_length": 443.6666666666667, + "completions/mean_length": 184.89056498209635, + "completions/mean_terminated_length": 177.82989705403645, + "completions/min_length": 53.56666666666667, + "completions/min_terminated_length": 53.56666666666667, + "entropy": 0.12675875040392082, + "epoch": 0.8521626856036152, + "eval/gt_acc_batch": 0.8036111454168956, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.33963245153427124, + "kd/policy_loss": 0.007787531719077379, + "kd/rkl_advantage_mean": 0.7681762390925239, + "kd/rkl_advantage_p95": 5.022990717490514, + "kd/rkl_advantage_std": 2.2184844106435775, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4787528956062735e-07, + "loss": 0.03115013043085734, + "num_tokens": 734114026.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111136277516, + "rewards/gt_logging_reward/std": 0.3889392460385958, + "sampling/importance_sampling_ratio/max": 1.9566850463549297, + "sampling/importance_sampling_ratio/mean": 0.9954413255055745, + "sampling/importance_sampling_ratio/min": 0.4417340432604154, + "sampling/sampling_logp_difference/max": 0.33248944878578185, + "sampling/sampling_logp_difference/mean": 0.00385664120161285, + "step": 22440, + "step_time": 8.128493446505551, + "student/entropy": 0.12675875040392082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777779015402, + "completions/max_length": 489.3, + "completions/max_terminated_length": 460.53333333333336, + "completions/mean_length": 187.16639963785806, + "completions/mean_terminated_length": 181.50589955647786, + "completions/min_length": 62.53333333333333, + "completions/min_terminated_length": 62.53333333333333, + "entropy": 0.13266368123392264, + "epoch": 0.8533019405308928, + "eval/gt_acc_batch": 0.7963889141877493, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3325611352920532, + "kd/policy_loss": 0.008609746536239981, + "kd/rkl_advantage_mean": 0.811208888143301, + "kd/rkl_advantage_p95": 5.078963627417882, + "kd/rkl_advantage_std": 2.246652549505234, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4673603463334976e-07, + "loss": 0.03443899154663086, + "num_tokens": 735086849.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888923327128, + "rewards/gt_logging_reward/std": 0.3939701954523722, + "sampling/importance_sampling_ratio/max": 1.8926754236221313, + "sampling/importance_sampling_ratio/mean": 0.9916149814923604, + "sampling/importance_sampling_ratio/min": 0.47476595391829807, + "sampling/sampling_logp_difference/max": 0.3210339307785034, + "sampling/sampling_logp_difference/mean": 0.004034489711436133, + "step": 22470, + "step_time": 8.100266399757432, + "student/entropy": 0.13266368123392264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029166668187826872, + "completions/max_length": 485.56666666666666, + "completions/max_terminated_length": 440.6666666666667, + "completions/mean_length": 188.68528696695964, + "completions/mean_terminated_length": 179.08577473958334, + "completions/min_length": 57.46666666666667, + "completions/min_terminated_length": 57.46666666666667, + "entropy": 0.13335021746655304, + "epoch": 0.8544411954581703, + "eval/gt_acc_batch": 0.792222241560618, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28452086448669434, + "kd/policy_loss": 0.008758650046850865, + "kd/rkl_advantage_mean": 0.7873229287815169, + "kd/rkl_advantage_p95": 5.027487107117971, + "kd/rkl_advantage_std": 2.225318942467372, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4559677970607223e-07, + "loss": 0.03503460089365641, + "num_tokens": 736064268.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222216924032, + "rewards/gt_logging_reward/std": 0.3977049599091212, + "sampling/importance_sampling_ratio/max": 1.9708264907201132, + "sampling/importance_sampling_ratio/mean": 1.0023731311162314, + "sampling/importance_sampling_ratio/min": 0.4492472931742668, + "sampling/sampling_logp_difference/max": 0.3002629081408183, + "sampling/sampling_logp_difference/mean": 0.004029122351979216, + "step": 22500, + "step_time": 8.042180602435838, + "student/entropy": 0.13335021746655304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02111111239840587, + "completions/max_length": 488.03333333333336, + "completions/max_terminated_length": 448.1666666666667, + "completions/mean_length": 187.7408437093099, + "completions/mean_terminated_length": 180.81082763671876, + "completions/min_length": 42.333333333333336, + "completions/min_terminated_length": 42.333333333333336, + "entropy": 0.12654674854129552, + "epoch": 0.8555804503854479, + "eval/gt_acc_batch": 0.8072222491105397, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23247447609901428, + "kd/policy_loss": 0.008197744206214944, + "kd/rkl_advantage_mean": 0.7917783304428061, + "kd/rkl_advantage_p95": 5.043321806192398, + "kd/rkl_advantage_std": 2.231949652234713, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4445752477879465e-07, + "loss": 0.0327909787495931, + "num_tokens": 737058023.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8072222232818603, + "rewards/gt_logging_reward/std": 0.3841112474600474, + "sampling/importance_sampling_ratio/max": 1.8878981709480285, + "sampling/importance_sampling_ratio/mean": 1.0032616257667542, + "sampling/importance_sampling_ratio/min": 0.4674414202570915, + "sampling/sampling_logp_difference/max": 0.379049026966095, + "sampling/sampling_logp_difference/mean": 0.0038838873617351053, + "step": 22530, + "step_time": 8.24105469052544, + "student/entropy": 0.12654674854129552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001111378273, + "completions/max_length": 495.53333333333336, + "completions/max_terminated_length": 452.8666666666667, + "completions/mean_length": 191.7586222330729, + "completions/mean_terminated_length": 184.6071782430013, + "completions/min_length": 55.8, + "completions/min_terminated_length": 55.8, + "entropy": 0.13138593280067046, + "epoch": 0.8567197053127255, + "eval/gt_acc_batch": 0.7833333512147268, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2906091511249542, + "kd/policy_loss": 0.008962466226269802, + "kd/rkl_advantage_mean": 0.7732627461353938, + "kd/rkl_advantage_p95": 5.078156393766403, + "kd/rkl_advantage_std": 2.24686561524868, + "kd/ssd_loss": 0.0, + "learning_rate": 1.433182698515171e-07, + "loss": 0.03584986925125122, + "num_tokens": 738062018.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7833333313465118, + "rewards/gt_logging_reward/std": 0.40062052607536314, + "sampling/importance_sampling_ratio/max": 1.907651436328888, + "sampling/importance_sampling_ratio/mean": 1.0028938094774882, + "sampling/importance_sampling_ratio/min": 0.46810534099737805, + "sampling/sampling_logp_difference/max": 0.3515829126040141, + "sampling/sampling_logp_difference/mean": 0.0040045262081548575, + "step": 22560, + "step_time": 8.218773230231212, + "student/entropy": 0.13138593280067046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030000001812974613, + "completions/max_length": 492.5, + "completions/max_terminated_length": 453.9, + "completions/mean_length": 194.8225107828776, + "completions/mean_terminated_length": 185.0391591389974, + "completions/min_length": 57.53333333333333, + "completions/min_terminated_length": 57.53333333333333, + "entropy": 0.1269713355228305, + "epoch": 0.8578589602400031, + "eval/gt_acc_batch": 0.7925000210603078, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23916542530059814, + "kd/policy_loss": 0.008523017290281132, + "kd/rkl_advantage_mean": 0.7542879135658344, + "kd/rkl_advantage_p95": 4.968574738502502, + "kd/rkl_advantage_std": 2.204097118973732, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4217901492423954e-07, + "loss": 0.034092068672180176, + "num_tokens": 739074187.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7924999992052714, + "rewards/gt_logging_reward/std": 0.3998308499654134, + "sampling/importance_sampling_ratio/max": 1.9216821233431498, + "sampling/importance_sampling_ratio/mean": 1.0049819707870484, + "sampling/importance_sampling_ratio/min": 0.4795647958914439, + "sampling/sampling_logp_difference/max": 0.3324305931727091, + "sampling/sampling_logp_difference/mean": 0.0038409030452991526, + "step": 22590, + "step_time": 8.170196252770257, + "student/entropy": 0.1269713355228305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02388889032105605, + "completions/max_length": 499.96666666666664, + "completions/max_terminated_length": 448.9, + "completions/mean_length": 193.6591781616211, + "completions/mean_terminated_length": 185.92354329427084, + "completions/min_length": 57.46666666666667, + "completions/min_terminated_length": 57.46666666666667, + "entropy": 0.12885711720834175, + "epoch": 0.8589982151672806, + "eval/gt_acc_batch": 0.8011111338933309, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23410849273204803, + "kd/policy_loss": 0.00862525042030029, + "kd/rkl_advantage_mean": 0.7329580425381815, + "kd/rkl_advantage_p95": 4.884479556481043, + "kd/rkl_advantage_std": 2.1987229426701864, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4103975999696198e-07, + "loss": 0.03450100421905518, + "num_tokens": 740082288.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8011111080646515, + "rewards/gt_logging_reward/std": 0.3924968302249908, + "sampling/importance_sampling_ratio/max": 1.9062953511873881, + "sampling/importance_sampling_ratio/mean": 0.999266900618871, + "sampling/importance_sampling_ratio/min": 0.44493191738923393, + "sampling/sampling_logp_difference/max": 0.31464346051216124, + "sampling/sampling_logp_difference/mean": 0.003918656660243869, + "step": 22620, + "step_time": 8.181687529632473, + "student/entropy": 0.12885711720834175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333454405268, + "completions/max_length": 476.9, + "completions/max_terminated_length": 444.6666666666667, + "completions/mean_length": 192.02167663574218, + "completions/mean_terminated_length": 185.1747589111328, + "completions/min_length": 54.5, + "completions/min_terminated_length": 54.5, + "entropy": 0.12966063097119332, + "epoch": 0.8601374700945582, + "eval/gt_acc_batch": 0.7950000206629435, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3890187740325928, + "kd/policy_loss": 0.008786219808583458, + "kd/rkl_advantage_mean": 0.7296668699942529, + "kd/rkl_advantage_p95": 4.898507034778595, + "kd/rkl_advantage_std": 2.1949948330720264, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3990050506968443e-07, + "loss": 0.035144881407419844, + "num_tokens": 741082526.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7949999948342641, + "rewards/gt_logging_reward/std": 0.39559268653392793, + "sampling/importance_sampling_ratio/max": 1.9263057708740234, + "sampling/importance_sampling_ratio/mean": 1.0015836954116821, + "sampling/importance_sampling_ratio/min": 0.47602372566858925, + "sampling/sampling_logp_difference/max": 0.32400619188944496, + "sampling/sampling_logp_difference/mean": 0.0039152637434502445, + "step": 22650, + "step_time": 8.323975195172048, + "student/entropy": 0.12966063097119332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02388889032105605, + "completions/max_length": 498.2, + "completions/max_terminated_length": 463.2, + "completions/mean_length": 188.03112030029297, + "completions/mean_terminated_length": 180.14672292073567, + "completions/min_length": 51.233333333333334, + "completions/min_terminated_length": 51.233333333333334, + "entropy": 0.13298477853337923, + "epoch": 0.8612767250218357, + "eval/gt_acc_batch": 0.782500022649765, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30567461252212524, + "kd/policy_loss": 0.008825844885238136, + "kd/rkl_advantage_mean": 0.8433149545143048, + "kd/rkl_advantage_p95": 5.172426209847132, + "kd/rkl_advantage_std": 2.2744755913813908, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3876125014240687e-07, + "loss": 0.035303378105163576, + "num_tokens": 742051870.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7824999988079071, + "rewards/gt_logging_reward/std": 0.4041748841603597, + "sampling/importance_sampling_ratio/max": 1.9203033169110617, + "sampling/importance_sampling_ratio/mean": 0.9997125784556071, + "sampling/importance_sampling_ratio/min": 0.46459635297457375, + "sampling/sampling_logp_difference/max": 0.31898285349210104, + "sampling/sampling_logp_difference/mean": 0.004035418449590603, + "step": 22680, + "step_time": 8.099874800767672, + "student/entropy": 0.13298477853337923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000001167257628, + "completions/max_length": 492.76666666666665, + "completions/max_terminated_length": 446.2, + "completions/mean_length": 182.72862040201824, + "completions/mean_terminated_length": 176.1713617960612, + "completions/min_length": 50.96666666666667, + "completions/min_terminated_length": 50.96666666666667, + "entropy": 0.13534270289043585, + "epoch": 0.8624159799491132, + "eval/gt_acc_batch": 0.7983333468437195, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2659091055393219, + "kd/policy_loss": 0.008991556313897793, + "kd/rkl_advantage_mean": 0.7731174814204375, + "kd/rkl_advantage_p95": 5.0061502655347185, + "kd/rkl_advantage_std": 2.253721491495768, + "kd/ssd_loss": 0.0, + "learning_rate": 1.376219952151293e-07, + "loss": 0.03596622149149577, + "num_tokens": 743011909.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7983333388964335, + "rewards/gt_logging_reward/std": 0.3950913369655609, + "sampling/importance_sampling_ratio/max": 1.9593561172485352, + "sampling/importance_sampling_ratio/mean": 1.004892881711324, + "sampling/importance_sampling_ratio/min": 0.46151103178660074, + "sampling/sampling_logp_difference/max": 0.3166197419166565, + "sampling/sampling_logp_difference/mean": 0.004075832377808789, + "step": 22710, + "step_time": 8.161384889101221, + "student/entropy": 0.13534270289043585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024722223666807017, + "completions/max_length": 473.9, + "completions/max_terminated_length": 451.1666666666667, + "completions/mean_length": 190.69556630452473, + "completions/mean_terminated_length": 182.83817291259766, + "completions/min_length": 60.5, + "completions/min_terminated_length": 60.5, + "entropy": 0.13217275626957417, + "epoch": 0.8635552348763909, + "eval/gt_acc_batch": 0.7955555737018585, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3010310232639313, + "kd/policy_loss": 0.008648102398728952, + "kd/rkl_advantage_mean": 0.8061075311775009, + "kd/rkl_advantage_p95": 5.063146662712097, + "kd/rkl_advantage_std": 2.2115997771422067, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3648274028785173e-07, + "loss": 0.03459240992863973, + "num_tokens": 743998573.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555578072866, + "rewards/gt_logging_reward/std": 0.3938512533903122, + "sampling/importance_sampling_ratio/max": 1.8555443167686463, + "sampling/importance_sampling_ratio/mean": 1.0006631433963775, + "sampling/importance_sampling_ratio/min": 0.4322515845298767, + "sampling/sampling_logp_difference/max": 0.3007625142733256, + "sampling/sampling_logp_difference/mean": 0.003977715449097256, + "step": 22740, + "step_time": 8.054252370401324, + "student/entropy": 0.13217275626957417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02388889032105605, + "completions/max_length": 479.9, + "completions/max_terminated_length": 439.3, + "completions/mean_length": 187.3636220296224, + "completions/mean_terminated_length": 179.3752197265625, + "completions/min_length": 54.96666666666667, + "completions/min_terminated_length": 54.96666666666667, + "entropy": 0.12611462449034055, + "epoch": 0.8646944898036684, + "eval/gt_acc_batch": 0.8094444751739502, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2863791584968567, + "kd/policy_loss": 0.007816589332651346, + "kd/rkl_advantage_mean": 0.7436263293027878, + "kd/rkl_advantage_p95": 5.002106481790543, + "kd/rkl_advantage_std": 2.2216884394486747, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3534348536057418e-07, + "loss": 0.03126635551452637, + "num_tokens": 744965314.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8094444493452708, + "rewards/gt_logging_reward/std": 0.385546279946963, + "sampling/importance_sampling_ratio/max": 1.9446592330932617, + "sampling/importance_sampling_ratio/mean": 1.0049631655216218, + "sampling/importance_sampling_ratio/min": 0.4615677704413732, + "sampling/sampling_logp_difference/max": 0.3129515280326208, + "sampling/sampling_logp_difference/mean": 0.0038275106344372034, + "step": 22770, + "step_time": 8.027029435528675, + "student/entropy": 0.12611462449034055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556491017341, + "completions/max_length": 488.5, + "completions/max_terminated_length": 462.3333333333333, + "completions/mean_length": 187.83084411621093, + "completions/mean_terminated_length": 182.75238342285155, + "completions/min_length": 59.2, + "completions/min_terminated_length": 59.2, + "entropy": 0.13078331767270962, + "epoch": 0.865833744730946, + "eval/gt_acc_batch": 0.8041666825612386, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24576346576213837, + "kd/policy_loss": 0.008285384303114067, + "kd/rkl_advantage_mean": 0.7870866217340032, + "kd/rkl_advantage_p95": 4.9916084210077925, + "kd/rkl_advantage_std": 2.207761217157046, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3420423043329662e-07, + "loss": 0.03314153949419658, + "num_tokens": 745947809.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8041666587193806, + "rewards/gt_logging_reward/std": 0.3853556881348292, + "sampling/importance_sampling_ratio/max": 1.9611984769503275, + "sampling/importance_sampling_ratio/mean": 1.001955791314443, + "sampling/importance_sampling_ratio/min": 0.4582179774840673, + "sampling/sampling_logp_difference/max": 0.3205002297957738, + "sampling/sampling_logp_difference/mean": 0.004002180183306336, + "step": 22800, + "step_time": 8.197869520298749, + "student/entropy": 0.13078331767270962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030277779667327802, + "completions/max_length": 485.6666666666667, + "completions/max_terminated_length": 439.8333333333333, + "completions/mean_length": 185.4747319539388, + "completions/mean_terminated_length": 175.55718739827475, + "completions/min_length": 51.53333333333333, + "completions/min_terminated_length": 51.53333333333333, + "entropy": 0.1318334046130379, + "epoch": 0.8669729996582235, + "eval/gt_acc_batch": 0.7816666801770528, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3620099425315857, + "kd/policy_loss": 0.008636449290982758, + "kd/rkl_advantage_mean": 0.749049085068206, + "kd/rkl_advantage_p95": 4.94884881178538, + "kd/rkl_advantage_std": 2.20999383131663, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3306497550601907e-07, + "loss": 0.03454579909642538, + "num_tokens": 746915806.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7816666662693024, + "rewards/gt_logging_reward/std": 0.40279511312643684, + "sampling/importance_sampling_ratio/max": 1.8047226667404175, + "sampling/importance_sampling_ratio/mean": 1.000716882944107, + "sampling/importance_sampling_ratio/min": 0.46378561755021414, + "sampling/sampling_logp_difference/max": 0.3257856428623199, + "sampling/sampling_logp_difference/mean": 0.004005224024876952, + "step": 22830, + "step_time": 7.9810386973248875, + "student/entropy": 0.1318334046130379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777778798093398, + "completions/max_length": 473.1333333333333, + "completions/max_terminated_length": 430.9, + "completions/mean_length": 184.97862091064454, + "completions/mean_terminated_length": 179.21016642252604, + "completions/min_length": 47.666666666666664, + "completions/min_terminated_length": 47.666666666666664, + "entropy": 0.12774022612720728, + "epoch": 0.8681122545855011, + "eval/gt_acc_batch": 0.8233333349227905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3894599676132202, + "kd/policy_loss": 0.008211941768725714, + "kd/rkl_advantage_mean": 0.6719420439951743, + "kd/rkl_advantage_p95": 4.7874577383200325, + "kd/rkl_advantage_std": 2.173890712857246, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3192572057874148e-07, + "loss": 0.032847766081492105, + "num_tokens": 747884785.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8233333309491475, + "rewards/gt_logging_reward/std": 0.3722184111674627, + "sampling/importance_sampling_ratio/max": 1.9391035914421082, + "sampling/importance_sampling_ratio/mean": 1.0083423793315887, + "sampling/importance_sampling_ratio/min": 0.4645162343978882, + "sampling/sampling_logp_difference/max": 0.3018129626909892, + "sampling/sampling_logp_difference/mean": 0.003905597678385675, + "step": 22860, + "step_time": 8.10379835596638, + "student/entropy": 0.12774022612720728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02861111263434092, + "completions/max_length": 499.5, + "completions/max_terminated_length": 456.8666666666667, + "completions/mean_length": 189.402788289388, + "completions/mean_terminated_length": 180.1240473429362, + "completions/min_length": 53.6, + "completions/min_terminated_length": 53.6, + "entropy": 0.12710884287953378, + "epoch": 0.8692515095127786, + "eval/gt_acc_batch": 0.8027777969837189, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3038082420825958, + "kd/policy_loss": 0.008547191692438597, + "kd/rkl_advantage_mean": 0.7638656681713959, + "kd/rkl_advantage_p95": 5.071392973264058, + "kd/rkl_advantage_std": 2.2542886257171633, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3078646565146393e-07, + "loss": 0.03418877124786377, + "num_tokens": 748874195.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8027777810891469, + "rewards/gt_logging_reward/std": 0.3877773478627205, + "sampling/importance_sampling_ratio/max": 1.9831329464912415, + "sampling/importance_sampling_ratio/mean": 1.002388346195221, + "sampling/importance_sampling_ratio/min": 0.4482503523429235, + "sampling/sampling_logp_difference/max": 0.3296577463547389, + "sampling/sampling_logp_difference/mean": 0.003844348209289213, + "step": 22890, + "step_time": 8.281567083243862, + "student/entropy": 0.12710884287953378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333467443784, + "completions/max_length": 478.46666666666664, + "completions/max_terminated_length": 446.46666666666664, + "completions/mean_length": 191.49806518554686, + "completions/mean_terminated_length": 184.01276601155598, + "completions/min_length": 55.03333333333333, + "completions/min_terminated_length": 55.03333333333333, + "entropy": 0.1259079256405433, + "epoch": 0.8703907644400563, + "eval/gt_acc_batch": 0.8033333579699199, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3195365369319916, + "kd/policy_loss": 0.008189297669256726, + "kd/rkl_advantage_mean": 0.6363264962720374, + "kd/rkl_advantage_p95": 4.780127602815628, + "kd/rkl_advantage_std": 2.1703596154848737, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2964721072418637e-07, + "loss": 0.032757192850112915, + "num_tokens": 749875284.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8033333321412405, + "rewards/gt_logging_reward/std": 0.3865125045180321, + "sampling/importance_sampling_ratio/max": 1.95700706243515, + "sampling/importance_sampling_ratio/mean": 1.0010215878486632, + "sampling/importance_sampling_ratio/min": 0.4595142036676407, + "sampling/sampling_logp_difference/max": 0.31296459833780926, + "sampling/sampling_logp_difference/mean": 0.0038274982711300255, + "step": 22920, + "step_time": 8.123934472636513, + "student/entropy": 0.1259079256405433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017777779015402, + "completions/max_length": 466.1666666666667, + "completions/max_terminated_length": 447.43333333333334, + "completions/mean_length": 187.82612101236978, + "completions/mean_terminated_length": 181.9598353068034, + "completions/min_length": 57.36666666666667, + "completions/min_terminated_length": 57.36666666666667, + "entropy": 0.12668142033119997, + "epoch": 0.8715300193673338, + "eval/gt_acc_batch": 0.806944465637207, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26339268684387207, + "kd/policy_loss": 0.008191281549322109, + "kd/rkl_advantage_mean": 0.6666274428367615, + "kd/rkl_advantage_p95": 4.831037294864655, + "kd/rkl_advantage_std": 2.1502163181702296, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2850795579690882e-07, + "loss": 0.032765124241511026, + "num_tokens": 750860850.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8069444437821706, + "rewards/gt_logging_reward/std": 0.3854315310716629, + "sampling/importance_sampling_ratio/max": 1.9626056631406148, + "sampling/importance_sampling_ratio/mean": 0.996525130669276, + "sampling/importance_sampling_ratio/min": 0.468292506535848, + "sampling/sampling_logp_difference/max": 0.3021794279416402, + "sampling/sampling_logp_difference/mean": 0.003850314075437685, + "step": 22950, + "step_time": 8.01469013816677, + "student/entropy": 0.12668142033119997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888889874021212, + "completions/max_length": 464.4, + "completions/max_terminated_length": 431.2, + "completions/mean_length": 182.20362040201823, + "completions/mean_terminated_length": 176.02877807617188, + "completions/min_length": 52.63333333333333, + "completions/min_terminated_length": 52.63333333333333, + "entropy": 0.12763134439786275, + "epoch": 0.8726692742946113, + "eval/gt_acc_batch": 0.8119444727897644, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3159853518009186, + "kd/policy_loss": 0.008042981130226204, + "kd/rkl_advantage_mean": 0.7860708992889461, + "kd/rkl_advantage_p95": 4.968589395284653, + "kd/rkl_advantage_std": 2.1899077316125233, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2736870086963126e-07, + "loss": 0.0321719229221344, + "num_tokens": 751812639.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8119444429874421, + "rewards/gt_logging_reward/std": 0.3829333007335663, + "sampling/importance_sampling_ratio/max": 1.7145963072776795, + "sampling/importance_sampling_ratio/mean": 0.9926984389623006, + "sampling/importance_sampling_ratio/min": 0.4637375553448995, + "sampling/sampling_logp_difference/max": 0.314682275056839, + "sampling/sampling_logp_difference/mean": 0.0039014135332157214, + "step": 22980, + "step_time": 7.858233055607221, + "student/entropy": 0.12763134439786275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02861111294478178, + "completions/max_length": 508.6666666666667, + "completions/max_terminated_length": 462.26666666666665, + "completions/mean_length": 192.06723225911458, + "completions/mean_terminated_length": 182.6823298136393, + "completions/min_length": 52.266666666666666, + "completions/min_terminated_length": 52.266666666666666, + "entropy": 0.1379728484277924, + "epoch": 0.8738085292218889, + "eval/gt_acc_batch": 0.7794444620609283, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2683645188808441, + "kd/policy_loss": 0.009489416866563261, + "kd/rkl_advantage_mean": 0.9328331373631954, + "kd/rkl_advantage_p95": 5.280851731697719, + "kd/rkl_advantage_std": 2.303202109535535, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2622944594235368e-07, + "loss": 0.03795766830444336, + "num_tokens": 752807569.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7794444402058919, + "rewards/gt_logging_reward/std": 0.40996646583080293, + "sampling/importance_sampling_ratio/max": 2.019237486521403, + "sampling/importance_sampling_ratio/mean": 0.9977734486262003, + "sampling/importance_sampling_ratio/min": 0.4443505257368088, + "sampling/sampling_logp_difference/max": 0.3163887023925781, + "sampling/sampling_logp_difference/mean": 0.004181384528055787, + "step": 23010, + "step_time": 8.375517830568908, + "student/entropy": 0.1379728484277924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023055556726952395, + "completions/max_length": 477.26666666666665, + "completions/max_terminated_length": 445.1333333333333, + "completions/mean_length": 186.9105662027995, + "completions/mean_terminated_length": 179.39688873291016, + "completions/min_length": 54.766666666666666, + "completions/min_terminated_length": 54.766666666666666, + "entropy": 0.13744439880053202, + "epoch": 0.8749477841491664, + "eval/gt_acc_batch": 0.7802778045336406, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2910434901714325, + "kd/policy_loss": 0.008973856673886378, + "kd/rkl_advantage_mean": 0.7343116017136102, + "kd/rkl_advantage_p95": 4.899587333202362, + "kd/rkl_advantage_std": 2.1812462468942004, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2509019101507615e-07, + "loss": 0.03589542706807455, + "num_tokens": 753776535.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7802777767181397, + "rewards/gt_logging_reward/std": 0.4041552593310674, + "sampling/importance_sampling_ratio/max": 1.8749382932980856, + "sampling/importance_sampling_ratio/mean": 0.9964870810508728, + "sampling/importance_sampling_ratio/min": 0.44657853841781614, + "sampling/sampling_logp_difference/max": 0.326977022488912, + "sampling/sampling_logp_difference/mean": 0.0041608799869815504, + "step": 23040, + "step_time": 8.076094415956565, + "student/entropy": 0.13744439880053202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667889803648, + "completions/max_length": 494.5, + "completions/max_terminated_length": 458.5, + "completions/mean_length": 186.9386210123698, + "completions/mean_terminated_length": 179.74625193277996, + "completions/min_length": 49.13333333333333, + "completions/min_terminated_length": 49.13333333333333, + "entropy": 0.1315788966914018, + "epoch": 0.876087039076444, + "eval/gt_acc_batch": 0.795555571715037, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3713463544845581, + "kd/policy_loss": 0.008503361391679695, + "kd/rkl_advantage_mean": 0.8242284173145891, + "kd/rkl_advantage_p95": 5.105216805140177, + "kd/rkl_advantage_std": 2.2501171121994656, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2395093608779857e-07, + "loss": 0.03401344617207845, + "num_tokens": 754750946.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555538336436, + "rewards/gt_logging_reward/std": 0.39735410710175834, + "sampling/importance_sampling_ratio/max": 1.9018062551816304, + "sampling/importance_sampling_ratio/mean": 1.0000363051891328, + "sampling/importance_sampling_ratio/min": 0.44314101040363313, + "sampling/sampling_logp_difference/max": 0.3336210668087006, + "sampling/sampling_logp_difference/mean": 0.004063480712162951, + "step": 23070, + "step_time": 8.120897257363929, + "student/entropy": 0.1315788966914018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02472222379098336, + "completions/max_length": 488.5, + "completions/max_terminated_length": 461.76666666666665, + "completions/mean_length": 191.97084350585936, + "completions/mean_terminated_length": 184.0526372273763, + "completions/min_length": 56.6, + "completions/min_terminated_length": 56.6, + "entropy": 0.1235576598594586, + "epoch": 0.8772262940037215, + "eval/gt_acc_batch": 0.8136111279328664, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31412848830223083, + "kd/policy_loss": 0.00814084980908471, + "kd/rkl_advantage_mean": 0.7211971155057351, + "kd/rkl_advantage_p95": 4.931626149018606, + "kd/rkl_advantage_std": 2.2074815442164737, + "kd/ssd_loss": 0.0, + "learning_rate": 1.22811681160521e-07, + "loss": 0.03256339828173319, + "num_tokens": 755750033.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813611102104187, + "rewards/gt_logging_reward/std": 0.37787968864043553, + "sampling/importance_sampling_ratio/max": 1.8633577783902486, + "sampling/importance_sampling_ratio/mean": 0.9996126095453898, + "sampling/importance_sampling_ratio/min": 0.4769335160652796, + "sampling/sampling_logp_difference/max": 0.3282018502553304, + "sampling/sampling_logp_difference/mean": 0.003831643184336523, + "step": 23100, + "step_time": 8.284613655266003, + "student/entropy": 0.1235576598594586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277778835346302, + "completions/max_length": 490.2, + "completions/max_terminated_length": 434.3666666666667, + "completions/mean_length": 184.14834340413412, + "completions/mean_terminated_length": 177.4874725341797, + "completions/min_length": 58.13333333333333, + "completions/min_terminated_length": 58.13333333333333, + "entropy": 0.13204672622183958, + "epoch": 0.8783655489309992, + "eval/gt_acc_batch": 0.8047222415606181, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3118095397949219, + "kd/policy_loss": 0.008502556944343572, + "kd/rkl_advantage_mean": 0.785019473110636, + "kd/rkl_advantage_p95": 5.044319146871567, + "kd/rkl_advantage_std": 2.221185443798701, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2167242623324346e-07, + "loss": 0.03401022752126058, + "num_tokens": 756711111.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8047222236792246, + "rewards/gt_logging_reward/std": 0.386617519458135, + "sampling/importance_sampling_ratio/max": 1.8386635065078736, + "sampling/importance_sampling_ratio/mean": 1.0054024199644724, + "sampling/importance_sampling_ratio/min": 0.43596192797025046, + "sampling/sampling_logp_difference/max": 0.32714031140009564, + "sampling/sampling_logp_difference/mean": 0.003959918658559521, + "step": 23130, + "step_time": 8.142099202092504, + "student/entropy": 0.13204672622183958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000109896064, + "completions/max_length": 473.3, + "completions/max_terminated_length": 442.53333333333336, + "completions/mean_length": 188.0261205037435, + "completions/mean_terminated_length": 182.4211196899414, + "completions/min_length": 57.7, + "completions/min_terminated_length": 57.7, + "entropy": 0.12690939251333475, + "epoch": 0.8795048038582767, + "eval/gt_acc_batch": 0.8013889094193777, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3181326687335968, + "kd/policy_loss": 0.007981227984419092, + "kd/rkl_advantage_mean": 0.7134026215722163, + "kd/rkl_advantage_p95": 4.92519794901212, + "kd/rkl_advantage_std": 2.1877118150393167, + "kd/ssd_loss": 0.0, + "learning_rate": 1.205331713059659e-07, + "loss": 0.03192491332689921, + "num_tokens": 757689589.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888895511627, + "rewards/gt_logging_reward/std": 0.388921661178271, + "sampling/importance_sampling_ratio/max": 1.950731341044108, + "sampling/importance_sampling_ratio/mean": 1.002543858687083, + "sampling/importance_sampling_ratio/min": 0.4581389933824539, + "sampling/sampling_logp_difference/max": 0.33347437580426537, + "sampling/sampling_logp_difference/mean": 0.003855933124820391, + "step": 23160, + "step_time": 7.966261979668828, + "student/entropy": 0.12690939251333475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333463718494, + "completions/max_length": 479.2, + "completions/max_terminated_length": 426.9, + "completions/mean_length": 179.63973134358724, + "completions/mean_terminated_length": 172.79292958577474, + "completions/min_length": 59.96666666666667, + "completions/min_terminated_length": 59.96666666666667, + "entropy": 0.13109578931083282, + "epoch": 0.8806440587855543, + "eval/gt_acc_batch": 0.7986111303170522, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31850823760032654, + "kd/policy_loss": 0.00883323613088578, + "kd/rkl_advantage_mean": 0.7740820974732439, + "kd/rkl_advantage_p95": 5.008342313766479, + "kd/rkl_advantage_std": 2.2178747882445653, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1939391637868832e-07, + "loss": 0.03533295392990112, + "num_tokens": 758633916.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7986111124356587, + "rewards/gt_logging_reward/std": 0.3878565872708956, + "sampling/importance_sampling_ratio/max": 1.929719050725301, + "sampling/importance_sampling_ratio/mean": 0.995940633614858, + "sampling/importance_sampling_ratio/min": 0.48022502263387046, + "sampling/sampling_logp_difference/max": 0.32048014005025227, + "sampling/sampling_logp_difference/mean": 0.00397167921376725, + "step": 23190, + "step_time": 7.953469023131765, + "student/entropy": 0.13109578931083282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021944445713112753, + "completions/max_length": 489.43333333333334, + "completions/max_terminated_length": 442.5, + "completions/mean_length": 195.04195556640624, + "completions/mean_terminated_length": 187.94127349853517, + "completions/min_length": 58.833333333333336, + "completions/min_terminated_length": 58.833333333333336, + "entropy": 0.12859247184048098, + "epoch": 0.8817833137128318, + "eval/gt_acc_batch": 0.7925000230471293, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29214024543762207, + "kd/policy_loss": 0.008587343197238322, + "kd/rkl_advantage_mean": 0.6732667512570818, + "kd/rkl_advantage_p95": 4.850631990035375, + "kd/rkl_advantage_std": 2.1883947253227234, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1825466145141077e-07, + "loss": 0.03434937795003255, + "num_tokens": 759643675.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7924999972184499, + "rewards/gt_logging_reward/std": 0.4015615314245224, + "sampling/importance_sampling_ratio/max": 1.8821366906166077, + "sampling/importance_sampling_ratio/mean": 1.0058015088240306, + "sampling/importance_sampling_ratio/min": 0.4771183336774508, + "sampling/sampling_logp_difference/max": 0.3369857887427012, + "sampling/sampling_logp_difference/mean": 0.003912599470155934, + "step": 23220, + "step_time": 8.177910016001745, + "student/entropy": 0.12859247184048098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014444445197780927, + "completions/max_length": 474.06666666666666, + "completions/max_terminated_length": 441.93333333333334, + "completions/mean_length": 183.99584350585937, + "completions/mean_terminated_length": 179.28905232747397, + "completions/min_length": 52.666666666666664, + "completions/min_terminated_length": 52.666666666666664, + "entropy": 0.13549641085167727, + "epoch": 0.8829225686401093, + "eval/gt_acc_batch": 0.7916666865348816, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3782370984554291, + "kd/policy_loss": 0.00875515874940902, + "kd/rkl_advantage_mean": 0.7392922422538201, + "kd/rkl_advantage_p95": 4.949899983406067, + "kd/rkl_advantage_std": 2.201300186912219, + "kd/ssd_loss": 0.0, + "learning_rate": 1.171154065241332e-07, + "loss": 0.03502063353856404, + "num_tokens": 760604476.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7916666666666666, + "rewards/gt_logging_reward/std": 0.39699329336484274, + "sampling/importance_sampling_ratio/max": 1.763451588153839, + "sampling/importance_sampling_ratio/mean": 0.994125618537267, + "sampling/importance_sampling_ratio/min": 0.46525353839000066, + "sampling/sampling_logp_difference/max": 0.3593585511048635, + "sampling/sampling_logp_difference/mean": 0.004090516013093293, + "step": 23250, + "step_time": 7.89857203453236, + "student/entropy": 0.13549641085167727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02500000180055698, + "completions/max_length": 489.9, + "completions/max_terminated_length": 461.23333333333335, + "completions/mean_length": 193.51195526123047, + "completions/mean_terminated_length": 185.50155080159504, + "completions/min_length": 49.266666666666666, + "completions/min_terminated_length": 49.266666666666666, + "entropy": 0.1317541171486179, + "epoch": 0.8840618235673869, + "eval/gt_acc_batch": 0.7925000190734863, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.307046502828598, + "kd/policy_loss": 0.008795026332760851, + "kd/rkl_advantage_mean": 0.7620453449587027, + "kd/rkl_advantage_p95": 5.0891047755877175, + "kd/rkl_advantage_std": 2.244570863246918, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1597615159685566e-07, + "loss": 0.03518010377883911, + "num_tokens": 761612135.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7924999972184499, + "rewards/gt_logging_reward/std": 0.39956229627132417, + "sampling/importance_sampling_ratio/max": 1.9159420768419901, + "sampling/importance_sampling_ratio/mean": 1.0006542305151622, + "sampling/importance_sampling_ratio/min": 0.4180464789271355, + "sampling/sampling_logp_difference/max": 0.3213007052739461, + "sampling/sampling_logp_difference/mean": 0.00399587731808424, + "step": 23280, + "step_time": 8.168324610532727, + "student/entropy": 0.1317541171486179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889051352938, + "completions/max_length": 486.43333333333334, + "completions/max_terminated_length": 454.06666666666666, + "completions/mean_length": 186.83751017252604, + "completions/mean_terminated_length": 178.27852579752604, + "completions/min_length": 50.86666666666667, + "completions/min_terminated_length": 50.86666666666667, + "entropy": 0.12874856740236282, + "epoch": 0.8852010784946645, + "eval/gt_acc_batch": 0.8075000226497651, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32310038805007935, + "kd/policy_loss": 0.008092415284287805, + "kd/rkl_advantage_mean": 0.8169227125744025, + "kd/rkl_advantage_p95": 5.1120341698328655, + "kd/rkl_advantage_std": 2.2435388614734015, + "kd/ssd_loss": 0.0, + "learning_rate": 1.148368966695781e-07, + "loss": 0.03236965934435527, + "num_tokens": 762592470.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8074999968210856, + "rewards/gt_logging_reward/std": 0.3868540346622467, + "sampling/importance_sampling_ratio/max": 1.8689161856969199, + "sampling/importance_sampling_ratio/mean": 0.995556092262268, + "sampling/importance_sampling_ratio/min": 0.4691173583269119, + "sampling/sampling_logp_difference/max": 0.32147066593170165, + "sampling/sampling_logp_difference/mean": 0.003921871647859613, + "step": 23310, + "step_time": 8.214100531528432, + "student/entropy": 0.12874856740236282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944445787618558, + "completions/max_length": 490.03333333333336, + "completions/max_terminated_length": 442.03333333333336, + "completions/mean_length": 190.2230662027995, + "completions/mean_terminated_length": 181.30670318603515, + "completions/min_length": 48.96666666666667, + "completions/min_terminated_length": 48.96666666666667, + "entropy": 0.12794590033590794, + "epoch": 0.8863403334219421, + "eval/gt_acc_batch": 0.7816666821638744, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30589011311531067, + "kd/policy_loss": 0.008202099396536747, + "kd/rkl_advantage_mean": 0.7609225244726986, + "kd/rkl_advantage_p95": 4.987123433748881, + "kd/rkl_advantage_std": 2.208218417565028, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1369764174230052e-07, + "loss": 0.03280839721361796, + "num_tokens": 763585753.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7816666642824809, + "rewards/gt_logging_reward/std": 0.40750241180260977, + "sampling/importance_sampling_ratio/max": 1.916506326198578, + "sampling/importance_sampling_ratio/mean": 0.9932536959648133, + "sampling/importance_sampling_ratio/min": 0.46115903158982596, + "sampling/sampling_logp_difference/max": 0.30962676405906675, + "sampling/sampling_logp_difference/mean": 0.0038949601196994386, + "step": 23340, + "step_time": 8.159490325363004, + "student/entropy": 0.12794590033590794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890352100133, + "completions/max_length": 494.76666666666665, + "completions/max_terminated_length": 450.96666666666664, + "completions/mean_length": 188.124733988444, + "completions/mean_terminated_length": 180.25436197916667, + "completions/min_length": 48.7, + "completions/min_terminated_length": 48.7, + "entropy": 0.1332431253666679, + "epoch": 0.8874795883492196, + "eval/gt_acc_batch": 0.8130555788675944, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2512551248073578, + "kd/policy_loss": 0.00858370353622983, + "kd/rkl_advantage_mean": 0.7310769268001119, + "kd/rkl_advantage_p95": 4.896017819643021, + "kd/rkl_advantage_std": 2.177577265103658, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1255838681502297e-07, + "loss": 0.034334818522135414, + "num_tokens": 764573898.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813055553038915, + "rewards/gt_logging_reward/std": 0.3828246742486954, + "sampling/importance_sampling_ratio/max": 1.7741247375806173, + "sampling/importance_sampling_ratio/mean": 0.9979776600996654, + "sampling/importance_sampling_ratio/min": 0.44155739843845365, + "sampling/sampling_logp_difference/max": 0.3160959601402283, + "sampling/sampling_logp_difference/mean": 0.00409540346202751, + "step": 23370, + "step_time": 8.30296038139204, + "student/entropy": 0.1332431253666679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024166668020188808, + "completions/max_length": 492.53333333333336, + "completions/max_terminated_length": 447.03333333333336, + "completions/mean_length": 187.49362131754557, + "completions/mean_terminated_length": 179.41869099934897, + "completions/min_length": 54.9, + "completions/min_terminated_length": 54.9, + "entropy": 0.13305013366043567, + "epoch": 0.8886188432764972, + "eval/gt_acc_batch": 0.8011111319065094, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.291963130235672, + "kd/policy_loss": 0.008593492650349314, + "kd/rkl_advantage_mean": 0.78143601672103, + "kd/rkl_advantage_p95": 5.004545233647028, + "kd/rkl_advantage_std": 2.194343473513921, + "kd/ssd_loss": 0.0, + "learning_rate": 1.114191318877454e-07, + "loss": 0.03437397480010986, + "num_tokens": 765552659.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.801111110051473, + "rewards/gt_logging_reward/std": 0.3912470549345016, + "sampling/importance_sampling_ratio/max": 1.893887428442637, + "sampling/importance_sampling_ratio/mean": 0.9967490951220195, + "sampling/importance_sampling_ratio/min": 0.45751746793588005, + "sampling/sampling_logp_difference/max": 0.3309444099664688, + "sampling/sampling_logp_difference/mean": 0.004061497103733321, + "step": 23400, + "step_time": 8.255273388834516, + "student/entropy": 0.13305013366043567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016666667473812898, + "completions/max_length": 472.6333333333333, + "completions/max_terminated_length": 446.93333333333334, + "completions/mean_length": 188.62334289550782, + "completions/mean_terminated_length": 183.34205373128256, + "completions/min_length": 59.46666666666667, + "completions/min_terminated_length": 59.46666666666667, + "entropy": 0.12337800289193789, + "epoch": 0.8897580982037747, + "eval/gt_acc_batch": 0.8155555744965871, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.272050678730011, + "kd/policy_loss": 0.008035271579865366, + "kd/rkl_advantage_mean": 0.7515330073734124, + "kd/rkl_advantage_p95": 4.978853589296341, + "kd/rkl_advantage_std": 2.204276688893636, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1027987696046786e-07, + "loss": 0.03214108149210612, + "num_tokens": 766540407.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8155555586020152, + "rewards/gt_logging_reward/std": 0.3792866716782252, + "sampling/importance_sampling_ratio/max": 1.8959392825762431, + "sampling/importance_sampling_ratio/mean": 0.9992969592412313, + "sampling/importance_sampling_ratio/min": 0.4805510888497035, + "sampling/sampling_logp_difference/max": 0.32385849157969154, + "sampling/sampling_logp_difference/mean": 0.0037831123685464264, + "step": 23430, + "step_time": 8.04136991656463, + "student/entropy": 0.12337800289193789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779536942642, + "completions/max_length": 493.8333333333333, + "completions/max_terminated_length": 453.1333333333333, + "completions/mean_length": 193.03028818766276, + "completions/mean_terminated_length": 184.0612340291341, + "completions/min_length": 50.56666666666667, + "completions/min_terminated_length": 50.56666666666667, + "entropy": 0.1322292722761631, + "epoch": 0.8908973531310523, + "eval/gt_acc_batch": 0.7825000246365865, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3117804229259491, + "kd/policy_loss": 0.008922240906395018, + "kd/rkl_advantage_mean": 0.7542698213209709, + "kd/rkl_advantage_p95": 4.9807889521121975, + "kd/rkl_advantage_std": 2.2299985736608505, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0914062203319029e-07, + "loss": 0.03568897247314453, + "num_tokens": 767542484.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7824999988079071, + "rewards/gt_logging_reward/std": 0.40217407047748566, + "sampling/importance_sampling_ratio/max": 1.8568728605906168, + "sampling/importance_sampling_ratio/mean": 1.0013722062110901, + "sampling/importance_sampling_ratio/min": 0.4250192736585935, + "sampling/sampling_logp_difference/max": 0.34292271931966145, + "sampling/sampling_logp_difference/mean": 0.004014841327443719, + "step": 23460, + "step_time": 8.214302424768297, + "student/entropy": 0.1322292722761631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016111112075547378, + "completions/max_length": 481.23333333333335, + "completions/max_terminated_length": 433.8333333333333, + "completions/mean_length": 187.6958429972331, + "completions/mean_terminated_length": 182.4075154622396, + "completions/min_length": 53.833333333333336, + "completions/min_terminated_length": 53.833333333333336, + "entropy": 0.12617952469736338, + "epoch": 0.8920366080583298, + "eval/gt_acc_batch": 0.8058333535989125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2548650801181793, + "kd/policy_loss": 0.00812690524229159, + "kd/rkl_advantage_mean": 0.7714449566633751, + "kd/rkl_advantage_p95": 4.9145223518212635, + "kd/rkl_advantage_std": 2.198826653758685, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0800136710591273e-07, + "loss": 0.03250762224197388, + "num_tokens": 768525333.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8058333377043406, + "rewards/gt_logging_reward/std": 0.38640056451161703, + "sampling/importance_sampling_ratio/max": 1.9118595878283182, + "sampling/importance_sampling_ratio/mean": 0.9879804313182831, + "sampling/importance_sampling_ratio/min": 0.4411916196346283, + "sampling/sampling_logp_difference/max": 0.30647833744684855, + "sampling/sampling_logp_difference/mean": 0.003921381640248, + "step": 23490, + "step_time": 8.102866832898387, + "student/entropy": 0.12617952469736338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944446098059415, + "completions/max_length": 483.43333333333334, + "completions/max_terminated_length": 446.53333333333336, + "completions/mean_length": 191.19695332845052, + "completions/mean_terminated_length": 182.44372965494793, + "completions/min_length": 55.5, + "completions/min_terminated_length": 55.5, + "entropy": 0.12614054686079423, + "epoch": 0.8931758629856074, + "eval/gt_acc_batch": 0.8069444636503855, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25433751940727234, + "kd/policy_loss": 0.008230253248863542, + "kd/rkl_advantage_mean": 0.7323411053667466, + "kd/rkl_advantage_p95": 4.915744103988012, + "kd/rkl_advantage_std": 2.164852320154508, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0686211217863516e-07, + "loss": 0.03292101224263509, + "num_tokens": 769516722.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8069444417953491, + "rewards/gt_logging_reward/std": 0.38519238034884135, + "sampling/importance_sampling_ratio/max": 1.845179311434428, + "sampling/importance_sampling_ratio/mean": 1.001533458630244, + "sampling/importance_sampling_ratio/min": 0.47524298826853434, + "sampling/sampling_logp_difference/max": 0.36615239779154457, + "sampling/sampling_logp_difference/mean": 0.003841124203366538, + "step": 23520, + "step_time": 8.132080155401491, + "student/entropy": 0.12614054686079423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667790462572, + "completions/max_length": 488.3, + "completions/max_terminated_length": 456.46666666666664, + "completions/mean_length": 189.2850118001302, + "completions/mean_terminated_length": 183.15941823323567, + "completions/min_length": 49.666666666666664, + "completions/min_terminated_length": 49.666666666666664, + "entropy": 0.1266453705728054, + "epoch": 0.894315117912885, + "eval/gt_acc_batch": 0.7933333655198415, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28384193778038025, + "kd/policy_loss": 0.007978556378899763, + "kd/rkl_advantage_mean": 0.7209117294599613, + "kd/rkl_advantage_p95": 4.8692471047242485, + "kd/rkl_advantage_std": 2.1787883301575977, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0572285725135762e-07, + "loss": 0.031914222240448, + "num_tokens": 770505852.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7933333317438761, + "rewards/gt_logging_reward/std": 0.3979853143294652, + "sampling/importance_sampling_ratio/max": 1.7859151005744933, + "sampling/importance_sampling_ratio/mean": 0.9967570801575979, + "sampling/importance_sampling_ratio/min": 0.4357630848884583, + "sampling/sampling_logp_difference/max": 0.32015870412190756, + "sampling/sampling_logp_difference/mean": 0.00392954039076964, + "step": 23550, + "step_time": 8.050195835426955, + "student/entropy": 0.1266453705728054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890445232392, + "completions/max_length": 506.26666666666665, + "completions/max_terminated_length": 463.76666666666665, + "completions/mean_length": 189.72834320068358, + "completions/mean_terminated_length": 181.82352650960286, + "completions/min_length": 53.266666666666666, + "completions/min_terminated_length": 53.266666666666666, + "entropy": 0.13573348081360262, + "epoch": 0.8954543728401625, + "eval/gt_acc_batch": 0.7813889106114705, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3088338077068329, + "kd/policy_loss": 0.00897150503199858, + "kd/rkl_advantage_mean": 0.7853486250232284, + "kd/rkl_advantage_p95": 5.014503516753515, + "kd/rkl_advantage_std": 2.220570707321167, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0458360232408005e-07, + "loss": 0.035886017481486, + "num_tokens": 771490210.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7813888867696126, + "rewards/gt_logging_reward/std": 0.4043851484855016, + "sampling/importance_sampling_ratio/max": 1.8822362860043844, + "sampling/importance_sampling_ratio/mean": 1.0025106867154439, + "sampling/importance_sampling_ratio/min": 0.46884009887774786, + "sampling/sampling_logp_difference/max": 0.30804698864618935, + "sampling/sampling_logp_difference/mean": 0.004097609128803015, + "step": 23580, + "step_time": 8.171456933397955, + "student/entropy": 0.13573348081360262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027777779381722212, + "completions/max_length": 480.8666666666667, + "completions/max_terminated_length": 442.6333333333333, + "completions/mean_length": 192.21417744954428, + "completions/mean_terminated_length": 183.2458302815755, + "completions/min_length": 47.93333333333333, + "completions/min_terminated_length": 47.93333333333333, + "entropy": 0.13117755074054002, + "epoch": 0.8965936277674401, + "eval/gt_acc_batch": 0.7872222463289896, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2844315469264984, + "kd/policy_loss": 0.008805507244930292, + "kd/rkl_advantage_mean": 0.7935517848003656, + "kd/rkl_advantage_p95": 4.979940607150396, + "kd/rkl_advantage_std": 2.178421734770139, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0344434739680248e-07, + "loss": 0.03522203763326009, + "num_tokens": 772486829.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7872222145398458, + "rewards/gt_logging_reward/std": 0.39905671576658885, + "sampling/importance_sampling_ratio/max": 1.939840265115102, + "sampling/importance_sampling_ratio/mean": 1.0023224731286366, + "sampling/importance_sampling_ratio/min": 0.47162244419256844, + "sampling/sampling_logp_difference/max": 0.33163593808809916, + "sampling/sampling_logp_difference/mean": 0.003988621251968046, + "step": 23610, + "step_time": 8.102807867936402, + "student/entropy": 0.13117755074054002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01611111229285598, + "completions/max_length": 468.76666666666665, + "completions/max_terminated_length": 434.96666666666664, + "completions/mean_length": 186.86612091064453, + "completions/mean_terminated_length": 181.6456502278646, + "completions/min_length": 54.266666666666666, + "completions/min_terminated_length": 54.266666666666666, + "entropy": 0.12615315094590188, + "epoch": 0.8977328826947176, + "eval/gt_acc_batch": 0.8191666841506958, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27189311385154724, + "kd/policy_loss": 0.007909120687205966, + "kd/rkl_advantage_mean": 0.7094635979117204, + "kd/rkl_advantage_p95": 4.767636255423228, + "kd/rkl_advantage_std": 2.1316077013810477, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0230509246952493e-07, + "loss": 0.031636488437652585, + "num_tokens": 773469195.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8191666682561238, + "rewards/gt_logging_reward/std": 0.36976704249779385, + "sampling/importance_sampling_ratio/max": 1.8388025005658468, + "sampling/importance_sampling_ratio/mean": 0.9993334670861562, + "sampling/importance_sampling_ratio/min": 0.4940034588177999, + "sampling/sampling_logp_difference/max": 0.31555574138959247, + "sampling/sampling_logp_difference/mean": 0.003834435824925701, + "step": 23640, + "step_time": 7.948519981171315, + "student/entropy": 0.12615315094590188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019444445644815764, + "completions/max_length": 476.0, + "completions/max_terminated_length": 455.7, + "completions/mean_length": 187.1372314453125, + "completions/mean_terminated_length": 180.84486083984376, + "completions/min_length": 49.5, + "completions/min_terminated_length": 49.5, + "entropy": 0.13058514626075823, + "epoch": 0.8988721376219952, + "eval/gt_acc_batch": 0.8119444688161214, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.307647705078125, + "kd/policy_loss": 0.008456412731902674, + "kd/rkl_advantage_mean": 0.7763529910395542, + "kd/rkl_advantage_p95": 5.081291961669922, + "kd/rkl_advantage_std": 2.2426914711793264, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0116583754224736e-07, + "loss": 0.03382565577824911, + "num_tokens": 774440297.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8119444449742635, + "rewards/gt_logging_reward/std": 0.38603309293588, + "sampling/importance_sampling_ratio/max": 1.90255552927653, + "sampling/importance_sampling_ratio/mean": 0.9967171986897786, + "sampling/importance_sampling_ratio/min": 0.45148248275121056, + "sampling/sampling_logp_difference/max": 0.3350133736928304, + "sampling/sampling_logp_difference/mean": 0.003976478373321394, + "step": 23670, + "step_time": 7.874063818599097, + "student/entropy": 0.13058514626075823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019444445458551247, + "completions/max_length": 488.1, + "completions/max_terminated_length": 454.46666666666664, + "completions/mean_length": 185.65945485432943, + "completions/mean_terminated_length": 179.20720876057942, + "completions/min_length": 61.63333333333333, + "completions/min_terminated_length": 61.63333333333333, + "entropy": 0.13355098962783812, + "epoch": 0.9000113925492728, + "eval/gt_acc_batch": 0.7922222375869751, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.40698111057281494, + "kd/policy_loss": 0.008877882072313999, + "kd/rkl_advantage_mean": 0.7841199447711309, + "kd/rkl_advantage_p95": 5.003775258858998, + "kd/rkl_advantage_std": 2.22268219490846, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0002658261496981e-07, + "loss": 0.035511533419291176, + "num_tokens": 775409743.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7922222236792247, + "rewards/gt_logging_reward/std": 0.3983675668636958, + "sampling/importance_sampling_ratio/max": 1.9016271432240803, + "sampling/importance_sampling_ratio/mean": 1.001073694229126, + "sampling/importance_sampling_ratio/min": 0.4980588068564733, + "sampling/sampling_logp_difference/max": 0.3543634951114655, + "sampling/sampling_logp_difference/mean": 0.0040837232333918415, + "step": 23700, + "step_time": 7.94702851936648, + "student/entropy": 0.13355098962783812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022500001142422357, + "completions/max_length": 484.4, + "completions/max_terminated_length": 449.3333333333333, + "completions/mean_length": 188.2966761271159, + "completions/mean_terminated_length": 180.95999755859376, + "completions/min_length": 54.56666666666667, + "completions/min_terminated_length": 54.56666666666667, + "entropy": 0.1267381079494953, + "epoch": 0.9011506474765504, + "eval/gt_acc_batch": 0.7991666853427887, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2846234440803528, + "kd/policy_loss": 0.00822012421558611, + "kd/rkl_advantage_mean": 0.6475600599505318, + "kd/rkl_advantage_p95": 4.8337732712427774, + "kd/rkl_advantage_std": 2.1716086586316425, + "kd/ssd_loss": 0.0, + "learning_rate": 9.888732768769224e-08, + "loss": 0.03288049300511678, + "num_tokens": 776388731.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7991666654745738, + "rewards/gt_logging_reward/std": 0.39278576771418255, + "sampling/importance_sampling_ratio/max": 1.873330811659495, + "sampling/importance_sampling_ratio/mean": 0.999523264169693, + "sampling/importance_sampling_ratio/min": 0.42483115941286087, + "sampling/sampling_logp_difference/max": 0.30087687969207766, + "sampling/sampling_logp_difference/mean": 0.003892188884007434, + "step": 23730, + "step_time": 7.9158067875629055, + "student/entropy": 0.1267381079494953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020277779114743075, + "completions/max_length": 480.9, + "completions/max_terminated_length": 449.26666666666665, + "completions/mean_length": 182.67278747558595, + "completions/mean_terminated_length": 175.970654296875, + "completions/min_length": 50.1, + "completions/min_terminated_length": 50.1, + "entropy": 0.1270023458947738, + "epoch": 0.9022899024038279, + "eval/gt_acc_batch": 0.8150000194708507, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3233932852745056, + "kd/policy_loss": 0.008069769232921924, + "kd/rkl_advantage_mean": 0.6946959936370453, + "kd/rkl_advantage_p95": 4.885032192866007, + "kd/rkl_advantage_std": 2.22254022359848, + "kd/ssd_loss": 0.0, + "learning_rate": 9.774807276041469e-08, + "loss": 0.03227907419204712, + "num_tokens": 777344753.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8149999996026357, + "rewards/gt_logging_reward/std": 0.3778323481480281, + "sampling/importance_sampling_ratio/max": 1.8723241567611695, + "sampling/importance_sampling_ratio/mean": 0.9986918608347575, + "sampling/importance_sampling_ratio/min": 0.4546799709399541, + "sampling/sampling_logp_difference/max": 0.3340025693178177, + "sampling/sampling_logp_difference/mean": 0.003962777298875153, + "step": 23760, + "step_time": 8.002034998271847, + "student/entropy": 0.1270023458947738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01944444573794802, + "completions/max_length": 484.0, + "completions/max_terminated_length": 446.46666666666664, + "completions/mean_length": 183.0741760253906, + "completions/mean_terminated_length": 176.61975708007813, + "completions/min_length": 56.333333333333336, + "completions/min_terminated_length": 56.333333333333336, + "entropy": 0.1321736787756284, + "epoch": 0.9034291573311054, + "eval/gt_acc_batch": 0.8013889014720916, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3071274757385254, + "kd/policy_loss": 0.008758024085545912, + "kd/rkl_advantage_mean": 0.7820529286051169, + "kd/rkl_advantage_p95": 4.968200701475143, + "kd/rkl_advantage_std": 2.2192766745885213, + "kd/ssd_loss": 0.0, + "learning_rate": 9.660881783313712e-08, + "loss": 0.035032097498575845, + "num_tokens": 778305316.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8013888875643412, + "rewards/gt_logging_reward/std": 0.3920508270462354, + "sampling/importance_sampling_ratio/max": 1.8845593492190043, + "sampling/importance_sampling_ratio/mean": 0.9999983290831248, + "sampling/importance_sampling_ratio/min": 0.4751937607924143, + "sampling/sampling_logp_difference/max": 0.3381643603245417, + "sampling/sampling_logp_difference/mean": 0.003992102846192817, + "step": 23790, + "step_time": 8.029779204102427, + "student/entropy": 0.1321736787756284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223803400994, + "completions/max_length": 500.8666666666667, + "completions/max_terminated_length": 465.7, + "completions/mean_length": 195.7247314453125, + "completions/mean_terminated_length": 186.19615936279297, + "completions/min_length": 55.36666666666667, + "completions/min_terminated_length": 55.36666666666667, + "entropy": 0.12782016259928544, + "epoch": 0.904568412258383, + "eval/gt_acc_batch": 0.7794444600741068, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24723078310489655, + "kd/policy_loss": 0.008521928042561437, + "kd/rkl_advantage_mean": 0.8118948372701804, + "kd/rkl_advantage_p95": 5.034744377930959, + "kd/rkl_advantage_std": 2.2229801625013352, + "kd/ssd_loss": 0.0, + "learning_rate": 9.546956290585956e-08, + "loss": 0.03408771355946859, + "num_tokens": 779307973.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7794444382190704, + "rewards/gt_logging_reward/std": 0.40750878949960073, + "sampling/importance_sampling_ratio/max": 1.8676459948221842, + "sampling/importance_sampling_ratio/mean": 0.999921691417694, + "sampling/importance_sampling_ratio/min": 0.44719278216362, + "sampling/sampling_logp_difference/max": 0.33270096182823183, + "sampling/sampling_logp_difference/mean": 0.0038990817808856565, + "step": 23820, + "step_time": 8.200439462598297, + "student/entropy": 0.12782016259928544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021666667827715477, + "completions/max_length": 475.1, + "completions/max_terminated_length": 426.1333333333333, + "completions/mean_length": 187.86112162272136, + "completions/mean_terminated_length": 180.65626831054686, + "completions/min_length": 57.166666666666664, + "completions/min_terminated_length": 57.166666666666664, + "entropy": 0.12658199910074472, + "epoch": 0.9057076671856605, + "eval/gt_acc_batch": 0.7969444692134857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2314251959323883, + "kd/policy_loss": 0.008088951010722666, + "kd/rkl_advantage_mean": 0.8036596624801556, + "kd/rkl_advantage_p95": 5.028963228066762, + "kd/rkl_advantage_std": 2.1908066819111505, + "kd/ssd_loss": 0.0, + "learning_rate": 9.433030797858201e-08, + "loss": 0.032355807224909466, + "num_tokens": 780282809.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7969444413979848, + "rewards/gt_logging_reward/std": 0.3957925816377004, + "sampling/importance_sampling_ratio/max": 1.866747999191284, + "sampling/importance_sampling_ratio/mean": 0.9975006441275279, + "sampling/importance_sampling_ratio/min": 0.4563299208879471, + "sampling/sampling_logp_difference/max": 0.3290259838104248, + "sampling/sampling_logp_difference/mean": 0.0038924424288173517, + "step": 23850, + "step_time": 7.912624507861135, + "student/entropy": 0.12658199910074472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026944446222235758, + "completions/max_length": 495.5, + "completions/max_terminated_length": 453.6666666666667, + "completions/mean_length": 193.49584096272787, + "completions/mean_terminated_length": 184.64491424560546, + "completions/min_length": 51.4, + "completions/min_terminated_length": 51.4, + "entropy": 0.12901725495855013, + "epoch": 0.9068469221129382, + "eval/gt_acc_batch": 0.7897222439448038, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.20007796585559845, + "kd/policy_loss": 0.00850366265901054, + "kd/rkl_advantage_mean": 0.8614704355597496, + "kd/rkl_advantage_p95": 5.234601287047068, + "kd/rkl_advantage_std": 2.2819776187340417, + "kd/ssd_loss": 0.0, + "learning_rate": 9.319105305130444e-08, + "loss": 0.03401464621225993, + "num_tokens": 781284706.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.789722220102946, + "rewards/gt_logging_reward/std": 0.39822153747081757, + "sampling/importance_sampling_ratio/max": 1.9900052467981975, + "sampling/importance_sampling_ratio/mean": 1.0023556172847747, + "sampling/importance_sampling_ratio/min": 0.4394782910744349, + "sampling/sampling_logp_difference/max": 0.3486208478609721, + "sampling/sampling_logp_difference/mean": 0.0039587310825785005, + "step": 23880, + "step_time": 8.201442767296491, + "student/entropy": 0.12901725495855013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016388889836768308, + "completions/max_length": 456.6666666666667, + "completions/max_terminated_length": 425.9, + "completions/mean_length": 181.38001022338867, + "completions/mean_terminated_length": 176.0780382792155, + "completions/min_length": 47.6, + "completions/min_terminated_length": 47.6, + "entropy": 0.12604821442315975, + "epoch": 0.9079861770402157, + "eval/gt_acc_batch": 0.8305555721124013, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.304845929145813, + "kd/policy_loss": 0.007890726769498238, + "kd/rkl_advantage_mean": 0.74743870832026, + "kd/rkl_advantage_p95": 4.945240527391434, + "kd/rkl_advantage_std": 2.1709065407514574, + "kd/ssd_loss": 0.0, + "learning_rate": 9.205179812402688e-08, + "loss": 0.031562906503677365, + "num_tokens": 782236338.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8305555621782938, + "rewards/gt_logging_reward/std": 0.3652407129605611, + "sampling/importance_sampling_ratio/max": 1.860983177026113, + "sampling/importance_sampling_ratio/mean": 1.0001076638698578, + "sampling/importance_sampling_ratio/min": 0.44710733691851295, + "sampling/sampling_logp_difference/max": 0.33841455380121865, + "sampling/sampling_logp_difference/mean": 0.003875758087572952, + "step": 23910, + "step_time": 7.709823411332521, + "student/entropy": 0.12604821442315975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444565102458, + "completions/max_length": 494.03333333333336, + "completions/max_terminated_length": 448.26666666666665, + "completions/mean_length": 188.48056335449218, + "completions/mean_terminated_length": 181.38579559326172, + "completions/min_length": 49.36666666666667, + "completions/min_terminated_length": 49.36666666666667, + "entropy": 0.12924063143630823, + "epoch": 0.9091254319674933, + "eval/gt_acc_batch": 0.8175000250339508, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3764802813529968, + "kd/policy_loss": 0.008610068874744078, + "kd/rkl_advantage_mean": 0.6941043509015192, + "kd/rkl_advantage_p95": 4.899414589007695, + "kd/rkl_advantage_std": 2.2093613783518475, + "kd/ssd_loss": 0.0, + "learning_rate": 9.091254319674931e-08, + "loss": 0.03444027900695801, + "num_tokens": 783231996.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8174999992052714, + "rewards/gt_logging_reward/std": 0.3803617646296819, + "sampling/importance_sampling_ratio/max": 1.9767000118891398, + "sampling/importance_sampling_ratio/mean": 1.0027135570844015, + "sampling/importance_sampling_ratio/min": 0.4536391923824946, + "sampling/sampling_logp_difference/max": 0.31942854324976605, + "sampling/sampling_logp_difference/mean": 0.003973771608434618, + "step": 23940, + "step_time": 8.185593083463027, + "student/entropy": 0.12924063143630823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03055555736646056, + "completions/max_length": 491.3, + "completions/max_terminated_length": 451.96666666666664, + "completions/mean_length": 190.96973368326823, + "completions/mean_terminated_length": 181.04071756998698, + "completions/min_length": 54.233333333333334, + "completions/min_terminated_length": 54.233333333333334, + "entropy": 0.1316263790552815, + "epoch": 0.9102646868947708, + "eval/gt_acc_batch": 0.8011111338933309, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26349952816963196, + "kd/policy_loss": 0.008730001186874385, + "kd/rkl_advantage_mean": 0.7578324956819416, + "kd/rkl_advantage_p95": 4.910652361313502, + "kd/rkl_advantage_std": 2.1888424227635066, + "kd/ssd_loss": 0.0, + "learning_rate": 8.977328826947177e-08, + "loss": 0.03492000500361125, + "num_tokens": 784229687.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8011111140251159, + "rewards/gt_logging_reward/std": 0.39369708200295767, + "sampling/importance_sampling_ratio/max": 1.9972583691279093, + "sampling/importance_sampling_ratio/mean": 1.000782020886739, + "sampling/importance_sampling_ratio/min": 0.45396350224812826, + "sampling/sampling_logp_difference/max": 0.33780250549316404, + "sampling/sampling_logp_difference/mean": 0.004032314824871719, + "step": 23970, + "step_time": 8.221964917665657, + "student/entropy": 0.1316263790552815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029722223927577337, + "completions/max_length": 490.43333333333334, + "completions/max_terminated_length": 464.5, + "completions/mean_length": 193.19862111409506, + "completions/mean_terminated_length": 183.675026957194, + "completions/min_length": 47.96666666666667, + "completions/min_terminated_length": 47.96666666666667, + "entropy": 0.14080434776842593, + "epoch": 0.9114039418220484, + "eval/gt_acc_batch": 0.7633333464463552, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27079567313194275, + "kd/policy_loss": 0.009601093892706558, + "kd/rkl_advantage_mean": 0.7961800189261946, + "kd/rkl_advantage_p95": 5.007323930660884, + "kd/rkl_advantage_std": 2.2040095210075377, + "kd/ssd_loss": 0.0, + "learning_rate": 8.86340333421942e-08, + "loss": 0.038404377301534016, + "num_tokens": 785228378.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7633333384990693, + "rewards/gt_logging_reward/std": 0.4196274613340696, + "sampling/importance_sampling_ratio/max": 1.9754541635513305, + "sampling/importance_sampling_ratio/mean": 0.9967398901780447, + "sampling/importance_sampling_ratio/min": 0.4341379741827647, + "sampling/sampling_logp_difference/max": 0.32603402932484943, + "sampling/sampling_logp_difference/mean": 0.004281433423360189, + "step": 24000, + "step_time": 8.131207675193824, + "student/entropy": 0.14080434776842593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025000001521160206, + "completions/max_length": 506.03333333333336, + "completions/max_terminated_length": 469.03333333333336, + "completions/mean_length": 189.5555648803711, + "completions/mean_terminated_length": 181.2795440673828, + "completions/min_length": 54.733333333333334, + "completions/min_terminated_length": 54.733333333333334, + "entropy": 0.13616626200576623, + "epoch": 0.9125431967493259, + "eval/gt_acc_batch": 0.7833333611488342, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24687303602695465, + "kd/policy_loss": 0.009002959602124367, + "kd/rkl_advantage_mean": 0.8708430830389261, + "kd/rkl_advantage_p95": 5.202428833643595, + "kd/rkl_advantage_std": 2.2701585680246352, + "kd/ssd_loss": 0.0, + "learning_rate": 8.749477841491663e-08, + "loss": 0.03601183891296387, + "num_tokens": 786211626.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7833333353201548, + "rewards/gt_logging_reward/std": 0.40647223591804504, + "sampling/importance_sampling_ratio/max": 1.9809089223543803, + "sampling/importance_sampling_ratio/mean": 0.9978289703528086, + "sampling/importance_sampling_ratio/min": 0.465154571334521, + "sampling/sampling_logp_difference/max": 0.3328208009401957, + "sampling/sampling_logp_difference/mean": 0.004184119077399373, + "step": 24030, + "step_time": 8.137740776697562, + "student/entropy": 0.13616626200576623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223499168953, + "completions/max_length": 501.06666666666666, + "completions/max_terminated_length": 453.03333333333336, + "completions/mean_length": 191.9058415730794, + "completions/mean_terminated_length": 185.41827290852865, + "completions/min_length": 54.96666666666667, + "completions/min_terminated_length": 54.96666666666667, + "entropy": 0.1336612952252229, + "epoch": 0.9136824516766034, + "eval/gt_acc_batch": 0.7777778029441833, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.37164852023124695, + "kd/policy_loss": 0.00917275834751005, + "kd/rkl_advantage_mean": 0.8903593073288599, + "kd/rkl_advantage_p95": 5.203114833434423, + "kd/rkl_advantage_std": 2.2665322492520015, + "kd/ssd_loss": 0.0, + "learning_rate": 8.635552348763908e-08, + "loss": 0.036691033840179445, + "num_tokens": 787200151.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7777777830759685, + "rewards/gt_logging_reward/std": 0.41261003812154134, + "sampling/importance_sampling_ratio/max": 1.868987508614858, + "sampling/importance_sampling_ratio/mean": 0.9985127210617065, + "sampling/importance_sampling_ratio/min": 0.44862388173739115, + "sampling/sampling_logp_difference/max": 0.3474954187870026, + "sampling/sampling_logp_difference/mean": 0.004056721576489508, + "step": 24060, + "step_time": 8.0661915777028, + "student/entropy": 0.1336612952252229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02194444558893641, + "completions/max_length": 488.53333333333336, + "completions/max_terminated_length": 445.06666666666666, + "completions/mean_length": 188.54584248860678, + "completions/mean_terminated_length": 181.3757746378581, + "completions/min_length": 58.93333333333333, + "completions/min_terminated_length": 58.93333333333333, + "entropy": 0.12421761391063531, + "epoch": 0.9148217066038811, + "eval/gt_acc_batch": 0.8116666932900747, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23088796436786652, + "kd/policy_loss": 0.008355669075778374, + "kd/rkl_advantage_mean": 0.8125200080374877, + "kd/rkl_advantage_p95": 5.07315554022789, + "kd/rkl_advantage_std": 2.2190392752488455, + "kd/ssd_loss": 0.0, + "learning_rate": 8.521626856036152e-08, + "loss": 0.03342267672220866, + "num_tokens": 788181548.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8116666674613953, + "rewards/gt_logging_reward/std": 0.38566667040189107, + "sampling/importance_sampling_ratio/max": 1.8316002130508422, + "sampling/importance_sampling_ratio/mean": 0.9992101530234019, + "sampling/importance_sampling_ratio/min": 0.478206401069959, + "sampling/sampling_logp_difference/max": 0.3250868101914724, + "sampling/sampling_logp_difference/mean": 0.003849405624593298, + "step": 24090, + "step_time": 7.9537887388287345, + "student/entropy": 0.12421761391063531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017222223151475192, + "completions/max_length": 484.53333333333336, + "completions/max_terminated_length": 450.93333333333334, + "completions/mean_length": 183.47723032633465, + "completions/mean_terminated_length": 177.88633473714194, + "completions/min_length": 58.86666666666667, + "completions/min_terminated_length": 58.86666666666667, + "entropy": 0.13513108311841884, + "epoch": 0.9159609615311586, + "eval/gt_acc_batch": 0.8008333583672841, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29545730352401733, + "kd/policy_loss": 0.008764328039251267, + "kd/rkl_advantage_mean": 0.7693331571916739, + "kd/rkl_advantage_p95": 4.984845026334127, + "kd/rkl_advantage_std": 2.2300729433695476, + "kd/ssd_loss": 0.0, + "learning_rate": 8.407701363308397e-08, + "loss": 0.03505731423695882, + "num_tokens": 789146722.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333345254263, + "rewards/gt_logging_reward/std": 0.3928915242354075, + "sampling/importance_sampling_ratio/max": 1.8894257426261902, + "sampling/importance_sampling_ratio/mean": 0.9994841953118642, + "sampling/importance_sampling_ratio/min": 0.44878021130959195, + "sampling/sampling_logp_difference/max": 0.3135238945484161, + "sampling/sampling_logp_difference/mean": 0.004165143558445076, + "step": 24120, + "step_time": 7.88628006609603, + "student/entropy": 0.13513108311841884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022222223381201427, + "completions/max_length": 487.1333333333333, + "completions/max_terminated_length": 440.23333333333335, + "completions/mean_length": 182.37417500813802, + "completions/mean_terminated_length": 174.99422454833984, + "completions/min_length": 53.2, + "completions/min_terminated_length": 53.2, + "entropy": 0.12878218721598386, + "epoch": 0.9171002164584362, + "eval/gt_acc_batch": 0.8025000194708506, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2513884902000427, + "kd/policy_loss": 0.007957524136872961, + "kd/rkl_advantage_mean": 0.7734642741580804, + "kd/rkl_advantage_p95": 4.935596162080765, + "kd/rkl_advantage_std": 2.1918361455202104, + "kd/ssd_loss": 0.0, + "learning_rate": 8.29377587058064e-08, + "loss": 0.031830100218455, + "num_tokens": 790096925.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8024999996026357, + "rewards/gt_logging_reward/std": 0.3855873907605807, + "sampling/importance_sampling_ratio/max": 1.9429988543192545, + "sampling/importance_sampling_ratio/mean": 0.9985186477502187, + "sampling/importance_sampling_ratio/min": 0.449700129032135, + "sampling/sampling_logp_difference/max": 0.35033594568570453, + "sampling/sampling_logp_difference/mean": 0.003937622462399304, + "step": 24150, + "step_time": 7.92726465529607, + "student/entropy": 0.12878218721598386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890389353038, + "completions/max_length": 497.3, + "completions/max_terminated_length": 453.1666666666667, + "completions/mean_length": 189.63139750162762, + "completions/mean_terminated_length": 180.96622670491536, + "completions/min_length": 47.333333333333336, + "completions/min_terminated_length": 47.333333333333336, + "entropy": 0.13250616223861775, + "epoch": 0.9182394713857137, + "eval/gt_acc_batch": 0.8036111374696095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34810641407966614, + "kd/policy_loss": 0.008769661973929033, + "kd/rkl_advantage_mean": 0.7697878775497278, + "kd/rkl_advantage_p95": 4.981754980484644, + "kd/rkl_advantage_std": 2.1975827028354007, + "kd/ssd_loss": 0.0, + "learning_rate": 8.179850377852884e-08, + "loss": 0.035078648726145426, + "num_tokens": 791091054.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111116409301, + "rewards/gt_logging_reward/std": 0.39145358502864835, + "sampling/importance_sampling_ratio/max": 1.8775548895200094, + "sampling/importance_sampling_ratio/mean": 0.9950630307197571, + "sampling/importance_sampling_ratio/min": 0.3956416661540667, + "sampling/sampling_logp_difference/max": 0.348774724205335, + "sampling/sampling_logp_difference/mean": 0.004057976448287566, + "step": 24180, + "step_time": 8.19715348695075, + "student/entropy": 0.13250616223861775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055557142943145, + "completions/max_length": 486.1333333333333, + "completions/max_terminated_length": 444.8, + "completions/mean_length": 193.95528717041014, + "completions/mean_terminated_length": 185.0251261393229, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.1290957521647215, + "epoch": 0.9193787263129913, + "eval/gt_acc_batch": 0.7950000206629435, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28387215733528137, + "kd/policy_loss": 0.00836574508381697, + "kd/rkl_advantage_mean": 0.7482448396272957, + "kd/rkl_advantage_p95": 5.060183203220367, + "kd/rkl_advantage_std": 2.261568104227384, + "kd/ssd_loss": 0.0, + "learning_rate": 8.065924885125127e-08, + "loss": 0.03346298535664876, + "num_tokens": 792099293.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7949999988079071, + "rewards/gt_logging_reward/std": 0.3953895553946495, + "sampling/importance_sampling_ratio/max": 1.8498003482818604, + "sampling/importance_sampling_ratio/mean": 0.9954717953999838, + "sampling/importance_sampling_ratio/min": 0.4513934224843979, + "sampling/sampling_logp_difference/max": 0.3205001493295034, + "sampling/sampling_logp_difference/mean": 0.003914256067946553, + "step": 24210, + "step_time": 8.059627126530783, + "student/entropy": 0.1290957521647215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02833333509042859, + "completions/max_length": 480.3333333333333, + "completions/max_terminated_length": 442.9, + "completions/mean_length": 186.40056584676105, + "completions/mean_terminated_length": 177.15359217325846, + "completions/min_length": 58.266666666666666, + "completions/min_terminated_length": 58.266666666666666, + "entropy": 0.13411923591047525, + "epoch": 0.9205179812402688, + "eval/gt_acc_batch": 0.7952778041362762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28909939527511597, + "kd/policy_loss": 0.008853009977610782, + "kd/rkl_advantage_mean": 0.7594434167879324, + "kd/rkl_advantage_p95": 5.03018672267596, + "kd/rkl_advantage_std": 2.2243890474239985, + "kd/ssd_loss": 0.0, + "learning_rate": 7.951999392397373e-08, + "loss": 0.03541203737258911, + "num_tokens": 793068439.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777763207753, + "rewards/gt_logging_reward/std": 0.3914156893889109, + "sampling/importance_sampling_ratio/max": 1.9108066836992899, + "sampling/importance_sampling_ratio/mean": 0.9997287193934122, + "sampling/importance_sampling_ratio/min": 0.4419751390814781, + "sampling/sampling_logp_difference/max": 0.3472080171108246, + "sampling/sampling_logp_difference/mean": 0.004082925726349155, + "step": 24240, + "step_time": 8.025126761002078, + "student/entropy": 0.13411923591047525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444578140974, + "completions/max_length": 489.96666666666664, + "completions/max_terminated_length": 444.6, + "completions/mean_length": 190.00223083496093, + "completions/mean_terminated_length": 182.16716664632162, + "completions/min_length": 54.766666666666666, + "completions/min_terminated_length": 54.766666666666666, + "entropy": 0.1296947694073121, + "epoch": 0.9216572361675465, + "eval/gt_acc_batch": 0.7955555895964305, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.310287207365036, + "kd/policy_loss": 0.008348648560543855, + "kd/rkl_advantage_mean": 0.7641405857012917, + "kd/rkl_advantage_p95": 4.962390444676081, + "kd/rkl_advantage_std": 2.2000066488981247, + "kd/ssd_loss": 0.0, + "learning_rate": 7.838073899669616e-08, + "loss": 0.033394598960876466, + "num_tokens": 794058743.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555518468221, + "rewards/gt_logging_reward/std": 0.3976501002907753, + "sampling/importance_sampling_ratio/max": 1.9544238805770875, + "sampling/importance_sampling_ratio/mean": 1.0007420202096304, + "sampling/importance_sampling_ratio/min": 0.44488295416037243, + "sampling/sampling_logp_difference/max": 0.340520699818929, + "sampling/sampling_logp_difference/mean": 0.0040046232752501965, + "step": 24270, + "step_time": 8.173836630471245, + "student/entropy": 0.1296947694073121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223890324432, + "completions/max_length": 491.8, + "completions/max_terminated_length": 461.26666666666665, + "completions/mean_length": 193.03889973958334, + "completions/mean_terminated_length": 184.26748708089193, + "completions/min_length": 56.56666666666667, + "completions/min_terminated_length": 56.56666666666667, + "entropy": 0.12809086559961239, + "epoch": 0.922796491094824, + "eval/gt_acc_batch": 0.7877778053283692, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23859140276908875, + "kd/policy_loss": 0.008671386803810795, + "kd/rkl_advantage_mean": 0.7436593965239202, + "kd/rkl_advantage_p95": 5.017655656735102, + "kd/rkl_advantage_std": 2.227001033226649, + "kd/ssd_loss": 0.0, + "learning_rate": 7.724148406941859e-08, + "loss": 0.03468554019927979, + "num_tokens": 795058347.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7877777755260468, + "rewards/gt_logging_reward/std": 0.39831776122252144, + "sampling/importance_sampling_ratio/max": 1.9549832741419475, + "sampling/importance_sampling_ratio/mean": 1.0001146833101908, + "sampling/importance_sampling_ratio/min": 0.41914865548412006, + "sampling/sampling_logp_difference/max": 0.39626035690307615, + "sampling/sampling_logp_difference/mean": 0.003894400356026987, + "step": 24300, + "step_time": 8.110658858840663, + "student/entropy": 0.12809086559961239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444562618931, + "completions/max_length": 471.46666666666664, + "completions/max_terminated_length": 440.8, + "completions/mean_length": 180.58639882405598, + "completions/mean_terminated_length": 172.3366673787435, + "completions/min_length": 52.2, + "completions/min_terminated_length": 52.2, + "entropy": 0.13257531964530547, + "epoch": 0.9239357460221016, + "eval/gt_acc_batch": 0.7947222371896108, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3346957266330719, + "kd/policy_loss": 0.009037291871694227, + "kd/rkl_advantage_mean": 0.7096733149606734, + "kd/rkl_advantage_p95": 4.84236698547999, + "kd/rkl_advantage_std": 2.1794380893309913, + "kd/ssd_loss": 0.0, + "learning_rate": 7.610222914214103e-08, + "loss": 0.03614916801452637, + "num_tokens": 796004642.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222212950389, + "rewards/gt_logging_reward/std": 0.39697710374991096, + "sampling/importance_sampling_ratio/max": 1.8644489924112955, + "sampling/importance_sampling_ratio/mean": 1.0002525786558787, + "sampling/importance_sampling_ratio/min": 0.43266439735889434, + "sampling/sampling_logp_difference/max": 0.3183053692181905, + "sampling/sampling_logp_difference/mean": 0.004057357312800983, + "step": 24330, + "step_time": 7.837908811730449, + "student/entropy": 0.13257531964530547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112559835116, + "completions/max_length": 480.23333333333335, + "completions/max_terminated_length": 441.06666666666666, + "completions/mean_length": 184.64278818766277, + "completions/mean_terminated_length": 176.7641143798828, + "completions/min_length": 53.266666666666666, + "completions/min_terminated_length": 53.266666666666666, + "entropy": 0.13107618348052105, + "epoch": 0.9250750009493791, + "eval/gt_acc_batch": 0.8094444632530212, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3053552508354187, + "kd/policy_loss": 0.00838627337361686, + "kd/rkl_advantage_mean": 0.780356620127956, + "kd/rkl_advantage_p95": 5.0758395314216616, + "kd/rkl_advantage_std": 2.253603415687879, + "kd/ssd_loss": 0.0, + "learning_rate": 7.496297421486348e-08, + "loss": 0.03354509671529134, + "num_tokens": 796974292.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8094444433848064, + "rewards/gt_logging_reward/std": 0.37785479923089343, + "sampling/importance_sampling_ratio/max": 1.9051557938257853, + "sampling/importance_sampling_ratio/mean": 1.0008688906828562, + "sampling/importance_sampling_ratio/min": 0.454911079009374, + "sampling/sampling_logp_difference/max": 0.3558944980303446, + "sampling/sampling_logp_difference/mean": 0.0039773758578424655, + "step": 24360, + "step_time": 8.201096722832881, + "student/entropy": 0.13107618348052105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020555556875964005, + "completions/max_length": 497.2, + "completions/max_terminated_length": 463.4, + "completions/mean_length": 190.2702875773112, + "completions/mean_terminated_length": 183.46165924072267, + "completions/min_length": 60.4, + "completions/min_terminated_length": 60.4, + "entropy": 0.13213521111756563, + "epoch": 0.9262142558766566, + "eval/gt_acc_batch": 0.7897222439448038, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2506757080554962, + "kd/policy_loss": 0.008648138827023406, + "kd/rkl_advantage_mean": 0.7827513482421636, + "kd/rkl_advantage_p95": 5.07906840244929, + "kd/rkl_advantage_std": 2.245951982339223, + "kd/ssd_loss": 0.0, + "learning_rate": 7.382371928758592e-08, + "loss": 0.03459256092707316, + "num_tokens": 797977865.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7897222260634105, + "rewards/gt_logging_reward/std": 0.40194974541664125, + "sampling/importance_sampling_ratio/max": 1.9685458461443583, + "sampling/importance_sampling_ratio/mean": 0.997860982020696, + "sampling/importance_sampling_ratio/min": 0.4405864412585894, + "sampling/sampling_logp_difference/max": 0.3488954842090607, + "sampling/sampling_logp_difference/mean": 0.004046107153408229, + "step": 24390, + "step_time": 8.335301085035704, + "student/entropy": 0.13213521111756563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777778965731462, + "completions/max_length": 493.96666666666664, + "completions/max_terminated_length": 456.76666666666665, + "completions/mean_length": 184.8872314453125, + "completions/mean_terminated_length": 177.47720692952473, + "completions/min_length": 52.733333333333334, + "completions/min_terminated_length": 52.733333333333334, + "entropy": 0.13279480542987585, + "epoch": 0.9273535108039342, + "eval/gt_acc_batch": 0.8005555828412374, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2746477723121643, + "kd/policy_loss": 0.008811842844200632, + "kd/rkl_advantage_mean": 0.7819192149986823, + "kd/rkl_advantage_p95": 4.989268231391907, + "kd/rkl_advantage_std": 2.2203943908214567, + "kd/ssd_loss": 0.0, + "learning_rate": 7.268446436030835e-08, + "loss": 0.035247377554575604, + "num_tokens": 798944491.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.800555557012558, + "rewards/gt_logging_reward/std": 0.38707745969295504, + "sampling/importance_sampling_ratio/max": 1.9298817992210389, + "sampling/importance_sampling_ratio/mean": 0.9952809711297353, + "sampling/importance_sampling_ratio/min": 0.46735452065865196, + "sampling/sampling_logp_difference/max": 0.3331974546114604, + "sampling/sampling_logp_difference/mean": 0.004064155369997024, + "step": 24420, + "step_time": 7.994101880199741, + "student/entropy": 0.13279480542987585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015833334407458703, + "completions/max_length": 478.6666666666667, + "completions/max_terminated_length": 445.93333333333334, + "completions/mean_length": 182.7969543457031, + "completions/mean_terminated_length": 177.5429911295573, + "completions/min_length": 50.96666666666667, + "completions/min_terminated_length": 50.96666666666667, + "entropy": 0.1280438815553983, + "epoch": 0.9284927657312118, + "eval/gt_acc_batch": 0.805833347638448, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2484748214483261, + "kd/policy_loss": 0.007990691585776706, + "kd/rkl_advantage_mean": 0.7109651803039014, + "kd/rkl_advantage_p95": 4.87907306154569, + "kd/rkl_advantage_std": 2.206076489885648, + "kd/ssd_loss": 0.0, + "learning_rate": 7.15452094330308e-08, + "loss": 0.031962768236796064, + "num_tokens": 799909648.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8058333337306977, + "rewards/gt_logging_reward/std": 0.38797997385263444, + "sampling/importance_sampling_ratio/max": 1.890613353252411, + "sampling/importance_sampling_ratio/mean": 0.9921106934547425, + "sampling/importance_sampling_ratio/min": 0.44995856086413066, + "sampling/sampling_logp_difference/max": 0.3203708529472351, + "sampling/sampling_logp_difference/mean": 0.00391869165468961, + "step": 24450, + "step_time": 7.935221141927953, + "student/entropy": 0.1280438815553983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0191666677283744, + "completions/max_length": 480.26666666666665, + "completions/max_terminated_length": 446.26666666666665, + "completions/mean_length": 189.16750895182292, + "completions/mean_terminated_length": 182.8718490600586, + "completions/min_length": 61.166666666666664, + "completions/min_terminated_length": 61.166666666666664, + "entropy": 0.12428132109344006, + "epoch": 0.9296320206584894, + "eval/gt_acc_batch": 0.7972222407658894, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27417561411857605, + "kd/policy_loss": 0.007864566820596034, + "kd/rkl_advantage_mean": 0.7553389204666019, + "kd/rkl_advantage_p95": 5.055232191085816, + "kd/rkl_advantage_std": 2.2313181350628537, + "kd/ssd_loss": 0.0, + "learning_rate": 7.040595450575323e-08, + "loss": 0.03145826657613118, + "num_tokens": 800899635.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7972222208976746, + "rewards/gt_logging_reward/std": 0.3851306393742561, + "sampling/importance_sampling_ratio/max": 1.9357712348302205, + "sampling/importance_sampling_ratio/mean": 0.9947208841641744, + "sampling/importance_sampling_ratio/min": 0.4542262921730677, + "sampling/sampling_logp_difference/max": 0.33078117767969767, + "sampling/sampling_logp_difference/mean": 0.003824336190397541, + "step": 24480, + "step_time": 8.041672167632107, + "student/entropy": 0.12428132109344006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027222223424663145, + "completions/max_length": 481.0, + "completions/max_terminated_length": 447.96666666666664, + "completions/mean_length": 186.6791753133138, + "completions/mean_terminated_length": 177.6014205932617, + "completions/min_length": 43.96666666666667, + "completions/min_terminated_length": 43.96666666666667, + "entropy": 0.13201345540583134, + "epoch": 0.9307712755857669, + "eval/gt_acc_batch": 0.7900000194708506, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2628467381000519, + "kd/policy_loss": 0.009080128222315883, + "kd/rkl_advantage_mean": 0.7401744809933006, + "kd/rkl_advantage_p95": 4.918387589852015, + "kd/rkl_advantage_std": 2.206055647134781, + "kd/ssd_loss": 0.0, + "learning_rate": 6.926669957847567e-08, + "loss": 0.03632051150004069, + "num_tokens": 801887408.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7899999996026357, + "rewards/gt_logging_reward/std": 0.396776453157266, + "sampling/importance_sampling_ratio/max": 1.9390531778335571, + "sampling/importance_sampling_ratio/mean": 0.9906928956508636, + "sampling/importance_sampling_ratio/min": 0.422007887562116, + "sampling/sampling_logp_difference/max": 0.30706318815549216, + "sampling/sampling_logp_difference/mean": 0.004028386529535055, + "step": 24510, + "step_time": 8.23317424923104, + "student/entropy": 0.13201345540583134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02500000090027849, + "completions/max_length": 483.93333333333334, + "completions/max_terminated_length": 449.6, + "completions/mean_length": 184.25084330240887, + "completions/mean_terminated_length": 176.05179951985676, + "completions/min_length": 55.53333333333333, + "completions/min_terminated_length": 55.53333333333333, + "entropy": 0.1298482804869612, + "epoch": 0.9319105305130445, + "eval/gt_acc_batch": 0.7872222463289896, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2328183799982071, + "kd/policy_loss": 0.008557815102782721, + "kd/rkl_advantage_mean": 0.8083499699831009, + "kd/rkl_advantage_p95": 5.007997518777847, + "kd/rkl_advantage_std": 2.1932487616936367, + "kd/ssd_loss": 0.0, + "learning_rate": 6.812744465119812e-08, + "loss": 0.034231261412302656, + "num_tokens": 802849951.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7872222244739533, + "rewards/gt_logging_reward/std": 0.4008211006720861, + "sampling/importance_sampling_ratio/max": 1.9704442779223124, + "sampling/importance_sampling_ratio/mean": 1.006645546356837, + "sampling/importance_sampling_ratio/min": 0.4473637893795967, + "sampling/sampling_logp_difference/max": 0.3264566500981649, + "sampling/sampling_logp_difference/mean": 0.004021703801117838, + "step": 24540, + "step_time": 7.893545758256611, + "student/entropy": 0.1298482804869612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02638889035830895, + "completions/max_length": 476.23333333333335, + "completions/max_terminated_length": 444.3333333333333, + "completions/mean_length": 194.45945434570314, + "completions/mean_terminated_length": 186.00371602376302, + "completions/min_length": 58.166666666666664, + "completions/min_terminated_length": 58.166666666666664, + "entropy": 0.12729665028552214, + "epoch": 0.933049785440322, + "eval/gt_acc_batch": 0.7997222443421682, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24804744124412537, + "kd/policy_loss": 0.008204941420505445, + "kd/rkl_advantage_mean": 0.7460974522633478, + "kd/rkl_advantage_p95": 4.990056389570237, + "kd/rkl_advantage_std": 2.21130576133728, + "kd/ssd_loss": 0.0, + "learning_rate": 6.698818972392055e-08, + "loss": 0.03281976977984111, + "num_tokens": 803857381.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7997222145398458, + "rewards/gt_logging_reward/std": 0.39104114919900895, + "sampling/importance_sampling_ratio/max": 1.9348055322964985, + "sampling/importance_sampling_ratio/mean": 1.0016352593898774, + "sampling/importance_sampling_ratio/min": 0.46754954059918724, + "sampling/sampling_logp_difference/max": 0.31488598783810934, + "sampling/sampling_logp_difference/mean": 0.003885683223294715, + "step": 24570, + "step_time": 8.026657565836407, + "student/entropy": 0.12729665028552214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01694444560756286, + "completions/max_length": 476.06666666666666, + "completions/max_terminated_length": 433.0, + "completions/mean_length": 183.76834360758463, + "completions/mean_terminated_length": 178.11383666992188, + "completions/min_length": 62.2, + "completions/min_terminated_length": 62.2, + "entropy": 0.12456653509289026, + "epoch": 0.9341890403675996, + "eval/gt_acc_batch": 0.8050000250339509, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2625117599964142, + "kd/policy_loss": 0.008142147552765286, + "kd/rkl_advantage_mean": 0.6941774322961768, + "kd/rkl_advantage_p95": 4.922849386930466, + "kd/rkl_advantage_std": 2.1999955624341965, + "kd/ssd_loss": 0.0, + "learning_rate": 6.584893479664299e-08, + "loss": 0.03256859381993612, + "num_tokens": 804824363.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.804999993244807, + "rewards/gt_logging_reward/std": 0.38858784834543864, + "sampling/importance_sampling_ratio/max": 1.8702336192131042, + "sampling/importance_sampling_ratio/mean": 0.9999607602755228, + "sampling/importance_sampling_ratio/min": 0.48782880008220675, + "sampling/sampling_logp_difference/max": 0.2991174499193827, + "sampling/sampling_logp_difference/mean": 0.003878850140608847, + "step": 24600, + "step_time": 7.896493811063313, + "student/entropy": 0.12456653509289026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333467443784, + "completions/max_length": 497.9, + "completions/max_terminated_length": 449.96666666666664, + "completions/mean_length": 186.63112131754556, + "completions/mean_terminated_length": 178.8535919189453, + "completions/min_length": 56.333333333333336, + "completions/min_terminated_length": 56.333333333333336, + "entropy": 0.13612669066836436, + "epoch": 0.9353282952948772, + "eval/gt_acc_batch": 0.7925000250339508, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28351742029190063, + "kd/policy_loss": 0.008878160048819457, + "kd/rkl_advantage_mean": 0.754980098331968, + "kd/rkl_advantage_p95": 5.0162323117256165, + "kd/rkl_advantage_std": 2.2303546160459518, + "kd/ssd_loss": 0.0, + "learning_rate": 6.470967986936544e-08, + "loss": 0.03551264603932699, + "num_tokens": 805793251.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7924999992052714, + "rewards/gt_logging_reward/std": 0.39557243287563326, + "sampling/importance_sampling_ratio/max": 1.9094221790631611, + "sampling/importance_sampling_ratio/mean": 1.0015060504277546, + "sampling/importance_sampling_ratio/min": 0.4497531841198603, + "sampling/sampling_logp_difference/max": 0.3101345539093018, + "sampling/sampling_logp_difference/mean": 0.004115506967840095, + "step": 24630, + "step_time": 8.077667048366857, + "student/entropy": 0.13612669066836436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02611111244186759, + "completions/max_length": 490.3666666666667, + "completions/max_terminated_length": 462.6666666666667, + "completions/mean_length": 187.8416763305664, + "completions/mean_terminated_length": 179.04737040201823, + "completions/min_length": 46.63333333333333, + "completions/min_terminated_length": 46.63333333333333, + "entropy": 0.14009424888839325, + "epoch": 0.9364675502221547, + "eval/gt_acc_batch": 0.784722238779068, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3100256621837616, + "kd/policy_loss": 0.009220946064063658, + "kd/rkl_advantage_mean": 0.8778209338677698, + "kd/rkl_advantage_p95": 5.231694370508194, + "kd/rkl_advantage_std": 2.2863646397988, + "kd/ssd_loss": 0.0, + "learning_rate": 6.357042494208788e-08, + "loss": 0.0368837833404541, + "num_tokens": 806773673.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7847222208976745, + "rewards/gt_logging_reward/std": 0.4038927664359411, + "sampling/importance_sampling_ratio/max": 1.991897968451182, + "sampling/importance_sampling_ratio/mean": 1.0021692295869191, + "sampling/importance_sampling_ratio/min": 0.41651338438193003, + "sampling/sampling_logp_difference/max": 0.35691334704558053, + "sampling/sampling_logp_difference/mean": 0.004245945120540758, + "step": 24660, + "step_time": 8.091892351696151, + "student/entropy": 0.14009424888839325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02250000151495139, + "completions/max_length": 487.96666666666664, + "completions/max_terminated_length": 450.56666666666666, + "completions/mean_length": 191.33167622884113, + "completions/mean_terminated_length": 184.0648956298828, + "completions/min_length": 54.1, + "completions/min_terminated_length": 54.1, + "entropy": 0.1305841570099195, + "epoch": 0.9376068051494323, + "eval/gt_acc_batch": 0.7786111275355021, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26745399832725525, + "kd/policy_loss": 0.008719816821394489, + "kd/rkl_advantage_mean": 0.7728871301437418, + "kd/rkl_advantage_p95": 5.0111926635106405, + "kd/rkl_advantage_std": 2.2147573113441466, + "kd/ssd_loss": 0.0, + "learning_rate": 6.243117001481031e-08, + "loss": 0.034879271189371744, + "num_tokens": 807761579.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7786111116409302, + "rewards/gt_logging_reward/std": 0.4103176732858022, + "sampling/importance_sampling_ratio/max": 1.933432682355245, + "sampling/importance_sampling_ratio/mean": 0.9951293845971425, + "sampling/importance_sampling_ratio/min": 0.45215099354585014, + "sampling/sampling_logp_difference/max": 0.3550235907236735, + "sampling/sampling_logp_difference/mean": 0.003992706226805846, + "step": 24690, + "step_time": 8.086319020169322, + "student/entropy": 0.1305841570099195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030277779481063288, + "completions/max_length": 476.0, + "completions/max_terminated_length": 434.6333333333333, + "completions/mean_length": 185.2533432006836, + "completions/mean_terminated_length": 175.2404993693034, + "completions/min_length": 50.63333333333333, + "completions/min_terminated_length": 50.63333333333333, + "entropy": 0.13155985604971648, + "epoch": 0.9387460600767098, + "eval/gt_acc_batch": 0.7991666793823242, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29941579699516296, + "kd/policy_loss": 0.008949914601786683, + "kd/rkl_advantage_mean": 0.7581661449434857, + "kd/rkl_advantage_p95": 4.904439270496368, + "kd/rkl_advantage_std": 2.1815010209878287, + "kd/ssd_loss": 0.0, + "learning_rate": 6.129191508753276e-08, + "loss": 0.035799654324849446, + "num_tokens": 808732651.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7991666674613953, + "rewards/gt_logging_reward/std": 0.3921600530544917, + "sampling/importance_sampling_ratio/max": 1.894372038046519, + "sampling/importance_sampling_ratio/mean": 0.9984656731287639, + "sampling/importance_sampling_ratio/min": 0.48466932972272236, + "sampling/sampling_logp_difference/max": 0.3439579923947652, + "sampling/sampling_logp_difference/mean": 0.004039739662160476, + "step": 24720, + "step_time": 8.047826841437685, + "student/entropy": 0.13155985604971648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018888889998197555, + "completions/max_length": 486.76666666666665, + "completions/max_terminated_length": 451.23333333333335, + "completions/mean_length": 191.569455464681, + "completions/mean_terminated_length": 185.39664052327473, + "completions/min_length": 55.9, + "completions/min_terminated_length": 55.9, + "entropy": 0.13240692317485808, + "epoch": 0.9398853150039874, + "eval/gt_acc_batch": 0.7844444632530212, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24432946741580963, + "kd/policy_loss": 0.008980188850546255, + "kd/rkl_advantage_mean": 0.6877846585586667, + "kd/rkl_advantage_p95": 4.842800623178482, + "kd/rkl_advantage_std": 2.1546165466308596, + "kd/ssd_loss": 0.0, + "learning_rate": 6.015266016025519e-08, + "loss": 0.03592075109481811, + "num_tokens": 809728637.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7844444413979849, + "rewards/gt_logging_reward/std": 0.40016779899597166, + "sampling/importance_sampling_ratio/max": 1.8770415385564168, + "sampling/importance_sampling_ratio/mean": 0.9985061764717102, + "sampling/importance_sampling_ratio/min": 0.42059174130360283, + "sampling/sampling_logp_difference/max": 0.29994409481684364, + "sampling/sampling_logp_difference/mean": 0.004023049461344878, + "step": 24750, + "step_time": 8.199638664635131, + "student/entropy": 0.13240692317485808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016944445421298346, + "completions/max_length": 479.1, + "completions/max_terminated_length": 450.8333333333333, + "completions/mean_length": 181.13556569417318, + "completions/mean_terminated_length": 175.58120930989583, + "completions/min_length": 53.166666666666664, + "completions/min_terminated_length": 53.166666666666664, + "entropy": 0.1324826224396626, + "epoch": 0.9410245699312649, + "eval/gt_acc_batch": 0.8036111334959666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24316681921482086, + "kd/policy_loss": 0.008656927667713414, + "kd/rkl_advantage_mean": 0.7674268210927645, + "kd/rkl_advantage_p95": 4.956534141302109, + "kd/rkl_advantage_std": 2.216938504576683, + "kd/ssd_loss": 0.0, + "learning_rate": 5.901340523297763e-08, + "loss": 0.034627715746561684, + "num_tokens": 810685925.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111096541086, + "rewards/gt_logging_reward/std": 0.38776003271341325, + "sampling/importance_sampling_ratio/max": 1.8172779560089112, + "sampling/importance_sampling_ratio/mean": 0.9998158633708953, + "sampling/importance_sampling_ratio/min": 0.5027601917584738, + "sampling/sampling_logp_difference/max": 0.3442122578620911, + "sampling/sampling_logp_difference/mean": 0.0040316622781877715, + "step": 24780, + "step_time": 7.979967062174304, + "student/entropy": 0.1324826224396626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025555556857337555, + "completions/max_length": 489.6333333333333, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 185.99362080891927, + "completions/mean_terminated_length": 177.4325932820638, + "completions/min_length": 56.666666666666664, + "completions/min_terminated_length": 56.666666666666664, + "entropy": 0.1282394488652547, + "epoch": 0.9421638248585426, + "eval/gt_acc_batch": 0.8033333599567414, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3410654664039612, + "kd/policy_loss": 0.008083765969301264, + "kd/rkl_advantage_mean": 0.8538721030888458, + "kd/rkl_advantage_p95": 5.138617493708929, + "kd/rkl_advantage_std": 2.259693310658137, + "kd/ssd_loss": 0.0, + "learning_rate": 5.7874150305700074e-08, + "loss": 0.03233507076899211, + "num_tokens": 811658334.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8033333361148834, + "rewards/gt_logging_reward/std": 0.3898633097608884, + "sampling/importance_sampling_ratio/max": 1.8815387845039369, + "sampling/importance_sampling_ratio/mean": 0.9928783456484477, + "sampling/importance_sampling_ratio/min": 0.42269068559010825, + "sampling/sampling_logp_difference/max": 0.3339910884698232, + "sampling/sampling_logp_difference/mean": 0.003935288397284845, + "step": 24810, + "step_time": 8.061000793993783, + "student/entropy": 0.1282394488652547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029444446259488662, + "completions/max_length": 493.6333333333333, + "completions/max_terminated_length": 457.3666666666667, + "completions/mean_length": 197.34334360758464, + "completions/mean_terminated_length": 187.9322280883789, + "completions/min_length": 58.5, + "completions/min_terminated_length": 58.5, + "entropy": 0.12375564215083917, + "epoch": 0.9433030797858201, + "eval/gt_acc_batch": 0.7741667052110036, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2412082552909851, + "kd/policy_loss": 0.008412272333710764, + "kd/rkl_advantage_mean": 0.7543401495243113, + "kd/rkl_advantage_p95": 4.991121147076289, + "kd/rkl_advantage_std": 2.1901863773663837, + "kd/ssd_loss": 0.0, + "learning_rate": 5.6734895378422505e-08, + "loss": 0.03364909092585246, + "num_tokens": 812677898.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7741666634877523, + "rewards/gt_logging_reward/std": 0.4098821312189102, + "sampling/importance_sampling_ratio/max": 1.9714237928390503, + "sampling/importance_sampling_ratio/mean": 1.0045884509881338, + "sampling/importance_sampling_ratio/min": 0.4727432663242022, + "sampling/sampling_logp_difference/max": 0.3472247004508972, + "sampling/sampling_logp_difference/mean": 0.003791413743359347, + "step": 24840, + "step_time": 8.12551270206944, + "student/entropy": 0.12375564215083917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.022777779214084148, + "completions/max_length": 493.0, + "completions/max_terminated_length": 464.2, + "completions/mean_length": 186.17389882405598, + "completions/mean_terminated_length": 178.67349650065105, + "completions/min_length": 51.06666666666667, + "completions/min_terminated_length": 51.06666666666667, + "entropy": 0.13023078398158153, + "epoch": 0.9444423347130977, + "eval/gt_acc_batch": 0.7883333504199982, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.26948243379592896, + "kd/policy_loss": 0.008595509852360314, + "kd/rkl_advantage_mean": 0.7534908684281011, + "kd/rkl_advantage_p95": 4.9672191619873045, + "kd/rkl_advantage_std": 2.2364750812451044, + "kd/ssd_loss": 0.0, + "learning_rate": 5.559564045114495e-08, + "loss": 0.03438203732172648, + "num_tokens": 813662620.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7883333305517832, + "rewards/gt_logging_reward/std": 0.40187323788801826, + "sampling/importance_sampling_ratio/max": 1.9222272356351218, + "sampling/importance_sampling_ratio/mean": 0.9968758583068847, + "sampling/importance_sampling_ratio/min": 0.432826758424441, + "sampling/sampling_logp_difference/max": 0.33258210817972816, + "sampling/sampling_logp_difference/mean": 0.003977641579695046, + "step": 24870, + "step_time": 8.219752277301934, + "student/entropy": 0.13023078398158153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.030000001781930526, + "completions/max_length": 501.7, + "completions/max_terminated_length": 462.23333333333335, + "completions/mean_length": 193.41695505777994, + "completions/mean_terminated_length": 183.54995422363282, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.1269750672702988, + "epoch": 0.9455815896403752, + "eval/gt_acc_batch": 0.7952778001626333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28251707553863525, + "kd/policy_loss": 0.008518616273067891, + "kd/rkl_advantage_mean": 0.7720505397766828, + "kd/rkl_advantage_p95": 5.044822671016058, + "kd/rkl_advantage_std": 2.236423121889432, + "kd/ssd_loss": 0.0, + "learning_rate": 5.445638552386739e-08, + "loss": 0.03407446543375651, + "num_tokens": 814676041.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7952777842680613, + "rewards/gt_logging_reward/std": 0.3983636399110158, + "sampling/importance_sampling_ratio/max": 1.8320323626200359, + "sampling/importance_sampling_ratio/mean": 0.9946711719036102, + "sampling/importance_sampling_ratio/min": 0.4405620962381363, + "sampling/sampling_logp_difference/max": 0.329513156414032, + "sampling/sampling_logp_difference/mean": 0.0038699347836275897, + "step": 24900, + "step_time": 8.296199215034722, + "student/entropy": 0.1269750672702988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334866911174, + "completions/max_length": 488.6, + "completions/max_terminated_length": 446.4, + "completions/mean_length": 186.58250986735027, + "completions/mean_terminated_length": 178.02739562988282, + "completions/min_length": 48.7, + "completions/min_terminated_length": 48.7, + "entropy": 0.13645621774097283, + "epoch": 0.9467208445676527, + "eval/gt_acc_batch": 0.7936111291249593, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3241773545742035, + "kd/policy_loss": 0.008837635333960255, + "kd/rkl_advantage_mean": 0.7753033785149455, + "kd/rkl_advantage_p95": 4.997669806083043, + "kd/rkl_advantage_std": 2.206843830148379, + "kd/ssd_loss": 0.0, + "learning_rate": 5.331713059658983e-08, + "loss": 0.03535054524739583, + "num_tokens": 815636402.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7936111112435659, + "rewards/gt_logging_reward/std": 0.3980326294898987, + "sampling/importance_sampling_ratio/max": 1.8778317093849182, + "sampling/importance_sampling_ratio/mean": 0.9986012697219848, + "sampling/importance_sampling_ratio/min": 0.44899581720431647, + "sampling/sampling_logp_difference/max": 0.34055788318316144, + "sampling/sampling_logp_difference/mean": 0.004154442483559251, + "step": 24930, + "step_time": 7.944392096399679, + "student/entropy": 0.13645621774097283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03055555730437239, + "completions/max_length": 488.43333333333334, + "completions/max_terminated_length": 452.03333333333336, + "completions/mean_length": 192.2047322591146, + "completions/mean_terminated_length": 182.10979665120442, + "completions/min_length": 56.56666666666667, + "completions/min_terminated_length": 56.56666666666667, + "entropy": 0.12518393745025, + "epoch": 0.9478600994949303, + "eval/gt_acc_batch": 0.7805555800596873, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2518783211708069, + "kd/policy_loss": 0.008176267067513739, + "kd/rkl_advantage_mean": 0.8226165916770697, + "kd/rkl_advantage_p95": 5.113885047038396, + "kd/rkl_advantage_std": 2.25910314420859, + "kd/ssd_loss": 0.0, + "learning_rate": 5.217787566931227e-08, + "loss": 0.03270507454872131, + "num_tokens": 816632515.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7805555542310079, + "rewards/gt_logging_reward/std": 0.40695204635461174, + "sampling/importance_sampling_ratio/max": 1.8491510192553202, + "sampling/importance_sampling_ratio/mean": 1.0000239650408427, + "sampling/importance_sampling_ratio/min": 0.5042829831441243, + "sampling/sampling_logp_difference/max": 0.335065354903539, + "sampling/sampling_logp_difference/mean": 0.0038080062872419755, + "step": 24960, + "step_time": 8.069969138635981, + "student/entropy": 0.12518393745025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02055555668969949, + "completions/max_length": 470.8333333333333, + "completions/max_terminated_length": 433.8333333333333, + "completions/mean_length": 182.3908426920573, + "completions/mean_terminated_length": 175.6163365681966, + "completions/min_length": 55.333333333333336, + "completions/min_terminated_length": 55.333333333333336, + "entropy": 0.13099185538788635, + "epoch": 0.9489993544222078, + "eval/gt_acc_batch": 0.8030555586020152, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.23619896173477173, + "kd/policy_loss": 0.008292750992889826, + "kd/rkl_advantage_mean": 0.7920535263294975, + "kd/rkl_advantage_p95": 4.982867064078649, + "kd/rkl_advantage_std": 2.2105432470639546, + "kd/ssd_loss": 0.0, + "learning_rate": 5.1038620742034706e-08, + "loss": 0.033171008030573525, + "num_tokens": 817585266.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555566151937, + "rewards/gt_logging_reward/std": 0.3934993237257004, + "sampling/importance_sampling_ratio/max": 1.842803911368052, + "sampling/importance_sampling_ratio/mean": 0.9949578245480856, + "sampling/importance_sampling_ratio/min": 0.42378868559996286, + "sampling/sampling_logp_difference/max": 0.3309706022342046, + "sampling/sampling_logp_difference/mean": 0.004035165944757561, + "step": 24990, + "step_time": 7.845307249266383, + "student/entropy": 0.13099185538788635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02777777916441361, + "completions/max_length": 482.6, + "completions/max_terminated_length": 450.6, + "completions/mean_length": 188.91500956217448, + "completions/mean_terminated_length": 179.74742584228517, + "completions/min_length": 53.93333333333333, + "completions/min_terminated_length": 53.93333333333333, + "entropy": 0.13674042597413064, + "epoch": 0.9501386093494855, + "eval/gt_acc_batch": 0.7983333607514699, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2626497149467468, + "kd/policy_loss": 0.00898749147503016, + "kd/rkl_advantage_mean": 0.7908262362703681, + "kd/rkl_advantage_p95": 5.016065907478333, + "kd/rkl_advantage_std": 2.201545310020447, + "kd/ssd_loss": 0.0, + "learning_rate": 4.989936581475715e-08, + "loss": 0.03594997326533, + "num_tokens": 818566872.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7983333309491475, + "rewards/gt_logging_reward/std": 0.3888216023643812, + "sampling/importance_sampling_ratio/max": 2.020621585845947, + "sampling/importance_sampling_ratio/mean": 0.9963590323925018, + "sampling/importance_sampling_ratio/min": 0.39448228627443316, + "sampling/sampling_logp_difference/max": 0.3104137102762858, + "sampling/sampling_logp_difference/mean": 0.004176092993778487, + "step": 25020, + "step_time": 8.060562322795159, + "student/entropy": 0.13674042597413064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01861111239219705, + "completions/max_length": 474.6333333333333, + "completions/max_terminated_length": 433.46666666666664, + "completions/mean_length": 184.36528727213542, + "completions/mean_terminated_length": 178.17527669270834, + "completions/min_length": 58.03333333333333, + "completions/min_terminated_length": 58.03333333333333, + "entropy": 0.12982334128270548, + "epoch": 0.951277864276763, + "eval/gt_acc_batch": 0.8075000226497651, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29888150095939636, + "kd/policy_loss": 0.008432998314189415, + "kd/rkl_advantage_mean": 0.695499346529444, + "kd/rkl_advantage_p95": 4.821553150812785, + "kd/rkl_advantage_std": 2.1806140055259067, + "kd/ssd_loss": 0.0, + "learning_rate": 4.876011088747958e-08, + "loss": 0.03373199701309204, + "num_tokens": 819525531.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8074999948342642, + "rewards/gt_logging_reward/std": 0.3828940689563751, + "sampling/importance_sampling_ratio/max": 1.8876761198043823, + "sampling/importance_sampling_ratio/mean": 1.0072312851746876, + "sampling/importance_sampling_ratio/min": 0.4735039174556732, + "sampling/sampling_logp_difference/max": 0.3200324167807897, + "sampling/sampling_logp_difference/mean": 0.0040294544693703456, + "step": 25050, + "step_time": 7.889603491432111, + "student/entropy": 0.12982334128270548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023888890352100133, + "completions/max_length": 490.2, + "completions/max_terminated_length": 450.6333333333333, + "completions/mean_length": 182.591677347819, + "completions/mean_terminated_length": 174.49136606852213, + "completions/min_length": 54.666666666666664, + "completions/min_terminated_length": 54.666666666666664, + "entropy": 0.1327617097645998, + "epoch": 0.9524171192040406, + "eval/gt_acc_batch": 0.7944444576899211, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31507647037506104, + "kd/policy_loss": 0.00863617088762112, + "kd/rkl_advantage_mean": 0.8222860315193733, + "kd/rkl_advantage_p95": 5.05715768734614, + "kd/rkl_advantage_std": 2.2318135609229404, + "kd/ssd_loss": 0.0, + "learning_rate": 4.7620855960202025e-08, + "loss": 0.034544686476389565, + "num_tokens": 820485069.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7944444417953491, + "rewards/gt_logging_reward/std": 0.39885089844465255, + "sampling/importance_sampling_ratio/max": 1.9525042494138083, + "sampling/importance_sampling_ratio/mean": 0.9930332481861115, + "sampling/importance_sampling_ratio/min": 0.46541177332401273, + "sampling/sampling_logp_difference/max": 0.3345559388399124, + "sampling/sampling_logp_difference/mean": 0.004076984811884662, + "step": 25080, + "step_time": 8.028514647929114, + "student/entropy": 0.1327617097645998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03583333548158407, + "completions/max_length": 485.56666666666666, + "completions/max_terminated_length": 454.96666666666664, + "completions/mean_length": 189.4269541422526, + "completions/mean_terminated_length": 177.6274429321289, + "completions/min_length": 53.8, + "completions/min_terminated_length": 53.8, + "entropy": 0.13320758274445932, + "epoch": 0.9535563741313181, + "eval/gt_acc_batch": 0.7816666901111603, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3119816482067108, + "kd/policy_loss": 0.008925532423503076, + "kd/rkl_advantage_mean": 0.7891461253787081, + "kd/rkl_advantage_p95": 5.03370398680369, + "kd/rkl_advantage_std": 2.22294635673364, + "kd/ssd_loss": 0.0, + "learning_rate": 4.648160103292446e-08, + "loss": 0.035702129205067955, + "num_tokens": 821467950.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7816666662693024, + "rewards/gt_logging_reward/std": 0.3988654578725497, + "sampling/importance_sampling_ratio/max": 1.9532179633776348, + "sampling/importance_sampling_ratio/mean": 0.9995966076850891, + "sampling/importance_sampling_ratio/min": 0.45515121122201285, + "sampling/sampling_logp_difference/max": 0.3302091062068939, + "sampling/sampling_logp_difference/mean": 0.004063619168785711, + "step": 25110, + "step_time": 8.038658925229296, + "student/entropy": 0.13320758274445932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.012222222797572613, + "completions/max_length": 458.3666666666667, + "completions/max_terminated_length": 430.3333333333333, + "completions/mean_length": 176.2438969930013, + "completions/mean_terminated_length": 172.21841430664062, + "completions/min_length": 55.4, + "completions/min_terminated_length": 55.4, + "entropy": 0.13083066393931705, + "epoch": 0.9546956290585957, + "eval/gt_acc_batch": 0.8113888998826345, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2782667279243469, + "kd/policy_loss": 0.008640762492238234, + "kd/rkl_advantage_mean": 0.7172781199454524, + "kd/rkl_advantage_p95": 4.959921995798747, + "kd/rkl_advantage_std": 2.221266954143842, + "kd/ssd_loss": 0.0, + "learning_rate": 4.534234610564691e-08, + "loss": 0.034563056627909344, + "num_tokens": 822398100.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8113888919353485, + "rewards/gt_logging_reward/std": 0.38282817900180816, + "sampling/importance_sampling_ratio/max": 1.9554167111714682, + "sampling/importance_sampling_ratio/mean": 1.0003062903881073, + "sampling/importance_sampling_ratio/min": 0.4829064577817917, + "sampling/sampling_logp_difference/max": 0.3442732453346252, + "sampling/sampling_logp_difference/mean": 0.004070508464549979, + "step": 25140, + "step_time": 7.744701725930402, + "student/entropy": 0.13083066393931705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444571932157, + "completions/max_length": 484.3, + "completions/max_terminated_length": 447.46666666666664, + "completions/mean_length": 188.47973225911457, + "completions/mean_terminated_length": 180.49605611165364, + "completions/min_length": 50.46666666666667, + "completions/min_terminated_length": 50.46666666666667, + "entropy": 0.13268559388816356, + "epoch": 0.9558348839858732, + "eval/gt_acc_batch": 0.7955555657545725, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3911365568637848, + "kd/policy_loss": 0.00865779813223829, + "kd/rkl_advantage_mean": 0.7199710711836815, + "kd/rkl_advantage_p95": 4.960261215766271, + "kd/rkl_advantage_std": 2.200184987982114, + "kd/ssd_loss": 0.0, + "learning_rate": 4.4203091178369344e-08, + "loss": 0.0346311887105306, + "num_tokens": 823377787.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7955555498600007, + "rewards/gt_logging_reward/std": 0.39419225255648294, + "sampling/importance_sampling_ratio/max": 1.997536019484202, + "sampling/importance_sampling_ratio/mean": 1.0060322244962057, + "sampling/importance_sampling_ratio/min": 0.4423445905248324, + "sampling/sampling_logp_difference/max": 0.3111525237560272, + "sampling/sampling_logp_difference/mean": 0.004049931885674596, + "step": 25170, + "step_time": 8.153587773335554, + "student/entropy": 0.13268559388816356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01972222337499261, + "completions/max_length": 475.26666666666665, + "completions/max_terminated_length": 437.2, + "completions/mean_length": 184.54000854492188, + "completions/mean_terminated_length": 177.9655314127604, + "completions/min_length": 59.233333333333334, + "completions/min_terminated_length": 59.233333333333334, + "entropy": 0.12393961225946744, + "epoch": 0.9569741389131508, + "eval/gt_acc_batch": 0.8052777926127116, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.25958582758903503, + "kd/policy_loss": 0.007926155956617246, + "kd/rkl_advantage_mean": 0.5950914551193516, + "kd/rkl_advantage_p95": 4.647977377971014, + "kd/rkl_advantage_std": 2.1287116328875224, + "kd/ssd_loss": 0.0, + "learning_rate": 4.306383625109179e-08, + "loss": 0.03170462449391683, + "num_tokens": 824337491.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8052777787049611, + "rewards/gt_logging_reward/std": 0.39072629312674206, + "sampling/importance_sampling_ratio/max": 1.8277850111325582, + "sampling/importance_sampling_ratio/mean": 1.0026935358842215, + "sampling/importance_sampling_ratio/min": 0.47143253187338513, + "sampling/sampling_logp_difference/max": 0.3106773793697357, + "sampling/sampling_logp_difference/mean": 0.0038217910410215457, + "step": 25200, + "step_time": 7.909526073031399, + "student/entropy": 0.12393961225946744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03333333497866988, + "completions/max_length": 492.2, + "completions/max_terminated_length": 463.06666666666666, + "completions/mean_length": 188.28334248860676, + "completions/mean_terminated_length": 177.1316380818685, + "completions/min_length": 50.266666666666666, + "completions/min_terminated_length": 50.266666666666666, + "entropy": 0.12831711278607447, + "epoch": 0.9581133938404284, + "eval/gt_acc_batch": 0.8036111315091451, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27893489599227905, + "kd/policy_loss": 0.008534739015158267, + "kd/rkl_advantage_mean": 0.7615919126042475, + "kd/rkl_advantage_p95": 5.017473479111989, + "kd/rkl_advantage_std": 2.225096133351326, + "kd/ssd_loss": 0.0, + "learning_rate": 4.1924581323814226e-08, + "loss": 0.034138961633046465, + "num_tokens": 825321175.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8036111076672872, + "rewards/gt_logging_reward/std": 0.39063728402058284, + "sampling/importance_sampling_ratio/max": 2.056057361761729, + "sampling/importance_sampling_ratio/mean": 0.9954114337762197, + "sampling/importance_sampling_ratio/min": 0.4280686954657237, + "sampling/sampling_logp_difference/max": 0.3311657468477885, + "sampling/sampling_logp_difference/mean": 0.0039273877395316955, + "step": 25230, + "step_time": 8.189894308732862, + "student/entropy": 0.12831711278607447 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02694444600492716, + "completions/max_length": 497.73333333333335, + "completions/max_terminated_length": 458.3, + "completions/mean_length": 193.13223164876302, + "completions/mean_terminated_length": 184.48110911051432, + "completions/min_length": 60.03333333333333, + "completions/min_terminated_length": 60.03333333333333, + "entropy": 0.12899899029483397, + "epoch": 0.9592526487677059, + "eval/gt_acc_batch": 0.7972222288449605, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29795926809310913, + "kd/policy_loss": 0.008532306456860777, + "kd/rkl_advantage_mean": 0.7856274671852589, + "kd/rkl_advantage_p95": 5.033992226918539, + "kd/rkl_advantage_std": 2.245484550793966, + "kd/ssd_loss": 0.0, + "learning_rate": 4.0785326396536663e-08, + "loss": 0.034129226207733156, + "num_tokens": 826317707.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7972222248713176, + "rewards/gt_logging_reward/std": 0.39417805075645446, + "sampling/importance_sampling_ratio/max": 1.9255762497584026, + "sampling/importance_sampling_ratio/mean": 1.00436971783638, + "sampling/importance_sampling_ratio/min": 0.46487816870212556, + "sampling/sampling_logp_difference/max": 0.32222875754038494, + "sampling/sampling_logp_difference/mean": 0.0039301298403491575, + "step": 25260, + "step_time": 8.08619044929801, + "student/entropy": 0.12899899029483397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.024444445905586084, + "completions/max_length": 488.2, + "completions/max_terminated_length": 441.9, + "completions/mean_length": 186.44417724609374, + "completions/mean_terminated_length": 178.3998041788737, + "completions/min_length": 51.86666666666667, + "completions/min_terminated_length": 51.86666666666667, + "entropy": 0.13008198930571477, + "epoch": 0.9603919036949835, + "eval/gt_acc_batch": 0.8005555709203084, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30398818850517273, + "kd/policy_loss": 0.008552576545237873, + "kd/rkl_advantage_mean": 0.7159406668215524, + "kd/rkl_advantage_p95": 4.859252285957337, + "kd/rkl_advantage_std": 2.1790125489234926, + "kd/ssd_loss": 0.0, + "learning_rate": 3.96460714692591e-08, + "loss": 0.03421030441919962, + "num_tokens": 827289914.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8005555629730224, + "rewards/gt_logging_reward/std": 0.39188215136528015, + "sampling/importance_sampling_ratio/max": 1.9239445328712463, + "sampling/importance_sampling_ratio/mean": 0.9947705447673798, + "sampling/importance_sampling_ratio/min": 0.45737740596135457, + "sampling/sampling_logp_difference/max": 0.3281169652938843, + "sampling/sampling_logp_difference/mean": 0.003963267725581924, + "step": 25290, + "step_time": 8.110874351026723, + "student/entropy": 0.13008198930571477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021388890190670886, + "completions/max_length": 477.8333333333333, + "completions/max_terminated_length": 441.8, + "completions/mean_length": 193.0925099690755, + "completions/mean_terminated_length": 186.2031723022461, + "completions/min_length": 55.56666666666667, + "completions/min_terminated_length": 55.56666666666667, + "entropy": 0.1319371560588479, + "epoch": 0.961531158622261, + "eval/gt_acc_batch": 0.7838889122009277, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3222583532333374, + "kd/policy_loss": 0.008579969646719594, + "kd/rkl_advantage_mean": 0.7255859084427356, + "kd/rkl_advantage_p95": 4.888201304276785, + "kd/rkl_advantage_std": 2.2006726801395415, + "kd/ssd_loss": 0.0, + "learning_rate": 3.850681654198154e-08, + "loss": 0.0343198815981547, + "num_tokens": 828281751.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7838888903458913, + "rewards/gt_logging_reward/std": 0.4039016664028168, + "sampling/importance_sampling_ratio/max": 1.921486492951711, + "sampling/importance_sampling_ratio/mean": 1.001198556025823, + "sampling/importance_sampling_ratio/min": 0.42929480771223705, + "sampling/sampling_logp_difference/max": 0.31789360344409945, + "sampling/sampling_logp_difference/mean": 0.004033027302163343, + "step": 25320, + "step_time": 8.043627736403142, + "student/entropy": 0.1319371560588479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.014722223145266373, + "completions/max_length": 463.73333333333335, + "completions/max_terminated_length": 430.1, + "completions/mean_length": 181.66695404052734, + "completions/mean_terminated_length": 176.85006052652994, + "completions/min_length": 53.766666666666666, + "completions/min_terminated_length": 53.766666666666666, + "entropy": 0.12740204868217309, + "epoch": 0.9626704135495386, + "eval/gt_acc_batch": 0.8152778009573619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29385823011398315, + "kd/policy_loss": 0.008285001137604315, + "kd/rkl_advantage_mean": 0.7118590155150741, + "kd/rkl_advantage_p95": 4.8768149276574455, + "kd/rkl_advantage_std": 2.1778985530138018, + "kd/ssd_loss": 0.0, + "learning_rate": 3.736756161470398e-08, + "loss": 0.033140007654825845, + "num_tokens": 829244280.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.815277781089147, + "rewards/gt_logging_reward/std": 0.37563092559576033, + "sampling/importance_sampling_ratio/max": 1.8216176788012186, + "sampling/importance_sampling_ratio/mean": 0.9986369848251343, + "sampling/importance_sampling_ratio/min": 0.4820076088110606, + "sampling/sampling_logp_difference/max": 0.3144509792327881, + "sampling/sampling_logp_difference/mean": 0.0039407208639507495, + "step": 25350, + "step_time": 7.98829766047808, + "student/entropy": 0.12740204868217309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028611112479120494, + "completions/max_length": 483.3666666666667, + "completions/max_terminated_length": 452.06666666666666, + "completions/mean_length": 195.40528869628906, + "completions/mean_terminated_length": 186.26628519694012, + "completions/min_length": 53.53333333333333, + "completions/min_terminated_length": 53.53333333333333, + "entropy": 0.130682162878414, + "epoch": 0.9638096684768162, + "eval/gt_acc_batch": 0.7750000278155009, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.32080820202827454, + "kd/policy_loss": 0.008627666622245064, + "kd/rkl_advantage_mean": 0.7892380439986785, + "kd/rkl_advantage_p95": 5.063161144653956, + "kd/rkl_advantage_std": 2.223318849007289, + "kd/ssd_loss": 0.0, + "learning_rate": 3.622830668742642e-08, + "loss": 0.03451066811879476, + "num_tokens": 830254403.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.774999996026357, + "rewards/gt_logging_reward/std": 0.40774583717187246, + "sampling/importance_sampling_ratio/max": 2.0273783206939697, + "sampling/importance_sampling_ratio/mean": 0.9962488373120626, + "sampling/importance_sampling_ratio/min": 0.4330617904663086, + "sampling/sampling_logp_difference/max": 0.3336852014064789, + "sampling/sampling_logp_difference/mean": 0.003982524806633592, + "step": 25380, + "step_time": 8.16838131583063, + "student/entropy": 0.130682162878414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016388889929900567, + "completions/max_length": 492.03333333333336, + "completions/max_terminated_length": 451.1666666666667, + "completions/mean_length": 183.53389943440754, + "completions/mean_terminated_length": 178.07000172932942, + "completions/min_length": 56.6, + "completions/min_terminated_length": 56.6, + "entropy": 0.12952319358785946, + "epoch": 0.9649489234040938, + "eval/gt_acc_batch": 0.7872222363948822, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2743333578109741, + "kd/policy_loss": 0.008752003345095242, + "kd/rkl_advantage_mean": 0.7564835985501607, + "kd/rkl_advantage_p95": 5.04572499593099, + "kd/rkl_advantage_std": 2.228487279017766, + "kd/ssd_loss": 0.0, + "learning_rate": 3.5089051760148864e-08, + "loss": 0.03500801722208659, + "num_tokens": 831211197.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7872222264607748, + "rewards/gt_logging_reward/std": 0.40619795421759286, + "sampling/importance_sampling_ratio/max": 1.9462404052416484, + "sampling/importance_sampling_ratio/mean": 1.0059245884418488, + "sampling/importance_sampling_ratio/min": 0.4865587999423345, + "sampling/sampling_logp_difference/max": 0.32678637504577634, + "sampling/sampling_logp_difference/mean": 0.003986581368371845, + "step": 25410, + "step_time": 8.071607213292737, + "student/entropy": 0.12952319358785946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019722223499168953, + "completions/max_length": 479.3, + "completions/max_terminated_length": 453.26666666666665, + "completions/mean_length": 185.22278747558593, + "completions/mean_terminated_length": 178.74498392740887, + "completions/min_length": 53.3, + "completions/min_terminated_length": 53.3, + "entropy": 0.133568988678356, + "epoch": 0.9660881783313713, + "eval/gt_acc_batch": 0.7963889042536417, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.36205175518989563, + "kd/policy_loss": 0.008795742188037062, + "kd/rkl_advantage_mean": 0.7411377023284634, + "kd/rkl_advantage_p95": 4.95534702539444, + "kd/rkl_advantage_std": 2.20900960067908, + "kd/ssd_loss": 0.0, + "learning_rate": 3.39497968328713e-08, + "loss": 0.03518296877543132, + "num_tokens": 832184031.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7963888903458913, + "rewards/gt_logging_reward/std": 0.3977812141180038, + "sampling/importance_sampling_ratio/max": 1.859949823220571, + "sampling/importance_sampling_ratio/mean": 1.0003622849782308, + "sampling/importance_sampling_ratio/min": 0.4547459751367569, + "sampling/sampling_logp_difference/max": 0.3435240606466929, + "sampling/sampling_logp_difference/mean": 0.004102508203747372, + "step": 25440, + "step_time": 7.96726013943262, + "student/entropy": 0.133568988678356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02944444582487146, + "completions/max_length": 492.73333333333335, + "completions/max_terminated_length": 442.1333333333333, + "completions/mean_length": 192.36167704264324, + "completions/mean_terminated_length": 182.72003580729168, + "completions/min_length": 45.1, + "completions/min_terminated_length": 45.1, + "entropy": 0.12862639868011078, + "epoch": 0.9672274332586488, + "eval/gt_acc_batch": 0.7769444723924, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.19486892223358154, + "kd/policy_loss": 0.008372719252171616, + "kd/rkl_advantage_mean": 0.7333988219499588, + "kd/rkl_advantage_p95": 4.972723340988159, + "kd/rkl_advantage_std": 2.1998937080303826, + "kd/ssd_loss": 0.0, + "learning_rate": 3.2810541905593746e-08, + "loss": 0.03349088033040364, + "num_tokens": 833191845.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7769444386164347, + "rewards/gt_logging_reward/std": 0.41019166161616644, + "sampling/importance_sampling_ratio/max": 1.9767670075098673, + "sampling/importance_sampling_ratio/mean": 0.9956629653771718, + "sampling/importance_sampling_ratio/min": 0.4242442155877749, + "sampling/sampling_logp_difference/max": 0.3230234483877818, + "sampling/sampling_logp_difference/mean": 0.003906546401170393, + "step": 25470, + "step_time": 8.332011923235648, + "student/entropy": 0.12862639868011078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02500000170742472, + "completions/max_length": 486.73333333333335, + "completions/max_terminated_length": 456.03333333333336, + "completions/mean_length": 193.88306579589843, + "completions/mean_terminated_length": 185.86634775797526, + "completions/min_length": 48.63333333333333, + "completions/min_terminated_length": 48.63333333333333, + "entropy": 0.1330043929318587, + "epoch": 0.9683666881859264, + "eval/gt_acc_batch": 0.7825000087420145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3081846833229065, + "kd/policy_loss": 0.008917039235044891, + "kd/rkl_advantage_mean": 0.7315323878700535, + "kd/rkl_advantage_p95": 4.918702753384908, + "kd/rkl_advantage_std": 2.196933067838351, + "kd/ssd_loss": 0.0, + "learning_rate": 3.1671286978316177e-08, + "loss": 0.035668158531188966, + "num_tokens": 834209000.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7824999968210856, + "rewards/gt_logging_reward/std": 0.40117308497428894, + "sampling/importance_sampling_ratio/max": 1.9461123148600261, + "sampling/importance_sampling_ratio/mean": 0.9954127887884776, + "sampling/importance_sampling_ratio/min": 0.47072348892688753, + "sampling/sampling_logp_difference/max": 0.3070799251397451, + "sampling/sampling_logp_difference/mean": 0.0040574379575749235, + "step": 25500, + "step_time": 8.458601801834691, + "student/entropy": 0.1330043929318587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013055556360632182, + "completions/max_length": 487.6, + "completions/max_terminated_length": 455.06666666666666, + "completions/mean_length": 183.0130645751953, + "completions/mean_terminated_length": 178.71727040608724, + "completions/min_length": 49.96666666666667, + "completions/min_terminated_length": 49.96666666666667, + "entropy": 0.13192408047616483, + "epoch": 0.9695059431132039, + "eval/gt_acc_batch": 0.7925000170866648, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3157636523246765, + "kd/policy_loss": 0.008837189377906422, + "kd/rkl_advantage_mean": 0.7337217430273691, + "kd/rkl_advantage_p95": 4.9609897593657175, + "kd/rkl_advantage_std": 2.20098641316096, + "kd/ssd_loss": 0.0, + "learning_rate": 3.053203205103862e-08, + "loss": 0.03534875710805257, + "num_tokens": 835169567.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7925000011920929, + "rewards/gt_logging_reward/std": 0.3976173902551333, + "sampling/importance_sampling_ratio/max": 1.8620247681935629, + "sampling/importance_sampling_ratio/mean": 0.999866267045339, + "sampling/importance_sampling_ratio/min": 0.4585867941379547, + "sampling/sampling_logp_difference/max": 0.31798043648401897, + "sampling/sampling_logp_difference/mean": 0.004014422923016052, + "step": 25530, + "step_time": 8.037151841366237, + "student/entropy": 0.13192408047616483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03611111296340823, + "completions/max_length": 484.73333333333335, + "completions/max_terminated_length": 444.03333333333336, + "completions/mean_length": 198.80834197998047, + "completions/mean_terminated_length": 187.2748026529948, + "completions/min_length": 55.56666666666667, + "completions/min_terminated_length": 55.56666666666667, + "entropy": 0.1325644062831998, + "epoch": 0.9706451980404815, + "eval/gt_acc_batch": 0.7744444708029429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.22444459795951843, + "kd/policy_loss": 0.009424723112412417, + "kd/rkl_advantage_mean": 0.7562341613695025, + "kd/rkl_advantage_p95": 4.920807061592738, + "kd/rkl_advantage_std": 2.183630327383677, + "kd/ssd_loss": 0.0, + "learning_rate": 2.9392777123761058e-08, + "loss": 0.037698888778686525, + "num_tokens": 836186213.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.774444446961085, + "rewards/gt_logging_reward/std": 0.40797437777121864, + "sampling/importance_sampling_ratio/max": 1.9879270156224569, + "sampling/importance_sampling_ratio/mean": 1.0029697219530742, + "sampling/importance_sampling_ratio/min": 0.4610103170077006, + "sampling/sampling_logp_difference/max": 0.30439485708872477, + "sampling/sampling_logp_difference/mean": 0.004002917751980324, + "step": 25560, + "step_time": 8.161123877001227, + "student/entropy": 0.1325644062831998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01972222328186035, + "completions/max_length": 476.53333333333336, + "completions/max_terminated_length": 447.76666666666665, + "completions/mean_length": 184.72723236083985, + "completions/mean_terminated_length": 178.18252207438152, + "completions/min_length": 49.6, + "completions/min_terminated_length": 49.6, + "entropy": 0.12961186543107034, + "epoch": 0.9717844529677591, + "eval/gt_acc_batch": 0.7983333508173625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.43864524364471436, + "kd/policy_loss": 0.00867119050041462, + "kd/rkl_advantage_mean": 0.6454908416916927, + "kd/rkl_advantage_p95": 4.6812934339046475, + "kd/rkl_advantage_std": 2.138492211699486, + "kd/ssd_loss": 0.0, + "learning_rate": 2.82535221964835e-08, + "loss": 0.03468476533889771, + "num_tokens": 837159487.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7983333349227906, + "rewards/gt_logging_reward/std": 0.39216124365727106, + "sampling/importance_sampling_ratio/max": 1.9744460344314576, + "sampling/importance_sampling_ratio/mean": 1.001758627096812, + "sampling/importance_sampling_ratio/min": 0.4741469740867615, + "sampling/sampling_logp_difference/max": 0.3270647287368774, + "sampling/sampling_logp_difference/mean": 0.003954213089309632, + "step": 25590, + "step_time": 8.148778123563776, + "student/entropy": 0.12961186543107034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.023611112621923287, + "completions/max_length": 490.46666666666664, + "completions/max_terminated_length": 445.6, + "completions/mean_length": 187.34612325032552, + "completions/mean_terminated_length": 179.43053741455077, + "completions/min_length": 51.6, + "completions/min_terminated_length": 51.6, + "entropy": 0.13513201499978703, + "epoch": 0.9729237078950367, + "eval/gt_acc_batch": 0.7983333468437195, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2579764425754547, + "kd/policy_loss": 0.008943776964830856, + "kd/rkl_advantage_mean": 0.7586506355553866, + "kd/rkl_advantage_p95": 5.039712723096212, + "kd/rkl_advantage_std": 2.2496157546838123, + "kd/ssd_loss": 0.0, + "learning_rate": 2.711426726920594e-08, + "loss": 0.03577511310577393, + "num_tokens": 838140149.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7983333349227906, + "rewards/gt_logging_reward/std": 0.3941521962483724, + "sampling/importance_sampling_ratio/max": 1.8266638278961183, + "sampling/importance_sampling_ratio/mean": 0.9985974629720052, + "sampling/importance_sampling_ratio/min": 0.456720499197642, + "sampling/sampling_logp_difference/max": 0.3177338182926178, + "sampling/sampling_logp_difference/mean": 0.004120384125659863, + "step": 25620, + "step_time": 8.228294184467329, + "student/entropy": 0.13513201499978703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025833334897955258, + "completions/max_length": 493.96666666666664, + "completions/max_terminated_length": 449.5, + "completions/mean_length": 193.77862294514975, + "completions/mean_terminated_length": 185.498393758138, + "completions/min_length": 58.1, + "completions/min_terminated_length": 58.1, + "entropy": 0.13006400186568498, + "epoch": 0.9740629628223142, + "eval/gt_acc_batch": 0.7797222415606181, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2566074728965759, + "kd/policy_loss": 0.009084195614559577, + "kd/rkl_advantage_mean": 0.8185616782555978, + "kd/rkl_advantage_p95": 5.181704910596212, + "kd/rkl_advantage_std": 2.278538206219673, + "kd/ssd_loss": 0.0, + "learning_rate": 2.5975012341928377e-08, + "loss": 0.03633677959442139, + "num_tokens": 839150816.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7797222256660461, + "rewards/gt_logging_reward/std": 0.41056076685587567, + "sampling/importance_sampling_ratio/max": 1.9577651778856913, + "sampling/importance_sampling_ratio/mean": 1.0018564105033874, + "sampling/importance_sampling_ratio/min": 0.45205502609411874, + "sampling/sampling_logp_difference/max": 0.30457824071248374, + "sampling/sampling_logp_difference/mean": 0.003944680692317585, + "step": 25650, + "step_time": 8.40865607560554, + "student/entropy": 0.13006400186568498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.016388889929900567, + "completions/max_length": 489.5, + "completions/max_terminated_length": 462.6, + "completions/mean_length": 193.68973134358723, + "completions/mean_terminated_length": 188.46128946940104, + "completions/min_length": 64.33333333333333, + "completions/min_terminated_length": 64.33333333333333, + "entropy": 0.12831056204934915, + "epoch": 0.9752022177495918, + "eval/gt_acc_batch": 0.7897222379843394, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.34627944231033325, + "kd/policy_loss": 0.008487316220998764, + "kd/rkl_advantage_mean": 0.6922600186429918, + "kd/rkl_advantage_p95": 4.841612490018209, + "kd/rkl_advantage_std": 2.179883915185928, + "kd/ssd_loss": 0.0, + "learning_rate": 2.4835757414650815e-08, + "loss": 0.03394926389058431, + "num_tokens": 840167579.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.789722228050232, + "rewards/gt_logging_reward/std": 0.4006514991323153, + "sampling/importance_sampling_ratio/max": 1.9565821528434753, + "sampling/importance_sampling_ratio/mean": 1.0069037179152172, + "sampling/importance_sampling_ratio/min": 0.4590534955263138, + "sampling/sampling_logp_difference/max": 0.33016475041707355, + "sampling/sampling_logp_difference/mean": 0.003921854930619399, + "step": 25680, + "step_time": 8.31306609630119, + "student/entropy": 0.12831056204934915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.020000000980993114, + "completions/max_length": 486.26666666666665, + "completions/max_terminated_length": 452.06666666666666, + "completions/mean_length": 186.80973256429036, + "completions/mean_terminated_length": 180.45391693115235, + "completions/min_length": 50.46666666666667, + "completions/min_terminated_length": 50.46666666666667, + "entropy": 0.13245037912080684, + "epoch": 0.9763414726768693, + "eval/gt_acc_batch": 0.7847222367922465, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31581375002861023, + "kd/policy_loss": 0.009109217122507592, + "kd/rkl_advantage_mean": 0.767375441469873, + "kd/rkl_advantage_p95": 5.05412505865097, + "kd/rkl_advantage_std": 2.2235380431016285, + "kd/ssd_loss": 0.0, + "learning_rate": 2.3696502487373256e-08, + "loss": 0.03643686771392822, + "num_tokens": 841159334.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7847222268581391, + "rewards/gt_logging_reward/std": 0.3974231307705243, + "sampling/importance_sampling_ratio/max": 1.9175263007481893, + "sampling/importance_sampling_ratio/mean": 0.9995357354482015, + "sampling/importance_sampling_ratio/min": 0.4588465323050817, + "sampling/sampling_logp_difference/max": 0.33755595088005064, + "sampling/sampling_logp_difference/mean": 0.004001803308104475, + "step": 25710, + "step_time": 8.36937265713544, + "student/entropy": 0.13245037912080684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01666666744276881, + "completions/max_length": 473.6666666666667, + "completions/max_terminated_length": 447.8333333333333, + "completions/mean_length": 185.72973175048827, + "completions/mean_terminated_length": 180.29439697265624, + "completions/min_length": 49.6, + "completions/min_terminated_length": 49.6, + "entropy": 0.12360023887207111, + "epoch": 0.9774807276041468, + "eval/gt_acc_batch": 0.8169444640477498, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2651395797729492, + "kd/policy_loss": 0.007767474247763554, + "kd/rkl_advantage_mean": 0.6748367723698417, + "kd/rkl_advantage_p95": 4.840807024637858, + "kd/rkl_advantage_std": 2.170597208539645, + "kd/ssd_loss": 0.0, + "learning_rate": 2.2557247560095696e-08, + "loss": 0.031069904565811157, + "num_tokens": 842131425.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8169444521268209, + "rewards/gt_logging_reward/std": 0.3762322689096133, + "sampling/importance_sampling_ratio/max": 1.8097192168235778, + "sampling/importance_sampling_ratio/mean": 0.9995728572209676, + "sampling/importance_sampling_ratio/min": 0.4578324576218923, + "sampling/sampling_logp_difference/max": 0.36437430381774905, + "sampling/sampling_logp_difference/mean": 0.0038115835981443525, + "step": 25740, + "step_time": 8.140447488966553, + "student/entropy": 0.12360023887207111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019166667790462572, + "completions/max_length": 489.06666666666666, + "completions/max_terminated_length": 455.7, + "completions/mean_length": 182.92362060546876, + "completions/mean_terminated_length": 176.5881825764974, + "completions/min_length": 56.266666666666666, + "completions/min_terminated_length": 56.266666666666666, + "entropy": 0.1334075702354312, + "epoch": 0.9786199825314245, + "eval/gt_acc_batch": 0.7947222371896108, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.289486289024353, + "kd/policy_loss": 0.008670657714052748, + "kd/rkl_advantage_mean": 0.7988404126216968, + "kd/rkl_advantage_p95": 5.024180165926615, + "kd/rkl_advantage_std": 2.2157385806242624, + "kd/ssd_loss": 0.0, + "learning_rate": 2.1417992632818137e-08, + "loss": 0.034682631492614746, + "num_tokens": 843087006.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222173213959, + "rewards/gt_logging_reward/std": 0.3954265962044398, + "sampling/importance_sampling_ratio/max": 1.8596965670585632, + "sampling/importance_sampling_ratio/mean": 1.0011861622333527, + "sampling/importance_sampling_ratio/min": 0.4236236706376076, + "sampling/sampling_logp_difference/max": 0.3128378947575887, + "sampling/sampling_logp_difference/mean": 0.004080408268297712, + "step": 25770, + "step_time": 8.09093707347056, + "student/entropy": 0.1334075702354312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02861111266538501, + "completions/max_length": 495.3666666666667, + "completions/max_terminated_length": 442.8333333333333, + "completions/mean_length": 188.39889831542968, + "completions/mean_terminated_length": 178.9198262532552, + "completions/min_length": 55.5, + "completions/min_terminated_length": 55.5, + "entropy": 0.1299333170677225, + "epoch": 0.979759237458702, + "eval/gt_acc_batch": 0.8044444759686787, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.31937655806541443, + "kd/policy_loss": 0.00869324939752308, + "kd/rkl_advantage_mean": 0.7774871803199251, + "kd/rkl_advantage_p95": 5.026236832141876, + "kd/rkl_advantage_std": 2.2389445920785267, + "kd/ssd_loss": 0.0, + "learning_rate": 2.0278737705540575e-08, + "loss": 0.034773000081380206, + "num_tokens": 844065122.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8044444402058919, + "rewards/gt_logging_reward/std": 0.38964470227559406, + "sampling/importance_sampling_ratio/max": 1.9585163593292236, + "sampling/importance_sampling_ratio/mean": 1.0083916326363882, + "sampling/importance_sampling_ratio/min": 0.44392294536034266, + "sampling/sampling_logp_difference/max": 0.3110794146855672, + "sampling/sampling_logp_difference/mean": 0.003963031976794203, + "step": 25800, + "step_time": 8.15649747736558, + "student/entropy": 0.1299333170677225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02416666786496838, + "completions/max_length": 477.8666666666667, + "completions/max_terminated_length": 452.9, + "completions/mean_length": 189.24445444742838, + "completions/mean_terminated_length": 181.42300618489583, + "completions/min_length": 52.233333333333334, + "completions/min_terminated_length": 52.233333333333334, + "entropy": 0.12997254468500613, + "epoch": 0.9808984923859796, + "eval/gt_acc_batch": 0.7947222411632537, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24728579819202423, + "kd/policy_loss": 0.008445959183154628, + "kd/rkl_advantage_mean": 0.7119853398452203, + "kd/rkl_advantage_p95": 4.874627381563187, + "kd/rkl_advantage_std": 2.174870640039444, + "kd/ssd_loss": 0.0, + "learning_rate": 1.9139482778263016e-08, + "loss": 0.03378383715947469, + "num_tokens": 845049298.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7947222252686819, + "rewards/gt_logging_reward/std": 0.39616652329762775, + "sampling/importance_sampling_ratio/max": 1.912439469496409, + "sampling/importance_sampling_ratio/mean": 0.9956080496311188, + "sampling/importance_sampling_ratio/min": 0.4217689990997314, + "sampling/sampling_logp_difference/max": 0.3144365211327871, + "sampling/sampling_logp_difference/mean": 0.003977785104264816, + "step": 25830, + "step_time": 8.105376634336427, + "student/entropy": 0.12997254468500613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.026388890327264866, + "completions/max_length": 480.6333333333333, + "completions/max_terminated_length": 448.53333333333336, + "completions/mean_length": 189.8400095621745, + "completions/mean_terminated_length": 181.2978775024414, + "completions/min_length": 52.96666666666667, + "completions/min_terminated_length": 52.96666666666667, + "entropy": 0.13036849020669858, + "epoch": 0.9820377473132571, + "eval/gt_acc_batch": 0.8008333563804626, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30574876070022583, + "kd/policy_loss": 0.0084376962215174, + "kd/rkl_advantage_mean": 0.6649701997327307, + "kd/rkl_advantage_p95": 4.843249678611755, + "kd/rkl_advantage_std": 2.1995602627595265, + "kd/ssd_loss": 0.0, + "learning_rate": 1.8000227850985456e-08, + "loss": 0.03375078439712524, + "num_tokens": 846051050.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8008333365122478, + "rewards/gt_logging_reward/std": 0.3911197200417519, + "sampling/importance_sampling_ratio/max": 1.8602129141489665, + "sampling/importance_sampling_ratio/mean": 0.9930752774079641, + "sampling/importance_sampling_ratio/min": 0.46860237618287404, + "sampling/sampling_logp_difference/max": 0.32267380952835084, + "sampling/sampling_logp_difference/mean": 0.0040006229964395365, + "step": 25860, + "step_time": 8.43520971473578, + "student/entropy": 0.13036849020669858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02444444578140974, + "completions/max_length": 488.9, + "completions/max_terminated_length": 459.1333333333333, + "completions/mean_length": 191.5130650838216, + "completions/mean_terminated_length": 183.56522572835286, + "completions/min_length": 56.333333333333336, + "completions/min_terminated_length": 56.333333333333336, + "entropy": 0.13120677849898735, + "epoch": 0.9831770022405347, + "eval/gt_acc_batch": 0.7836111346880595, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.315908282995224, + "kd/policy_loss": 0.008970911831905444, + "kd/rkl_advantage_mean": 0.7408635417620341, + "kd/rkl_advantage_p95": 4.904705486694971, + "kd/rkl_advantage_std": 2.1858988871177036, + "kd/ssd_loss": 0.0, + "learning_rate": 1.6860972923707897e-08, + "loss": 0.03588364521662394, + "num_tokens": 847032721.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7836111108462016, + "rewards/gt_logging_reward/std": 0.40685406227906545, + "sampling/importance_sampling_ratio/max": 1.9558974186579385, + "sampling/importance_sampling_ratio/mean": 1.0013761301835378, + "sampling/importance_sampling_ratio/min": 0.48268304268519086, + "sampling/sampling_logp_difference/max": 0.32509916126728056, + "sampling/sampling_logp_difference/mean": 0.004024115313465397, + "step": 25890, + "step_time": 8.13915358936841, + "student/entropy": 0.13120677849898735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018611112236976625, + "completions/max_length": 473.5, + "completions/max_terminated_length": 430.4, + "completions/mean_length": 179.9275115966797, + "completions/mean_terminated_length": 173.85843251546223, + "completions/min_length": 52.266666666666666, + "completions/min_terminated_length": 52.266666666666666, + "entropy": 0.1267509805659453, + "epoch": 0.9843162571678122, + "eval/gt_acc_batch": 0.8350000222524007, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3241766691207886, + "kd/policy_loss": 0.007837413342591996, + "kd/rkl_advantage_mean": 0.715189221004645, + "kd/rkl_advantage_p95": 4.838110377391179, + "kd/rkl_advantage_std": 2.1672684381405514, + "kd/ssd_loss": 0.0, + "learning_rate": 1.572171799643033e-08, + "loss": 0.031349651018778485, + "num_tokens": 847983404.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8349999984105428, + "rewards/gt_logging_reward/std": 0.35999320149421693, + "sampling/importance_sampling_ratio/max": 1.9078854282697042, + "sampling/importance_sampling_ratio/mean": 0.9985920468966166, + "sampling/importance_sampling_ratio/min": 0.47632539172967275, + "sampling/sampling_logp_difference/max": 0.3207974314689636, + "sampling/sampling_logp_difference/mean": 0.003906527161598205, + "step": 25920, + "step_time": 7.95295362896189, + "student/entropy": 0.1267509805659453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015555556335796913, + "completions/max_length": 481.6333333333333, + "completions/max_terminated_length": 448.4, + "completions/mean_length": 183.0652877807617, + "completions/mean_terminated_length": 177.93433024088543, + "completions/min_length": 51.03333333333333, + "completions/min_terminated_length": 51.03333333333333, + "entropy": 0.13137134729574124, + "epoch": 0.9854555120950899, + "eval/gt_acc_batch": 0.8027777969837189, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30164381861686707, + "kd/policy_loss": 0.008437655022135004, + "kd/rkl_advantage_mean": 0.7230048411215345, + "kd/rkl_advantage_p95": 4.961972419420878, + "kd/rkl_advantage_std": 2.2239978432655336, + "kd/ssd_loss": 0.0, + "learning_rate": 1.4582463069152774e-08, + "loss": 0.03375062147776286, + "num_tokens": 848958231.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8027777731418609, + "rewards/gt_logging_reward/std": 0.3892322599887848, + "sampling/importance_sampling_ratio/max": 1.9103217482566834, + "sampling/importance_sampling_ratio/mean": 0.9940680921077728, + "sampling/importance_sampling_ratio/min": 0.43847170571486155, + "sampling/sampling_logp_difference/max": 0.3145290851593018, + "sampling/sampling_logp_difference/mean": 0.004032742255367339, + "step": 25950, + "step_time": 8.174920318300913, + "student/entropy": 0.13137134729574124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021111112367361785, + "completions/max_length": 491.5, + "completions/max_terminated_length": 449.8, + "completions/mean_length": 187.0666753133138, + "completions/mean_terminated_length": 180.1588348388672, + "completions/min_length": 54.43333333333333, + "completions/min_terminated_length": 54.43333333333333, + "entropy": 0.1301615135744214, + "epoch": 0.9865947670223674, + "eval/gt_acc_batch": 0.7886111418406169, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2689487338066101, + "kd/policy_loss": 0.008552788170830657, + "kd/rkl_advantage_mean": 0.7168316151636343, + "kd/rkl_advantage_p95": 4.809187984466552, + "kd/rkl_advantage_std": 2.1598029802242915, + "kd/ssd_loss": 0.0, + "learning_rate": 1.3443208141875213e-08, + "loss": 0.03421115477879842, + "num_tokens": 849925999.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7886111120382945, + "rewards/gt_logging_reward/std": 0.39979895850022634, + "sampling/importance_sampling_ratio/max": 1.8350385069847106, + "sampling/importance_sampling_ratio/mean": 0.992460717757543, + "sampling/importance_sampling_ratio/min": 0.46694736878077187, + "sampling/sampling_logp_difference/max": 0.34262816508611044, + "sampling/sampling_logp_difference/mean": 0.004015247283192972, + "step": 25980, + "step_time": 8.111343131936156, + "student/entropy": 0.1301615135744214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.019444445768992107, + "completions/max_length": 476.26666666666665, + "completions/max_terminated_length": 446.43333333333334, + "completions/mean_length": 182.17639872233073, + "completions/mean_terminated_length": 175.74290110270184, + "completions/min_length": 48.36666666666667, + "completions/min_terminated_length": 48.36666666666667, + "entropy": 0.12609328714509804, + "epoch": 0.9877340219496449, + "eval/gt_acc_batch": 0.8136111219724019, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.29485341906547546, + "kd/policy_loss": 0.008153392117431697, + "kd/rkl_advantage_mean": 0.72918068356812, + "kd/rkl_advantage_p95": 4.981149832407634, + "kd/rkl_advantage_std": 2.223110869526863, + "kd/ssd_loss": 0.0, + "learning_rate": 1.2303953214597652e-08, + "loss": 0.03261356751124064, + "num_tokens": 850885322.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813611114025116, + "rewards/gt_logging_reward/std": 0.383774197101593, + "sampling/importance_sampling_ratio/max": 1.8540683786074321, + "sampling/importance_sampling_ratio/mean": 0.9976613938808441, + "sampling/importance_sampling_ratio/min": 0.4809098000327746, + "sampling/sampling_logp_difference/max": 0.3186839004357656, + "sampling/sampling_logp_difference/mean": 0.003838386681551735, + "step": 26010, + "step_time": 8.073742181133518, + "student/entropy": 0.12609328714509804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0169444455144306, + "completions/max_length": 455.1333333333333, + "completions/max_terminated_length": 426.1, + "completions/mean_length": 183.83584238688152, + "completions/mean_terminated_length": 178.28578389485676, + "completions/min_length": 58.4, + "completions/min_terminated_length": 58.4, + "entropy": 0.1264139175415039, + "epoch": 0.9888732768769225, + "eval/gt_acc_batch": 0.7883333484331767, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2913793921470642, + "kd/policy_loss": 0.0083192681777291, + "kd/rkl_advantage_mean": 0.6752849956443242, + "kd/rkl_advantage_p95": 4.787274132172267, + "kd/rkl_advantage_std": 2.1445546338955563, + "kd/ssd_loss": 0.0, + "learning_rate": 1.1164698287320091e-08, + "loss": 0.033277078469594316, + "num_tokens": 851850435.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7883333285649617, + "rewards/gt_logging_reward/std": 0.3997439170877139, + "sampling/importance_sampling_ratio/max": 1.859102157751719, + "sampling/importance_sampling_ratio/mean": 0.9983133157094319, + "sampling/importance_sampling_ratio/min": 0.4347180128097534, + "sampling/sampling_logp_difference/max": 0.31340410709381106, + "sampling/sampling_logp_difference/mean": 0.003875847921396295, + "step": 26040, + "step_time": 7.882410497805298, + "student/entropy": 0.1264139175415039 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0188888899050653, + "completions/max_length": 469.8, + "completions/max_terminated_length": 435.6666666666667, + "completions/mean_length": 183.3408447265625, + "completions/mean_terminated_length": 177.20124359130858, + "completions/min_length": 53.3, + "completions/min_terminated_length": 53.3, + "entropy": 0.1300936117147406, + "epoch": 0.9900125318042, + "eval/gt_acc_batch": 0.7925000190734863, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2765967845916748, + "kd/policy_loss": 0.00893237089427809, + "kd/rkl_advantage_mean": 0.6858150751019517, + "kd/rkl_advantage_p95": 4.789360505342484, + "kd/rkl_advantage_std": 2.137502234180768, + "kd/ssd_loss": 0.0, + "learning_rate": 1.0025443360042532e-08, + "loss": 0.035729479789733884, + "num_tokens": 852813422.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7924999992052714, + "rewards/gt_logging_reward/std": 0.3983038117488225, + "sampling/importance_sampling_ratio/max": 1.8832035700480143, + "sampling/importance_sampling_ratio/mean": 1.0029024620850882, + "sampling/importance_sampling_ratio/min": 0.47241399983565013, + "sampling/sampling_logp_difference/max": 0.3255273868640264, + "sampling/sampling_logp_difference/mean": 0.003997542647023996, + "step": 26070, + "step_time": 7.918748533410447, + "student/entropy": 0.1300936117147406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02333333473652601, + "completions/max_length": 469.73333333333335, + "completions/max_terminated_length": 434.2, + "completions/mean_length": 186.2983428955078, + "completions/mean_terminated_length": 178.56804453531902, + "completions/min_length": 57.766666666666666, + "completions/min_terminated_length": 57.766666666666666, + "entropy": 0.12867186795920132, + "epoch": 0.9911517867314776, + "eval/gt_acc_batch": 0.8083333472410837, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.390809029340744, + "kd/policy_loss": 0.008630790755463143, + "kd/rkl_advantage_mean": 0.7488794637843966, + "kd/rkl_advantage_p95": 4.955602953831355, + "kd/rkl_advantage_std": 2.202052347858747, + "kd/ssd_loss": 0.0, + "learning_rate": 8.886188432764971e-09, + "loss": 0.034523165225982665, + "num_tokens": 853786024.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8083333392937978, + "rewards/gt_logging_reward/std": 0.38258339365323385, + "sampling/importance_sampling_ratio/max": 1.8806252559026082, + "sampling/importance_sampling_ratio/mean": 0.9985775391260783, + "sampling/importance_sampling_ratio/min": 0.39658739368120827, + "sampling/sampling_logp_difference/max": 0.34525205691655475, + "sampling/sampling_logp_difference/mean": 0.003973177308216691, + "step": 26100, + "step_time": 7.850210814535967, + "student/entropy": 0.12867186795920132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0202777790526549, + "completions/max_length": 470.56666666666666, + "completions/max_terminated_length": 446.53333333333336, + "completions/mean_length": 192.93084208170572, + "completions/mean_terminated_length": 186.38890991210937, + "completions/min_length": 55.46666666666667, + "completions/min_terminated_length": 55.46666666666667, + "entropy": 0.12089140995716055, + "epoch": 0.9922910416587551, + "eval/gt_acc_batch": 0.809166685740153, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.21763113141059875, + "kd/policy_loss": 0.00786741889314726, + "kd/rkl_advantage_mean": 0.7188729146495462, + "kd/rkl_advantage_p95": 4.885275481144587, + "kd/rkl_advantage_std": 2.158831504980723, + "kd/ssd_loss": 0.0, + "learning_rate": 7.74693350548741e-09, + "loss": 0.031469676891962686, + "num_tokens": 854787423.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8091666718324025, + "rewards/gt_logging_reward/std": 0.3849528248111407, + "sampling/importance_sampling_ratio/max": 1.8665737191836038, + "sampling/importance_sampling_ratio/mean": 1.0004808187484742, + "sampling/importance_sampling_ratio/min": 0.45080735931793847, + "sampling/sampling_logp_difference/max": 0.35430472493171694, + "sampling/sampling_logp_difference/mean": 0.0037062507200365264, + "step": 26130, + "step_time": 7.993042539756668, + "student/entropy": 0.12089140995716055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01750000085060795, + "completions/max_length": 460.6666666666667, + "completions/max_terminated_length": 431.3333333333333, + "completions/mean_length": 180.97000834147136, + "completions/mean_terminated_length": 175.1681889851888, + "completions/min_length": 55.96666666666667, + "completions/min_terminated_length": 55.96666666666667, + "entropy": 0.12800014751652877, + "epoch": 0.9934302965860328, + "eval/gt_acc_batch": 0.813055564959844, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.27454864978790283, + "kd/policy_loss": 0.008255690512790655, + "kd/rkl_advantage_mean": 0.7758802431325118, + "kd/rkl_advantage_p95": 4.995607282718023, + "kd/rkl_advantage_std": 2.202400025725365, + "kd/ssd_loss": 0.0, + "learning_rate": 6.607678578209851e-09, + "loss": 0.03302276134490967, + "num_tokens": 855747827.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.813055553038915, + "rewards/gt_logging_reward/std": 0.3846996436516444, + "sampling/importance_sampling_ratio/max": 1.9079003930091858, + "sampling/importance_sampling_ratio/mean": 1.0022391974925995, + "sampling/importance_sampling_ratio/min": 0.4842998261253039, + "sampling/sampling_logp_difference/max": 0.32255615790685016, + "sampling/sampling_logp_difference/mean": 0.003932675944330792, + "step": 26160, + "step_time": 7.902928676539644, + "student/entropy": 0.12800014751652877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02166666785875956, + "completions/max_length": 476.76666666666665, + "completions/max_terminated_length": 438.23333333333335, + "completions/mean_length": 185.26834208170573, + "completions/mean_terminated_length": 178.12603454589845, + "completions/min_length": 52.6, + "completions/min_terminated_length": 52.6, + "entropy": 0.1309782262891531, + "epoch": 0.9945695515133103, + "eval/gt_acc_batch": 0.7983333468437195, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.24374276399612427, + "kd/policy_loss": 0.008675073692575096, + "kd/rkl_advantage_mean": 0.8372010676811139, + "kd/rkl_advantage_p95": 5.139206876357396, + "kd/rkl_advantage_std": 2.2301428854465484, + "kd/ssd_loss": 0.0, + "learning_rate": 5.4684236509322896e-09, + "loss": 0.034700290362040205, + "num_tokens": 856714585.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.798333328962326, + "rewards/gt_logging_reward/std": 0.3951241299510002, + "sampling/importance_sampling_ratio/max": 1.8785622477531434, + "sampling/importance_sampling_ratio/mean": 1.0008698006470997, + "sampling/importance_sampling_ratio/min": 0.45275761783123014, + "sampling/sampling_logp_difference/max": 0.3333398203055064, + "sampling/sampling_logp_difference/mean": 0.00402181150081257, + "step": 26190, + "step_time": 7.946380482030993, + "student/entropy": 0.1309782262891531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.028055557142943145, + "completions/max_length": 486.4, + "completions/max_terminated_length": 452.56666666666666, + "completions/mean_length": 193.12167612711588, + "completions/mean_terminated_length": 183.9206756591797, + "completions/min_length": 54.53333333333333, + "completions/min_terminated_length": 54.53333333333333, + "entropy": 0.13113699809958537, + "epoch": 0.9957088064405879, + "eval/gt_acc_batch": 0.7805555780728658, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.28879261016845703, + "kd/policy_loss": 0.008728436174957702, + "kd/rkl_advantage_mean": 0.7380654470374187, + "kd/rkl_advantage_p95": 4.967583119869232, + "kd/rkl_advantage_std": 2.2158943941195806, + "kd/ssd_loss": 0.0, + "learning_rate": 4.3291687236547296e-09, + "loss": 0.03491374254226685, + "num_tokens": 857716311.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7805555522441864, + "rewards/gt_logging_reward/std": 0.40181585351626076, + "sampling/importance_sampling_ratio/max": 1.884137252966563, + "sampling/importance_sampling_ratio/mean": 1.000909827152888, + "sampling/importance_sampling_ratio/min": 0.43389966239531835, + "sampling/sampling_logp_difference/max": 0.3700602054595947, + "sampling/sampling_logp_difference/mean": 0.003988236056951185, + "step": 26220, + "step_time": 8.093219568100176, + "student/entropy": 0.13113699809958537 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013333334090809027, + "completions/max_length": 471.96666666666664, + "completions/max_terminated_length": 433.6333333333333, + "completions/mean_length": 178.25500946044923, + "completions/mean_terminated_length": 173.84633229573566, + "completions/min_length": 56.2, + "completions/min_terminated_length": 56.2, + "entropy": 0.13539091056833666, + "epoch": 0.9968480613678654, + "eval/gt_acc_batch": 0.8030555764834086, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.35537031292915344, + "kd/policy_loss": 0.008484880897837381, + "kd/rkl_advantage_mean": 0.8018611467909068, + "kd/rkl_advantage_p95": 5.036621918280919, + "kd/rkl_advantage_std": 2.2405777255694073, + "kd/ssd_loss": 0.0, + "learning_rate": 3.189913796377169e-09, + "loss": 0.033939528465271, + "num_tokens": 858665373.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.8030555526415507, + "rewards/gt_logging_reward/std": 0.39192101260026296, + "sampling/importance_sampling_ratio/max": 1.876450788974762, + "sampling/importance_sampling_ratio/mean": 0.998519241809845, + "sampling/importance_sampling_ratio/min": 0.45547755261262257, + "sampling/sampling_logp_difference/max": 0.30508578817049664, + "sampling/sampling_logp_difference/mean": 0.0040684374592577415, + "step": 26250, + "step_time": 8.077394647765322, + "student/entropy": 0.13539091056833666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.018333334165314834, + "completions/max_length": 485.0, + "completions/max_terminated_length": 460.3666666666667, + "completions/mean_length": 184.33417612711588, + "completions/mean_terminated_length": 178.0758814493815, + "completions/min_length": 51.13333333333333, + "completions/min_terminated_length": 51.13333333333333, + "entropy": 0.13116911152998606, + "epoch": 0.9979873162951429, + "eval/gt_acc_batch": 0.7977778057257334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.3382938802242279, + "kd/policy_loss": 0.008836072588261837, + "kd/rkl_advantage_mean": 0.786857272281001, + "kd/rkl_advantage_p95": 4.997472576300303, + "kd/rkl_advantage_std": 2.2221710364023846, + "kd/ssd_loss": 0.0, + "learning_rate": 2.050658869099609e-09, + "loss": 0.03534429868062337, + "num_tokens": 859624080.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.797777779897054, + "rewards/gt_logging_reward/std": 0.3975469579299291, + "sampling/importance_sampling_ratio/max": 1.8823233803113302, + "sampling/importance_sampling_ratio/mean": 1.0015786170959473, + "sampling/importance_sampling_ratio/min": 0.49061890244483947, + "sampling/sampling_logp_difference/max": 0.3357905725638072, + "sampling/sampling_logp_difference/mean": 0.004006391111761332, + "step": 26280, + "step_time": 8.099503362265144, + "student/entropy": 0.13116911152998606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02083333448196451, + "completions/max_length": 484.03333333333336, + "completions/max_terminated_length": 442.8333333333333, + "completions/mean_length": 189.11084391276043, + "completions/mean_terminated_length": 182.35631612141927, + "completions/min_length": 52.733333333333334, + "completions/min_terminated_length": 52.733333333333334, + "entropy": 0.13059442571053903, + "epoch": 0.9991265712224205, + "eval/gt_acc_batch": 0.7827777981758117, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.30764850974082947, + "kd/policy_loss": 0.008549473979898418, + "kd/rkl_advantage_mean": 0.7253896731262406, + "kd/rkl_advantage_p95": 4.902711906035742, + "kd/rkl_advantage_std": 2.189149084687233, + "kd/ssd_loss": 0.0, + "learning_rate": 9.114039418220484e-10, + "loss": 0.034197898705800374, + "num_tokens": 860620687.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/gt_logging_reward/mean": 0.7827777802944184, + "rewards/gt_logging_reward/std": 0.40299503107865653, + "sampling/importance_sampling_ratio/max": 1.898701306184133, + "sampling/importance_sampling_ratio/mean": 0.9932234068711598, + "sampling/importance_sampling_ratio/min": 0.41570016046365105, + "sampling/sampling_logp_difference/max": 0.3144602735837301, + "sampling/sampling_logp_difference/mean": 0.003965286064582566, + "step": 26310, + "step_time": 8.335269565691124, + "student/entropy": 0.13059442571053903 + } + ], + "logging_steps": 30, + "max_steps": 26333, + "num_input_tokens_seen": 861353169, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 10, + "trial_name": null, + "trial_params": null +}