{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 500, "global_step": 26333, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027500001216928163, "completions/max_length": 490.56666666666666, "completions/max_terminated_length": 461.4, "completions/mean_length": 188.9975112915039, "completions/mean_terminated_length": 180.10747375488282, "completions/min_length": 43.4, "completions/min_terminated_length": 43.4, "entropy": 0.2029941453287999, "epoch": 0.0011392549272775604, "eval/gt_acc_batch": 0.7313889125982921, "frac_reward_zero_std": 1.0, "grad_norm": 0.3513047993183136, "kd/policy_loss": 0.013840306313553204, "kd/rkl_advantage_mean": 0.24764675879268908, "kd/rkl_advantage_p95": 3.8125994205474854, "kd/rkl_advantage_std": 2.0226369639237722, "kd/ssd_loss": 0.0, "learning_rate": 9.98898720236965e-07, "loss": 0.05536123116811117, "num_tokens": 980727.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7313888847827912, "rewards/gt_logging_reward/std": 0.43746271431446077, "sampling/importance_sampling_ratio/max": 1.9902352770169576, "sampling/importance_sampling_ratio/mean": 1.0039433201154073, "sampling/importance_sampling_ratio/min": 0.4575092285871506, "sampling/sampling_logp_difference/max": 0.27222853899002075, "sampling/sampling_logp_difference/mean": 0.004479383436652521, "step": 30, "step_time": 7.885652181366458, "student/entropy": 0.2029941453287999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722224300106365, "completions/max_length": 503.46666666666664, "completions/max_terminated_length": 465.2, "completions/mean_length": 195.39556681315105, "completions/mean_terminated_length": 185.8046076456706, "completions/min_length": 45.5, "completions/min_terminated_length": 45.5, "entropy": 0.19655973594635726, "epoch": 0.002278509854555121, "eval/gt_acc_batch": 0.723888905843099, "frac_reward_zero_std": 1.0, "grad_norm": 0.47802847623825073, "kd/policy_loss": 0.013037882768549025, "kd/rkl_advantage_mean": 0.3268238168520232, "kd/rkl_advantage_p95": 3.9106563359498976, "kd/rkl_advantage_std": 2.0042735169331234, "kd/ssd_loss": 0.0, "learning_rate": 9.977594653096876e-07, "loss": 0.05215153694152832, "num_tokens": 1983783.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.723888885974884, "rewards/gt_logging_reward/std": 0.43908915122350056, "sampling/importance_sampling_ratio/max": 1.9542729576428732, "sampling/importance_sampling_ratio/mean": 0.99688547650973, "sampling/importance_sampling_ratio/min": 0.4774436811606089, "sampling/sampling_logp_difference/max": 0.2812481959660848, "sampling/sampling_logp_difference/mean": 0.004415741958655417, "step": 60, "step_time": 7.903740958504689, "student/entropy": 0.19655973594635726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030277779418975116, "completions/max_length": 495.23333333333335, "completions/max_terminated_length": 461.1666666666667, "completions/mean_length": 195.65695444742838, "completions/mean_terminated_length": 185.7668248494466, "completions/min_length": 47.06666666666667, "completions/min_terminated_length": 47.06666666666667, "entropy": 0.18201517096410194, "epoch": 0.0034177647818326813, "eval/gt_acc_batch": 0.7461111307144165, "frac_reward_zero_std": 1.0, "grad_norm": 0.28869009017944336, "kd/policy_loss": 0.012130236094041418, "kd/rkl_advantage_mean": 0.27249475416998997, "kd/rkl_advantage_p95": 3.8112845798333486, "kd/rkl_advantage_std": 1.9830802311499913, "kd/ssd_loss": 0.0, "learning_rate": 9.9662021038241e-07, "loss": 0.04852093855539958, "num_tokens": 2993028.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7461111108462016, "rewards/gt_logging_reward/std": 0.4324429521958033, "sampling/importance_sampling_ratio/max": 1.985810387134552, "sampling/importance_sampling_ratio/mean": 0.9974713385105133, "sampling/importance_sampling_ratio/min": 0.4217121745149294, "sampling/sampling_logp_difference/max": 0.3127308944861094, "sampling/sampling_logp_difference/mean": 0.004246281879022718, "step": 90, "step_time": 8.030283141244825, "student/entropy": 0.18201517096410194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334668229023, "completions/max_length": 475.8666666666667, "completions/max_terminated_length": 437.6666666666667, "completions/mean_length": 184.11834309895832, "completions/mean_terminated_length": 177.2191925048828, "completions/min_length": 47.96666666666667, "completions/min_terminated_length": 47.96666666666667, "entropy": 0.1702964631219705, "epoch": 0.004557019709110242, "eval/gt_acc_batch": 0.7655555764834087, "frac_reward_zero_std": 1.0, "grad_norm": 0.2791326642036438, "kd/policy_loss": 0.011101469253965964, "kd/rkl_advantage_mean": 0.29705981047203145, "kd/rkl_advantage_p95": 3.889968055486679, "kd/rkl_advantage_std": 1.9908931851387024, "kd/ssd_loss": 0.0, "learning_rate": 9.954809554551323e-07, "loss": 0.04440587361653646, "num_tokens": 3966214.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7655555605888367, "rewards/gt_logging_reward/std": 0.4159337828556697, "sampling/importance_sampling_ratio/max": 1.946328834692637, "sampling/importance_sampling_ratio/mean": 1.0006588041782378, "sampling/importance_sampling_ratio/min": 0.47153339087963103, "sampling/sampling_logp_difference/max": 0.2840041796366374, "sampling/sampling_logp_difference/mean": 0.004205367582229277, "step": 120, "step_time": 7.771810823880757, "student/entropy": 0.1702964631219705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016666667411724727, "completions/max_length": 483.93333333333334, "completions/max_terminated_length": 458.8666666666667, "completions/mean_length": 187.3025115966797, "completions/mean_terminated_length": 181.9511276245117, "completions/min_length": 53.1, "completions/min_terminated_length": 53.1, "entropy": 0.17139346152544022, "epoch": 0.005696274636387802, "eval/gt_acc_batch": 0.7683333476384481, "frac_reward_zero_std": 1.0, "grad_norm": 0.28446802496910095, "kd/policy_loss": 0.011136007619400819, "kd/rkl_advantage_mean": 0.30794541554835936, "kd/rkl_advantage_p95": 3.914899532000224, "kd/rkl_advantage_std": 2.0140340367952985, "kd/ssd_loss": 0.0, "learning_rate": 9.943417005278546e-07, "loss": 0.044544029235839847, "num_tokens": 4941935.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7683333297570546, "rewards/gt_logging_reward/std": 0.41198851565519967, "sampling/importance_sampling_ratio/max": 1.8517481009165446, "sampling/importance_sampling_ratio/mean": 0.9994672973950703, "sampling/importance_sampling_ratio/min": 0.3872102633118629, "sampling/sampling_logp_difference/max": 0.27680769165356955, "sampling/sampling_logp_difference/mean": 0.004217145894654095, "step": 150, "step_time": 7.67959278972509, "student/entropy": 0.17139346152544022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333448196451, "completions/max_length": 483.3333333333333, "completions/max_terminated_length": 439.8, "completions/mean_length": 190.27195383707684, "completions/mean_terminated_length": 183.5467737833659, "completions/min_length": 49.56666666666667, "completions/min_terminated_length": 49.56666666666667, "entropy": 0.15777629570414622, "epoch": 0.0068355295636653626, "eval/gt_acc_batch": 0.7630555729071299, "frac_reward_zero_std": 1.0, "grad_norm": 0.3181924819946289, "kd/policy_loss": 0.011039282781227181, "kd/rkl_advantage_mean": 0.3438058839334796, "kd/rkl_advantage_p95": 4.038741926352183, "kd/rkl_advantage_std": 2.029553493857384, "kd/ssd_loss": 0.0, "learning_rate": 9.932024456005772e-07, "loss": 0.044157135486602786, "num_tokens": 5927162.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.763055553038915, "rewards/gt_logging_reward/std": 0.41541487773259483, "sampling/importance_sampling_ratio/max": 1.995250121752421, "sampling/importance_sampling_ratio/mean": 1.0013793885707856, "sampling/importance_sampling_ratio/min": 0.46634476880232495, "sampling/sampling_logp_difference/max": 0.26239971319834393, "sampling/sampling_logp_difference/mean": 0.003986104313905041, "step": 180, "step_time": 7.824581230175681, "student/entropy": 0.15777629570414622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018333334382623433, "completions/max_length": 483.43333333333334, "completions/max_terminated_length": 433.03333333333336, "completions/mean_length": 189.06250915527343, "completions/mean_terminated_length": 183.01174875895182, "completions/min_length": 54.06666666666667, "completions/min_terminated_length": 54.06666666666667, "entropy": 0.16184666175395251, "epoch": 0.007974784490942924, "eval/gt_acc_batch": 0.757500022649765, "frac_reward_zero_std": 1.0, "grad_norm": 0.31827056407928467, "kd/policy_loss": 0.011123116644254576, "kd/rkl_advantage_mean": 0.3897760251381745, "kd/rkl_advantage_p95": 4.137957958380381, "kd/rkl_advantage_std": 2.025299736857414, "kd/ssd_loss": 0.0, "learning_rate": 9.920631906732996e-07, "loss": 0.04449246724446614, "num_tokens": 6905203.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.75750000278155, "rewards/gt_logging_reward/std": 0.4231238047281901, "sampling/importance_sampling_ratio/max": 1.8898478945096333, "sampling/importance_sampling_ratio/mean": 0.9948788781960806, "sampling/importance_sampling_ratio/min": 0.4167682041724523, "sampling/sampling_logp_difference/max": 0.2742826998233795, "sampling/sampling_logp_difference/mean": 0.004086840362288058, "step": 210, "step_time": 7.628092233848292, "student/entropy": 0.16184666175395251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890128582715, "completions/max_length": 485.1333333333333, "completions/max_terminated_length": 431.56666666666666, "completions/mean_length": 187.1519536336263, "completions/mean_terminated_length": 180.2242421468099, "completions/min_length": 44.43333333333333, "completions/min_terminated_length": 44.43333333333333, "entropy": 0.15420423752317827, "epoch": 0.009114039418220483, "eval/gt_acc_batch": 0.7691666821638743, "frac_reward_zero_std": 1.0, "grad_norm": 0.29642724990844727, "kd/policy_loss": 0.01012023645453155, "kd/rkl_advantage_mean": 0.44419000793326024, "kd/rkl_advantage_p95": 4.182680708169937, "kd/rkl_advantage_std": 2.0236889561017355, "kd/ssd_loss": 0.0, "learning_rate": 9.909239357460222e-07, "loss": 0.04048094352086385, "num_tokens": 7871902.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7691666682561239, "rewards/gt_logging_reward/std": 0.41583429972330727, "sampling/importance_sampling_ratio/max": 1.9532995144526164, "sampling/importance_sampling_ratio/mean": 0.9994506537914276, "sampling/importance_sampling_ratio/min": 0.463630481561025, "sampling/sampling_logp_difference/max": 0.28302475611368816, "sampling/sampling_logp_difference/mean": 0.004033791902475059, "step": 240, "step_time": 7.651990339788608, "student/entropy": 0.15420423752317827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055557422339915, "completions/max_length": 499.3, "completions/max_terminated_length": 466.2, "completions/mean_length": 193.70028889973958, "completions/mean_terminated_length": 184.71808115641275, "completions/min_length": 50.4, "completions/min_terminated_length": 50.4, "entropy": 0.1553508721292019, "epoch": 0.010253294345498045, "eval/gt_acc_batch": 0.7497222383817037, "frac_reward_zero_std": 1.0, "grad_norm": 0.4439185857772827, "kd/policy_loss": 0.010542551445541903, "kd/rkl_advantage_mean": 0.3405587593772604, "kd/rkl_advantage_p95": 4.016158455610276, "kd/rkl_advantage_std": 2.0188363720973332, "kd/ssd_loss": 0.0, "learning_rate": 9.897846808187445e-07, "loss": 0.04217021465301514, "num_tokens": 8872327.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7497222205003102, "rewards/gt_logging_reward/std": 0.42170828878879546, "sampling/importance_sampling_ratio/max": 1.8455493092536925, "sampling/importance_sampling_ratio/mean": 1.0015559593836467, "sampling/importance_sampling_ratio/min": 0.45648245016733807, "sampling/sampling_logp_difference/max": 0.32204410433769226, "sampling/sampling_logp_difference/mean": 0.004114525183103979, "step": 270, "step_time": 7.861769156728405, "student/entropy": 0.1553508721292019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556677281857, "completions/max_length": 457.8666666666667, "completions/max_terminated_length": 434.3, "completions/mean_length": 177.24584350585937, "completions/mean_terminated_length": 172.0651112874349, "completions/min_length": 59.03333333333333, "completions/min_terminated_length": 59.03333333333333, "entropy": 0.15365676948179802, "epoch": 0.011392549272775604, "eval/gt_acc_batch": 0.779722253481547, "frac_reward_zero_std": 1.0, "grad_norm": 0.33860477805137634, "kd/policy_loss": 0.010421268160765369, "kd/rkl_advantage_mean": 0.40006949002854525, "kd/rkl_advantage_p95": 4.114296293258667, "kd/rkl_advantage_std": 2.0460882812738417, "kd/ssd_loss": 0.0, "learning_rate": 9.88645425891467e-07, "loss": 0.04168507258097331, "num_tokens": 9805596.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7797222276528676, "rewards/gt_logging_reward/std": 0.4046100532015165, "sampling/importance_sampling_ratio/max": 1.769742524623871, "sampling/importance_sampling_ratio/mean": 0.99597261150678, "sampling/importance_sampling_ratio/min": 0.4837028334538142, "sampling/sampling_logp_difference/max": 0.29774684508641563, "sampling/sampling_logp_difference/mean": 0.00405015309030811, "step": 300, "step_time": 7.470101887814235, "student/entropy": 0.15365676948179802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055557484428086, "completions/max_length": 489.3666666666667, "completions/max_terminated_length": 461.6, "completions/mean_length": 194.38917694091796, "completions/mean_terminated_length": 185.24178924560547, "completions/min_length": 44.06666666666667, "completions/min_terminated_length": 44.06666666666667, "entropy": 0.15853289403021337, "epoch": 0.012531804200053166, "eval/gt_acc_batch": 0.7630555768807729, "frac_reward_zero_std": 1.0, "grad_norm": 0.2567886710166931, "kd/policy_loss": 0.011201375493934998, "kd/rkl_advantage_mean": 0.4160057204309851, "kd/rkl_advantage_p95": 4.156571886936823, "kd/rkl_advantage_std": 2.04227860669295, "kd/ssd_loss": 0.0, "learning_rate": 9.875061709641895e-07, "loss": 0.044805502891540526, "num_tokens": 10812701.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.763055557012558, "rewards/gt_logging_reward/std": 0.41593413253625233, "sampling/importance_sampling_ratio/max": 1.88770885070165, "sampling/importance_sampling_ratio/mean": 1.0014776666959126, "sampling/importance_sampling_ratio/min": 0.43548253178596497, "sampling/sampling_logp_difference/max": 0.33984741965929666, "sampling/sampling_logp_difference/mean": 0.004208573054832717, "step": 330, "step_time": 7.850916166669534, "student/entropy": 0.15853289403021337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017222223120431104, "completions/max_length": 479.0, "completions/max_terminated_length": 443.3333333333333, "completions/mean_length": 189.57362009684246, "completions/mean_terminated_length": 183.9298589070638, "completions/min_length": 55.333333333333336, "completions/min_terminated_length": 55.333333333333336, "entropy": 0.13943199117978414, "epoch": 0.013671059127330725, "eval/gt_acc_batch": 0.7850000301996867, "frac_reward_zero_std": 1.0, "grad_norm": 0.2700779139995575, "kd/policy_loss": 0.00939162780996412, "kd/rkl_advantage_mean": 0.3628113689133897, "kd/rkl_advantage_p95": 4.056593650579453, "kd/rkl_advantage_std": 2.0081779569387437, "kd/ssd_loss": 0.0, "learning_rate": 9.863669160369118e-07, "loss": 0.037566510836283366, "num_tokens": 11807766.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7850000043710073, "rewards/gt_logging_reward/std": 0.4038076023260752, "sampling/importance_sampling_ratio/max": 1.8830128192901612, "sampling/importance_sampling_ratio/mean": 1.000617935260137, "sampling/importance_sampling_ratio/min": 0.4885498543580373, "sampling/sampling_logp_difference/max": 0.32463822364807127, "sampling/sampling_logp_difference/mean": 0.003892183490097523, "step": 360, "step_time": 7.804262704406089, "student/entropy": 0.13943199117978414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.031111113106211025, "completions/max_length": 489.3333333333333, "completions/max_terminated_length": 447.93333333333334, "completions/mean_length": 190.9025095621745, "completions/mean_terminated_length": 180.78719685872395, "completions/min_length": 53.93333333333333, "completions/min_terminated_length": 53.93333333333333, "entropy": 0.15333793237805365, "epoch": 0.014810314054608286, "eval/gt_acc_batch": 0.7547222435474396, "frac_reward_zero_std": 1.0, "grad_norm": 0.29113906621932983, "kd/policy_loss": 0.010763291265660276, "kd/rkl_advantage_mean": 0.41609783358095837, "kd/rkl_advantage_p95": 4.173465975125631, "kd/rkl_advantage_std": 2.028749202688535, "kd/ssd_loss": 0.0, "learning_rate": 9.852276611096342e-07, "loss": 0.043053166071573896, "num_tokens": 12794863.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7547222236792247, "rewards/gt_logging_reward/std": 0.4254201134045919, "sampling/importance_sampling_ratio/max": 1.8480279962221782, "sampling/importance_sampling_ratio/mean": 0.9982978244622548, "sampling/importance_sampling_ratio/min": 0.45347995261351265, "sampling/sampling_logp_difference/max": 0.3424540042877197, "sampling/sampling_logp_difference/mean": 0.00413392271536092, "step": 390, "step_time": 7.738586896366906, "student/entropy": 0.15333793237805365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014166667498648167, "completions/max_length": 473.6666666666667, "completions/max_terminated_length": 445.93333333333334, "completions/mean_length": 184.82306416829428, "completions/mean_terminated_length": 180.31780141194662, "completions/min_length": 55.56666666666667, "completions/min_terminated_length": 55.56666666666667, "entropy": 0.14132583755999803, "epoch": 0.015949568981885848, "eval/gt_acc_batch": 0.7955555737018585, "frac_reward_zero_std": 1.0, "grad_norm": 0.29947349429130554, "kd/policy_loss": 0.009550300537375734, "kd/rkl_advantage_mean": 0.3828863142931368, "kd/rkl_advantage_p95": 4.193539774417877, "kd/rkl_advantage_std": 2.0660841882228853, "kd/ssd_loss": 0.0, "learning_rate": 9.840884061823566e-07, "loss": 0.03820120890935262, "num_tokens": 13754402.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.795555563767751, "rewards/gt_logging_reward/std": 0.3925007184346517, "sampling/importance_sampling_ratio/max": 1.875376558303833, "sampling/importance_sampling_ratio/mean": 1.0012089172999064, "sampling/importance_sampling_ratio/min": 0.4797952324151993, "sampling/sampling_logp_difference/max": 0.3346335828304291, "sampling/sampling_logp_difference/mean": 0.003995176893658936, "step": 420, "step_time": 7.599777976481709, "student/entropy": 0.14132583755999803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722224175930022, "completions/max_length": 492.1333333333333, "completions/max_terminated_length": 444.6, "completions/mean_length": 191.62806599934896, "completions/mean_terminated_length": 181.9193094889323, "completions/min_length": 45.43333333333333, "completions/min_terminated_length": 45.43333333333333, "entropy": 0.14340751251826683, "epoch": 0.017088823909163407, "eval/gt_acc_batch": 0.7650000075499217, "frac_reward_zero_std": 1.0, "grad_norm": 0.2561100721359253, "kd/policy_loss": 0.010143733047880233, "kd/rkl_advantage_mean": 0.37719604197579126, "kd/rkl_advantage_p95": 4.20889135201772, "kd/rkl_advantage_std": 2.0824067334334058, "kd/ssd_loss": 0.0, "learning_rate": 9.829491512550791e-07, "loss": 0.04057493209838867, "num_tokens": 14752295.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7650000015894572, "rewards/gt_logging_reward/std": 0.41605375409126283, "sampling/importance_sampling_ratio/max": 2.015727432568868, "sampling/importance_sampling_ratio/mean": 1.005115513006846, "sampling/importance_sampling_ratio/min": 0.45442722340424857, "sampling/sampling_logp_difference/max": 0.39062931934992473, "sampling/sampling_logp_difference/mean": 0.004068104084581137, "step": 450, "step_time": 7.94331601227168, "student/entropy": 0.14340751251826683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668020188808, "completions/max_length": 493.26666666666665, "completions/max_terminated_length": 450.8666666666667, "completions/mean_length": 194.21556549072267, "completions/mean_terminated_length": 186.7259063720703, "completions/min_length": 56.46666666666667, "completions/min_terminated_length": 56.46666666666667, "entropy": 0.1411548318962256, "epoch": 0.018228078836440967, "eval/gt_acc_batch": 0.7563889106114705, "frac_reward_zero_std": 1.0, "grad_norm": 0.2622755169868469, "kd/policy_loss": 0.009793725827087959, "kd/rkl_advantage_mean": 0.37990102749705934, "kd/rkl_advantage_p95": 4.182383513450622, "kd/rkl_advantage_std": 2.075037932395935, "kd/ssd_loss": 0.0, "learning_rate": 9.818098963278015e-07, "loss": 0.03917490641276042, "num_tokens": 15752231.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7563888907432557, "rewards/gt_logging_reward/std": 0.4220193445682526, "sampling/importance_sampling_ratio/max": 1.9249449451764424, "sampling/importance_sampling_ratio/mean": 0.9980144182840983, "sampling/importance_sampling_ratio/min": 0.48571751912434896, "sampling/sampling_logp_difference/max": 0.3790138145287832, "sampling/sampling_logp_difference/mean": 0.004046919693549474, "step": 480, "step_time": 7.863551015628036, "student/entropy": 0.1411548318962256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029444445949047805, "completions/max_length": 488.8666666666667, "completions/max_terminated_length": 456.26666666666665, "completions/mean_length": 188.29223175048827, "completions/mean_terminated_length": 178.7053227742513, "completions/min_length": 50.36666666666667, "completions/min_terminated_length": 50.36666666666667, "entropy": 0.14622472779204448, "epoch": 0.019367333763718526, "eval/gt_acc_batch": 0.7755555848280589, "frac_reward_zero_std": 1.0, "grad_norm": 0.2487870752811432, "kd/policy_loss": 0.009903502669961503, "kd/rkl_advantage_mean": 0.4074699590482245, "kd/rkl_advantage_p95": 4.28292181690534, "kd/rkl_advantage_std": 2.1078390469153723, "kd/ssd_loss": 0.0, "learning_rate": 9.80670641400524e-07, "loss": 0.03961400985717774, "num_tokens": 16732571.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7755555510520935, "rewards/gt_logging_reward/std": 0.4081752300262451, "sampling/importance_sampling_ratio/max": 1.8574137608210246, "sampling/importance_sampling_ratio/mean": 0.9997616767883301, "sampling/importance_sampling_ratio/min": 0.3839781532684962, "sampling/sampling_logp_difference/max": 0.3663038671016693, "sampling/sampling_logp_difference/mean": 0.004202438510643939, "step": 510, "step_time": 7.860085966651483, "student/entropy": 0.14622472779204448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556658655404, "completions/max_length": 474.46666666666664, "completions/max_terminated_length": 442.6666666666667, "completions/mean_length": 181.78084309895834, "completions/mean_terminated_length": 175.08450063069662, "completions/min_length": 46.13333333333333, "completions/min_terminated_length": 46.13333333333333, "entropy": 0.1426143292337656, "epoch": 0.02050658869099609, "eval/gt_acc_batch": 0.7880555788675944, "frac_reward_zero_std": 1.0, "grad_norm": 0.3439692556858063, "kd/policy_loss": 0.009800962308266511, "kd/rkl_advantage_mean": 0.4476227140519768, "kd/rkl_advantage_p95": 4.288417659203211, "kd/rkl_advantage_std": 2.079409404595693, "kd/ssd_loss": 0.0, "learning_rate": 9.795313864732464e-07, "loss": 0.03920385042826335, "num_tokens": 17691302.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7880555470784505, "rewards/gt_logging_reward/std": 0.3955184519290924, "sampling/importance_sampling_ratio/max": 1.9108129064242045, "sampling/importance_sampling_ratio/mean": 1.002501873175303, "sampling/importance_sampling_ratio/min": 0.463145304719607, "sampling/sampling_logp_difference/max": 0.3244284232457479, "sampling/sampling_logp_difference/mean": 0.004124870981710652, "step": 540, "step_time": 7.847898686913928, "student/entropy": 0.1426143292337656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01805555643513799, "completions/max_length": 473.6333333333333, "completions/max_terminated_length": 431.03333333333336, "completions/mean_length": 183.23861999511718, "completions/mean_terminated_length": 177.39219919840494, "completions/min_length": 57.06666666666667, "completions/min_terminated_length": 57.06666666666667, "entropy": 0.13455340353151163, "epoch": 0.02164584361827365, "eval/gt_acc_batch": 0.7950000246365865, "frac_reward_zero_std": 1.0, "grad_norm": 0.27403488755226135, "kd/policy_loss": 0.009416112274629995, "kd/rkl_advantage_mean": 0.36873630223696335, "kd/rkl_advantage_p95": 4.148748598496119, "kd/rkl_advantage_std": 2.0706773926814397, "kd/ssd_loss": 0.0, "learning_rate": 9.783921315459688e-07, "loss": 0.03766444524129232, "num_tokens": 18651465.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7950000027815501, "rewards/gt_logging_reward/std": 0.39625293711821236, "sampling/importance_sampling_ratio/max": 1.9384885907173157, "sampling/importance_sampling_ratio/mean": 1.0050499478975932, "sampling/importance_sampling_ratio/min": 0.47230272988478345, "sampling/sampling_logp_difference/max": 0.4102041006088257, "sampling/sampling_logp_difference/mean": 0.004004332531864444, "step": 570, "step_time": 7.69368868387537, "student/entropy": 0.13455340353151163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556969096264, "completions/max_length": 485.53333333333336, "completions/max_terminated_length": 442.03333333333336, "completions/mean_length": 186.77945454915366, "completions/mean_terminated_length": 180.0146275838216, "completions/min_length": 58.766666666666666, "completions/min_terminated_length": 58.766666666666666, "entropy": 0.13927260829756657, "epoch": 0.02278509854555121, "eval/gt_acc_batch": 0.7733333388964335, "frac_reward_zero_std": 1.0, "grad_norm": 0.24254149198532104, "kd/policy_loss": 0.010074080315340931, "kd/rkl_advantage_mean": 0.4601244098206128, "kd/rkl_advantage_p95": 4.289574770132701, "kd/rkl_advantage_std": 2.0753382553656894, "kd/ssd_loss": 0.0, "learning_rate": 9.772528766186914e-07, "loss": 0.040296324094136554, "num_tokens": 19627471.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7733333329359691, "rewards/gt_logging_reward/std": 0.4112900694211324, "sampling/importance_sampling_ratio/max": 1.96339408159256, "sampling/importance_sampling_ratio/mean": 1.0031494975090027, "sampling/importance_sampling_ratio/min": 0.46231493453184763, "sampling/sampling_logp_difference/max": 0.36232064565022787, "sampling/sampling_logp_difference/mean": 0.0041303595139955485, "step": 600, "step_time": 7.854809297442746, "student/entropy": 0.13927260829756657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444571932157, "completions/max_length": 484.56666666666666, "completions/max_terminated_length": 463.46666666666664, "completions/mean_length": 190.37056681315104, "completions/mean_terminated_length": 182.43284352620444, "completions/min_length": 57.833333333333336, "completions/min_terminated_length": 57.833333333333336, "entropy": 0.13945921019961435, "epoch": 0.02392435347282877, "eval/gt_acc_batch": 0.7700000087420146, "frac_reward_zero_std": 1.0, "grad_norm": 0.2700849175453186, "kd/policy_loss": 0.009955698476793865, "kd/rkl_advantage_mean": 0.44710651903102794, "kd/rkl_advantage_p95": 4.316534155607224, "kd/rkl_advantage_std": 2.066954290866852, "kd/ssd_loss": 0.0, "learning_rate": 9.761136216914138e-07, "loss": 0.03982279300689697, "num_tokens": 20611565.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7700000007947286, "rewards/gt_logging_reward/std": 0.411551421880722, "sampling/importance_sampling_ratio/max": 1.82971484263738, "sampling/importance_sampling_ratio/mean": 0.9980163673559824, "sampling/importance_sampling_ratio/min": 0.43132879535357155, "sampling/sampling_logp_difference/max": 0.37539899349212646, "sampling/sampling_logp_difference/mean": 0.0041238917348285515, "step": 630, "step_time": 7.880207194733278, "student/entropy": 0.13945921019961435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001080334186, "completions/max_length": 482.3333333333333, "completions/max_terminated_length": 447.5, "completions/mean_length": 192.34917653401692, "completions/mean_terminated_length": 185.05050862630208, "completions/min_length": 56.93333333333333, "completions/min_terminated_length": 56.93333333333333, "entropy": 0.13604440626998743, "epoch": 0.02506360840010633, "eval/gt_acc_batch": 0.7391666869322459, "frac_reward_zero_std": 1.0, "grad_norm": 0.26972514390945435, "kd/policy_loss": 0.00971394965502744, "kd/rkl_advantage_mean": 0.43573887475261774, "kd/rkl_advantage_p95": 4.294135334094365, "kd/rkl_advantage_std": 2.0947425991296766, "kd/ssd_loss": 0.0, "learning_rate": 9.749743667641363e-07, "loss": 0.038855799039204914, "num_tokens": 21611230.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7391666650772095, "rewards/gt_logging_reward/std": 0.43218535085519155, "sampling/importance_sampling_ratio/max": 1.986191991964976, "sampling/importance_sampling_ratio/mean": 1.0037957549095153, "sampling/importance_sampling_ratio/min": 0.44674121886491774, "sampling/sampling_logp_difference/max": 0.35540513396263124, "sampling/sampling_logp_difference/mean": 0.00408477804157883, "step": 660, "step_time": 7.846261426092436, "student/entropy": 0.13604440626998743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777912716071, "completions/max_length": 500.43333333333334, "completions/max_terminated_length": 452.93333333333334, "completions/mean_length": 194.68112131754557, "completions/mean_terminated_length": 186.46819712320965, "completions/min_length": 50.03333333333333, "completions/min_terminated_length": 50.03333333333333, "entropy": 0.1426631100475788, "epoch": 0.02620286332738389, "eval/gt_acc_batch": 0.7677777945995331, "frac_reward_zero_std": 1.0, "grad_norm": 0.3072415590286255, "kd/policy_loss": 0.009705559837554271, "kd/rkl_advantage_mean": 0.4660925720915354, "kd/rkl_advantage_p95": 4.405044122536977, "kd/rkl_advantage_std": 2.099466252326965, "kd/ssd_loss": 0.0, "learning_rate": 9.738351118368587e-07, "loss": 0.03882224162419637, "num_tokens": 22627594.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7677777787049611, "rewards/gt_logging_reward/std": 0.41670821110407513, "sampling/importance_sampling_ratio/max": 1.9280497749646506, "sampling/importance_sampling_ratio/mean": 0.9964009046554565, "sampling/importance_sampling_ratio/min": 0.43074339826901753, "sampling/sampling_logp_difference/max": 0.37814860542615253, "sampling/sampling_logp_difference/mean": 0.004066226266634961, "step": 690, "step_time": 8.091164884564932, "student/entropy": 0.1426631100475788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01722222330669562, "completions/max_length": 484.9, "completions/max_terminated_length": 442.3, "completions/mean_length": 182.7969533284505, "completions/mean_terminated_length": 177.06156311035156, "completions/min_length": 51.06666666666667, "completions/min_terminated_length": 51.06666666666667, "entropy": 0.1351904353747765, "epoch": 0.02734211825466145, "eval/gt_acc_batch": 0.7869444529215495, "frac_reward_zero_std": 1.0, "grad_norm": 0.2781446874141693, "kd/policy_loss": 0.009253040524587657, "kd/rkl_advantage_mean": 0.45914874530086913, "kd/rkl_advantage_p95": 4.394913913806279, "kd/rkl_advantage_std": 2.1079026718934375, "kd/ssd_loss": 0.0, "learning_rate": 9.72695856909581e-07, "loss": 0.03701215982437134, "num_tokens": 23577199.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.786944446961085, "rewards/gt_logging_reward/std": 0.4002072314421336, "sampling/importance_sampling_ratio/max": 1.7663983821868896, "sampling/importance_sampling_ratio/mean": 0.995485124985377, "sampling/importance_sampling_ratio/min": 0.4424729789296786, "sampling/sampling_logp_difference/max": 0.3894407510757446, "sampling/sampling_logp_difference/mean": 0.0040714646767204005, "step": 720, "step_time": 7.662151861124827, "student/entropy": 0.1351904353747765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.011666667430351179, "completions/max_length": 479.96666666666664, "completions/max_terminated_length": 444.1666666666667, "completions/mean_length": 185.07445526123047, "completions/mean_terminated_length": 181.31953125, "completions/min_length": 53.7, "completions/min_terminated_length": 53.7, "entropy": 0.1302966583520174, "epoch": 0.028481373181939013, "eval/gt_acc_batch": 0.8038889050483704, "frac_reward_zero_std": 1.0, "grad_norm": 0.22736547887325287, "kd/policy_loss": 0.00871235299273394, "kd/rkl_advantage_mean": 0.4638107375203011, "kd/rkl_advantage_p95": 4.355591390530268, "kd/rkl_advantage_std": 2.0973064690828322, "kd/ssd_loss": 0.0, "learning_rate": 9.715566019823034e-07, "loss": 0.03484941323598226, "num_tokens": 24545275.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8038888931274414, "rewards/gt_logging_reward/std": 0.38952807982762655, "sampling/importance_sampling_ratio/max": 1.9141941865285237, "sampling/importance_sampling_ratio/mean": 1.0008125305175781, "sampling/importance_sampling_ratio/min": 0.4044308945536613, "sampling/sampling_logp_difference/max": 0.3755131463209788, "sampling/sampling_logp_difference/mean": 0.003983167395927012, "step": 750, "step_time": 7.773384385940153, "student/entropy": 0.1302966583520174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01555555642892917, "completions/max_length": 496.8333333333333, "completions/max_terminated_length": 452.3, "completions/mean_length": 187.51250915527345, "completions/mean_terminated_length": 182.51211496988932, "completions/min_length": 58.06666666666667, "completions/min_terminated_length": 58.06666666666667, "entropy": 0.13024521954357623, "epoch": 0.029620628109216573, "eval/gt_acc_batch": 0.7900000194708506, "frac_reward_zero_std": 1.0, "grad_norm": 0.27903661131858826, "kd/policy_loss": 0.008959814921642344, "kd/rkl_advantage_mean": 0.47128258938901124, "kd/rkl_advantage_p95": 4.425841889778773, "kd/rkl_advantage_std": 2.125308553377787, "kd/ssd_loss": 0.0, "learning_rate": 9.70417347055026e-07, "loss": 0.03583926757176717, "num_tokens": 25526264.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7900000055631001, "rewards/gt_logging_reward/std": 0.3976499944925308, "sampling/importance_sampling_ratio/max": 1.8693394263585408, "sampling/importance_sampling_ratio/mean": 0.9993054568767548, "sampling/importance_sampling_ratio/min": 0.46665342648824054, "sampling/sampling_logp_difference/max": 0.3991669982671738, "sampling/sampling_logp_difference/mean": 0.004013178536357979, "step": 780, "step_time": 8.0328287954481, "student/entropy": 0.13024521954357623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778965731462, "completions/max_length": 479.46666666666664, "completions/max_terminated_length": 447.3333333333333, "completions/mean_length": 187.7672317504883, "completions/mean_terminated_length": 180.35535685221353, "completions/min_length": 56.86666666666667, "completions/min_terminated_length": 56.86666666666667, "entropy": 0.13549417797476054, "epoch": 0.030759883036494132, "eval/gt_acc_batch": 0.7911111315091451, "frac_reward_zero_std": 1.0, "grad_norm": 0.2940017282962799, "kd/policy_loss": 0.009195210505276918, "kd/rkl_advantage_mean": 0.4105372965617183, "kd/rkl_advantage_p95": 4.2755749722321825, "kd/rkl_advantage_std": 2.097934796412786, "kd/ssd_loss": 0.0, "learning_rate": 9.692780921277484e-07, "loss": 0.036780846118927, "num_tokens": 26507650.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7911111136277517, "rewards/gt_logging_reward/std": 0.3993778293331464, "sampling/importance_sampling_ratio/max": 1.8894100228945414, "sampling/importance_sampling_ratio/mean": 1.0030367096265158, "sampling/importance_sampling_ratio/min": 0.436696927746137, "sampling/sampling_logp_difference/max": 0.37654449939727785, "sampling/sampling_logp_difference/mean": 0.004130651662126184, "step": 810, "step_time": 7.920025414938573, "student/entropy": 0.13549417797476054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555662761132, "completions/max_length": 471.43333333333334, "completions/max_terminated_length": 442.93333333333334, "completions/mean_length": 188.99862213134764, "completions/mean_terminated_length": 182.22716420491537, "completions/min_length": 59.06666666666667, "completions/min_terminated_length": 59.06666666666667, "entropy": 0.13113776929676532, "epoch": 0.031899137963771695, "eval/gt_acc_batch": 0.7977777898311615, "frac_reward_zero_std": 1.0, "grad_norm": 0.3167026937007904, "kd/policy_loss": 0.008760214527137577, "kd/rkl_advantage_mean": 0.49579665608083207, "kd/rkl_advantage_p95": 4.406716956694921, "kd/rkl_advantage_std": 2.100488053758939, "kd/ssd_loss": 0.0, "learning_rate": 9.68138837200471e-07, "loss": 0.03504086335500081, "num_tokens": 27493437.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.797777775923411, "rewards/gt_logging_reward/std": 0.3927893136938413, "sampling/importance_sampling_ratio/max": 1.8255353331565858, "sampling/importance_sampling_ratio/mean": 0.9953386028607686, "sampling/importance_sampling_ratio/min": 0.4383034035563469, "sampling/sampling_logp_difference/max": 0.41399598916371666, "sampling/sampling_logp_difference/mean": 0.0040854938871537645, "step": 840, "step_time": 7.795423277063916, "student/entropy": 0.13113776929676532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112466702857, "completions/max_length": 471.8333333333333, "completions/max_terminated_length": 431.1, "completions/mean_length": 191.52001037597657, "completions/mean_terminated_length": 183.71764577229817, "completions/min_length": 63.36666666666667, "completions/min_terminated_length": 63.36666666666667, "entropy": 0.13077728723486265, "epoch": 0.033038392891049255, "eval/gt_acc_batch": 0.7825000246365865, "frac_reward_zero_std": 1.0, "grad_norm": 0.34009861946105957, "kd/policy_loss": 0.009060778591083363, "kd/rkl_advantage_mean": 0.3144032231997699, "kd/rkl_advantage_p95": 4.078970756133398, "kd/rkl_advantage_std": 2.047230197985967, "kd/ssd_loss": 0.0, "learning_rate": 9.669995822731933e-07, "loss": 0.036243116855621337, "num_tokens": 28489541.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7825000007947286, "rewards/gt_logging_reward/std": 0.40601938863595327, "sampling/importance_sampling_ratio/max": 1.9155193527539571, "sampling/importance_sampling_ratio/mean": 1.0023401161034902, "sampling/importance_sampling_ratio/min": 0.4083817412455877, "sampling/sampling_logp_difference/max": 0.39253672361373904, "sampling/sampling_logp_difference/mean": 0.004067242743136982, "step": 870, "step_time": 7.803381375456229, "student/entropy": 0.13077728723486265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026111112628132104, "completions/max_length": 491.76666666666665, "completions/max_terminated_length": 456.3666666666667, "completions/mean_length": 190.58778737386066, "completions/mean_terminated_length": 182.11893056233723, "completions/min_length": 56.333333333333336, "completions/min_terminated_length": 56.333333333333336, "entropy": 0.13490459906558197, "epoch": 0.034177647818326815, "eval/gt_acc_batch": 0.7894444664319357, "frac_reward_zero_std": 1.0, "grad_norm": 0.25146594643592834, "kd/policy_loss": 0.00953551210113801, "kd/rkl_advantage_mean": 0.4179676368056486, "kd/rkl_advantage_p95": 4.329463563362757, "kd/rkl_advantage_std": 2.101443925499916, "kd/ssd_loss": 0.0, "learning_rate": 9.658603273459157e-07, "loss": 0.03814204533894857, "num_tokens": 29474625.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7894444465637207, "rewards/gt_logging_reward/std": 0.4031149516503016, "sampling/importance_sampling_ratio/max": 1.977028783162435, "sampling/importance_sampling_ratio/mean": 1.0031423171361287, "sampling/importance_sampling_ratio/min": 0.4506333082914352, "sampling/sampling_logp_difference/max": 0.3928524414698283, "sampling/sampling_logp_difference/mean": 0.0041915961386015015, "step": 900, "step_time": 7.931726507640755, "student/entropy": 0.13490459906558197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223468124866, "completions/max_length": 478.2, "completions/max_terminated_length": 431.8, "completions/mean_length": 190.4394561767578, "completions/mean_terminated_length": 184.04926554361978, "completions/min_length": 57.96666666666667, "completions/min_terminated_length": 57.96666666666667, "entropy": 0.1333870379254222, "epoch": 0.035316902745604374, "eval/gt_acc_batch": 0.795000026623408, "frac_reward_zero_std": 1.0, "grad_norm": 0.2882496118545532, "kd/policy_loss": 0.00902173495075355, "kd/rkl_advantage_mean": 0.4238787419628352, "kd/rkl_advantage_p95": 4.3318038562933605, "kd/rkl_advantage_std": 2.094679540395737, "kd/ssd_loss": 0.0, "learning_rate": 9.647210724186382e-07, "loss": 0.03608693679173788, "num_tokens": 30476663.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7950000007947285, "rewards/gt_logging_reward/std": 0.3967317114273707, "sampling/importance_sampling_ratio/max": 1.8449629624684651, "sampling/importance_sampling_ratio/mean": 0.9999599854151408, "sampling/importance_sampling_ratio/min": 0.4286300371090571, "sampling/sampling_logp_difference/max": 0.38531355063120526, "sampling/sampling_logp_difference/mean": 0.004047162450539569, "step": 930, "step_time": 7.952911384574448, "student/entropy": 0.1333870379254222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018055556590358415, "completions/max_length": 493.73333333333335, "completions/max_terminated_length": 451.9, "completions/mean_length": 186.55139719645183, "completions/mean_terminated_length": 180.5032943725586, "completions/min_length": 45.43333333333333, "completions/min_terminated_length": 45.43333333333333, "entropy": 0.12980590853840113, "epoch": 0.036456157672881934, "eval/gt_acc_batch": 0.7861111402511597, "frac_reward_zero_std": 1.0, "grad_norm": 0.24491049349308014, "kd/policy_loss": 0.008555913060748328, "kd/rkl_advantage_mean": 0.4483588564898431, "kd/rkl_advantage_p95": 4.356897507111231, "kd/rkl_advantage_std": 2.1074685782194136, "kd/ssd_loss": 0.0, "learning_rate": 9.635818174913606e-07, "loss": 0.03422364393870036, "num_tokens": 31453608.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111124356588, "rewards/gt_logging_reward/std": 0.4054222196340561, "sampling/importance_sampling_ratio/max": 1.772775900363922, "sampling/importance_sampling_ratio/mean": 0.9887248973051707, "sampling/importance_sampling_ratio/min": 0.4422304277618726, "sampling/sampling_logp_difference/max": 0.37954145272572837, "sampling/sampling_logp_difference/mean": 0.004126150587884088, "step": 960, "step_time": 7.903867139919506, "student/entropy": 0.12980590853840113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500000974784293, "completions/max_length": 482.8333333333333, "completions/max_terminated_length": 450.26666666666665, "completions/mean_length": 181.49723103841146, "completions/mean_terminated_length": 175.67654774983723, "completions/min_length": 54.06666666666667, "completions/min_terminated_length": 54.06666666666667, "entropy": 0.13112400906781355, "epoch": 0.03759541260015949, "eval/gt_acc_batch": 0.7855555713176727, "frac_reward_zero_std": 1.0, "grad_norm": 0.24488502740859985, "kd/policy_loss": 0.009096235865339015, "kd/rkl_advantage_mean": 0.5158614825302114, "kd/rkl_advantage_p95": 4.516036679347356, "kd/rkl_advantage_std": 2.1424601902564366, "kd/ssd_loss": 0.0, "learning_rate": 9.62442562564083e-07, "loss": 0.03638494809468587, "num_tokens": 32405342.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555593967438, "rewards/gt_logging_reward/std": 0.4048471430937449, "sampling/importance_sampling_ratio/max": 1.9286951581637064, "sampling/importance_sampling_ratio/mean": 1.0020315488179525, "sampling/importance_sampling_ratio/min": 0.38562664290269216, "sampling/sampling_logp_difference/max": 0.3808856993913651, "sampling/sampling_logp_difference/mean": 0.00424522323689113, "step": 990, "step_time": 7.735459811434461, "student/entropy": 0.13112400906781355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.033888890563199915, "completions/max_length": 494.3333333333333, "completions/max_terminated_length": 455.43333333333334, "completions/mean_length": 194.93556467692056, "completions/mean_terminated_length": 183.98558756510417, "completions/min_length": 56.766666666666666, "completions/min_terminated_length": 56.766666666666666, "entropy": 0.13406364638358353, "epoch": 0.03873466752743705, "eval/gt_acc_batch": 0.7783333520094554, "frac_reward_zero_std": 1.0, "grad_norm": 0.3306242823600769, "kd/policy_loss": 0.009381091454997659, "kd/rkl_advantage_mean": 0.5222811039498386, "kd/rkl_advantage_p95": 4.4965941429138185, "kd/rkl_advantage_std": 2.122713221112887, "kd/ssd_loss": 0.0, "learning_rate": 9.613033076368056e-07, "loss": 0.03752436637878418, "num_tokens": 33410774.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.778333330154419, "rewards/gt_logging_reward/std": 0.40906290809313456, "sampling/importance_sampling_ratio/max": 1.9052444219589233, "sampling/importance_sampling_ratio/mean": 0.9960593402385711, "sampling/importance_sampling_ratio/min": 0.40988250970840456, "sampling/sampling_logp_difference/max": 0.39618947307268776, "sampling/sampling_logp_difference/mean": 0.004212169861420989, "step": 1020, "step_time": 7.983822884177789, "student/entropy": 0.13406364638358353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334928999346, "completions/max_length": 500.96666666666664, "completions/max_terminated_length": 465.8333333333333, "completions/mean_length": 188.27556508382162, "completions/mean_terminated_length": 179.7653615315755, "completions/min_length": 53.2, "completions/min_terminated_length": 53.2, "entropy": 0.13963014855980874, "epoch": 0.03987392245471462, "eval/gt_acc_batch": 0.7869444688161215, "frac_reward_zero_std": 1.0, "grad_norm": 0.2319197952747345, "kd/policy_loss": 0.009476867906050757, "kd/rkl_advantage_mean": 0.5123576496417324, "kd/rkl_advantage_p95": 4.489940039316813, "kd/rkl_advantage_std": 2.1511889706055323, "kd/ssd_loss": 0.0, "learning_rate": 9.60164052709528e-07, "loss": 0.03790746927261353, "num_tokens": 34392310.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.786944446961085, "rewards/gt_logging_reward/std": 0.4031680484612783, "sampling/importance_sampling_ratio/max": 1.868604266643524, "sampling/importance_sampling_ratio/mean": 1.0038973490397136, "sampling/importance_sampling_ratio/min": 0.43526746531327565, "sampling/sampling_logp_difference/max": 0.3888191262880961, "sampling/sampling_logp_difference/mean": 0.004248286543103556, "step": 1050, "step_time": 8.008410968938067, "student/entropy": 0.13963014855980874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.035833335326363645, "completions/max_length": 499.0, "completions/max_terminated_length": 454.8666666666667, "completions/mean_length": 191.2777872721354, "completions/mean_terminated_length": 179.53827718098958, "completions/min_length": 59.1, "completions/min_terminated_length": 59.1, "entropy": 0.13994152272741, "epoch": 0.04101317738199218, "eval/gt_acc_batch": 0.7677777985731761, "frac_reward_zero_std": 1.0, "grad_norm": 0.2719656825065613, "kd/policy_loss": 0.009729890661158909, "kd/rkl_advantage_mean": 0.5908375907844553, "kd/rkl_advantage_p95": 4.625113179286321, "kd/rkl_advantage_std": 2.1743407130241392, "kd/ssd_loss": 0.0, "learning_rate": 9.590247977822503e-07, "loss": 0.03891956806182861, "num_tokens": 35380126.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7677777747313181, "rewards/gt_logging_reward/std": 0.4150000015894572, "sampling/importance_sampling_ratio/max": 1.9340392827987671, "sampling/importance_sampling_ratio/mean": 0.9947075486183167, "sampling/importance_sampling_ratio/min": 0.4014507070183754, "sampling/sampling_logp_difference/max": 0.43332671423753105, "sampling/sampling_logp_difference/mean": 0.004417717627560099, "step": 1080, "step_time": 7.928483367478475, "student/entropy": 0.13994152272741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334680646657, "completions/max_length": 495.1333333333333, "completions/max_terminated_length": 442.73333333333335, "completions/mean_length": 187.20528920491537, "completions/mean_terminated_length": 178.68617553710936, "completions/min_length": 57.4, "completions/min_terminated_length": 57.4, "entropy": 0.1327054822196563, "epoch": 0.04215243230926974, "eval/gt_acc_batch": 0.7869444708029429, "frac_reward_zero_std": 1.0, "grad_norm": 0.3031732738018036, "kd/policy_loss": 0.009455308564550553, "kd/rkl_advantage_mean": 0.5121178429263333, "kd/rkl_advantage_p95": 4.560929008324941, "kd/rkl_advantage_std": 2.174414282043775, "kd/ssd_loss": 0.0, "learning_rate": 9.578855428549729e-07, "loss": 0.037821229298909506, "num_tokens": 36369729.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444410006206, "rewards/gt_logging_reward/std": 0.4028739700714747, "sampling/importance_sampling_ratio/max": 1.825038989384969, "sampling/importance_sampling_ratio/mean": 0.9981901089350382, "sampling/importance_sampling_ratio/min": 0.4426324799656868, "sampling/sampling_logp_difference/max": 0.3975507736206055, "sampling/sampling_logp_difference/mean": 0.004209040904728075, "step": 1110, "step_time": 8.02577666247574, "student/entropy": 0.1327054822196563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02750000183780988, "completions/max_length": 481.56666666666666, "completions/max_terminated_length": 453.3333333333333, "completions/mean_length": 186.12584279378254, "completions/mean_terminated_length": 177.0022232055664, "completions/min_length": 50.7, "completions/min_terminated_length": 50.7, "entropy": 0.13475909202049177, "epoch": 0.0432916872365473, "eval/gt_acc_batch": 0.782500026623408, "frac_reward_zero_std": 1.0, "grad_norm": 0.2897556722164154, "kd/policy_loss": 0.009416640375275166, "kd/rkl_advantage_mean": 0.4255038768130665, "kd/rkl_advantage_p95": 4.313121604919433, "kd/rkl_advantage_std": 2.1042641043663024, "kd/ssd_loss": 0.0, "learning_rate": 9.567462879276952e-07, "loss": 0.03766656716664632, "num_tokens": 37340934.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7825000007947286, "rewards/gt_logging_reward/std": 0.408221623301506, "sampling/importance_sampling_ratio/max": 1.8776241938273113, "sampling/importance_sampling_ratio/mean": 0.9951621373494466, "sampling/importance_sampling_ratio/min": 0.41814109484354656, "sampling/sampling_logp_difference/max": 0.39207719365755717, "sampling/sampling_logp_difference/mean": 0.004229770484380424, "step": 1140, "step_time": 7.784824705496431, "student/entropy": 0.13475909202049177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014722222990045945, "completions/max_length": 472.6666666666667, "completions/max_terminated_length": 444.06666666666666, "completions/mean_length": 182.723898824056, "completions/mean_terminated_length": 177.81996307373046, "completions/min_length": 49.36666666666667, "completions/min_terminated_length": 49.36666666666667, "entropy": 0.1366841400663058, "epoch": 0.04443094216382486, "eval/gt_acc_batch": 0.7875000178813935, "frac_reward_zero_std": 1.0, "grad_norm": 0.26732006669044495, "kd/policy_loss": 0.009866254712687806, "kd/rkl_advantage_mean": 0.38594752827387613, "kd/rkl_advantage_p95": 4.242750970522563, "kd/rkl_advantage_std": 2.09828284184138, "kd/ssd_loss": 0.0, "learning_rate": 9.556070330004176e-07, "loss": 0.039465022087097165, "num_tokens": 38306932.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7875, "rewards/gt_logging_reward/std": 0.40186246037483214, "sampling/importance_sampling_ratio/max": 2.042272484302521, "sampling/importance_sampling_ratio/mean": 1.0045514742533366, "sampling/importance_sampling_ratio/min": 0.40624864101409913, "sampling/sampling_logp_difference/max": 0.4059317191441854, "sampling/sampling_logp_difference/mean": 0.004361911839805543, "step": 1170, "step_time": 7.696019343108249, "student/entropy": 0.1366841400663058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444586833318, "completions/max_length": 488.9, "completions/max_terminated_length": 450.1333333333333, "completions/mean_length": 190.08334452311198, "completions/mean_terminated_length": 182.97108154296876, "completions/min_length": 52.13333333333333, "completions/min_terminated_length": 52.13333333333333, "entropy": 0.13806713689118624, "epoch": 0.04557019709110242, "eval/gt_acc_batch": 0.7733333528041839, "frac_reward_zero_std": 1.0, "grad_norm": 0.2650133967399597, "kd/policy_loss": 0.010004761141802494, "kd/rkl_advantage_mean": 0.5138223921860724, "kd/rkl_advantage_p95": 4.495609273513158, "kd/rkl_advantage_std": 2.143854647874832, "kd/ssd_loss": 0.0, "learning_rate": 9.544677780731402e-07, "loss": 0.04001904328664144, "num_tokens": 39297368.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7733333309491476, "rewards/gt_logging_reward/std": 0.4099621762832006, "sampling/importance_sampling_ratio/max": 1.9180695295333863, "sampling/importance_sampling_ratio/mean": 1.0017491082350414, "sampling/importance_sampling_ratio/min": 0.4176797722776731, "sampling/sampling_logp_difference/max": 0.4539886236190796, "sampling/sampling_logp_difference/mean": 0.00429296368577828, "step": 1200, "step_time": 8.0293332656535, "student/entropy": 0.13806713689118624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0213888899423182, "completions/max_length": 494.8333333333333, "completions/max_terminated_length": 455.7, "completions/mean_length": 186.1877873738607, "completions/mean_terminated_length": 179.12924346923828, "completions/min_length": 60.43333333333333, "completions/min_terminated_length": 60.43333333333333, "entropy": 0.13894468024373055, "epoch": 0.04670945201837998, "eval/gt_acc_batch": 0.7758333484331766, "frac_reward_zero_std": 1.0, "grad_norm": 0.28859245777130127, "kd/policy_loss": 0.009930307212440918, "kd/rkl_advantage_mean": 0.5055101167488222, "kd/rkl_advantage_p95": 4.523312179247538, "kd/rkl_advantage_std": 2.176307301719983, "kd/ssd_loss": 0.0, "learning_rate": 9.533285231458625e-07, "loss": 0.039721238613128665, "num_tokens": 40271628.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7758333325386048, "rewards/gt_logging_reward/std": 0.41134084860483805, "sampling/importance_sampling_ratio/max": 1.9202594876289367, "sampling/importance_sampling_ratio/mean": 0.9987711807092031, "sampling/importance_sampling_ratio/min": 0.40990047256151835, "sampling/sampling_logp_difference/max": 0.3990109403928121, "sampling/sampling_logp_difference/mean": 0.004414699777650337, "step": 1230, "step_time": 7.970608511924123, "student/entropy": 0.13894468024373055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667734583218, "completions/max_length": 461.93333333333334, "completions/max_terminated_length": 425.6333333333333, "completions/mean_length": 185.57806599934895, "completions/mean_terminated_length": 178.4705352783203, "completions/min_length": 58.766666666666666, "completions/min_terminated_length": 58.766666666666666, "entropy": 0.1305646586542328, "epoch": 0.04784870694565754, "eval/gt_acc_batch": 0.799444462855657, "frac_reward_zero_std": 1.0, "grad_norm": 0.23523734509944916, "kd/policy_loss": 0.00875447181169875, "kd/rkl_advantage_mean": 0.4655029031060015, "kd/rkl_advantage_p95": 4.329164098699888, "kd/rkl_advantage_std": 2.086276273926099, "kd/ssd_loss": 0.0, "learning_rate": 9.52189268218585e-07, "loss": 0.03501788377761841, "num_tokens": 41244957.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7994444410006205, "rewards/gt_logging_reward/std": 0.3884798968831698, "sampling/importance_sampling_ratio/max": 1.881559463342031, "sampling/importance_sampling_ratio/mean": 0.9924729486306508, "sampling/importance_sampling_ratio/min": 0.42593758801619214, "sampling/sampling_logp_difference/max": 0.39916374286015827, "sampling/sampling_logp_difference/mean": 0.004212799986513953, "step": 1260, "step_time": 7.658384246181231, "student/entropy": 0.1305646586542328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778704961143, "completions/max_length": 481.4, "completions/max_terminated_length": 448.9, "completions/mean_length": 190.30445404052733, "completions/mean_terminated_length": 184.59811197916667, "completions/min_length": 53.8, "completions/min_terminated_length": 53.8, "entropy": 0.13205954780181248, "epoch": 0.0489879618729351, "eval/gt_acc_batch": 0.7675000230471293, "frac_reward_zero_std": 1.0, "grad_norm": 0.2250290960073471, "kd/policy_loss": 0.009586874818584572, "kd/rkl_advantage_mean": 0.3927052370427797, "kd/rkl_advantage_p95": 4.232746241490046, "kd/rkl_advantage_std": 2.0877994179725645, "kd/ssd_loss": 0.0, "learning_rate": 9.510500132913074e-07, "loss": 0.038347506523132326, "num_tokens": 42223237.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7674999992052715, "rewards/gt_logging_reward/std": 0.41773091355959574, "sampling/importance_sampling_ratio/max": 1.9180840929349263, "sampling/importance_sampling_ratio/mean": 1.004743750890096, "sampling/importance_sampling_ratio/min": 0.4335288678606351, "sampling/sampling_logp_difference/max": 0.3657384693622589, "sampling/sampling_logp_difference/mean": 0.0042173663464685285, "step": 1290, "step_time": 7.754713985145402, "student/entropy": 0.13205954780181248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890345891316, "completions/max_length": 480.5, "completions/max_terminated_length": 443.3, "completions/mean_length": 183.62750905354818, "completions/mean_terminated_length": 176.38719533284504, "completions/min_length": 53.5, "completions/min_terminated_length": 53.5, "entropy": 0.1295868429665764, "epoch": 0.05012721680021266, "eval/gt_acc_batch": 0.7997222403685252, "frac_reward_zero_std": 1.0, "grad_norm": 0.24079741537570953, "kd/policy_loss": 0.009047037681254248, "kd/rkl_advantage_mean": 0.4350222146138549, "kd/rkl_advantage_p95": 4.302380998929341, "kd/rkl_advantage_std": 2.096010450522105, "kd/ssd_loss": 0.0, "learning_rate": 9.499107583640299e-07, "loss": 0.036188149452209474, "num_tokens": 43188016.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7997222205003103, "rewards/gt_logging_reward/std": 0.39699907501538595, "sampling/importance_sampling_ratio/max": 1.9416816631952922, "sampling/importance_sampling_ratio/mean": 1.001844576994578, "sampling/importance_sampling_ratio/min": 0.43477134058872857, "sampling/sampling_logp_difference/max": 0.40198151270548504, "sampling/sampling_logp_difference/mean": 0.004196803597733378, "step": 1320, "step_time": 7.808941108019401, "student/entropy": 0.1295868429665764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01861111226802071, "completions/max_length": 483.8, "completions/max_terminated_length": 446.73333333333335, "completions/mean_length": 182.73889923095703, "completions/mean_terminated_length": 176.54483337402343, "completions/min_length": 46.86666666666667, "completions/min_terminated_length": 46.86666666666667, "entropy": 0.1320499664793412, "epoch": 0.05126647172749022, "eval/gt_acc_batch": 0.7905555685361226, "frac_reward_zero_std": 1.0, "grad_norm": 0.24322107434272766, "kd/policy_loss": 0.0089542455971241, "kd/rkl_advantage_mean": 0.4264907380488391, "kd/rkl_advantage_p95": 4.335220678647359, "kd/rkl_advantage_std": 2.097256257136663, "kd/ssd_loss": 0.0, "learning_rate": 9.487715034367523e-07, "loss": 0.03581698338190715, "num_tokens": 44151020.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7905555605888367, "rewards/gt_logging_reward/std": 0.4011397918065389, "sampling/importance_sampling_ratio/max": 1.8292698740959168, "sampling/importance_sampling_ratio/mean": 1.0026962379614512, "sampling/importance_sampling_ratio/min": 0.42409441620111465, "sampling/sampling_logp_difference/max": 0.44165732661883034, "sampling/sampling_logp_difference/mean": 0.004296521128465732, "step": 1350, "step_time": 7.874784729979001, "student/entropy": 0.1320499664793412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334804823003, "completions/max_length": 502.9, "completions/max_terminated_length": 460.3333333333333, "completions/mean_length": 192.06556447347006, "completions/mean_terminated_length": 183.61007843017578, "completions/min_length": 54.63333333333333, "completions/min_terminated_length": 54.63333333333333, "entropy": 0.13089757189154624, "epoch": 0.05240572665476778, "eval/gt_acc_batch": 0.7880555748939514, "frac_reward_zero_std": 1.0, "grad_norm": 0.29934316873550415, "kd/policy_loss": 0.009186981463183959, "kd/rkl_advantage_mean": 0.5305200241971761, "kd/rkl_advantage_p95": 4.562812473376592, "kd/rkl_advantage_std": 2.1609819849332172, "kd/ssd_loss": 0.0, "learning_rate": 9.476322485094748e-07, "loss": 0.03674793243408203, "num_tokens": 45160104.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7880555589993795, "rewards/gt_logging_reward/std": 0.40707863569259645, "sampling/importance_sampling_ratio/max": 1.970625642935435, "sampling/importance_sampling_ratio/mean": 1.001666647195816, "sampling/importance_sampling_ratio/min": 0.42452289164066315, "sampling/sampling_logp_difference/max": 0.4395647903283437, "sampling/sampling_logp_difference/mean": 0.004168329395664235, "step": 1380, "step_time": 8.31072727527159, "student/entropy": 0.13089757189154624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779630074897, "completions/max_length": 492.6333333333333, "completions/max_terminated_length": 446.1, "completions/mean_length": 191.01917673746746, "completions/mean_terminated_length": 182.02425842285157, "completions/min_length": 56.03333333333333, "completions/min_terminated_length": 56.03333333333333, "entropy": 0.13712634996821482, "epoch": 0.05354498158204534, "eval/gt_acc_batch": 0.7752777934074402, "frac_reward_zero_std": 1.0, "grad_norm": 0.2746981084346771, "kd/policy_loss": 0.009554964965597415, "kd/rkl_advantage_mean": 0.5726941334704558, "kd/rkl_advantage_p95": 4.600284363826116, "kd/rkl_advantage_std": 2.1604993959267933, "kd/ssd_loss": 0.0, "learning_rate": 9.464929935821972e-07, "loss": 0.03821985721588135, "num_tokens": 46145733.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7752777715524037, "rewards/gt_logging_reward/std": 0.4115516314903895, "sampling/importance_sampling_ratio/max": 1.9517006039619447, "sampling/importance_sampling_ratio/mean": 0.9982248087724049, "sampling/importance_sampling_ratio/min": 0.42693432917197544, "sampling/sampling_logp_difference/max": 0.411659969886144, "sampling/sampling_logp_difference/mean": 0.004319315419221917, "step": 1410, "step_time": 8.012428873990817, "student/entropy": 0.13712634996821482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112435658773, "completions/max_length": 474.73333333333335, "completions/max_terminated_length": 446.56666666666666, "completions/mean_length": 189.61028747558595, "completions/mean_terminated_length": 182.14378458658854, "completions/min_length": 54.266666666666666, "completions/min_terminated_length": 54.266666666666666, "entropy": 0.13169130471845467, "epoch": 0.0546842365093229, "eval/gt_acc_batch": 0.7775000075499217, "frac_reward_zero_std": 1.0, "grad_norm": 0.331046998500824, "kd/policy_loss": 0.0092339646575662, "kd/rkl_advantage_mean": 0.41531487298004016, "kd/rkl_advantage_p95": 4.238137894868851, "kd/rkl_advantage_std": 2.0813306232293445, "kd/ssd_loss": 0.0, "learning_rate": 9.453537386549196e-07, "loss": 0.036935861905415854, "num_tokens": 47128386.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7774999976158142, "rewards/gt_logging_reward/std": 0.4047900716463725, "sampling/importance_sampling_ratio/max": 1.9343948403994242, "sampling/importance_sampling_ratio/mean": 0.9925498286883037, "sampling/importance_sampling_ratio/min": 0.41420688678820927, "sampling/sampling_logp_difference/max": 0.4047835429509481, "sampling/sampling_logp_difference/mean": 0.004293958866037428, "step": 1440, "step_time": 7.746310222602915, "student/entropy": 0.13169130471845467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0269444459117949, "completions/max_length": 493.1666666666667, "completions/max_terminated_length": 446.7, "completions/mean_length": 186.8672322591146, "completions/mean_terminated_length": 177.87122751871746, "completions/min_length": 54.2, "completions/min_terminated_length": 54.2, "entropy": 0.12774553907414277, "epoch": 0.05582349143660046, "eval/gt_acc_batch": 0.7861111362775167, "frac_reward_zero_std": 1.0, "grad_norm": 0.24788865447044373, "kd/policy_loss": 0.008305798439929882, "kd/rkl_advantage_mean": 0.4606620704755187, "kd/rkl_advantage_p95": 4.401072019338608, "kd/rkl_advantage_std": 2.1347546468178433, "kd/ssd_loss": 0.0, "learning_rate": 9.442144837276421e-07, "loss": 0.03322318991025289, "num_tokens": 48114532.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111104488373, "rewards/gt_logging_reward/std": 0.4019089033206304, "sampling/importance_sampling_ratio/max": 1.8723795930544536, "sampling/importance_sampling_ratio/mean": 0.9937098880608877, "sampling/importance_sampling_ratio/min": 0.39819308519363406, "sampling/sampling_logp_difference/max": 0.3934435208638509, "sampling/sampling_logp_difference/mean": 0.004123895149677992, "step": 1470, "step_time": 7.986871154994393, "student/entropy": 0.12774553907414277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028611112323900064, "completions/max_length": 485.6333333333333, "completions/max_terminated_length": 461.8666666666667, "completions/mean_length": 189.5636199951172, "completions/mean_terminated_length": 180.39380950927733, "completions/min_length": 55.766666666666666, "completions/min_terminated_length": 55.766666666666666, "entropy": 0.1362609348570307, "epoch": 0.05696274636387803, "eval/gt_acc_batch": 0.7936111211776733, "frac_reward_zero_std": 1.0, "grad_norm": 0.24917960166931152, "kd/policy_loss": 0.0094911340798717, "kd/rkl_advantage_mean": 0.4859275811783543, "kd/rkl_advantage_p95": 4.420587690671285, "kd/rkl_advantage_std": 2.1099185824394224, "kd/ssd_loss": 0.0, "learning_rate": 9.430752288003644e-07, "loss": 0.03796453475952148, "num_tokens": 49105249.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111152172088, "rewards/gt_logging_reward/std": 0.4004330108563105, "sampling/importance_sampling_ratio/max": 1.912660280863444, "sampling/importance_sampling_ratio/mean": 1.0031083742777507, "sampling/importance_sampling_ratio/min": 0.3929472347100576, "sampling/sampling_logp_difference/max": 0.4075843930244446, "sampling/sampling_logp_difference/mean": 0.004371285112574697, "step": 1500, "step_time": 7.849856868712232, "student/entropy": 0.1362609348570307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001483907302, "completions/max_length": 488.8, "completions/max_terminated_length": 439.1, "completions/mean_length": 182.81306533813478, "completions/mean_terminated_length": 175.29583511352538, "completions/min_length": 65.2, "completions/min_terminated_length": 65.2, "entropy": 0.12687780776371557, "epoch": 0.058102001291155586, "eval/gt_acc_batch": 0.7933333615461985, "frac_reward_zero_std": 1.0, "grad_norm": 0.29639458656311035, "kd/policy_loss": 0.008688496709025155, "kd/rkl_advantage_mean": 0.47396734592039136, "kd/rkl_advantage_p95": 4.453593985239665, "kd/rkl_advantage_std": 2.133880740404129, "kd/ssd_loss": 0.0, "learning_rate": 9.41935973873087e-07, "loss": 0.03475399017333984, "num_tokens": 50066408.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7933333277702331, "rewards/gt_logging_reward/std": 0.39841359555721284, "sampling/importance_sampling_ratio/max": 1.9006261746088664, "sampling/importance_sampling_ratio/mean": 0.9991104125976562, "sampling/importance_sampling_ratio/min": 0.4092116753260295, "sampling/sampling_logp_difference/max": 0.3977258066336314, "sampling/sampling_logp_difference/mean": 0.00413516325255235, "step": 1530, "step_time": 7.887949022060881, "student/entropy": 0.12687780776371557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0269444459117949, "completions/max_length": 484.93333333333334, "completions/max_terminated_length": 452.53333333333336, "completions/mean_length": 193.22639973958334, "completions/mean_terminated_length": 184.7485336303711, "completions/min_length": 55.233333333333334, "completions/min_terminated_length": 55.233333333333334, "entropy": 0.13641904791196188, "epoch": 0.059241256218433146, "eval/gt_acc_batch": 0.783611136674881, "frac_reward_zero_std": 1.0, "grad_norm": 0.30261698365211487, "kd/policy_loss": 0.009951253417724123, "kd/rkl_advantage_mean": 0.4300662286424389, "kd/rkl_advantage_p95": 4.312958325942358, "kd/rkl_advantage_std": 2.113129373391469, "kd/ssd_loss": 0.0, "learning_rate": 9.407967189458094e-07, "loss": 0.039805010954538984, "num_tokens": 51071663.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7836111068725586, "rewards/gt_logging_reward/std": 0.4057087163130442, "sampling/importance_sampling_ratio/max": 1.9647137880325318, "sampling/importance_sampling_ratio/mean": 1.0016080379486083, "sampling/importance_sampling_ratio/min": 0.4159156357248624, "sampling/sampling_logp_difference/max": 0.428339276711146, "sampling/sampling_logp_difference/mean": 0.004306183770919839, "step": 1560, "step_time": 7.867474790868195, "student/entropy": 0.13641904791196188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01777777892226974, "completions/max_length": 476.06666666666666, "completions/max_terminated_length": 432.03333333333336, "completions/mean_length": 185.55361989339193, "completions/mean_terminated_length": 179.80084025065105, "completions/min_length": 50.266666666666666, "completions/min_terminated_length": 50.266666666666666, "entropy": 0.12798440338422853, "epoch": 0.060380511145710705, "eval/gt_acc_batch": 0.8094444612661997, "frac_reward_zero_std": 1.0, "grad_norm": 0.23594437539577484, "kd/policy_loss": 0.00869161959659929, "kd/rkl_advantage_mean": 0.4818067514473417, "kd/rkl_advantage_p95": 4.415740209817886, "kd/rkl_advantage_std": 2.0991670499245325, "kd/ssd_loss": 0.0, "learning_rate": 9.396574640185319e-07, "loss": 0.03476647535959879, "num_tokens": 52043656.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8094444473584493, "rewards/gt_logging_reward/std": 0.38414963483810427, "sampling/importance_sampling_ratio/max": 1.8557328422864279, "sampling/importance_sampling_ratio/mean": 0.9989226559797922, "sampling/importance_sampling_ratio/min": 0.421547802289327, "sampling/sampling_logp_difference/max": 0.4241507808367411, "sampling/sampling_logp_difference/mean": 0.004141134621265034, "step": 1590, "step_time": 7.829480991147769, "student/entropy": 0.12798440338422853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556857337555, "completions/max_length": 494.96666666666664, "completions/max_terminated_length": 456.26666666666665, "completions/mean_length": 192.9372334798177, "completions/mean_terminated_length": 184.76953887939453, "completions/min_length": 56.13333333333333, "completions/min_terminated_length": 56.13333333333333, "entropy": 0.13910383880138397, "epoch": 0.061519766072988265, "eval/gt_acc_batch": 0.7761111319065094, "frac_reward_zero_std": 1.0, "grad_norm": 0.1981118768453598, "kd/policy_loss": 0.009668007772415877, "kd/rkl_advantage_mean": 0.502127900859341, "kd/rkl_advantage_p95": 4.536231740315755, "kd/rkl_advantage_std": 2.156936464707057, "kd/ssd_loss": 0.0, "learning_rate": 9.385182090912542e-07, "loss": 0.038672037919362384, "num_tokens": 53037478.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7761111080646514, "rewards/gt_logging_reward/std": 0.4119443287452062, "sampling/importance_sampling_ratio/max": 1.924413557847341, "sampling/importance_sampling_ratio/mean": 1.0025873442490896, "sampling/importance_sampling_ratio/min": 0.3829710433880488, "sampling/sampling_logp_difference/max": 0.4407365103562673, "sampling/sampling_logp_difference/mean": 0.004381851029271881, "step": 1620, "step_time": 7.88754960373432, "student/entropy": 0.13910383880138397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556397885084, "completions/max_length": 464.1666666666667, "completions/max_terminated_length": 434.56666666666666, "completions/mean_length": 189.06473185221355, "completions/mean_terminated_length": 184.08736216227214, "completions/min_length": 61.266666666666666, "completions/min_terminated_length": 61.266666666666666, "entropy": 0.12107625082135201, "epoch": 0.06265902100026582, "eval/gt_acc_batch": 0.8227778017520905, "frac_reward_zero_std": 1.0, "grad_norm": 0.2539253234863281, "kd/policy_loss": 0.007921620974472413, "kd/rkl_advantage_mean": 0.39888008338360426, "kd/rkl_advantage_p95": 4.2554416954517365, "kd/rkl_advantage_std": 2.0922290345033008, "kd/ssd_loss": 0.0, "learning_rate": 9.373789541639767e-07, "loss": 0.03168648680051168, "num_tokens": 54024903.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8227777779102325, "rewards/gt_logging_reward/std": 0.37089019318421684, "sampling/importance_sampling_ratio/max": 1.8643449624379476, "sampling/importance_sampling_ratio/mean": 0.9945861160755157, "sampling/importance_sampling_ratio/min": 0.38786589801311494, "sampling/sampling_logp_difference/max": 0.450235507885615, "sampling/sampling_logp_difference/mean": 0.004055990069173276, "step": 1650, "step_time": 7.6910857483861035, "student/entropy": 0.12107625082135201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030833335189769665, "completions/max_length": 479.96666666666664, "completions/max_terminated_length": 432.46666666666664, "completions/mean_length": 189.30112050374348, "completions/mean_terminated_length": 179.13788604736328, "completions/min_length": 56.333333333333336, "completions/min_terminated_length": 56.333333333333336, "entropy": 0.13177461369584004, "epoch": 0.06379827592754339, "eval/gt_acc_batch": 0.7755555788675944, "frac_reward_zero_std": 1.0, "grad_norm": 0.21538788080215454, "kd/policy_loss": 0.009356924623716623, "kd/rkl_advantage_mean": 0.4908350238886972, "kd/rkl_advantage_p95": 4.434452267487844, "kd/rkl_advantage_std": 2.1075219531853993, "kd/ssd_loss": 0.0, "learning_rate": 9.362396992366992e-07, "loss": 0.037427695592244466, "num_tokens": 55017155.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7755555609862009, "rewards/gt_logging_reward/std": 0.41068689624468485, "sampling/importance_sampling_ratio/max": 2.0372203906377155, "sampling/importance_sampling_ratio/mean": 1.0019435207049052, "sampling/importance_sampling_ratio/min": 0.39773240784804026, "sampling/sampling_logp_difference/max": 0.43994247118631996, "sampling/sampling_logp_difference/mean": 0.004310186843698224, "step": 1680, "step_time": 7.9412532573643455, "student/entropy": 0.13177461369584004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890575617552, "completions/max_length": 500.26666666666665, "completions/max_terminated_length": 457.43333333333334, "completions/mean_length": 190.44639790852864, "completions/mean_terminated_length": 181.8815887451172, "completions/min_length": 52.733333333333334, "completions/min_terminated_length": 52.733333333333334, "entropy": 0.13354260058452685, "epoch": 0.06493753085482094, "eval/gt_acc_batch": 0.8072222391764323, "frac_reward_zero_std": 1.0, "grad_norm": 0.2548181712627411, "kd/policy_loss": 0.008769306695709626, "kd/rkl_advantage_mean": 0.5298034142547597, "kd/rkl_advantage_p95": 4.522440363963445, "kd/rkl_advantage_std": 2.1655157496531805, "kd/ssd_loss": 0.0, "learning_rate": 9.351004443094216e-07, "loss": 0.035077230135599775, "num_tokens": 56005258.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222272555033, "rewards/gt_logging_reward/std": 0.38567766845226287, "sampling/importance_sampling_ratio/max": 2.0129003802935284, "sampling/importance_sampling_ratio/mean": 0.997453773021698, "sampling/importance_sampling_ratio/min": 0.4268814002474149, "sampling/sampling_logp_difference/max": 0.43880842526753744, "sampling/sampling_logp_difference/mean": 0.004218905186280608, "step": 1710, "step_time": 7.921825190389063, "student/entropy": 0.13354260058452685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777894089619, "completions/max_length": 467.3666666666667, "completions/max_terminated_length": 425.3333333333333, "completions/mean_length": 183.33445383707684, "completions/mean_terminated_length": 174.98929901123046, "completions/min_length": 53.86666666666667, "completions/min_terminated_length": 53.86666666666667, "entropy": 0.13233609708646935, "epoch": 0.06607678578209851, "eval/gt_acc_batch": 0.8047222455342611, "frac_reward_zero_std": 1.0, "grad_norm": 0.3584451675415039, "kd/policy_loss": 0.009240899370828023, "kd/rkl_advantage_mean": 0.45176814281536887, "kd/rkl_advantage_p95": 4.389229263861974, "kd/rkl_advantage_std": 2.137334558367729, "kd/ssd_loss": 0.0, "learning_rate": 9.339611893821441e-07, "loss": 0.03696360190709432, "num_tokens": 56964294.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222197055817, "rewards/gt_logging_reward/std": 0.38385874032974243, "sampling/importance_sampling_ratio/max": 1.9180762767791748, "sampling/importance_sampling_ratio/mean": 1.0078616797924043, "sampling/importance_sampling_ratio/min": 0.40352218051751454, "sampling/sampling_logp_difference/max": 0.3903858184814453, "sampling/sampling_logp_difference/mean": 0.004334763833321631, "step": 1740, "step_time": 7.5688447172753515, "student/entropy": 0.13233609708646935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779350678124, "completions/max_length": 481.5, "completions/max_terminated_length": 439.6333333333333, "completions/mean_length": 190.05862121582032, "completions/mean_terminated_length": 181.03280232747395, "completions/min_length": 53.166666666666664, "completions/min_terminated_length": 53.166666666666664, "entropy": 0.13602967746555805, "epoch": 0.06721604070937606, "eval/gt_acc_batch": 0.7786111255486806, "frac_reward_zero_std": 1.0, "grad_norm": 0.2277403324842453, "kd/policy_loss": 0.009074522543232887, "kd/rkl_advantage_mean": 0.5186608690749078, "kd/rkl_advantage_p95": 4.538462978601456, "kd/rkl_advantage_std": 2.1583961407343546, "kd/ssd_loss": 0.0, "learning_rate": 9.328219344548665e-07, "loss": 0.036298092206319174, "num_tokens": 57949609.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7786111116409302, "rewards/gt_logging_reward/std": 0.4105398029088974, "sampling/importance_sampling_ratio/max": 1.9610918800036112, "sampling/importance_sampling_ratio/mean": 0.9964329997698466, "sampling/importance_sampling_ratio/min": 0.4393785903851191, "sampling/sampling_logp_difference/max": 0.4592077414194743, "sampling/sampling_logp_difference/mean": 0.00437441060009102, "step": 1770, "step_time": 7.899257109093014, "student/entropy": 0.13602967746555805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334668229023, "completions/max_length": 477.8333333333333, "completions/max_terminated_length": 445.26666666666665, "completions/mean_length": 187.36834309895832, "completions/mean_terminated_length": 180.5821965535482, "completions/min_length": 55.96666666666667, "completions/min_terminated_length": 55.96666666666667, "entropy": 0.13195363817115624, "epoch": 0.06835529563665363, "eval/gt_acc_batch": 0.7947222431500752, "frac_reward_zero_std": 1.0, "grad_norm": 0.24217137694358826, "kd/policy_loss": 0.008946204841292153, "kd/rkl_advantage_mean": 0.5176153003548583, "kd/rkl_advantage_p95": 4.5280272424221035, "kd/rkl_advantage_std": 2.135515178243319, "kd/ssd_loss": 0.0, "learning_rate": 9.316826795275889e-07, "loss": 0.03578481674194336, "num_tokens": 58922367.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222212950389, "rewards/gt_logging_reward/std": 0.3924879292647044, "sampling/importance_sampling_ratio/max": 1.8849701523780822, "sampling/importance_sampling_ratio/mean": 0.9979813675085704, "sampling/importance_sampling_ratio/min": 0.39969506512085595, "sampling/sampling_logp_difference/max": 0.41199642022450766, "sampling/sampling_logp_difference/mean": 0.004335533788738151, "step": 1800, "step_time": 7.9093199571361765, "student/entropy": 0.13195363817115624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02222222356746594, "completions/max_length": 483.53333333333336, "completions/max_terminated_length": 454.23333333333335, "completions/mean_length": 188.1958435058594, "completions/mean_terminated_length": 180.89789326985678, "completions/min_length": 53.43333333333333, "completions/min_terminated_length": 53.43333333333333, "entropy": 0.1322110606978337, "epoch": 0.0694945505639312, "eval/gt_acc_batch": 0.7866666813691457, "frac_reward_zero_std": 1.0, "grad_norm": 0.2800222635269165, "kd/policy_loss": 0.00927802122120435, "kd/rkl_advantage_mean": 0.5515845583130916, "kd/rkl_advantage_p95": 4.65053463379542, "kd/rkl_advantage_std": 2.1627071569363276, "kd/ssd_loss": 0.0, "learning_rate": 9.305434246003113e-07, "loss": 0.03711208502451579, "num_tokens": 59909456.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666714350382, "rewards/gt_logging_reward/std": 0.40107011000315346, "sampling/importance_sampling_ratio/max": 1.899915599822998, "sampling/importance_sampling_ratio/mean": 1.0009732723236084, "sampling/importance_sampling_ratio/min": 0.41238957742849985, "sampling/sampling_logp_difference/max": 0.4790292024612427, "sampling/sampling_logp_difference/mean": 0.004299720167182386, "step": 1830, "step_time": 7.9273036416581215, "student/entropy": 0.1322110606978337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02916666828095913, "completions/max_length": 497.43333333333334, "completions/max_terminated_length": 461.03333333333336, "completions/mean_length": 188.6047337849935, "completions/mean_terminated_length": 179.02967020670573, "completions/min_length": 49.733333333333334, "completions/min_terminated_length": 49.733333333333334, "entropy": 0.134600345355769, "epoch": 0.07063380549120875, "eval/gt_acc_batch": 0.7805555661519369, "frac_reward_zero_std": 1.0, "grad_norm": 0.31848758459091187, "kd/policy_loss": 0.0094726511859335, "kd/rkl_advantage_mean": 0.45058512369869275, "kd/rkl_advantage_p95": 4.371852391958237, "kd/rkl_advantage_std": 2.1258409549792607, "kd/ssd_loss": 0.0, "learning_rate": 9.294041696730338e-07, "loss": 0.03789060910542806, "num_tokens": 60898121.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7805555582046508, "rewards/gt_logging_reward/std": 0.4102844427029292, "sampling/importance_sampling_ratio/max": 2.022121028105418, "sampling/importance_sampling_ratio/mean": 1.0026258409023285, "sampling/importance_sampling_ratio/min": 0.4274846265713374, "sampling/sampling_logp_difference/max": 0.44315466086069744, "sampling/sampling_logp_difference/mean": 0.0043998717175175745, "step": 1860, "step_time": 7.966170869457225, "student/entropy": 0.134600345355769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890476276476, "completions/max_length": 494.26666666666665, "completions/max_terminated_length": 461.5, "completions/mean_length": 189.09417775472005, "completions/mean_terminated_length": 181.22430674235025, "completions/min_length": 55.666666666666664, "completions/min_terminated_length": 55.666666666666664, "entropy": 0.12791626217464605, "epoch": 0.07177306041848631, "eval/gt_acc_batch": 0.7988889118035635, "frac_reward_zero_std": 1.0, "grad_norm": 0.23677761852741241, "kd/policy_loss": 0.008572325997132186, "kd/rkl_advantage_mean": 0.5044254133788247, "kd/rkl_advantage_p95": 4.585406603415807, "kd/rkl_advantage_std": 2.1961194117863974, "kd/ssd_loss": 0.0, "learning_rate": 9.282649147457561e-07, "loss": 0.03428930838902791, "num_tokens": 61886188.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7988888919353485, "rewards/gt_logging_reward/std": 0.3947013681133588, "sampling/importance_sampling_ratio/max": 1.8421831369400024, "sampling/importance_sampling_ratio/mean": 0.9961995999018352, "sampling/importance_sampling_ratio/min": 0.4116328462958336, "sampling/sampling_logp_difference/max": 0.41308249831199645, "sampling/sampling_logp_difference/mean": 0.0042181566124781964, "step": 1890, "step_time": 7.958910997126562, "student/entropy": 0.12791626217464605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001198301712, "completions/max_length": 495.8666666666667, "completions/max_terminated_length": 453.2, "completions/mean_length": 190.811398824056, "completions/mean_terminated_length": 184.44028065999348, "completions/min_length": 59.5, "completions/min_terminated_length": 59.5, "entropy": 0.12937359996140002, "epoch": 0.07291231534576387, "eval/gt_acc_batch": 0.7880555868148804, "frac_reward_zero_std": 1.0, "grad_norm": 0.2507643401622772, "kd/policy_loss": 0.008979624465185528, "kd/rkl_advantage_mean": 0.48593370406500375, "kd/rkl_advantage_p95": 4.4564838886260985, "kd/rkl_advantage_std": 2.106365994612376, "kd/ssd_loss": 0.0, "learning_rate": 9.271256598184787e-07, "loss": 0.03591849803924561, "num_tokens": 62879117.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7880555550257365, "rewards/gt_logging_reward/std": 0.40570049385229745, "sampling/importance_sampling_ratio/max": 1.871559989452362, "sampling/importance_sampling_ratio/mean": 0.9994694511095683, "sampling/importance_sampling_ratio/min": 0.4205407828092575, "sampling/sampling_logp_difference/max": 0.4400892317295074, "sampling/sampling_logp_difference/mean": 0.00428825127116094, "step": 1920, "step_time": 7.8791185051668435, "student/entropy": 0.12937359996140002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03250000181918343, "completions/max_length": 501.56666666666666, "completions/max_terminated_length": 450.8, "completions/mean_length": 194.31778717041016, "completions/mean_terminated_length": 183.59711151123048, "completions/min_length": 57.166666666666664, "completions/min_terminated_length": 57.166666666666664, "entropy": 0.12474996813883384, "epoch": 0.07405157027304143, "eval/gt_acc_batch": 0.7783333619435628, "frac_reward_zero_std": 1.0, "grad_norm": 0.23270298540592194, "kd/policy_loss": 0.008680731562587122, "kd/rkl_advantage_mean": 0.5235075515462085, "kd/rkl_advantage_p95": 4.505733462174733, "kd/rkl_advantage_std": 2.106735266248385, "kd/ssd_loss": 0.0, "learning_rate": 9.259864048912012e-07, "loss": 0.03472293217976888, "num_tokens": 63888741.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7783333341280619, "rewards/gt_logging_reward/std": 0.4085713585217794, "sampling/importance_sampling_ratio/max": 1.9575131773948669, "sampling/importance_sampling_ratio/mean": 1.0029576261838278, "sampling/importance_sampling_ratio/min": 0.4283009946346283, "sampling/sampling_logp_difference/max": 0.4523381094137828, "sampling/sampling_logp_difference/mean": 0.004111053065086404, "step": 1950, "step_time": 8.10444961649288, "student/entropy": 0.12474996813883384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02416666808227698, "completions/max_length": 472.8333333333333, "completions/max_terminated_length": 455.3666666666667, "completions/mean_length": 192.26556549072265, "completions/mean_terminated_length": 184.59081370035807, "completions/min_length": 56.46666666666667, "completions/min_terminated_length": 56.46666666666667, "entropy": 0.13155975739161174, "epoch": 0.07519082520031899, "eval/gt_acc_batch": 0.7775000174840291, "frac_reward_zero_std": 1.0, "grad_norm": 0.28483787178993225, "kd/policy_loss": 0.009256099399256831, "kd/rkl_advantage_mean": 0.39859505960096914, "kd/rkl_advantage_p95": 4.260186217228571, "kd/rkl_advantage_std": 2.093841141462326, "kd/ssd_loss": 0.0, "learning_rate": 9.248471499639236e-07, "loss": 0.037024402618408205, "num_tokens": 64877249.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7775000015894572, "rewards/gt_logging_reward/std": 0.40315716216961545, "sampling/importance_sampling_ratio/max": 1.8879090468088786, "sampling/importance_sampling_ratio/mean": 1.0004876752694447, "sampling/importance_sampling_ratio/min": 0.3881937583287557, "sampling/sampling_logp_difference/max": 0.4138429323832194, "sampling/sampling_logp_difference/mean": 0.004333026086290677, "step": 1980, "step_time": 7.837078059573347, "student/entropy": 0.13155975739161174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01805555655931433, "completions/max_length": 486.4, "completions/max_terminated_length": 447.43333333333334, "completions/mean_length": 187.77889709472657, "completions/mean_terminated_length": 181.87198944091796, "completions/min_length": 60.56666666666667, "completions/min_terminated_length": 60.56666666666667, "entropy": 0.13058917224407196, "epoch": 0.07633008012759655, "eval/gt_acc_batch": 0.7944444497426351, "frac_reward_zero_std": 1.0, "grad_norm": 0.2517739236354828, "kd/policy_loss": 0.009205438285910835, "kd/rkl_advantage_mean": 0.465671230328735, "kd/rkl_advantage_p95": 4.359647141893705, "kd/rkl_advantage_std": 2.0911360065142315, "kd/ssd_loss": 0.0, "learning_rate": 9.23707895036646e-07, "loss": 0.036821754773457845, "num_tokens": 65861029.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7944444417953491, "rewards/gt_logging_reward/std": 0.4000958055257797, "sampling/importance_sampling_ratio/max": 1.9076964298884074, "sampling/importance_sampling_ratio/mean": 0.9961376329263051, "sampling/importance_sampling_ratio/min": 0.4364798714717229, "sampling/sampling_logp_difference/max": 0.443558531999588, "sampling/sampling_logp_difference/mean": 0.004243813408538699, "step": 2010, "step_time": 7.965713440747155, "student/entropy": 0.13058917224407196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0277777794127663, "completions/max_length": 468.0, "completions/max_terminated_length": 437.7, "completions/mean_length": 191.1175099690755, "completions/mean_terminated_length": 181.93260192871094, "completions/min_length": 57.96666666666667, "completions/min_terminated_length": 57.96666666666667, "entropy": 0.13233683096865814, "epoch": 0.0774693350548741, "eval/gt_acc_batch": 0.782500022649765, "frac_reward_zero_std": 1.0, "grad_norm": 0.2849023938179016, "kd/policy_loss": 0.008967649209080265, "kd/rkl_advantage_mean": 0.5760604279736677, "kd/rkl_advantage_p95": 4.565486985445022, "kd/rkl_advantage_std": 2.1440737187862395, "kd/ssd_loss": 0.0, "learning_rate": 9.225686401093684e-07, "loss": 0.03587059577306111, "num_tokens": 66850452.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7825000007947286, "rewards/gt_logging_reward/std": 0.4072064975897471, "sampling/importance_sampling_ratio/max": 1.9765862186749776, "sampling/importance_sampling_ratio/mean": 1.000109883149465, "sampling/importance_sampling_ratio/min": 0.43475004931290945, "sampling/sampling_logp_difference/max": 0.4214184045791626, "sampling/sampling_logp_difference/mean": 0.004355892795138061, "step": 2040, "step_time": 7.868182047569038, "student/entropy": 0.13233683096865814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667790462572, "completions/max_length": 482.06666666666666, "completions/max_terminated_length": 451.23333333333335, "completions/mean_length": 185.57001088460285, "completions/mean_terminated_length": 179.32699432373047, "completions/min_length": 56.86666666666667, "completions/min_terminated_length": 56.86666666666667, "entropy": 0.1306519312163194, "epoch": 0.07860858998215167, "eval/gt_acc_batch": 0.7930555721124013, "frac_reward_zero_std": 1.0, "grad_norm": 0.22426144778728485, "kd/policy_loss": 0.008937614793345953, "kd/rkl_advantage_mean": 0.4725372999285658, "kd/rkl_advantage_p95": 4.467708683013916, "kd/rkl_advantage_std": 2.1575969139734905, "kd/ssd_loss": 0.0, "learning_rate": 9.214293851820909e-07, "loss": 0.03575046459833781, "num_tokens": 67821464.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555562178294, "rewards/gt_logging_reward/std": 0.3979779620965322, "sampling/importance_sampling_ratio/max": 1.9353002866109212, "sampling/importance_sampling_ratio/mean": 0.9949861109256745, "sampling/importance_sampling_ratio/min": 0.37038032660881676, "sampling/sampling_logp_difference/max": 0.43952191472053526, "sampling/sampling_logp_difference/mean": 0.004331361634346346, "step": 2070, "step_time": 7.758045650382216, "student/entropy": 0.1306519312163194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668175409238, "completions/max_length": 490.76666666666665, "completions/max_terminated_length": 454.7, "completions/mean_length": 192.8047317504883, "completions/mean_terminated_length": 184.96310577392578, "completions/min_length": 57.36666666666667, "completions/min_terminated_length": 57.36666666666667, "entropy": 0.13108663304398457, "epoch": 0.07974784490942924, "eval/gt_acc_batch": 0.7961111307144165, "frac_reward_zero_std": 1.0, "grad_norm": 0.31515657901763916, "kd/policy_loss": 0.00930598470925664, "kd/rkl_advantage_mean": 0.557286356560265, "kd/rkl_advantage_p95": 4.571281848351161, "kd/rkl_advantage_std": 2.1340308596690494, "kd/ssd_loss": 0.0, "learning_rate": 9.202901302548133e-07, "loss": 0.03722394307454427, "num_tokens": 68823425.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111088593801, "rewards/gt_logging_reward/std": 0.39907184143861135, "sampling/importance_sampling_ratio/max": 1.985925034681956, "sampling/importance_sampling_ratio/mean": 1.000217455625534, "sampling/importance_sampling_ratio/min": 0.4095621605714162, "sampling/sampling_logp_difference/max": 0.4749327818552653, "sampling/sampling_logp_difference/mean": 0.004257549671456218, "step": 2100, "step_time": 7.872133801978392, "student/entropy": 0.13108663304398457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333448196451, "completions/max_length": 478.43333333333334, "completions/max_terminated_length": 444.6, "completions/mean_length": 186.3211196899414, "completions/mean_terminated_length": 179.52101186116536, "completions/min_length": 60.06666666666667, "completions/min_terminated_length": 60.06666666666667, "entropy": 0.13381838761270046, "epoch": 0.08088709983670679, "eval/gt_acc_batch": 0.7872222443421681, "frac_reward_zero_std": 1.0, "grad_norm": 0.2583805322647095, "kd/policy_loss": 0.009222612485367184, "kd/rkl_advantage_mean": 0.5216386028099805, "kd/rkl_advantage_p95": 4.520686920483907, "kd/rkl_advantage_std": 2.1466214428345363, "kd/ssd_loss": 0.0, "learning_rate": 9.191508753275358e-07, "loss": 0.03689045906066894, "num_tokens": 69794789.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7872222224871318, "rewards/gt_logging_reward/std": 0.4005415221055349, "sampling/importance_sampling_ratio/max": 1.853285543123881, "sampling/importance_sampling_ratio/mean": 0.9982494692007701, "sampling/importance_sampling_ratio/min": 0.41366642514864604, "sampling/sampling_logp_difference/max": 0.43381335337956745, "sampling/sampling_logp_difference/mean": 0.0044001693604514, "step": 2130, "step_time": 7.756042513608311, "student/entropy": 0.13381838761270046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01805555643513799, "completions/max_length": 472.8, "completions/max_terminated_length": 442.6666666666667, "completions/mean_length": 179.66750996907552, "completions/mean_terminated_length": 173.6427241007487, "completions/min_length": 51.46666666666667, "completions/min_terminated_length": 51.46666666666667, "entropy": 0.12885212413966657, "epoch": 0.08202635476398436, "eval/gt_acc_batch": 0.810833356777827, "frac_reward_zero_std": 1.0, "grad_norm": 0.3199542760848999, "kd/policy_loss": 0.008512443996733055, "kd/rkl_advantage_mean": 0.5504339669598266, "kd/rkl_advantage_p95": 4.5083488365014395, "kd/rkl_advantage_std": 2.1296819110711414, "kd/ssd_loss": 0.0, "learning_rate": 9.180116204002582e-07, "loss": 0.03404978116353353, "num_tokens": 70740232.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8108333349227905, "rewards/gt_logging_reward/std": 0.3798377806941668, "sampling/importance_sampling_ratio/max": 1.8914802432060243, "sampling/importance_sampling_ratio/mean": 1.002708731094996, "sampling/importance_sampling_ratio/min": 0.44367477695147195, "sampling/sampling_logp_difference/max": 0.4106687476237615, "sampling/sampling_logp_difference/mean": 0.004249967130211492, "step": 2160, "step_time": 7.752001272196261, "student/entropy": 0.12885212413966657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444446060806514, "completions/max_length": 485.46666666666664, "completions/max_terminated_length": 459.1666666666667, "completions/mean_length": 188.53945566813152, "completions/mean_terminated_length": 180.44600779215494, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "entropy": 0.12847886104136705, "epoch": 0.08316560969126191, "eval/gt_acc_batch": 0.791111143430074, "frac_reward_zero_std": 1.0, "grad_norm": 0.38358309864997864, "kd/policy_loss": 0.008954476113043104, "kd/rkl_advantage_mean": 0.5600560375644515, "kd/rkl_advantage_p95": 4.63530224164327, "kd/rkl_advantage_std": 2.1912126739819846, "kd/ssd_loss": 0.0, "learning_rate": 9.168723654729806e-07, "loss": 0.03581790924072266, "num_tokens": 71730190.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7911111096541087, "rewards/gt_logging_reward/std": 0.3988018115361532, "sampling/importance_sampling_ratio/max": 1.9492147564888, "sampling/importance_sampling_ratio/mean": 1.0023767451445262, "sampling/importance_sampling_ratio/min": 0.4121244788169861, "sampling/sampling_logp_difference/max": 0.40340741475423175, "sampling/sampling_logp_difference/mean": 0.0042830877316494785, "step": 2190, "step_time": 8.082188894727732, "student/entropy": 0.12847886104136705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223381201427, "completions/max_length": 475.56666666666666, "completions/max_terminated_length": 441.3, "completions/mean_length": 183.92556508382162, "completions/mean_terminated_length": 176.56497243245443, "completions/min_length": 57.9, "completions/min_terminated_length": 57.9, "entropy": 0.1243789363031586, "epoch": 0.08430486461853948, "eval/gt_acc_batch": 0.8094444751739502, "frac_reward_zero_std": 1.0, "grad_norm": 0.2651103138923645, "kd/policy_loss": 0.008534347357150788, "kd/rkl_advantage_mean": 0.5753608885531624, "kd/rkl_advantage_p95": 4.659590444962183, "kd/rkl_advantage_std": 2.164811690648397, "kd/ssd_loss": 0.0, "learning_rate": 9.15733110545703e-07, "loss": 0.0341373880704244, "num_tokens": 72697154.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8094444433848064, "rewards/gt_logging_reward/std": 0.3843872144818306, "sampling/importance_sampling_ratio/max": 1.9825315276781719, "sampling/importance_sampling_ratio/mean": 1.0088355084260305, "sampling/importance_sampling_ratio/min": 0.43042065054178236, "sampling/sampling_logp_difference/max": 0.4561076323191325, "sampling/sampling_logp_difference/mean": 0.004204935526164869, "step": 2220, "step_time": 7.821717067854479, "student/entropy": 0.1243789363031586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445936630168, "completions/max_length": 487.7, "completions/max_terminated_length": 451.93333333333334, "completions/mean_length": 185.12139739990235, "completions/mean_terminated_length": 176.955441792806, "completions/min_length": 52.8, "completions/min_terminated_length": 52.8, "entropy": 0.129484648630023, "epoch": 0.08544411954581703, "eval/gt_acc_batch": 0.8050000270207723, "frac_reward_zero_std": 1.0, "grad_norm": 0.3025919198989868, "kd/policy_loss": 0.008739575621439144, "kd/rkl_advantage_mean": 0.6011017846642062, "kd/rkl_advantage_p95": 4.68336132367452, "kd/rkl_advantage_std": 2.198494995633761, "kd/ssd_loss": 0.0, "learning_rate": 9.145938556184255e-07, "loss": 0.03495830694834391, "num_tokens": 73667079.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8050000031789144, "rewards/gt_logging_reward/std": 0.38644814242919284, "sampling/importance_sampling_ratio/max": 1.9399807453155518, "sampling/importance_sampling_ratio/mean": 1.0035587271054587, "sampling/importance_sampling_ratio/min": 0.4109939277172089, "sampling/sampling_logp_difference/max": 0.4741142471631368, "sampling/sampling_logp_difference/mean": 0.004305462857397894, "step": 2250, "step_time": 7.9964593221782705, "student/entropy": 0.129484648630023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223753730456, "completions/max_length": 488.6333333333333, "completions/max_terminated_length": 459.06666666666666, "completions/mean_length": 188.71112111409505, "completions/mean_terminated_length": 181.4258071899414, "completions/min_length": 59.333333333333336, "completions/min_terminated_length": 59.333333333333336, "entropy": 0.13618510228892167, "epoch": 0.0865833744730946, "eval/gt_acc_batch": 0.7858333607514699, "frac_reward_zero_std": 1.0, "grad_norm": 0.26028093695640564, "kd/policy_loss": 0.008713893995930751, "kd/rkl_advantage_mean": 0.5672770070649373, "kd/rkl_advantage_p95": 4.659412030378977, "kd/rkl_advantage_std": 2.1844313472509382, "kd/ssd_loss": 0.0, "learning_rate": 9.134546006911479e-07, "loss": 0.03485557238260905, "num_tokens": 74641191.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.785833336909612, "rewards/gt_logging_reward/std": 0.4077138970295588, "sampling/importance_sampling_ratio/max": 1.8542234619458517, "sampling/importance_sampling_ratio/mean": 0.9892552455266317, "sampling/importance_sampling_ratio/min": 0.39006411482890446, "sampling/sampling_logp_difference/max": 0.4547816276550293, "sampling/sampling_logp_difference/mean": 0.004381965977760653, "step": 2280, "step_time": 7.927389533216289, "student/entropy": 0.13618510228892167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01500000087544322, "completions/max_length": 477.7, "completions/max_terminated_length": 452.0, "completions/mean_length": 184.48667755126954, "completions/mean_terminated_length": 179.61041819254558, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "entropy": 0.12824787913511196, "epoch": 0.08772262940037216, "eval/gt_acc_batch": 0.8016666909058888, "frac_reward_zero_std": 1.0, "grad_norm": 0.2790432572364807, "kd/policy_loss": 0.00864245427850013, "kd/rkl_advantage_mean": 0.5562232778407633, "kd/rkl_advantage_p95": 4.596868836879731, "kd/rkl_advantage_std": 2.1463759650786716, "kd/ssd_loss": 0.0, "learning_rate": 9.123153457638704e-07, "loss": 0.03456981976826986, "num_tokens": 75601087.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.801666667064031, "rewards/gt_logging_reward/std": 0.3928664356470108, "sampling/importance_sampling_ratio/max": 1.9825396259625754, "sampling/importance_sampling_ratio/mean": 1.0014914254347482, "sampling/importance_sampling_ratio/min": 0.40905782481034597, "sampling/sampling_logp_difference/max": 0.4472511967023214, "sampling/sampling_logp_difference/mean": 0.004358638504830499, "step": 2310, "step_time": 7.795646638136047, "student/entropy": 0.12824787913511196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0247222236978511, "completions/max_length": 483.03333333333336, "completions/max_terminated_length": 448.6666666666667, "completions/mean_length": 187.61445337931315, "completions/mean_terminated_length": 179.74809036254882, "completions/min_length": 60.266666666666666, "completions/min_terminated_length": 60.266666666666666, "entropy": 0.1265005715812246, "epoch": 0.08886188432764972, "eval/gt_acc_batch": 0.7919444700082143, "frac_reward_zero_std": 1.0, "grad_norm": 0.2810114324092865, "kd/policy_loss": 0.008615959022426978, "kd/rkl_advantage_mean": 0.4957778957167951, "kd/rkl_advantage_p95": 4.550545448064804, "kd/rkl_advantage_std": 2.162119206786156, "kd/ssd_loss": 0.0, "learning_rate": 9.111760908365929e-07, "loss": 0.03446383476257324, "num_tokens": 76594507.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7919444421927134, "rewards/gt_logging_reward/std": 0.400081005692482, "sampling/importance_sampling_ratio/max": 1.8127265691757202, "sampling/importance_sampling_ratio/mean": 0.9961942970752716, "sampling/importance_sampling_ratio/min": 0.4229888717333476, "sampling/sampling_logp_difference/max": 0.37418672839800515, "sampling/sampling_logp_difference/mean": 0.00419707950980713, "step": 2340, "step_time": 8.22678322234812, "student/entropy": 0.1265005715812246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112652967375, "completions/max_length": 476.26666666666665, "completions/max_terminated_length": 430.73333333333335, "completions/mean_length": 182.51112060546876, "completions/mean_terminated_length": 174.89916585286457, "completions/min_length": 52.96666666666667, "completions/min_terminated_length": 52.96666666666667, "entropy": 0.1289219186330835, "epoch": 0.09000113925492728, "eval/gt_acc_batch": 0.8111111303170522, "frac_reward_zero_std": 1.0, "grad_norm": 0.28333938121795654, "kd/policy_loss": 0.00867459579797772, "kd/rkl_advantage_mean": 0.5293283195312445, "kd/rkl_advantage_p95": 4.567687068382899, "kd/rkl_advantage_std": 2.1652725537618003, "kd/ssd_loss": 0.0, "learning_rate": 9.100368359093152e-07, "loss": 0.03469838301340739, "num_tokens": 77552115.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8111111144224803, "rewards/gt_logging_reward/std": 0.3824382558465004, "sampling/importance_sampling_ratio/max": 1.895455006758372, "sampling/importance_sampling_ratio/mean": 1.003621083498001, "sampling/importance_sampling_ratio/min": 0.39692553877830505, "sampling/sampling_logp_difference/max": 0.4748138646284739, "sampling/sampling_logp_difference/mean": 0.00436567230305324, "step": 2370, "step_time": 7.9887411564898985, "student/entropy": 0.1289219186330835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02722222398345669, "completions/max_length": 481.9, "completions/max_terminated_length": 442.6333333333333, "completions/mean_length": 186.71334330240884, "completions/mean_terminated_length": 177.7746617635091, "completions/min_length": 55.5, "completions/min_terminated_length": 55.5, "entropy": 0.13743510966499647, "epoch": 0.09114039418220483, "eval/gt_acc_batch": 0.7658333500226339, "frac_reward_zero_std": 1.0, "grad_norm": 0.27445101737976074, "kd/policy_loss": 0.009643352860196804, "kd/rkl_advantage_mean": 0.5966234654188156, "kd/rkl_advantage_p95": 4.704431922237078, "kd/rkl_advantage_std": 2.176137657960256, "kd/ssd_loss": 0.0, "learning_rate": 9.088975809820377e-07, "loss": 0.038573408126831056, "num_tokens": 78512619.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7658333301544189, "rewards/gt_logging_reward/std": 0.41176370630661646, "sampling/importance_sampling_ratio/max": 1.8626338322957356, "sampling/importance_sampling_ratio/mean": 0.9989446997642517, "sampling/importance_sampling_ratio/min": 0.41120772461096444, "sampling/sampling_logp_difference/max": 0.4143080830574036, "sampling/sampling_logp_difference/mean": 0.004561686042385797, "step": 2400, "step_time": 7.828006503769817, "student/entropy": 0.13743510966499647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.034166668790082136, "completions/max_length": 487.26666666666665, "completions/max_terminated_length": 460.2, "completions/mean_length": 195.2527877807617, "completions/mean_terminated_length": 184.14296112060546, "completions/min_length": 55.9, "completions/min_terminated_length": 55.9, "entropy": 0.13822718095034361, "epoch": 0.0922796491094824, "eval/gt_acc_batch": 0.7561111251513163, "frac_reward_zero_std": 1.0, "grad_norm": 0.3636430501937866, "kd/policy_loss": 0.00965895431387859, "kd/rkl_advantage_mean": 0.638688262825599, "kd/rkl_advantage_p95": 4.699793750047684, "kd/rkl_advantage_std": 2.1665239999691646, "kd/ssd_loss": 0.0, "learning_rate": 9.077583260547601e-07, "loss": 0.03863582213719686, "num_tokens": 79515505.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7561111132303874, "rewards/gt_logging_reward/std": 0.42501756449540457, "sampling/importance_sampling_ratio/max": 1.9506949583689372, "sampling/importance_sampling_ratio/mean": 0.9988558570543925, "sampling/importance_sampling_ratio/min": 0.37841168691714605, "sampling/sampling_logp_difference/max": 0.4947968681653341, "sampling/sampling_logp_difference/mean": 0.004528520093299448, "step": 2430, "step_time": 8.054076588549652, "student/entropy": 0.13822718095034361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028888891140619915, "completions/max_length": 496.8666666666667, "completions/max_terminated_length": 459.3333333333333, "completions/mean_length": 197.1725107828776, "completions/mean_terminated_length": 187.96822814941407, "completions/min_length": 55.666666666666664, "completions/min_terminated_length": 55.666666666666664, "entropy": 0.1277294119199117, "epoch": 0.09341890403675995, "eval/gt_acc_batch": 0.7830555677413941, "frac_reward_zero_std": 1.0, "grad_norm": 0.25965452194213867, "kd/policy_loss": 0.009027881823324908, "kd/rkl_advantage_mean": 0.5254877035661291, "kd/rkl_advantage_p95": 4.502410932381948, "kd/rkl_advantage_std": 2.106247592965762, "kd/ssd_loss": 0.0, "learning_rate": 9.066190711274826e-07, "loss": 0.03611152569452922, "num_tokens": 80530478.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7830555538336436, "rewards/gt_logging_reward/std": 0.40618504385153453, "sampling/importance_sampling_ratio/max": 1.971566092967987, "sampling/importance_sampling_ratio/mean": 0.9997199316819508, "sampling/importance_sampling_ratio/min": 0.41501439809799195, "sampling/sampling_logp_difference/max": 0.4329114278157552, "sampling/sampling_logp_difference/mean": 0.0042702968465164306, "step": 2460, "step_time": 8.195509743981528, "student/entropy": 0.1277294119199117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02222222356746594, "completions/max_length": 488.23333333333335, "completions/max_terminated_length": 445.1333333333333, "completions/mean_length": 184.1086201985677, "completions/mean_terminated_length": 176.73757985432943, "completions/min_length": 55.7, "completions/min_terminated_length": 55.7, "entropy": 0.13043890030433733, "epoch": 0.09455815896403752, "eval/gt_acc_batch": 0.8186111390590668, "frac_reward_zero_std": 1.0, "grad_norm": 0.2930452823638916, "kd/policy_loss": 0.00854595378623344, "kd/rkl_advantage_mean": 0.5432878953715166, "kd/rkl_advantage_p95": 4.610193755229314, "kd/rkl_advantage_std": 2.1820219000180563, "kd/ssd_loss": 0.0, "learning_rate": 9.054798162002049e-07, "loss": 0.03418382008870443, "num_tokens": 81491389.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8186111092567444, "rewards/gt_logging_reward/std": 0.38035005231698354, "sampling/importance_sampling_ratio/max": 1.9551400581995646, "sampling/importance_sampling_ratio/mean": 0.9915592789649963, "sampling/importance_sampling_ratio/min": 0.3871398086349169, "sampling/sampling_logp_difference/max": 0.5042222360769908, "sampling/sampling_logp_difference/mean": 0.00438570948317647, "step": 2490, "step_time": 7.838864750756572, "student/entropy": 0.13043890030433733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02833333474894365, "completions/max_length": 506.7, "completions/max_terminated_length": 462.7, "completions/mean_length": 196.07445576985677, "completions/mean_terminated_length": 186.93275299072266, "completions/min_length": 56.56666666666667, "completions/min_terminated_length": 56.56666666666667, "entropy": 0.12967719994485377, "epoch": 0.09569741389131509, "eval/gt_acc_batch": 0.7811111430327098, "frac_reward_zero_std": 1.0, "grad_norm": 0.26822954416275024, "kd/policy_loss": 0.009079055338709925, "kd/rkl_advantage_mean": 0.6136037112524112, "kd/rkl_advantage_p95": 4.723098788658778, "kd/rkl_advantage_std": 2.187207872668902, "kd/ssd_loss": 0.0, "learning_rate": 9.043405612729275e-07, "loss": 0.03631621996561686, "num_tokens": 82498705.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7811111092567444, "rewards/gt_logging_reward/std": 0.40334372719128925, "sampling/importance_sampling_ratio/max": 1.9551493604977925, "sampling/importance_sampling_ratio/mean": 0.9996428032716115, "sampling/importance_sampling_ratio/min": 0.4240936776002248, "sampling/sampling_logp_difference/max": 0.44321104685465496, "sampling/sampling_logp_difference/mean": 0.004373557625027994, "step": 2520, "step_time": 8.132418876424587, "student/entropy": 0.12967719994485377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018055556683490674, "completions/max_length": 470.6666666666667, "completions/max_terminated_length": 418.1333333333333, "completions/mean_length": 182.11056416829427, "completions/mean_terminated_length": 176.12134958902996, "completions/min_length": 53.233333333333334, "completions/min_terminated_length": 53.233333333333334, "entropy": 0.1342774836346507, "epoch": 0.09683666881859264, "eval/gt_acc_batch": 0.7858333587646484, "frac_reward_zero_std": 1.0, "grad_norm": 0.24099889397621155, "kd/policy_loss": 0.009323345495310302, "kd/rkl_advantage_mean": 0.4866545496857725, "kd/rkl_advantage_p95": 4.470307143529256, "kd/rkl_advantage_std": 2.136853490273158, "kd/ssd_loss": 0.0, "learning_rate": 9.0320130634565e-07, "loss": 0.0372933824857076, "num_tokens": 83457847.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7858333249886831, "rewards/gt_logging_reward/std": 0.39770562052726743, "sampling/importance_sampling_ratio/max": 1.9260886589686075, "sampling/importance_sampling_ratio/mean": 0.9984665671984355, "sampling/importance_sampling_ratio/min": 0.40174026240905125, "sampling/sampling_logp_difference/max": 0.5191780765851338, "sampling/sampling_logp_difference/mean": 0.004489704562971989, "step": 2550, "step_time": 7.829716186807491, "student/entropy": 0.1342774836346507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018055556745578846, "completions/max_length": 477.53333333333336, "completions/max_terminated_length": 444.03333333333336, "completions/mean_length": 185.80945434570313, "completions/mean_terminated_length": 180.01238962809245, "completions/min_length": 54.63333333333333, "completions/min_terminated_length": 54.63333333333333, "entropy": 0.1341927339633306, "epoch": 0.0979759237458702, "eval/gt_acc_batch": 0.7916666865348816, "frac_reward_zero_std": 1.0, "grad_norm": 0.2536690831184387, "kd/policy_loss": 0.009400840136610593, "kd/rkl_advantage_mean": 0.5568274569697678, "kd/rkl_advantage_p95": 4.577443089087804, "kd/rkl_advantage_std": 2.138621539870898, "kd/ssd_loss": 0.0, "learning_rate": 9.020620514183723e-07, "loss": 0.037603362401326494, "num_tokens": 84434537.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7916666746139527, "rewards/gt_logging_reward/std": 0.39548064519961673, "sampling/importance_sampling_ratio/max": 1.9541515866915384, "sampling/importance_sampling_ratio/mean": 1.0015944302082063, "sampling/importance_sampling_ratio/min": 0.3928253730138143, "sampling/sampling_logp_difference/max": 0.4888993302981059, "sampling/sampling_logp_difference/mean": 0.004441790600928168, "step": 2580, "step_time": 7.883022433325338, "student/entropy": 0.1341927339633306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026111112566043933, "completions/max_length": 483.46666666666664, "completions/max_terminated_length": 446.43333333333334, "completions/mean_length": 190.1341776529948, "completions/mean_terminated_length": 181.83374837239583, "completions/min_length": 55.36666666666667, "completions/min_terminated_length": 55.36666666666667, "entropy": 0.1284243743866682, "epoch": 0.09911517867314776, "eval/gt_acc_batch": 0.773333348830541, "frac_reward_zero_std": 1.0, "grad_norm": 0.2707442045211792, "kd/policy_loss": 0.008845166879473254, "kd/rkl_advantage_mean": 0.46266511380672454, "kd/rkl_advantage_p95": 4.441837922732035, "kd/rkl_advantage_std": 2.1361097087462744, "kd/ssd_loss": 0.0, "learning_rate": 9.009227964910948e-07, "loss": 0.03538066546122233, "num_tokens": 85431156.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7733333329359691, "rewards/gt_logging_reward/std": 0.4092830717563629, "sampling/importance_sampling_ratio/max": 1.9331445773442586, "sampling/importance_sampling_ratio/mean": 0.9974872887134552, "sampling/importance_sampling_ratio/min": 0.40437979201475777, "sampling/sampling_logp_difference/max": 0.4334670344988505, "sampling/sampling_logp_difference/mean": 0.004338604576575259, "step": 2610, "step_time": 8.039176417258568, "student/entropy": 0.1284243743866682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016666667660077412, "completions/max_length": 480.96666666666664, "completions/max_terminated_length": 437.6, "completions/mean_length": 185.89695383707684, "completions/mean_terminated_length": 180.45635935465495, "completions/min_length": 57.5, "completions/min_terminated_length": 57.5, "entropy": 0.13133317635705075, "epoch": 0.10025443360042532, "eval/gt_acc_batch": 0.7988889018694559, "frac_reward_zero_std": 1.0, "grad_norm": 0.33449363708496094, "kd/policy_loss": 0.008953722327714787, "kd/rkl_advantage_mean": 0.5451364499206345, "kd/rkl_advantage_p95": 4.603030594189962, "kd/rkl_advantage_std": 2.145874415834745, "kd/ssd_loss": 0.0, "learning_rate": 8.997835415638172e-07, "loss": 0.03581489721934001, "num_tokens": 86410369.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7988888919353485, "rewards/gt_logging_reward/std": 0.3921315347154935, "sampling/importance_sampling_ratio/max": 1.918578024705251, "sampling/importance_sampling_ratio/mean": 1.007682885726293, "sampling/importance_sampling_ratio/min": 0.4189666564265887, "sampling/sampling_logp_difference/max": 0.40926923155784606, "sampling/sampling_logp_difference/mean": 0.00427223058262219, "step": 2640, "step_time": 7.882418575227105, "student/entropy": 0.13133317635705075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018611111957579852, "completions/max_length": 493.6333333333333, "completions/max_terminated_length": 468.43333333333334, "completions/mean_length": 190.97223205566405, "completions/mean_terminated_length": 184.96208699544272, "completions/min_length": 63.36666666666667, "completions/min_terminated_length": 63.36666666666667, "entropy": 0.13132069098452728, "epoch": 0.10139368852770288, "eval/gt_acc_batch": 0.7741667012373606, "frac_reward_zero_std": 1.0, "grad_norm": 0.25463974475860596, "kd/policy_loss": 0.009242772068440293, "kd/rkl_advantage_mean": 0.5823033522814512, "kd/rkl_advantage_p95": 4.650602527459463, "kd/rkl_advantage_std": 2.153051367402077, "kd/ssd_loss": 0.0, "learning_rate": 8.986442866365396e-07, "loss": 0.036971092224121094, "num_tokens": 87391933.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7741666694482168, "rewards/gt_logging_reward/std": 0.4141622046629588, "sampling/importance_sampling_ratio/max": 1.9754610776901245, "sampling/importance_sampling_ratio/mean": 1.0021492660045623, "sampling/importance_sampling_ratio/min": 0.3904266655445099, "sampling/sampling_logp_difference/max": 0.4780644436677297, "sampling/sampling_logp_difference/mean": 0.004398190408634642, "step": 2670, "step_time": 7.954217938353152, "student/entropy": 0.13132069098452728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223604718842, "completions/max_length": 486.26666666666665, "completions/max_terminated_length": 453.8666666666667, "completions/mean_length": 188.47417653401692, "completions/mean_terminated_length": 180.43152720133463, "completions/min_length": 55.333333333333336, "completions/min_terminated_length": 55.333333333333336, "entropy": 0.12492078511665265, "epoch": 0.10253294345498044, "eval/gt_acc_batch": 0.8030555705229442, "frac_reward_zero_std": 1.0, "grad_norm": 0.244247704744339, "kd/policy_loss": 0.00832951344588461, "kd/rkl_advantage_mean": 0.5099521154072135, "kd/rkl_advantage_p95": 4.483334050575892, "kd/rkl_advantage_std": 2.1368101994196573, "kd/ssd_loss": 0.0, "learning_rate": 8.975050317092621e-07, "loss": 0.033318054676055905, "num_tokens": 88359576.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555526415507, "rewards/gt_logging_reward/std": 0.3893969794114431, "sampling/importance_sampling_ratio/max": 2.0530823787053425, "sampling/importance_sampling_ratio/mean": 1.009898457924525, "sampling/importance_sampling_ratio/min": 0.4220739091436068, "sampling/sampling_logp_difference/max": 0.43813617825508117, "sampling/sampling_logp_difference/mean": 0.004195696270714203, "step": 2700, "step_time": 7.917494105997806, "student/entropy": 0.12492078511665265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019444445458551247, "completions/max_length": 490.46666666666664, "completions/max_terminated_length": 443.7, "completions/mean_length": 186.33612111409505, "completions/mean_terminated_length": 180.01663157145182, "completions/min_length": 59.63333333333333, "completions/min_terminated_length": 59.63333333333333, "entropy": 0.12733064573258163, "epoch": 0.10367219838225801, "eval/gt_acc_batch": 0.793888908624649, "frac_reward_zero_std": 1.0, "grad_norm": 0.26950204372406006, "kd/policy_loss": 0.008816314496410389, "kd/rkl_advantage_mean": 0.6282241777982562, "kd/rkl_advantage_p95": 4.679370059569677, "kd/rkl_advantage_std": 2.134985715150833, "kd/ssd_loss": 0.0, "learning_rate": 8.963657767819846e-07, "loss": 0.03526525497436524, "num_tokens": 89326578.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7938888887564342, "rewards/gt_logging_reward/std": 0.39312157879273096, "sampling/importance_sampling_ratio/max": 1.9670658230781555, "sampling/importance_sampling_ratio/mean": 1.0060882647832234, "sampling/importance_sampling_ratio/min": 0.4259149859348933, "sampling/sampling_logp_difference/max": 0.43084608713785805, "sampling/sampling_logp_difference/mean": 0.004235842498019338, "step": 2730, "step_time": 7.977963447582442, "student/entropy": 0.12733064573258163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016666667660077412, "completions/max_length": 487.3333333333333, "completions/max_terminated_length": 447.1333333333333, "completions/mean_length": 188.76028849283855, "completions/mean_terminated_length": 183.2480962117513, "completions/min_length": 61.2, "completions/min_terminated_length": 61.2, "entropy": 0.1279523404315114, "epoch": 0.10481145330953556, "eval/gt_acc_batch": 0.8111111402511597, "frac_reward_zero_std": 1.0, "grad_norm": 0.2640644609928131, "kd/policy_loss": 0.00848029184077556, "kd/rkl_advantage_mean": 0.5692271318286657, "kd/rkl_advantage_p95": 4.649699354171753, "kd/rkl_advantage_std": 2.159418644507726, "kd/ssd_loss": 0.0, "learning_rate": 8.952265218547069e-07, "loss": 0.03392117023468018, "num_tokens": 90299411.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8111111124356588, "rewards/gt_logging_reward/std": 0.38412694732348124, "sampling/importance_sampling_ratio/max": 1.8974849939346314, "sampling/importance_sampling_ratio/mean": 0.9985783139864604, "sampling/importance_sampling_ratio/min": 0.4559721330801646, "sampling/sampling_logp_difference/max": 0.4708087642987569, "sampling/sampling_logp_difference/mean": 0.004288840813872715, "step": 2760, "step_time": 7.826985008506259, "student/entropy": 0.1279523404315114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334513008596, "completions/max_length": 482.06666666666666, "completions/max_terminated_length": 455.46666666666664, "completions/mean_length": 183.5077870686849, "completions/mean_terminated_length": 176.67318369547527, "completions/min_length": 54.63333333333333, "completions/min_terminated_length": 54.63333333333333, "entropy": 0.1293431148553888, "epoch": 0.10595070823681313, "eval/gt_acc_batch": 0.7947222451368968, "frac_reward_zero_std": 1.0, "grad_norm": 0.334206223487854, "kd/policy_loss": 0.009016885379484545, "kd/rkl_advantage_mean": 0.5772790737605343, "kd/rkl_advantage_p95": 4.689591989914576, "kd/rkl_advantage_std": 2.191928951938947, "kd/ssd_loss": 0.0, "learning_rate": 8.940872669274294e-07, "loss": 0.03606754938761393, "num_tokens": 91257703.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222193082174, "rewards/gt_logging_reward/std": 0.3933519000808398, "sampling/importance_sampling_ratio/max": 1.8908846735954286, "sampling/importance_sampling_ratio/mean": 0.9974142014980316, "sampling/importance_sampling_ratio/min": 0.3986753324667613, "sampling/sampling_logp_difference/max": 0.46421138842900594, "sampling/sampling_logp_difference/mean": 0.0043472337691734236, "step": 2790, "step_time": 7.86610577335426, "student/entropy": 0.1293431148553888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001390775046, "completions/max_length": 487.26666666666665, "completions/max_terminated_length": 461.3666666666667, "completions/mean_length": 185.4466756184896, "completions/mean_terminated_length": 178.0888427734375, "completions/min_length": 47.2, "completions/min_terminated_length": 47.2, "entropy": 0.1341731216137608, "epoch": 0.10708996316409068, "eval/gt_acc_batch": 0.7841666758060455, "frac_reward_zero_std": 1.0, "grad_norm": 0.288561075925827, "kd/policy_loss": 0.009394078220551213, "kd/rkl_advantage_mean": 0.5921128570723037, "kd/rkl_advantage_p95": 4.710111363728841, "kd/rkl_advantage_std": 2.186418343583743, "kd/ssd_loss": 0.0, "learning_rate": 8.929480120001519e-07, "loss": 0.03757631778717041, "num_tokens": 92217671.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.784166673819224, "rewards/gt_logging_reward/std": 0.4062733858823776, "sampling/importance_sampling_ratio/max": 1.91035106976827, "sampling/importance_sampling_ratio/mean": 0.9994420568148296, "sampling/importance_sampling_ratio/min": 0.37972700546185173, "sampling/sampling_logp_difference/max": 0.4596379160881042, "sampling/sampling_logp_difference/mean": 0.004473197739571333, "step": 2820, "step_time": 7.825856989622116, "student/entropy": 0.1341731216137608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444578140974, "completions/max_length": 484.3, "completions/max_terminated_length": 453.23333333333335, "completions/mean_length": 192.96056569417317, "completions/mean_terminated_length": 185.20463053385416, "completions/min_length": 55.7, "completions/min_terminated_length": 55.7, "entropy": 0.1357055296500524, "epoch": 0.10822921809136825, "eval/gt_acc_batch": 0.7722222407658895, "frac_reward_zero_std": 1.0, "grad_norm": 0.29710280895233154, "kd/policy_loss": 0.009339032826634746, "kd/rkl_advantage_mean": 0.6207599075511098, "kd/rkl_advantage_p95": 4.786616762479146, "kd/rkl_advantage_std": 2.1960125962893167, "kd/ssd_loss": 0.0, "learning_rate": 8.918087570728742e-07, "loss": 0.03735613028208415, "num_tokens": 93229705.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.772222222884496, "rewards/gt_logging_reward/std": 0.41294146378835045, "sampling/importance_sampling_ratio/max": 1.9031977852185566, "sampling/importance_sampling_ratio/mean": 1.0009443660577138, "sampling/importance_sampling_ratio/min": 0.40976707090934117, "sampling/sampling_logp_difference/max": 0.44607590039571127, "sampling/sampling_logp_difference/mean": 0.004281426329786579, "step": 2850, "step_time": 8.19971856539681, "student/entropy": 0.1357055296500524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02583333496004343, "completions/max_length": 487.76666666666665, "completions/max_terminated_length": 454.1666666666667, "completions/mean_length": 185.3658426920573, "completions/mean_terminated_length": 176.65553232828776, "completions/min_length": 57.43333333333333, "completions/min_terminated_length": 57.43333333333333, "entropy": 0.12650156350185474, "epoch": 0.1093684730186458, "eval/gt_acc_batch": 0.7933333516120911, "frac_reward_zero_std": 1.0, "grad_norm": 0.3176569640636444, "kd/policy_loss": 0.008896940671062718, "kd/rkl_advantage_mean": 0.5651173517418404, "kd/rkl_advantage_p95": 4.694011131922404, "kd/rkl_advantage_std": 2.2001018931468326, "kd/ssd_loss": 0.0, "learning_rate": 8.906695021455968e-07, "loss": 0.03558776378631592, "num_tokens": 94189790.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7933333337306976, "rewards/gt_logging_reward/std": 0.39701737960179645, "sampling/importance_sampling_ratio/max": 1.968362327416738, "sampling/importance_sampling_ratio/mean": 1.0021819472312927, "sampling/importance_sampling_ratio/min": 0.44424338738123575, "sampling/sampling_logp_difference/max": 0.4665427347024282, "sampling/sampling_logp_difference/mean": 0.004246408158602814, "step": 2880, "step_time": 7.841231532418169, "student/entropy": 0.12650156350185474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01666666759798924, "completions/max_length": 462.8666666666667, "completions/max_terminated_length": 441.1, "completions/mean_length": 188.96973215738933, "completions/mean_terminated_length": 183.48166198730468, "completions/min_length": 49.3, "completions/min_terminated_length": 49.3, "entropy": 0.1263409563029806, "epoch": 0.11050772794592337, "eval/gt_acc_batch": 0.8027778049310048, "frac_reward_zero_std": 1.0, "grad_norm": 0.3114893138408661, "kd/policy_loss": 0.008268149144714698, "kd/rkl_advantage_mean": 0.46100068429659585, "kd/rkl_advantage_p95": 4.421410803000132, "kd/rkl_advantage_std": 2.1155777404705685, "kd/ssd_loss": 0.0, "learning_rate": 8.895302472183192e-07, "loss": 0.03307259678840637, "num_tokens": 95174321.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8027777771155039, "rewards/gt_logging_reward/std": 0.38886961390574776, "sampling/importance_sampling_ratio/max": 1.8528289556503297, "sampling/importance_sampling_ratio/mean": 0.9955129702885945, "sampling/importance_sampling_ratio/min": 0.4217707778016726, "sampling/sampling_logp_difference/max": 0.4762453258037567, "sampling/sampling_logp_difference/mean": 0.004242754727602005, "step": 2910, "step_time": 7.914536713626391, "student/entropy": 0.1263409563029806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777779151995976, "completions/max_length": 480.7, "completions/max_terminated_length": 444.2, "completions/mean_length": 188.05973103841146, "completions/mean_terminated_length": 180.63280639648437, "completions/min_length": 60.03333333333333, "completions/min_terminated_length": 60.03333333333333, "entropy": 0.12963624962915976, "epoch": 0.11164698287320092, "eval/gt_acc_batch": 0.7877777953942616, "frac_reward_zero_std": 1.0, "grad_norm": 0.25547686219215393, "kd/policy_loss": 0.0091092527921622, "kd/rkl_advantage_mean": 0.5409802442067303, "kd/rkl_advantage_p95": 4.570285373926163, "kd/rkl_advantage_std": 2.1494710832834243, "kd/ssd_loss": 0.0, "learning_rate": 8.883909922910417e-07, "loss": 0.03643701871236165, "num_tokens": 96146608.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7877777775128683, "rewards/gt_logging_reward/std": 0.4004997327923775, "sampling/importance_sampling_ratio/max": 1.9795308709144592, "sampling/importance_sampling_ratio/mean": 1.003355618317922, "sampling/importance_sampling_ratio/min": 0.4325621897975604, "sampling/sampling_logp_difference/max": 0.4222909927368164, "sampling/sampling_logp_difference/mean": 0.004409500785792868, "step": 2940, "step_time": 7.845258369828419, "student/entropy": 0.12963624962915976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029444446011135977, "completions/max_length": 497.6, "completions/max_terminated_length": 452.73333333333335, "completions/mean_length": 186.3055653889974, "completions/mean_terminated_length": 176.55448506673176, "completions/min_length": 47.7, "completions/min_terminated_length": 47.7, "entropy": 0.12982990108430387, "epoch": 0.11278623780047849, "eval/gt_acc_batch": 0.7850000242392222, "frac_reward_zero_std": 1.0, "grad_norm": 0.30163827538490295, "kd/policy_loss": 0.009107812658961241, "kd/rkl_advantage_mean": 0.5615044108572571, "kd/rkl_advantage_p95": 4.723702734708786, "kd/rkl_advantage_std": 2.21654703716437, "kd/ssd_loss": 0.0, "learning_rate": 8.87251737363764e-07, "loss": 0.0364312489827474, "num_tokens": 97119532.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7850000023841858, "rewards/gt_logging_reward/std": 0.405627104640007, "sampling/importance_sampling_ratio/max": 1.9664215246836345, "sampling/importance_sampling_ratio/mean": 1.0001489380995432, "sampling/importance_sampling_ratio/min": 0.38202548027038574, "sampling/sampling_logp_difference/max": 0.46672728657722473, "sampling/sampling_logp_difference/mean": 0.004370495770126581, "step": 2970, "step_time": 8.081611769630884, "student/entropy": 0.12982990108430387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779381722212, "completions/max_length": 481.8333333333333, "completions/max_terminated_length": 457.8333333333333, "completions/mean_length": 192.47112019856772, "completions/mean_terminated_length": 183.46061197916666, "completions/min_length": 56.233333333333334, "completions/min_terminated_length": 56.233333333333334, "entropy": 0.13391398737827936, "epoch": 0.11392549272775605, "eval/gt_acc_batch": 0.7908333520094554, "frac_reward_zero_std": 1.0, "grad_norm": 0.31095436215400696, "kd/policy_loss": 0.009304822529278075, "kd/rkl_advantage_mean": 0.553014601006483, "kd/rkl_advantage_p95": 4.591026719411214, "kd/rkl_advantage_std": 2.149878619114558, "kd/ssd_loss": 0.0, "learning_rate": 8.861124824364865e-07, "loss": 0.03721928596496582, "num_tokens": 98114596.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7908333361148834, "rewards/gt_logging_reward/std": 0.39829020301500956, "sampling/importance_sampling_ratio/max": 1.9340099612871806, "sampling/importance_sampling_ratio/mean": 0.9997600456078847, "sampling/importance_sampling_ratio/min": 0.3979719529549281, "sampling/sampling_logp_difference/max": 0.47996949354807533, "sampling/sampling_logp_difference/mean": 0.0044729281449690465, "step": 3000, "step_time": 7.91545682354675, "student/entropy": 0.13391398737827936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667852550744, "completions/max_length": 485.1, "completions/max_terminated_length": 454.96666666666664, "completions/mean_length": 189.883620707194, "completions/mean_terminated_length": 183.66282297770184, "completions/min_length": 57.13333333333333, "completions/min_terminated_length": 57.13333333333333, "entropy": 0.12644131345053514, "epoch": 0.1150647476550336, "eval/gt_acc_batch": 0.7922222554683686, "frac_reward_zero_std": 1.0, "grad_norm": 0.2448025792837143, "kd/policy_loss": 0.008910949272103608, "kd/rkl_advantage_mean": 0.5127500921060952, "kd/rkl_advantage_p95": 4.604007281859716, "kd/rkl_advantage_std": 2.1944486021995546, "kd/ssd_loss": 0.0, "learning_rate": 8.849732275092089e-07, "loss": 0.035643792152404784, "num_tokens": 99099281.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222236792247, "rewards/gt_logging_reward/std": 0.3964786320924759, "sampling/importance_sampling_ratio/max": 1.9348843892415364, "sampling/importance_sampling_ratio/mean": 1.0020498156547546, "sampling/importance_sampling_ratio/min": 0.4202634369333585, "sampling/sampling_logp_difference/max": 0.41174968481063845, "sampling/sampling_logp_difference/mean": 0.0042539237843205535, "step": 3030, "step_time": 7.904931049289492, "student/entropy": 0.12644131345053514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01833333435157935, "completions/max_length": 476.03333333333336, "completions/max_terminated_length": 444.53333333333336, "completions/mean_length": 187.8555669148763, "completions/mean_terminated_length": 181.8478001912435, "completions/min_length": 60.06666666666667, "completions/min_terminated_length": 60.06666666666667, "entropy": 0.12160934892793497, "epoch": 0.11620400258231117, "eval/gt_acc_batch": 0.8072222431500753, "frac_reward_zero_std": 1.0, "grad_norm": 0.265241801738739, "kd/policy_loss": 0.008438106981338933, "kd/rkl_advantage_mean": 0.5222618892245615, "kd/rkl_advantage_p95": 4.517312947909037, "kd/rkl_advantage_std": 2.1241929054260256, "kd/ssd_loss": 0.0, "learning_rate": 8.838339725819313e-07, "loss": 0.03375242551167806, "num_tokens": 100085817.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222272555033, "rewards/gt_logging_reward/std": 0.38527297377586367, "sampling/importance_sampling_ratio/max": 1.937837302684784, "sampling/importance_sampling_ratio/mean": 1.0022961934407553, "sampling/importance_sampling_ratio/min": 0.4192844410737356, "sampling/sampling_logp_difference/max": 0.4281455318133036, "sampling/sampling_logp_difference/mean": 0.004147628229111433, "step": 3060, "step_time": 7.935295636769539, "student/entropy": 0.12160934892793497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01861111211280028, "completions/max_length": 486.06666666666666, "completions/max_terminated_length": 452.0, "completions/mean_length": 189.7658432006836, "completions/mean_terminated_length": 183.73353932698566, "completions/min_length": 64.63333333333334, "completions/min_terminated_length": 64.63333333333334, "entropy": 0.12537016390512387, "epoch": 0.11734325750958872, "eval/gt_acc_batch": 0.7930555840333303, "frac_reward_zero_std": 1.0, "grad_norm": 0.388371080160141, "kd/policy_loss": 0.0087150711818443, "kd/rkl_advantage_mean": 0.47350176858211246, "kd/rkl_advantage_p95": 4.398908207813899, "kd/rkl_advantage_std": 2.095533405741056, "kd/ssd_loss": 0.0, "learning_rate": 8.826947176546539e-07, "loss": 0.03486028512318929, "num_tokens": 101077710.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555522441864, "rewards/gt_logging_reward/std": 0.4004454096158346, "sampling/importance_sampling_ratio/max": 1.9231550137201945, "sampling/importance_sampling_ratio/mean": 0.9995372414588928, "sampling/importance_sampling_ratio/min": 0.41753091613451637, "sampling/sampling_logp_difference/max": 0.4601218303044637, "sampling/sampling_logp_difference/mean": 0.004274066219416757, "step": 3090, "step_time": 8.022318035088635, "student/entropy": 0.12537016390512387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666668076068162, "completions/max_length": 497.6666666666667, "completions/max_terminated_length": 435.1666666666667, "completions/mean_length": 185.15723164876303, "completions/mean_terminated_length": 177.98362681070964, "completions/min_length": 56.86666666666667, "completions/min_terminated_length": 56.86666666666667, "entropy": 0.12795597265164058, "epoch": 0.11848251243686629, "eval/gt_acc_batch": 0.7991666793823242, "frac_reward_zero_std": 1.0, "grad_norm": 0.28930187225341797, "kd/policy_loss": 0.008806529788610835, "kd/rkl_advantage_mean": 0.55240562291195, "kd/rkl_advantage_p95": 4.598730285962422, "kd/rkl_advantage_std": 2.1570199698209764, "kd/ssd_loss": 0.0, "learning_rate": 8.815554627273763e-07, "loss": 0.03522611856460571, "num_tokens": 102047124.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7991666634877522, "rewards/gt_logging_reward/std": 0.3915647322932879, "sampling/importance_sampling_ratio/max": 2.00159508784612, "sampling/importance_sampling_ratio/mean": 0.9988702615102132, "sampling/importance_sampling_ratio/min": 0.34816933423280716, "sampling/sampling_logp_difference/max": 0.4654865086078644, "sampling/sampling_logp_difference/mean": 0.004354878274413446, "step": 3120, "step_time": 7.986426722312657, "student/entropy": 0.12795597265164058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445998718343, "completions/max_length": 467.73333333333335, "completions/max_terminated_length": 438.43333333333334, "completions/mean_length": 180.44500986735025, "completions/mean_terminated_length": 172.2989242553711, "completions/min_length": 52.9, "completions/min_terminated_length": 52.9, "entropy": 0.1321094830830892, "epoch": 0.11962176736414384, "eval/gt_acc_batch": 0.8136111279328664, "frac_reward_zero_std": 1.0, "grad_norm": 0.2480178028345108, "kd/policy_loss": 0.008777907448044667, "kd/rkl_advantage_mean": 0.5383117169917871, "kd/rkl_advantage_p95": 4.536651798089346, "kd/rkl_advantage_std": 2.1649298737446467, "kd/ssd_loss": 0.0, "learning_rate": 8.804162078000987e-07, "loss": 0.03511163393656413, "num_tokens": 102990126.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813611110051473, "rewards/gt_logging_reward/std": 0.3783667981624603, "sampling/importance_sampling_ratio/max": 1.908983329931895, "sampling/importance_sampling_ratio/mean": 0.9979601184527079, "sampling/importance_sampling_ratio/min": 0.410832779109478, "sampling/sampling_logp_difference/max": 0.4460064470767975, "sampling/sampling_logp_difference/mean": 0.004480631245921056, "step": 3150, "step_time": 7.703203705182144, "student/entropy": 0.1321094830830892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556720743575, "completions/max_length": 481.8666666666667, "completions/max_terminated_length": 447.4, "completions/mean_length": 190.03112182617187, "completions/mean_terminated_length": 183.36178588867188, "completions/min_length": 51.53333333333333, "completions/min_terminated_length": 51.53333333333333, "entropy": 0.13199725324908892, "epoch": 0.12076102229142141, "eval/gt_acc_batch": 0.7861111342906952, "frac_reward_zero_std": 1.0, "grad_norm": 0.26870352029800415, "kd/policy_loss": 0.009176970007441316, "kd/rkl_advantage_mean": 0.5988997920105855, "kd/rkl_advantage_p95": 4.655586703618368, "kd/rkl_advantage_std": 2.149715925256411, "kd/ssd_loss": 0.0, "learning_rate": 8.792769528728211e-07, "loss": 0.03670788606007894, "num_tokens": 103973710.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111124356588, "rewards/gt_logging_reward/std": 0.4016463900605837, "sampling/importance_sampling_ratio/max": 2.050091246763865, "sampling/importance_sampling_ratio/mean": 1.0013919691244761, "sampling/importance_sampling_ratio/min": 0.3806546022494634, "sampling/sampling_logp_difference/max": 0.4668731908003489, "sampling/sampling_logp_difference/mean": 0.004434997580635051, "step": 3180, "step_time": 7.865223933236363, "student/entropy": 0.13199725324908892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778835346302, "completions/max_length": 485.3333333333333, "completions/max_terminated_length": 448.3333333333333, "completions/mean_length": 188.7697321573893, "completions/mean_terminated_length": 182.18024648030598, "completions/min_length": 57.8, "completions/min_terminated_length": 57.8, "entropy": 0.13337789283444484, "epoch": 0.12190027721869898, "eval/gt_acc_batch": 0.7766666869322459, "frac_reward_zero_std": 1.0, "grad_norm": 0.30028414726257324, "kd/policy_loss": 0.009340588449655721, "kd/rkl_advantage_mean": 0.5577432734891772, "kd/rkl_advantage_p95": 4.56463151772817, "kd/rkl_advantage_std": 2.1621671726306277, "kd/ssd_loss": 0.0, "learning_rate": 8.781376979455436e-07, "loss": 0.03736236095428467, "num_tokens": 104950185.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7766666650772095, "rewards/gt_logging_reward/std": 0.4093223611513774, "sampling/importance_sampling_ratio/max": 2.022914628187815, "sampling/importance_sampling_ratio/mean": 1.0096119443575542, "sampling/importance_sampling_ratio/min": 0.4428839564323425, "sampling/sampling_logp_difference/max": 0.42456708550453187, "sampling/sampling_logp_difference/mean": 0.004422038040744761, "step": 3210, "step_time": 7.90896871748846, "student/entropy": 0.13337789283444484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890041659276, "completions/max_length": 485.53333333333336, "completions/max_terminated_length": 453.3666666666667, "completions/mean_length": 193.2819559733073, "completions/mean_terminated_length": 185.54942982991537, "completions/min_length": 48.4, "completions/min_terminated_length": 48.4, "entropy": 0.1277780519798398, "epoch": 0.12303953214597653, "eval/gt_acc_batch": 0.7797222395737966, "frac_reward_zero_std": 1.0, "grad_norm": 0.3451780080795288, "kd/policy_loss": 0.008912410268870493, "kd/rkl_advantage_mean": 0.5381518165425708, "kd/rkl_advantage_p95": 4.678116275866826, "kd/rkl_advantage_std": 2.1909276922543843, "kd/ssd_loss": 0.0, "learning_rate": 8.769984430182659e-07, "loss": 0.03564964532852173, "num_tokens": 105968432.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7797222276528676, "rewards/gt_logging_reward/std": 0.4090959757566452, "sampling/importance_sampling_ratio/max": 1.9940569917360942, "sampling/importance_sampling_ratio/mean": 1.0004310230414073, "sampling/importance_sampling_ratio/min": 0.41443876127401985, "sampling/sampling_logp_difference/max": 0.4682272712389628, "sampling/sampling_logp_difference/mean": 0.004273979055384795, "step": 3240, "step_time": 8.241763534043761, "student/entropy": 0.1277780519798398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890389353038, "completions/max_length": 488.3, "completions/max_terminated_length": 453.73333333333335, "completions/mean_length": 190.14417673746746, "completions/mean_terminated_length": 181.58126220703124, "completions/min_length": 60.0, "completions/min_terminated_length": 60.0, "entropy": 0.13247901070863008, "epoch": 0.1241787870732541, "eval/gt_acc_batch": 0.787500015894572, "frac_reward_zero_std": 1.0, "grad_norm": 0.30201953649520874, "kd/policy_loss": 0.009239215487226223, "kd/rkl_advantage_mean": 0.6205948137290155, "kd/rkl_advantage_p95": 4.720363096396128, "kd/rkl_advantage_std": 2.166735730568568, "kd/ssd_loss": 0.0, "learning_rate": 8.758591880909884e-07, "loss": 0.0369568665822347, "num_tokens": 106968887.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7875, "rewards/gt_logging_reward/std": 0.39916381339232127, "sampling/importance_sampling_ratio/max": 1.9217944264411926, "sampling/importance_sampling_ratio/mean": 0.9941265602906545, "sampling/importance_sampling_ratio/min": 0.4073998168110847, "sampling/sampling_logp_difference/max": 0.43743375738461815, "sampling/sampling_logp_difference/mean": 0.004402770381420851, "step": 3270, "step_time": 8.298590180262302, "student/entropy": 0.13247901070863008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030555557428548732, "completions/max_length": 497.76666666666665, "completions/max_terminated_length": 471.6333333333333, "completions/mean_length": 194.0466766357422, "completions/mean_terminated_length": 184.24701131184895, "completions/min_length": 49.4, "completions/min_terminated_length": 49.4, "entropy": 0.1351915016149481, "epoch": 0.12531804200053165, "eval/gt_acc_batch": 0.7686111191908519, "frac_reward_zero_std": 1.0, "grad_norm": 0.2448456883430481, "kd/policy_loss": 0.00939669026993215, "kd/rkl_advantage_mean": 0.5985618131545682, "kd/rkl_advantage_p95": 4.730362848440806, "kd/rkl_advantage_std": 2.1924953987201055, "kd/ssd_loss": 0.0, "learning_rate": 8.747199331637109e-07, "loss": 0.037586760520935056, "num_tokens": 107972199.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7686111172040303, "rewards/gt_logging_reward/std": 0.41108149737119676, "sampling/importance_sampling_ratio/max": 1.9616144100824993, "sampling/importance_sampling_ratio/mean": 1.0011666397253671, "sampling/importance_sampling_ratio/min": 0.42680716862281165, "sampling/sampling_logp_difference/max": 0.4030303080876668, "sampling/sampling_logp_difference/mean": 0.004450731181229154, "step": 3300, "step_time": 8.085356685208778, "student/entropy": 0.1351915016149481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277778909852108, "completions/max_length": 489.26666666666665, "completions/max_terminated_length": 448.4, "completions/mean_length": 186.99139963785808, "completions/mean_terminated_length": 178.57846323649088, "completions/min_length": 55.233333333333334, "completions/min_terminated_length": 55.233333333333334, "entropy": 0.13238962894926468, "epoch": 0.12645729692780922, "eval/gt_acc_batch": 0.7877777814865112, "frac_reward_zero_std": 1.0, "grad_norm": 0.2529484033584595, "kd/policy_loss": 0.008619312464725226, "kd/rkl_advantage_mean": 0.6833821083574245, "kd/rkl_advantage_p95": 4.863298573096594, "kd/rkl_advantage_std": 2.2282499402761458, "kd/ssd_loss": 0.0, "learning_rate": 8.735806782364334e-07, "loss": 0.03447724978129069, "num_tokens": 108948720.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7877777874469757, "rewards/gt_logging_reward/std": 0.40146307051181795, "sampling/importance_sampling_ratio/max": 1.9197857896486918, "sampling/importance_sampling_ratio/mean": 0.9870763500531514, "sampling/importance_sampling_ratio/min": 0.43445735772450766, "sampling/sampling_logp_difference/max": 0.4601202170054118, "sampling/sampling_logp_difference/mean": 0.00447044720252355, "step": 3330, "step_time": 8.037400667757417, "student/entropy": 0.13238962894926468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.036944446184982854, "completions/max_length": 482.1, "completions/max_terminated_length": 452.06666666666666, "completions/mean_length": 188.72723286946615, "completions/mean_terminated_length": 176.61573028564453, "completions/min_length": 46.733333333333334, "completions/min_terminated_length": 46.733333333333334, "entropy": 0.13653970242788394, "epoch": 0.12759655185508678, "eval/gt_acc_batch": 0.7863889078299204, "frac_reward_zero_std": 1.0, "grad_norm": 0.23837272822856903, "kd/policy_loss": 0.009488712132830793, "kd/rkl_advantage_mean": 0.5568264831788838, "kd/rkl_advantage_p95": 4.644654782613118, "kd/rkl_advantage_std": 2.1731474498907724, "kd/ssd_loss": 0.0, "learning_rate": 8.724414233091558e-07, "loss": 0.03795484701792399, "num_tokens": 109936050.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.786388882001241, "rewards/gt_logging_reward/std": 0.40324352582295736, "sampling/importance_sampling_ratio/max": 2.0410133918126423, "sampling/importance_sampling_ratio/mean": 1.0044897417227427, "sampling/importance_sampling_ratio/min": 0.40215475906928383, "sampling/sampling_logp_difference/max": 0.45568832953770955, "sampling/sampling_logp_difference/mean": 0.004498862063822647, "step": 3360, "step_time": 7.999358086194843, "student/entropy": 0.13653970242788394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668020188808, "completions/max_length": 490.6666666666667, "completions/max_terminated_length": 445.96666666666664, "completions/mean_length": 183.08889821370443, "completions/mean_terminated_length": 175.2589314778646, "completions/min_length": 54.46666666666667, "completions/min_terminated_length": 54.46666666666667, "entropy": 0.12626711533715326, "epoch": 0.12873580678236432, "eval/gt_acc_batch": 0.8111111283302307, "frac_reward_zero_std": 1.0, "grad_norm": 0.26094377040863037, "kd/policy_loss": 0.008689720319428792, "kd/rkl_advantage_mean": 0.5621983289747732, "kd/rkl_advantage_p95": 4.551067258914312, "kd/rkl_advantage_std": 2.139119583368301, "kd/ssd_loss": 0.0, "learning_rate": 8.713021683818782e-07, "loss": 0.0347588857014974, "num_tokens": 110895850.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8111111164093018, "rewards/gt_logging_reward/std": 0.37525113026301066, "sampling/importance_sampling_ratio/max": 1.8438990791638692, "sampling/importance_sampling_ratio/mean": 1.0001079320907593, "sampling/importance_sampling_ratio/min": 0.4195111900568008, "sampling/sampling_logp_difference/max": 0.4220865488052368, "sampling/sampling_logp_difference/mean": 0.0042311886325478556, "step": 3390, "step_time": 7.98834578403815, "student/entropy": 0.12626711533715326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030555557211240134, "completions/max_length": 482.6666666666667, "completions/max_terminated_length": 452.26666666666665, "completions/mean_length": 194.74445343017578, "completions/mean_terminated_length": 185.01844329833983, "completions/min_length": 52.86666666666667, "completions/min_terminated_length": 52.86666666666667, "entropy": 0.1302453612908721, "epoch": 0.1298750617096419, "eval/gt_acc_batch": 0.768888904651006, "frac_reward_zero_std": 1.0, "grad_norm": 0.24697674810886383, "kd/policy_loss": 0.008794553405217205, "kd/rkl_advantage_mean": 0.5280657878456017, "kd/rkl_advantage_p95": 4.533943629264831, "kd/rkl_advantage_std": 2.146011910835902, "kd/ssd_loss": 0.0, "learning_rate": 8.701629134546007e-07, "loss": 0.035178216298421223, "num_tokens": 111907426.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7688888907432556, "rewards/gt_logging_reward/std": 0.4114924043416977, "sampling/importance_sampling_ratio/max": 1.933761723836263, "sampling/importance_sampling_ratio/mean": 0.9991441210110982, "sampling/importance_sampling_ratio/min": 0.3814587106307348, "sampling/sampling_logp_difference/max": 0.44338545004526775, "sampling/sampling_logp_difference/mean": 0.004357165734594067, "step": 3420, "step_time": 8.202975747094024, "student/entropy": 0.1302453612908721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334513008596, "completions/max_length": 481.2, "completions/max_terminated_length": 457.6, "completions/mean_length": 189.1997309366862, "completions/mean_terminated_length": 182.47071024576823, "completions/min_length": 58.5, "completions/min_terminated_length": 58.5, "entropy": 0.1301593027388056, "epoch": 0.13101431663691945, "eval/gt_acc_batch": 0.7936111370722453, "frac_reward_zero_std": 1.0, "grad_norm": 0.2319832295179367, "kd/policy_loss": 0.008673139702295884, "kd/rkl_advantage_mean": 0.5553556760462622, "kd/rkl_advantage_p95": 4.539742328723272, "kd/rkl_advantage_std": 2.1206376055876413, "kd/ssd_loss": 0.0, "learning_rate": 8.69023658527323e-07, "loss": 0.03469256560007731, "num_tokens": 112898833.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111072699229, "rewards/gt_logging_reward/std": 0.39466269115606945, "sampling/importance_sampling_ratio/max": 1.8678322672843932, "sampling/importance_sampling_ratio/mean": 0.9977221747239431, "sampling/importance_sampling_ratio/min": 0.4058547422289848, "sampling/sampling_logp_difference/max": 0.37442789872487386, "sampling/sampling_logp_difference/mean": 0.004325135479060312, "step": 3450, "step_time": 8.108620758506003, "student/entropy": 0.1301593027388056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001396983863, "completions/max_length": 481.6, "completions/max_terminated_length": 451.23333333333335, "completions/mean_length": 194.38084309895834, "completions/mean_terminated_length": 186.57857767740884, "completions/min_length": 52.9, "completions/min_terminated_length": 52.9, "entropy": 0.13400991975019375, "epoch": 0.13215357156419702, "eval/gt_acc_batch": 0.7869444708029429, "frac_reward_zero_std": 1.0, "grad_norm": 0.24049454927444458, "kd/policy_loss": 0.008904942965212588, "kd/rkl_advantage_mean": 0.5046597116005918, "kd/rkl_advantage_p95": 4.539095809062322, "kd/rkl_advantage_std": 2.115022851030032, "kd/ssd_loss": 0.0, "learning_rate": 8.678844036000456e-07, "loss": 0.035619775454203285, "num_tokens": 113904988.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444489479065, "rewards/gt_logging_reward/std": 0.3916092832883199, "sampling/importance_sampling_ratio/max": 1.9139215151468914, "sampling/importance_sampling_ratio/mean": 0.998860776424408, "sampling/importance_sampling_ratio/min": 0.4372386733690898, "sampling/sampling_logp_difference/max": 0.4241244912147522, "sampling/sampling_logp_difference/mean": 0.004352133977226913, "step": 3480, "step_time": 8.119304089977716, "student/entropy": 0.13400991975019375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779065072538, "completions/max_length": 485.96666666666664, "completions/max_terminated_length": 445.6, "completions/mean_length": 188.73556518554688, "completions/mean_terminated_length": 180.48362833658854, "completions/min_length": 53.06666666666667, "completions/min_terminated_length": 53.06666666666667, "entropy": 0.13260286673903465, "epoch": 0.1332928264914746, "eval/gt_acc_batch": 0.7891666869322459, "frac_reward_zero_std": 1.0, "grad_norm": 0.2622111439704895, "kd/policy_loss": 0.008634296084831778, "kd/rkl_advantage_mean": 0.537520574987866, "kd/rkl_advantage_p95": 4.569740730524063, "kd/rkl_advantage_std": 2.146898920337359, "kd/ssd_loss": 0.0, "learning_rate": 8.66745148672768e-07, "loss": 0.03453719218571981, "num_tokens": 114883076.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7891666670640309, "rewards/gt_logging_reward/std": 0.4042519986629486, "sampling/importance_sampling_ratio/max": 1.9484352310498556, "sampling/importance_sampling_ratio/mean": 0.9981636106967926, "sampling/importance_sampling_ratio/min": 0.41927977204322814, "sampling/sampling_logp_difference/max": 0.45506683985392254, "sampling/sampling_logp_difference/mean": 0.004316694235118727, "step": 3510, "step_time": 8.016562550724483, "student/entropy": 0.13260286673903465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01944444583108028, "completions/max_length": 484.0, "completions/max_terminated_length": 437.6666666666667, "completions/mean_length": 184.0636205037435, "completions/mean_terminated_length": 177.8388478597005, "completions/min_length": 51.1, "completions/min_terminated_length": 51.1, "entropy": 0.13540159004429977, "epoch": 0.13443208141875213, "eval/gt_acc_batch": 0.7802778085072836, "frac_reward_zero_std": 1.0, "grad_norm": 0.25266408920288086, "kd/policy_loss": 0.00931189118224817, "kd/rkl_advantage_mean": 0.5511935097941508, "kd/rkl_advantage_p95": 4.618194834391276, "kd/rkl_advantage_std": 2.173896497488022, "kd/ssd_loss": 0.0, "learning_rate": 8.656058937454904e-07, "loss": 0.037247570355733235, "num_tokens": 115839081.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7802777787049612, "rewards/gt_logging_reward/std": 0.4015271455049515, "sampling/importance_sampling_ratio/max": 1.9464171489079793, "sampling/importance_sampling_ratio/mean": 0.9974754492441813, "sampling/importance_sampling_ratio/min": 0.4063983072837194, "sampling/sampling_logp_difference/max": 0.4629483828941981, "sampling/sampling_logp_difference/mean": 0.0045270572726925215, "step": 3540, "step_time": 7.8362607803506155, "student/entropy": 0.13540159004429977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334513008596, "completions/max_length": 485.1333333333333, "completions/max_terminated_length": 450.8666666666667, "completions/mean_length": 189.6013977050781, "completions/mean_terminated_length": 182.6419642130534, "completions/min_length": 57.266666666666666, "completions/min_terminated_length": 57.266666666666666, "entropy": 0.13055024159451326, "epoch": 0.1355713363460297, "eval/gt_acc_batch": 0.7788889149824778, "frac_reward_zero_std": 1.0, "grad_norm": 0.23441430926322937, "kd/policy_loss": 0.009325501035588483, "kd/rkl_advantage_mean": 0.5786990081658587, "kd/rkl_advantage_p95": 4.636624004443487, "kd/rkl_advantage_std": 2.1888610114653906, "kd/ssd_loss": 0.0, "learning_rate": 8.644666388182128e-07, "loss": 0.037301997343699135, "num_tokens": 116823806.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7788888871669769, "rewards/gt_logging_reward/std": 0.4113510290781657, "sampling/importance_sampling_ratio/max": 1.9666248321533204, "sampling/importance_sampling_ratio/mean": 0.9968909382820129, "sampling/importance_sampling_ratio/min": 0.3710144822796186, "sampling/sampling_logp_difference/max": 0.4799356559912364, "sampling/sampling_logp_difference/mean": 0.004412760701961815, "step": 3570, "step_time": 7.977315610387207, "student/entropy": 0.13055024159451326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777918924888, "completions/max_length": 499.2, "completions/max_terminated_length": 458.96666666666664, "completions/mean_length": 186.19112141927084, "completions/mean_terminated_length": 177.73036295572916, "completions/min_length": 53.96666666666667, "completions/min_terminated_length": 53.96666666666667, "entropy": 0.12850381328413885, "epoch": 0.13671059127330726, "eval/gt_acc_batch": 0.7952777922153473, "frac_reward_zero_std": 1.0, "grad_norm": 0.32929104566574097, "kd/policy_loss": 0.008902148574513073, "kd/rkl_advantage_mean": 0.6130951015899578, "kd/rkl_advantage_p95": 4.75553472439448, "kd/rkl_advantage_std": 2.1760454813639325, "kd/ssd_loss": 0.0, "learning_rate": 8.633273838909353e-07, "loss": 0.035608593622843424, "num_tokens": 117793478.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777763207753, "rewards/gt_logging_reward/std": 0.397961688041687, "sampling/importance_sampling_ratio/max": 1.8282899936040242, "sampling/importance_sampling_ratio/mean": 0.9934767206509908, "sampling/importance_sampling_ratio/min": 0.39634723713000614, "sampling/sampling_logp_difference/max": 0.4875128984451294, "sampling/sampling_logp_difference/mean": 0.004293988971039653, "step": 3600, "step_time": 8.183716369051641, "student/entropy": 0.12850381328413885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.033055557621022066, "completions/max_length": 501.03333333333336, "completions/max_terminated_length": 464.3333333333333, "completions/mean_length": 192.74139862060548, "completions/mean_terminated_length": 182.01934763590495, "completions/min_length": 54.96666666666667, "completions/min_terminated_length": 54.96666666666667, "entropy": 0.13772923015058042, "epoch": 0.13784984620058482, "eval/gt_acc_batch": 0.7919444680213928, "frac_reward_zero_std": 1.0, "grad_norm": 0.4877491593360901, "kd/policy_loss": 0.009148767363512889, "kd/rkl_advantage_mean": 0.6430709361952419, "kd/rkl_advantage_p95": 4.856299110253652, "kd/rkl_advantage_std": 2.2608909249305724, "kd/ssd_loss": 0.0, "learning_rate": 8.621881289636577e-07, "loss": 0.03659507433573405, "num_tokens": 118792947.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7919444461663564, "rewards/gt_logging_reward/std": 0.3969426900148392, "sampling/importance_sampling_ratio/max": 1.9295590281486512, "sampling/importance_sampling_ratio/mean": 0.9980403006076812, "sampling/importance_sampling_ratio/min": 0.4249534060557683, "sampling/sampling_logp_difference/max": 0.4415054082870483, "sampling/sampling_logp_difference/mean": 0.004379628342576325, "step": 3630, "step_time": 8.211108769231942, "student/entropy": 0.13772923015058042 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02861111310000221, "completions/max_length": 492.3, "completions/max_terminated_length": 462.53333333333336, "completions/mean_length": 193.63917643229166, "completions/mean_terminated_length": 184.320046488444, "completions/min_length": 61.766666666666666, "completions/min_terminated_length": 61.766666666666666, "entropy": 0.1250526058798035, "epoch": 0.1389891011278624, "eval/gt_acc_batch": 0.7861111283302307, "frac_reward_zero_std": 1.0, "grad_norm": 0.2157025784254074, "kd/policy_loss": 0.008474511333042755, "kd/rkl_advantage_mean": 0.5564098631807913, "kd/rkl_advantage_p95": 4.690655622879664, "kd/rkl_advantage_std": 2.177954180041949, "kd/ssd_loss": 0.0, "learning_rate": 8.610488740363801e-07, "loss": 0.03389804760615031, "num_tokens": 119793912.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111144224803, "rewards/gt_logging_reward/std": 0.4022726277510325, "sampling/importance_sampling_ratio/max": 1.9469571709632874, "sampling/importance_sampling_ratio/mean": 0.9901086886723837, "sampling/importance_sampling_ratio/min": 0.4241572543978691, "sampling/sampling_logp_difference/max": 0.4286548952261607, "sampling/sampling_logp_difference/mean": 0.004182238814731439, "step": 3660, "step_time": 8.055642103306793, "student/entropy": 0.1250526058798035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445905586084, "completions/max_length": 474.56666666666666, "completions/max_terminated_length": 448.43333333333334, "completions/mean_length": 194.18306477864584, "completions/mean_terminated_length": 186.5422332763672, "completions/min_length": 58.166666666666664, "completions/min_terminated_length": 58.166666666666664, "entropy": 0.12626226376742125, "epoch": 0.14012835605513993, "eval/gt_acc_batch": 0.7863889118035634, "frac_reward_zero_std": 1.0, "grad_norm": 0.23186156153678894, "kd/policy_loss": 0.008528966208299, "kd/rkl_advantage_mean": 0.541322452078263, "kd/rkl_advantage_p95": 4.5639243940512335, "kd/rkl_advantage_std": 2.13153993388017, "kd/ssd_loss": 0.0, "learning_rate": 8.599096191091027e-07, "loss": 0.03411586284637451, "num_tokens": 120812619.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7863888919353486, "rewards/gt_logging_reward/std": 0.4029636343320211, "sampling/importance_sampling_ratio/max": 1.8730605880419413, "sampling/importance_sampling_ratio/mean": 0.9944485267003377, "sampling/importance_sampling_ratio/min": 0.43048641632000606, "sampling/sampling_logp_difference/max": 0.4312378684679667, "sampling/sampling_logp_difference/mean": 0.004237973092434307, "step": 3690, "step_time": 8.109928004796773, "student/entropy": 0.12626226376742125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445998718343, "completions/max_length": 499.7, "completions/max_terminated_length": 461.9, "completions/mean_length": 192.49445444742838, "completions/mean_terminated_length": 184.62982177734375, "completions/min_length": 55.666666666666664, "completions/min_terminated_length": 55.666666666666664, "entropy": 0.12696559578180314, "epoch": 0.1412676109824175, "eval/gt_acc_batch": 0.8119444588820139, "frac_reward_zero_std": 1.0, "grad_norm": 0.3019994795322418, "kd/policy_loss": 0.008440619639198605, "kd/rkl_advantage_mean": 0.6472775445940594, "kd/rkl_advantage_p95": 4.821461766958237, "kd/rkl_advantage_std": 2.227101605137189, "kd/ssd_loss": 0.0, "learning_rate": 8.58770364181825e-07, "loss": 0.03376247882843018, "num_tokens": 121803511.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.811944454908371, "rewards/gt_logging_reward/std": 0.3849032700061798, "sampling/importance_sampling_ratio/max": 1.9297804196675619, "sampling/importance_sampling_ratio/mean": 0.9985670487085978, "sampling/importance_sampling_ratio/min": 0.439083065589269, "sampling/sampling_logp_difference/max": 0.45107114712397256, "sampling/sampling_logp_difference/mean": 0.004259158484637737, "step": 3720, "step_time": 8.133531683667874, "student/entropy": 0.12696559578180314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777779120951892, "completions/max_length": 494.43333333333334, "completions/max_terminated_length": 465.76666666666665, "completions/mean_length": 195.31028798421224, "completions/mean_terminated_length": 188.24877014160157, "completions/min_length": 53.13333333333333, "completions/min_terminated_length": 53.13333333333333, "entropy": 0.12976726250102122, "epoch": 0.14240686590969506, "eval/gt_acc_batch": 0.7769444713989894, "frac_reward_zero_std": 1.0, "grad_norm": 0.19312074780464172, "kd/policy_loss": 0.0087610934007292, "kd/rkl_advantage_mean": 0.5740872770547867, "kd/rkl_advantage_p95": 4.656116048494975, "kd/rkl_advantage_std": 2.151873853802681, "kd/ssd_loss": 0.0, "learning_rate": 8.576311092545475e-07, "loss": 0.03504437208175659, "num_tokens": 122818020.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7769444445768993, "rewards/gt_logging_reward/std": 0.40281115571657816, "sampling/importance_sampling_ratio/max": 1.8301790634791055, "sampling/importance_sampling_ratio/mean": 0.9951868911584218, "sampling/importance_sampling_ratio/min": 0.4119135975837708, "sampling/sampling_logp_difference/max": 0.45305673281351727, "sampling/sampling_logp_difference/mean": 0.004307411580036084, "step": 3750, "step_time": 8.086361345136538, "student/entropy": 0.12976726250102122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028888890861223142, "completions/max_length": 484.8666666666667, "completions/max_terminated_length": 464.8666666666667, "completions/mean_length": 190.31167653401693, "completions/mean_terminated_length": 180.82097574869792, "completions/min_length": 52.96666666666667, "completions/min_terminated_length": 52.96666666666667, "entropy": 0.13808030740668376, "epoch": 0.14354612083697263, "eval/gt_acc_batch": 0.7783333559830984, "frac_reward_zero_std": 1.0, "grad_norm": 0.2946498394012451, "kd/policy_loss": 0.00938511304363298, "kd/rkl_advantage_mean": 0.5895082773718362, "kd/rkl_advantage_p95": 4.561515768369039, "kd/rkl_advantage_std": 2.1249737332264584, "kd/ssd_loss": 0.0, "learning_rate": 8.564918543272699e-07, "loss": 0.03754044771194458, "num_tokens": 123804726.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7783333381017049, "rewards/gt_logging_reward/std": 0.4050659328699112, "sampling/importance_sampling_ratio/max": 1.971240222454071, "sampling/importance_sampling_ratio/mean": 0.9934919357299805, "sampling/importance_sampling_ratio/min": 0.3956456502278646, "sampling/sampling_logp_difference/max": 0.4436298648516337, "sampling/sampling_logp_difference/mean": 0.00435984677169472, "step": 3780, "step_time": 7.9574565231023975, "student/entropy": 0.13808030740668376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001012037198, "completions/max_length": 469.46666666666664, "completions/max_terminated_length": 439.1666666666667, "completions/mean_length": 187.12167714436848, "completions/mean_terminated_length": 180.75076700846355, "completions/min_length": 57.166666666666664, "completions/min_terminated_length": 57.166666666666664, "entropy": 0.13251610522468885, "epoch": 0.14468537576425017, "eval/gt_acc_batch": 0.7975000282128651, "frac_reward_zero_std": 1.0, "grad_norm": 0.41638410091400146, "kd/policy_loss": 0.00902582513129649, "kd/rkl_advantage_mean": 0.5862570943310856, "kd/rkl_advantage_p95": 4.606290302673975, "kd/rkl_advantage_std": 2.1336263755957288, "kd/ssd_loss": 0.0, "learning_rate": 8.553525993999924e-07, "loss": 0.0361033042271932, "num_tokens": 124779356.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7975000003973644, "rewards/gt_logging_reward/std": 0.392922568321228, "sampling/importance_sampling_ratio/max": 1.9322321573893229, "sampling/importance_sampling_ratio/mean": 0.9996098061402638, "sampling/importance_sampling_ratio/min": 0.4147029717763265, "sampling/sampling_logp_difference/max": 0.42061358292897544, "sampling/sampling_logp_difference/mean": 0.004438786565636595, "step": 3810, "step_time": 7.876054451447756, "student/entropy": 0.13251610522468885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01694444545234243, "completions/max_length": 479.8333333333333, "completions/max_terminated_length": 436.53333333333336, "completions/mean_length": 183.59612121582032, "completions/mean_terminated_length": 177.94090220133464, "completions/min_length": 57.766666666666666, "completions/min_terminated_length": 57.766666666666666, "entropy": 0.12859336622059345, "epoch": 0.14582463069152773, "eval/gt_acc_batch": 0.8013889054457347, "frac_reward_zero_std": 1.0, "grad_norm": 0.3059675395488739, "kd/policy_loss": 0.008941694678893934, "kd/rkl_advantage_mean": 0.5909876501653344, "kd/rkl_advantage_p95": 4.609255439043045, "kd/rkl_advantage_std": 2.1475826650857925, "kd/ssd_loss": 0.0, "learning_rate": 8.542133444727147e-07, "loss": 0.035766780376434326, "num_tokens": 125736798.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888895511627, "rewards/gt_logging_reward/std": 0.3891641512513161, "sampling/importance_sampling_ratio/max": 1.900791331132253, "sampling/importance_sampling_ratio/mean": 1.0055002550284067, "sampling/importance_sampling_ratio/min": 0.44749155392249423, "sampling/sampling_logp_difference/max": 0.40686410466829936, "sampling/sampling_logp_difference/mean": 0.0042761899453277385, "step": 3840, "step_time": 7.885675118297028, "student/entropy": 0.12859336622059345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018333334444711604, "completions/max_length": 476.26666666666665, "completions/max_terminated_length": 441.6333333333333, "completions/mean_length": 187.06084289550782, "completions/mean_terminated_length": 181.1723856608073, "completions/min_length": 64.6, "completions/min_terminated_length": 64.6, "entropy": 0.12036727238446474, "epoch": 0.1469638856188053, "eval/gt_acc_batch": 0.8288889050483703, "frac_reward_zero_std": 1.0, "grad_norm": 0.21585385501384735, "kd/policy_loss": 0.007807196591359874, "kd/rkl_advantage_mean": 0.5831190113152843, "kd/rkl_advantage_p95": 4.754276239871979, "kd/rkl_advantage_std": 2.2196255713701247, "kd/ssd_loss": 0.0, "learning_rate": 8.530740895454373e-07, "loss": 0.031228786706924437, "num_tokens": 126713065.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8288888931274414, "rewards/gt_logging_reward/std": 0.36962173581123353, "sampling/importance_sampling_ratio/max": 1.8702951073646545, "sampling/importance_sampling_ratio/mean": 0.9955123623212179, "sampling/importance_sampling_ratio/min": 0.424912183980147, "sampling/sampling_logp_difference/max": 0.4764505346616109, "sampling/sampling_logp_difference/mean": 0.004113429140609999, "step": 3870, "step_time": 7.873756031416511, "student/entropy": 0.12036727238446474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03000000175088644, "completions/max_length": 482.56666666666666, "completions/max_terminated_length": 448.03333333333336, "completions/mean_length": 192.46139882405598, "completions/mean_terminated_length": 182.69854532877605, "completions/min_length": 51.36666666666667, "completions/min_terminated_length": 51.36666666666667, "entropy": 0.12716003296275932, "epoch": 0.14810314054608287, "eval/gt_acc_batch": 0.7825000147024791, "frac_reward_zero_std": 1.0, "grad_norm": 0.2505664527416229, "kd/policy_loss": 0.008980624920999011, "kd/rkl_advantage_mean": 0.6183612528412292, "kd/rkl_advantage_p95": 4.813882062832515, "kd/rkl_advantage_std": 2.2132724155982335, "kd/ssd_loss": 0.0, "learning_rate": 8.519348346181598e-07, "loss": 0.035922507445017494, "num_tokens": 127717358.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7825000047683716, "rewards/gt_logging_reward/std": 0.4025372336308161, "sampling/importance_sampling_ratio/max": 1.9249574780464171, "sampling/importance_sampling_ratio/mean": 1.0055257042249044, "sampling/importance_sampling_ratio/min": 0.40461665342251457, "sampling/sampling_logp_difference/max": 0.4361312707265218, "sampling/sampling_logp_difference/mean": 0.004229902957255642, "step": 3900, "step_time": 8.010640232280517, "student/entropy": 0.12716003296275932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01777777907749017, "completions/max_length": 490.1333333333333, "completions/max_terminated_length": 456.4, "completions/mean_length": 183.76695454915364, "completions/mean_terminated_length": 177.86695861816406, "completions/min_length": 46.9, "completions/min_terminated_length": 46.9, "entropy": 0.13071482709298532, "epoch": 0.14924239547336043, "eval/gt_acc_batch": 0.8030555685361226, "frac_reward_zero_std": 1.0, "grad_norm": 0.21536394953727722, "kd/policy_loss": 0.0085913473352169, "kd/rkl_advantage_mean": 0.6167706399302308, "kd/rkl_advantage_p95": 4.794722936550776, "kd/rkl_advantage_std": 2.2393792817989984, "kd/ssd_loss": 0.0, "learning_rate": 8.507955796908821e-07, "loss": 0.034365391731262206, "num_tokens": 128679967.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555546283722, "rewards/gt_logging_reward/std": 0.3914008448521296, "sampling/importance_sampling_ratio/max": 1.887408939997355, "sampling/importance_sampling_ratio/mean": 1.0004679143428803, "sampling/importance_sampling_ratio/min": 0.44575444211562476, "sampling/sampling_logp_difference/max": 0.4034588098526001, "sampling/sampling_logp_difference/mean": 0.004369302784713606, "step": 3930, "step_time": 7.921837930567563, "student/entropy": 0.13071482709298532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015000000906487307, "completions/max_length": 482.9, "completions/max_terminated_length": 441.53333333333336, "completions/mean_length": 183.66584243774415, "completions/mean_terminated_length": 178.80771814982097, "completions/min_length": 51.3, "completions/min_terminated_length": 51.3, "entropy": 0.1309664336964488, "epoch": 0.15038165040063797, "eval/gt_acc_batch": 0.7900000214576721, "frac_reward_zero_std": 1.0, "grad_norm": 0.2550804018974304, "kd/policy_loss": 0.008917909204804649, "kd/rkl_advantage_mean": 0.5318890278460457, "kd/rkl_advantage_p95": 4.5937477350234985, "kd/rkl_advantage_std": 2.1663916617631913, "kd/ssd_loss": 0.0, "learning_rate": 8.496563247636046e-07, "loss": 0.035671643416086835, "num_tokens": 129644308.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7900000015894572, "rewards/gt_logging_reward/std": 0.3956454336643219, "sampling/importance_sampling_ratio/max": 1.888447646299998, "sampling/importance_sampling_ratio/mean": 1.0059825201829276, "sampling/importance_sampling_ratio/min": 0.41258290559053423, "sampling/sampling_logp_difference/max": 0.4695527990659078, "sampling/sampling_logp_difference/mean": 0.004362455933975677, "step": 3960, "step_time": 7.947355321375653, "student/entropy": 0.1309664336964488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333482965827, "completions/max_length": 493.0, "completions/max_terminated_length": 447.76666666666665, "completions/mean_length": 187.94500885009765, "completions/mean_terminated_length": 180.33023732503256, "completions/min_length": 56.733333333333334, "completions/min_terminated_length": 56.733333333333334, "entropy": 0.1282873999948303, "epoch": 0.15152090532791554, "eval/gt_acc_batch": 0.7961111307144165, "frac_reward_zero_std": 1.0, "grad_norm": 0.2680627107620239, "kd/policy_loss": 0.008808373221351454, "kd/rkl_advantage_mean": 0.6283982713085909, "kd/rkl_advantage_p95": 4.722524823745092, "kd/rkl_advantage_std": 2.1692258606354398, "kd/ssd_loss": 0.0, "learning_rate": 8.48517069836327e-07, "loss": 0.035233497619628906, "num_tokens": 130626662.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111088593801, "rewards/gt_logging_reward/std": 0.39642598827679953, "sampling/importance_sampling_ratio/max": 1.9171944936116536, "sampling/importance_sampling_ratio/mean": 0.9993879159291585, "sampling/importance_sampling_ratio/min": 0.4389844780166944, "sampling/sampling_logp_difference/max": 0.4655621826648712, "sampling/sampling_logp_difference/mean": 0.004273882888567945, "step": 3990, "step_time": 8.047604063828476, "student/entropy": 0.1282873999948303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778866390386, "completions/max_length": 492.4, "completions/max_terminated_length": 459.5, "completions/mean_length": 189.8327880859375, "completions/mean_terminated_length": 183.2701665242513, "completions/min_length": 53.166666666666664, "completions/min_terminated_length": 53.166666666666664, "entropy": 0.1299325656145811, "epoch": 0.1526601602551931, "eval/gt_acc_batch": 0.7847222487131754, "frac_reward_zero_std": 1.0, "grad_norm": 0.24568936228752136, "kd/policy_loss": 0.009243122106029962, "kd/rkl_advantage_mean": 0.5238471609850724, "kd/rkl_advantage_p95": 4.631309457619985, "kd/rkl_advantage_std": 2.19391452173392, "kd/ssd_loss": 0.0, "learning_rate": 8.473778149090494e-07, "loss": 0.03697249094645182, "num_tokens": 131620908.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7847222208976745, "rewards/gt_logging_reward/std": 0.40395907759666444, "sampling/importance_sampling_ratio/max": 1.9699184934298197, "sampling/importance_sampling_ratio/mean": 1.0056901971499126, "sampling/importance_sampling_ratio/min": 0.4527740334471067, "sampling/sampling_logp_difference/max": 0.4292052914698919, "sampling/sampling_logp_difference/mean": 0.004380018962547183, "step": 4020, "step_time": 8.108341849094723, "student/entropy": 0.1299325656145811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223660598197, "completions/max_length": 493.0, "completions/max_terminated_length": 453.3666666666667, "completions/mean_length": 190.1541763305664, "completions/mean_terminated_length": 183.0502151489258, "completions/min_length": 60.666666666666664, "completions/min_terminated_length": 60.666666666666664, "entropy": 0.13141200070579848, "epoch": 0.15379941518247067, "eval/gt_acc_batch": 0.8022222379843394, "frac_reward_zero_std": 1.0, "grad_norm": 0.2678760290145874, "kd/policy_loss": 0.008843089612976958, "kd/rkl_advantage_mean": 0.6553081595959763, "kd/rkl_advantage_p95": 4.819316520293554, "kd/rkl_advantage_std": 2.190024987856547, "kd/ssd_loss": 0.0, "learning_rate": 8.462385599817718e-07, "loss": 0.035372364521026614, "num_tokens": 132607495.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8022222260634104, "rewards/gt_logging_reward/std": 0.39186418056488037, "sampling/importance_sampling_ratio/max": 1.9844186027844748, "sampling/importance_sampling_ratio/mean": 1.0023646175861358, "sampling/importance_sampling_ratio/min": 0.4091393311818441, "sampling/sampling_logp_difference/max": 0.4160718103249868, "sampling/sampling_logp_difference/mean": 0.004390641332914432, "step": 4050, "step_time": 8.02225600798071, "student/entropy": 0.13141200070579848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334680646657, "completions/max_length": 481.76666666666665, "completions/max_terminated_length": 452.46666666666664, "completions/mean_length": 185.7327870686849, "completions/mean_terminated_length": 177.23232676188152, "completions/min_length": 47.9, "completions/min_terminated_length": 47.9, "entropy": 0.12927385376145442, "epoch": 0.1549386701097482, "eval/gt_acc_batch": 0.7952777922153473, "frac_reward_zero_std": 1.0, "grad_norm": 0.3615736961364746, "kd/policy_loss": 0.008323279265702392, "kd/rkl_advantage_mean": 0.4607571441990634, "kd/rkl_advantage_p95": 4.408920383453369, "kd/rkl_advantage_std": 2.1355039536952973, "kd/ssd_loss": 0.0, "learning_rate": 8.450993050544944e-07, "loss": 0.03329311807950338, "num_tokens": 133578677.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777802944183, "rewards/gt_logging_reward/std": 0.3934675469994545, "sampling/importance_sampling_ratio/max": 1.9203877607981363, "sampling/importance_sampling_ratio/mean": 0.9961386839548747, "sampling/importance_sampling_ratio/min": 0.40807383954524995, "sampling/sampling_logp_difference/max": 0.431478151679039, "sampling/sampling_logp_difference/mean": 0.004261350080681344, "step": 4080, "step_time": 7.902688853178794, "student/entropy": 0.12927385376145442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334643393756, "completions/max_length": 495.0, "completions/max_terminated_length": 464.6, "completions/mean_length": 192.14778747558594, "completions/mean_terminated_length": 184.61016031901042, "completions/min_length": 51.5, "completions/min_terminated_length": 51.5, "entropy": 0.13545831721276044, "epoch": 0.15607792503702578, "eval/gt_acc_batch": 0.7855555752913157, "frac_reward_zero_std": 1.0, "grad_norm": 0.2718941569328308, "kd/policy_loss": 0.009231335087679326, "kd/rkl_advantage_mean": 0.6478290696628392, "kd/rkl_advantage_p95": 4.811893941958745, "kd/rkl_advantage_std": 2.183758536974589, "kd/ssd_loss": 0.0, "learning_rate": 8.439600501272167e-07, "loss": 0.03692533572514852, "num_tokens": 134578281.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555574099223, "rewards/gt_logging_reward/std": 0.4042228748401006, "sampling/importance_sampling_ratio/max": 1.9205192049344382, "sampling/importance_sampling_ratio/mean": 0.9998274505138397, "sampling/importance_sampling_ratio/min": 0.40738620758056643, "sampling/sampling_logp_difference/max": 0.43125993212064107, "sampling/sampling_logp_difference/mean": 0.0044761765748262405, "step": 4110, "step_time": 8.082483251288068, "student/entropy": 0.13545831721276044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223530213037, "completions/max_length": 485.43333333333334, "completions/max_terminated_length": 459.3333333333333, "completions/mean_length": 190.36112111409506, "completions/mean_terminated_length": 183.98695017496746, "completions/min_length": 47.7, "completions/min_terminated_length": 47.7, "entropy": 0.12813850914438565, "epoch": 0.15721717996430334, "eval/gt_acc_batch": 0.7922222375869751, "frac_reward_zero_std": 1.0, "grad_norm": 0.28710395097732544, "kd/policy_loss": 0.008951588733665024, "kd/rkl_advantage_mean": 0.5302395096087518, "kd/rkl_advantage_p95": 4.6482244769732155, "kd/rkl_advantage_std": 2.1745707402626673, "kd/ssd_loss": 0.0, "learning_rate": 8.428207951999392e-07, "loss": 0.03580636183420817, "num_tokens": 135567621.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222216924032, "rewards/gt_logging_reward/std": 0.4008854548136393, "sampling/importance_sampling_ratio/max": 1.9571528633435566, "sampling/importance_sampling_ratio/mean": 0.9992006381352743, "sampling/importance_sampling_ratio/min": 0.41946633607149125, "sampling/sampling_logp_difference/max": 0.42390459378560386, "sampling/sampling_logp_difference/mean": 0.004311849487324556, "step": 4140, "step_time": 8.069770083529875, "student/entropy": 0.12813850914438565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223474333683, "completions/max_length": 487.4, "completions/max_terminated_length": 442.9, "completions/mean_length": 187.0519546508789, "completions/mean_terminated_length": 179.7940892537435, "completions/min_length": 51.13333333333333, "completions/min_terminated_length": 51.13333333333333, "entropy": 0.13038908429443835, "epoch": 0.1583564348915809, "eval/gt_acc_batch": 0.7930555740992228, "frac_reward_zero_std": 1.0, "grad_norm": 0.3027611970901489, "kd/policy_loss": 0.009152335289400071, "kd/rkl_advantage_mean": 0.6023480845615268, "kd/rkl_advantage_p95": 4.736371819178263, "kd/rkl_advantage_std": 2.195637721816699, "kd/ssd_loss": 0.0, "learning_rate": 8.416815402726617e-07, "loss": 0.036609347661336264, "num_tokens": 136534576.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555562178294, "rewards/gt_logging_reward/std": 0.39814168711503345, "sampling/importance_sampling_ratio/max": 1.9007872621218362, "sampling/importance_sampling_ratio/mean": 1.004242604970932, "sampling/importance_sampling_ratio/min": 0.3928728853662809, "sampling/sampling_logp_difference/max": 0.4371136208375295, "sampling/sampling_logp_difference/mean": 0.0043490045160676045, "step": 4170, "step_time": 8.01301650107683, "student/entropy": 0.13038908429443835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02777777922650178, "completions/max_length": 498.73333333333335, "completions/max_terminated_length": 461.1333333333333, "completions/mean_length": 194.81389872233072, "completions/mean_terminated_length": 185.9985778808594, "completions/min_length": 49.43333333333333, "completions/min_terminated_length": 49.43333333333333, "entropy": 0.13223536722362042, "epoch": 0.15949568981885848, "eval/gt_acc_batch": 0.786388901869456, "frac_reward_zero_std": 1.0, "grad_norm": 0.2800457179546356, "kd/policy_loss": 0.009084603279673805, "kd/rkl_advantage_mean": 0.6758867027238011, "kd/rkl_advantage_p95": 4.853655767440796, "kd/rkl_advantage_std": 2.2197127838929496, "kd/ssd_loss": 0.0, "learning_rate": 8.40542285345384e-07, "loss": 0.03633841673533122, "num_tokens": 137537498.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7863888879617055, "rewards/gt_logging_reward/std": 0.4031791776418686, "sampling/importance_sampling_ratio/max": 1.9349221388498943, "sampling/importance_sampling_ratio/mean": 0.9986820340156555, "sampling/importance_sampling_ratio/min": 0.40646717250347136, "sampling/sampling_logp_difference/max": 0.4193293869495392, "sampling/sampling_logp_difference/mean": 0.004404727757597963, "step": 4200, "step_time": 8.124026103814442, "student/entropy": 0.13223536722362042 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01694444545234243, "completions/max_length": 484.43333333333334, "completions/max_terminated_length": 447.6666666666667, "completions/mean_length": 184.49473368326824, "completions/mean_terminated_length": 178.87979125976562, "completions/min_length": 50.8, "completions/min_terminated_length": 50.8, "entropy": 0.12642300768444936, "epoch": 0.16063494474613602, "eval/gt_acc_batch": 0.7991666853427887, "frac_reward_zero_std": 1.0, "grad_norm": 0.29206791520118713, "kd/policy_loss": 0.00826465521628658, "kd/rkl_advantage_mean": 0.6074105527950451, "kd/rkl_advantage_p95": 4.779417123397192, "kd/rkl_advantage_std": 2.213130287329356, "kd/ssd_loss": 0.0, "learning_rate": 8.394030304181065e-07, "loss": 0.033058623472849526, "num_tokens": 138500719.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7991666714350383, "rewards/gt_logging_reward/std": 0.39380384186903633, "sampling/importance_sampling_ratio/max": 1.9179837147394816, "sampling/importance_sampling_ratio/mean": 0.9942876120408376, "sampling/importance_sampling_ratio/min": 0.3885172242919604, "sampling/sampling_logp_difference/max": 0.48046698570251467, "sampling/sampling_logp_difference/mean": 0.004306765731113652, "step": 4230, "step_time": 7.8829748247362055, "student/entropy": 0.12642300768444936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555684491992, "completions/max_length": 488.2, "completions/max_terminated_length": 447.8, "completions/mean_length": 191.7305643717448, "completions/mean_terminated_length": 185.01238403320312, "completions/min_length": 54.2, "completions/min_terminated_length": 54.2, "entropy": 0.12226870047549407, "epoch": 0.16177419967341358, "eval/gt_acc_batch": 0.8033333599567414, "frac_reward_zero_std": 1.0, "grad_norm": 0.23413728177547455, "kd/policy_loss": 0.008420986034131299, "kd/rkl_advantage_mean": 0.5809727735662212, "kd/rkl_advantage_p95": 4.705149080355962, "kd/rkl_advantage_std": 2.1833930631478626, "kd/ssd_loss": 0.0, "learning_rate": 8.38263775490829e-07, "loss": 0.03368393977483113, "num_tokens": 139496805.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8033333361148834, "rewards/gt_logging_reward/std": 0.3885658378402392, "sampling/importance_sampling_ratio/max": 1.9447064081827798, "sampling/importance_sampling_ratio/mean": 1.0011164784431457, "sampling/importance_sampling_ratio/min": 0.4261832709113757, "sampling/sampling_logp_difference/max": 0.4207845350106557, "sampling/sampling_logp_difference/mean": 0.004083939998721083, "step": 4260, "step_time": 8.008160975850963, "student/entropy": 0.12226870047549407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02277777905886372, "completions/max_length": 494.1333333333333, "completions/max_terminated_length": 448.43333333333334, "completions/mean_length": 186.33889872233073, "completions/mean_terminated_length": 178.69863637288412, "completions/min_length": 55.8, "completions/min_terminated_length": 55.8, "entropy": 0.13278882515927157, "epoch": 0.16291345460069115, "eval/gt_acc_batch": 0.7769444783528646, "frac_reward_zero_std": 1.0, "grad_norm": 0.25176528096199036, "kd/policy_loss": 0.009169622797829409, "kd/rkl_advantage_mean": 0.7123029805719853, "kd/rkl_advantage_p95": 5.012585739294688, "kd/rkl_advantage_std": 2.2778198520342507, "kd/ssd_loss": 0.0, "learning_rate": 8.371245205635515e-07, "loss": 0.036678496996561685, "num_tokens": 140462601.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7769444465637207, "rewards/gt_logging_reward/std": 0.41087177296479543, "sampling/importance_sampling_ratio/max": 1.9165485938390097, "sampling/importance_sampling_ratio/mean": 0.9935587366422017, "sampling/importance_sampling_ratio/min": 0.3750995397567749, "sampling/sampling_logp_difference/max": 0.43510931730270386, "sampling/sampling_logp_difference/mean": 0.004375011147931218, "step": 4290, "step_time": 7.892824964035147, "student/entropy": 0.13278882515927157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445837289096, "completions/max_length": 477.6, "completions/max_terminated_length": 446.3, "completions/mean_length": 187.93251037597656, "completions/mean_terminated_length": 180.8996129353841, "completions/min_length": 50.766666666666666, "completions/min_terminated_length": 50.766666666666666, "entropy": 0.12988774714370568, "epoch": 0.16405270952796872, "eval/gt_acc_batch": 0.7933333436648051, "frac_reward_zero_std": 1.0, "grad_norm": 0.22875063121318817, "kd/policy_loss": 0.008804961365725224, "kd/rkl_advantage_mean": 0.6138166089231769, "kd/rkl_advantage_p95": 4.789567377169927, "kd/rkl_advantage_std": 2.200960459311803, "kd/ssd_loss": 0.0, "learning_rate": 8.359852656362738e-07, "loss": 0.03521984418233236, "num_tokens": 141446630.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7933333297570546, "rewards/gt_logging_reward/std": 0.3907527590791384, "sampling/importance_sampling_ratio/max": 1.9711983283360799, "sampling/importance_sampling_ratio/mean": 0.9954597314198812, "sampling/importance_sampling_ratio/min": 0.3752659817536672, "sampling/sampling_logp_difference/max": 0.4387907862663269, "sampling/sampling_logp_difference/mean": 0.004335878975689411, "step": 4320, "step_time": 7.991025264386553, "student/entropy": 0.12988774714370568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001459072035, "completions/max_length": 491.06666666666666, "completions/max_terminated_length": 458.06666666666666, "completions/mean_length": 189.28084208170574, "completions/mean_terminated_length": 180.9113774617513, "completions/min_length": 47.6, "completions/min_terminated_length": 47.6, "entropy": 0.12740826072792213, "epoch": 0.16519196445524628, "eval/gt_acc_batch": 0.7855555752913157, "frac_reward_zero_std": 1.0, "grad_norm": 0.2615356743335724, "kd/policy_loss": 0.00887118478034002, "kd/rkl_advantage_mean": 0.6155977550971632, "kd/rkl_advantage_p95": 4.726803463697434, "kd/rkl_advantage_std": 2.1859089503685634, "kd/ssd_loss": 0.0, "learning_rate": 8.348460107089963e-07, "loss": 0.03548474709192912, "num_tokens": 142428105.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555534362793, "rewards/gt_logging_reward/std": 0.4063624233007431, "sampling/importance_sampling_ratio/max": 2.0196414868036907, "sampling/importance_sampling_ratio/mean": 1.0054883539676667, "sampling/importance_sampling_ratio/min": 0.4323948656519254, "sampling/sampling_logp_difference/max": 0.4197516103585561, "sampling/sampling_logp_difference/mean": 0.004196122894063592, "step": 4350, "step_time": 7.913747670012526, "student/entropy": 0.12740826072792213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010000000645716985, "completions/max_length": 457.5, "completions/max_terminated_length": 433.1666666666667, "completions/mean_length": 177.16917622884114, "completions/mean_terminated_length": 173.8779037475586, "completions/min_length": 56.53333333333333, "completions/min_terminated_length": 56.53333333333333, "entropy": 0.13573877370605866, "epoch": 0.16633121938252382, "eval/gt_acc_batch": 0.812500019868215, "frac_reward_zero_std": 1.0, "grad_norm": 0.22958894073963165, "kd/policy_loss": 0.008992421204069009, "kd/rkl_advantage_mean": 0.5670546897066137, "kd/rkl_advantage_p95": 4.600766197840373, "kd/rkl_advantage_std": 2.1719899584849673, "kd/ssd_loss": 0.0, "learning_rate": 8.337067557817187e-07, "loss": 0.03596968650817871, "num_tokens": 143354218.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8125000019868215, "rewards/gt_logging_reward/std": 0.38306012799342476, "sampling/importance_sampling_ratio/max": 1.9170852541923522, "sampling/importance_sampling_ratio/mean": 0.9930353462696075, "sampling/importance_sampling_ratio/min": 0.4210829511284828, "sampling/sampling_logp_difference/max": 0.4191339612007141, "sampling/sampling_logp_difference/mean": 0.004544446376773218, "step": 4380, "step_time": 7.561804490520929, "student/entropy": 0.13573877370605866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02944444613531232, "completions/max_length": 492.73333333333335, "completions/max_terminated_length": 448.4, "completions/mean_length": 193.28695475260417, "completions/mean_terminated_length": 183.55145517985025, "completions/min_length": 53.833333333333336, "completions/min_terminated_length": 53.833333333333336, "entropy": 0.13779268302023412, "epoch": 0.1674704743098014, "eval/gt_acc_batch": 0.770000018676122, "frac_reward_zero_std": 1.0, "grad_norm": 0.3444743752479553, "kd/policy_loss": 0.009203816397348418, "kd/rkl_advantage_mean": 0.6024982201556365, "kd/rkl_advantage_p95": 4.741541596253713, "kd/rkl_advantage_std": 2.183046145240466, "kd/ssd_loss": 0.0, "learning_rate": 8.325675008544411e-07, "loss": 0.03681526978810628, "num_tokens": 144355939.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7700000007947286, "rewards/gt_logging_reward/std": 0.4161013712485631, "sampling/importance_sampling_ratio/max": 1.9570293307304383, "sampling/importance_sampling_ratio/mean": 0.9965164522329967, "sampling/importance_sampling_ratio/min": 0.4027052313089371, "sampling/sampling_logp_difference/max": 0.44581857323646545, "sampling/sampling_logp_difference/mean": 0.0043686490428323545, "step": 4410, "step_time": 8.040070042076211, "student/entropy": 0.13779268302023412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944445880750813, "completions/max_length": 494.3333333333333, "completions/max_terminated_length": 453.1, "completions/mean_length": 191.8630635579427, "completions/mean_terminated_length": 183.09041697184244, "completions/min_length": 51.96666666666667, "completions/min_terminated_length": 51.96666666666667, "entropy": 0.13141766488552092, "epoch": 0.16860972923707895, "eval/gt_acc_batch": 0.7813889145851135, "frac_reward_zero_std": 1.0, "grad_norm": 0.26691102981567383, "kd/policy_loss": 0.008583428520554055, "kd/rkl_advantage_mean": 0.6079949674468177, "kd/rkl_advantage_p95": 4.76937315662702, "kd/rkl_advantage_std": 2.1792185723781587, "kd/ssd_loss": 0.0, "learning_rate": 8.314282459271636e-07, "loss": 0.03433371384938558, "num_tokens": 145344862.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7813888927300771, "rewards/gt_logging_reward/std": 0.40287031133969625, "sampling/importance_sampling_ratio/max": 1.9123770952224732, "sampling/importance_sampling_ratio/mean": 0.9939290483792623, "sampling/importance_sampling_ratio/min": 0.39903106689453127, "sampling/sampling_logp_difference/max": 0.4310739199320475, "sampling/sampling_logp_difference/mean": 0.004372928656327228, "step": 4440, "step_time": 8.000834671012125, "student/entropy": 0.13141766488552092 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.033611113329728444, "completions/max_length": 492.8333333333333, "completions/max_terminated_length": 450.1, "completions/mean_length": 194.15056610107422, "completions/mean_terminated_length": 183.2870641072591, "completions/min_length": 55.53333333333333, "completions/min_terminated_length": 55.53333333333333, "entropy": 0.12776162804414828, "epoch": 0.16974898416435652, "eval/gt_acc_batch": 0.7922222554683686, "frac_reward_zero_std": 1.0, "grad_norm": 0.3164127469062805, "kd/policy_loss": 0.008474199530125286, "kd/rkl_advantage_mean": 0.5693696306397518, "kd/rkl_advantage_p95": 4.676901431878408, "kd/rkl_advantage_std": 2.188996205727259, "kd/ssd_loss": 0.0, "learning_rate": 8.302889909998861e-07, "loss": 0.03389679988225301, "num_tokens": 146339292.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222216924032, "rewards/gt_logging_reward/std": 0.3943330317735672, "sampling/importance_sampling_ratio/max": 1.9623850226402282, "sampling/importance_sampling_ratio/mean": 0.9955548346042633, "sampling/importance_sampling_ratio/min": 0.4032886683940887, "sampling/sampling_logp_difference/max": 0.43698521653811134, "sampling/sampling_logp_difference/mean": 0.004246901821655532, "step": 4470, "step_time": 7.984031063651977, "student/entropy": 0.12776162804414828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778934687375, "completions/max_length": 485.7, "completions/max_terminated_length": 454.5, "completions/mean_length": 189.89195404052734, "completions/mean_terminated_length": 182.57731119791666, "completions/min_length": 56.266666666666666, "completions/min_terminated_length": 56.266666666666666, "entropy": 0.1351021594678362, "epoch": 0.17088823909163406, "eval/gt_acc_batch": 0.8066666781902313, "frac_reward_zero_std": 1.0, "grad_norm": 0.19093075394630432, "kd/policy_loss": 0.009064448933349923, "kd/rkl_advantage_mean": 0.5618911135088032, "kd/rkl_advantage_p95": 4.63934986392657, "kd/rkl_advantage_std": 2.1560906539360682, "kd/ssd_loss": 0.0, "learning_rate": 8.291497360726085e-07, "loss": 0.03625779946645101, "num_tokens": 147328519.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8066666642824809, "rewards/gt_logging_reward/std": 0.3878040427962939, "sampling/importance_sampling_ratio/max": 1.9465405106544496, "sampling/importance_sampling_ratio/mean": 0.9980142474174499, "sampling/importance_sampling_ratio/min": 0.40284324834744134, "sampling/sampling_logp_difference/max": 0.43215983907381694, "sampling/sampling_logp_difference/mean": 0.0043777971994131805, "step": 4500, "step_time": 7.985646306048148, "student/entropy": 0.1351021594678362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02000000113621354, "completions/max_length": 485.6333333333333, "completions/max_terminated_length": 438.93333333333334, "completions/mean_length": 182.3072311401367, "completions/mean_terminated_length": 175.67564697265624, "completions/min_length": 52.1, "completions/min_terminated_length": 52.1, "entropy": 0.12640664496769508, "epoch": 0.17202749401891163, "eval/gt_acc_batch": 0.8147222518920898, "frac_reward_zero_std": 1.0, "grad_norm": 0.28761154413223267, "kd/policy_loss": 0.008413757100546112, "kd/rkl_advantage_mean": 0.6445614118284235, "kd/rkl_advantage_p95": 4.846730122963588, "kd/rkl_advantage_std": 2.2158480187257132, "kd/ssd_loss": 0.0, "learning_rate": 8.280104811453309e-07, "loss": 0.0336550235748291, "num_tokens": 148288633.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8147222201029459, "rewards/gt_logging_reward/std": 0.37879129548867546, "sampling/importance_sampling_ratio/max": 1.935811444123586, "sampling/importance_sampling_ratio/mean": 1.0003867129484811, "sampling/importance_sampling_ratio/min": 0.44180815716584526, "sampling/sampling_logp_difference/max": 0.41475554505983986, "sampling/sampling_logp_difference/mean": 0.004239704886761804, "step": 4530, "step_time": 7.797652539199529, "student/entropy": 0.12640664496769508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0233333347675701, "completions/max_length": 483.8, "completions/max_terminated_length": 432.56666666666666, "completions/mean_length": 181.18473103841146, "completions/mean_terminated_length": 173.3445841471354, "completions/min_length": 51.6, "completions/min_terminated_length": 51.6, "entropy": 0.13005811826636393, "epoch": 0.1731667489461892, "eval/gt_acc_batch": 0.7952778061230977, "frac_reward_zero_std": 1.0, "grad_norm": 0.30359604954719543, "kd/policy_loss": 0.008982894301880151, "kd/rkl_advantage_mean": 0.5897338472306728, "kd/rkl_advantage_p95": 4.751080181201299, "kd/rkl_advantage_std": 2.2319200158119203, "kd/ssd_loss": 0.0, "learning_rate": 8.268712262180534e-07, "loss": 0.03593157927195231, "num_tokens": 149239658.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777743339539, "rewards/gt_logging_reward/std": 0.39392486413319905, "sampling/importance_sampling_ratio/max": 1.9605566104253134, "sampling/importance_sampling_ratio/mean": 1.0029195149739583, "sampling/importance_sampling_ratio/min": 0.37723441670338315, "sampling/sampling_logp_difference/max": 0.39552374680836994, "sampling/sampling_logp_difference/mean": 0.004370854158575336, "step": 4560, "step_time": 7.811583073999888, "student/entropy": 0.13005811826636393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001390775046, "completions/max_length": 488.26666666666665, "completions/max_terminated_length": 455.73333333333335, "completions/mean_length": 186.7961217244466, "completions/mean_terminated_length": 179.4837855021159, "completions/min_length": 50.13333333333333, "completions/min_terminated_length": 50.13333333333333, "entropy": 0.12826346643269063, "epoch": 0.17430600387346676, "eval/gt_acc_batch": 0.7963888982931773, "frac_reward_zero_std": 1.0, "grad_norm": 0.3308059275150299, "kd/policy_loss": 0.008675741654587909, "kd/rkl_advantage_mean": 0.5662101903949709, "kd/rkl_advantage_p95": 4.757735512653986, "kd/rkl_advantage_std": 2.1863811870416003, "kd/ssd_loss": 0.0, "learning_rate": 8.257319712907757e-07, "loss": 0.03470296462376912, "num_tokens": 150218876.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888903458913, "rewards/gt_logging_reward/std": 0.39548701445261636, "sampling/importance_sampling_ratio/max": 1.978613305091858, "sampling/importance_sampling_ratio/mean": 1.001912667353948, "sampling/importance_sampling_ratio/min": 0.4289726952711741, "sampling/sampling_logp_difference/max": 0.3773029585679372, "sampling/sampling_logp_difference/mean": 0.0043018937731782595, "step": 4590, "step_time": 7.9483325641679885, "student/entropy": 0.12826346643269063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333460614085, "completions/max_length": 497.3, "completions/max_terminated_length": 462.3333333333333, "completions/mean_length": 185.18917643229167, "completions/mean_terminated_length": 178.24716186523438, "completions/min_length": 49.86666666666667, "completions/min_terminated_length": 49.86666666666667, "entropy": 0.12332344210396211, "epoch": 0.17544525880074432, "eval/gt_acc_batch": 0.818888908624649, "frac_reward_zero_std": 1.0, "grad_norm": 0.2777160108089447, "kd/policy_loss": 0.007984639207522074, "kd/rkl_advantage_mean": 0.5584135146966825, "kd/rkl_advantage_p95": 4.640872142712275, "kd/rkl_advantage_std": 2.1811570515235266, "kd/ssd_loss": 0.0, "learning_rate": 8.245927163634982e-07, "loss": 0.03193855285644531, "num_tokens": 151194301.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8188888847827911, "rewards/gt_logging_reward/std": 0.37655321657657626, "sampling/importance_sampling_ratio/max": 1.9578959067662558, "sampling/importance_sampling_ratio/mean": 1.003688915570577, "sampling/importance_sampling_ratio/min": 0.40716680934031807, "sampling/sampling_logp_difference/max": 0.4396363536516825, "sampling/sampling_logp_difference/mean": 0.004185476053195695, "step": 4620, "step_time": 8.16773157971523, "student/entropy": 0.12332344210396211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01583333412806193, "completions/max_length": 471.43333333333334, "completions/max_terminated_length": 445.3666666666667, "completions/mean_length": 186.65750986735026, "completions/mean_terminated_length": 181.48958943684895, "completions/min_length": 55.9, "completions/min_terminated_length": 55.9, "entropy": 0.13256355784833432, "epoch": 0.17658451372802186, "eval/gt_acc_batch": 0.8116666853427887, "frac_reward_zero_std": 1.0, "grad_norm": 0.2945660948753357, "kd/policy_loss": 0.008556259871693329, "kd/rkl_advantage_mean": 0.5929759226273745, "kd/rkl_advantage_p95": 4.641309656699499, "kd/rkl_advantage_std": 2.1446397483348845, "kd/ssd_loss": 0.0, "learning_rate": 8.234534614362207e-07, "loss": 0.034225038687388104, "num_tokens": 152164468.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8116666694482167, "rewards/gt_logging_reward/std": 0.38389231959978737, "sampling/importance_sampling_ratio/max": 1.9554205338160198, "sampling/importance_sampling_ratio/mean": 1.0028269708156585, "sampling/importance_sampling_ratio/min": 0.4107947478691737, "sampling/sampling_logp_difference/max": 0.3855579455693563, "sampling/sampling_logp_difference/mean": 0.0042993779371803004, "step": 4650, "step_time": 7.799338809668552, "student/entropy": 0.13256355784833432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01750000116104881, "completions/max_length": 482.6666666666667, "completions/max_terminated_length": 449.4, "completions/mean_length": 179.59223225911458, "completions/mean_terminated_length": 173.6505579630534, "completions/min_length": 52.03333333333333, "completions/min_terminated_length": 52.03333333333333, "entropy": 0.13241675111154716, "epoch": 0.17772376865529943, "eval/gt_acc_batch": 0.8013889054457347, "frac_reward_zero_std": 1.0, "grad_norm": 0.3724587857723236, "kd/policy_loss": 0.008851472563886393, "kd/rkl_advantage_mean": 0.7153036874408524, "kd/rkl_advantage_p95": 4.879216810067494, "kd/rkl_advantage_std": 2.219032023350398, "kd/ssd_loss": 0.0, "learning_rate": 8.223142065089432e-07, "loss": 0.03540589014689128, "num_tokens": 153106456.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888935248057, "rewards/gt_logging_reward/std": 0.39207198719183606, "sampling/importance_sampling_ratio/max": 1.9501164158185322, "sampling/importance_sampling_ratio/mean": 1.0024066289265952, "sampling/importance_sampling_ratio/min": 0.4250838026404381, "sampling/sampling_logp_difference/max": 0.43630882700284324, "sampling/sampling_logp_difference/mean": 0.004398288282876213, "step": 4680, "step_time": 7.852808898683482, "student/entropy": 0.13241675111154716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018333334382623433, "completions/max_length": 474.96666666666664, "completions/max_terminated_length": 422.9, "completions/mean_length": 178.6702891031901, "completions/mean_terminated_length": 172.5624547322591, "completions/min_length": 52.5, "completions/min_terminated_length": 52.5, "entropy": 0.12636554582665363, "epoch": 0.178863023582577, "eval/gt_acc_batch": 0.80916668176651, "frac_reward_zero_std": 1.0, "grad_norm": 0.2682987451553345, "kd/policy_loss": 0.008390730277945598, "kd/rkl_advantage_mean": 0.6116872578820524, "kd/rkl_advantage_p95": 4.742968197663625, "kd/rkl_advantage_std": 2.1827247778574628, "kd/ssd_loss": 0.0, "learning_rate": 8.211749515816655e-07, "loss": 0.03356292247772217, "num_tokens": 154051573.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8091666658719381, "rewards/gt_logging_reward/std": 0.38704575300216676, "sampling/importance_sampling_ratio/max": 1.899699910481771, "sampling/importance_sampling_ratio/mean": 0.9954484442869822, "sampling/importance_sampling_ratio/min": 0.40983395278453827, "sampling/sampling_logp_difference/max": 0.41518606344858805, "sampling/sampling_logp_difference/mean": 0.004248332441784441, "step": 4710, "step_time": 7.796969199638503, "student/entropy": 0.12636554582665363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500000881652038, "completions/max_length": 481.1333333333333, "completions/max_terminated_length": 451.03333333333336, "completions/mean_length": 187.83417561848958, "completions/mean_terminated_length": 182.183327738444, "completions/min_length": 60.36666666666667, "completions/min_terminated_length": 60.36666666666667, "entropy": 0.12451310840745768, "epoch": 0.18000227850985456, "eval/gt_acc_batch": 0.8002777973810832, "frac_reward_zero_std": 1.0, "grad_norm": 0.22770988941192627, "kd/policy_loss": 0.00824097120591129, "kd/rkl_advantage_mean": 0.6182008556555957, "kd/rkl_advantage_p95": 4.792526137828827, "kd/rkl_advantage_std": 2.1992188721895216, "kd/ssd_loss": 0.0, "learning_rate": 8.20035696654388e-07, "loss": 0.032963887850443525, "num_tokens": 155027120.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8002777794996897, "rewards/gt_logging_reward/std": 0.38704984535773596, "sampling/importance_sampling_ratio/max": 1.9299711863199869, "sampling/importance_sampling_ratio/mean": 1.0019956847031912, "sampling/importance_sampling_ratio/min": 0.4091160347064336, "sampling/sampling_logp_difference/max": 0.4355523645877838, "sampling/sampling_logp_difference/mean": 0.004151805676519871, "step": 4740, "step_time": 7.793366843543481, "student/entropy": 0.12451310840745768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668020188808, "completions/max_length": 495.0, "completions/max_terminated_length": 458.1, "completions/mean_length": 191.9569549560547, "completions/mean_terminated_length": 184.2638147989909, "completions/min_length": 61.36666666666667, "completions/min_terminated_length": 61.36666666666667, "entropy": 0.12749640084803104, "epoch": 0.1811415334371321, "eval/gt_acc_batch": 0.787500015894572, "frac_reward_zero_std": 1.0, "grad_norm": 0.23528142273426056, "kd/policy_loss": 0.008920859351443747, "kd/rkl_advantage_mean": 0.5840230805566534, "kd/rkl_advantage_p95": 4.680854390064876, "kd/rkl_advantage_std": 2.173820400238037, "kd/ssd_loss": 0.0, "learning_rate": 8.188964417271105e-07, "loss": 0.0356834332148234, "num_tokens": 156021021.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7874999980131785, "rewards/gt_logging_reward/std": 0.39909369697173436, "sampling/importance_sampling_ratio/max": 2.029611035188039, "sampling/importance_sampling_ratio/mean": 0.9984595775604248, "sampling/importance_sampling_ratio/min": 0.43955254058043164, "sampling/sampling_logp_difference/max": 0.4505197246869405, "sampling/sampling_logp_difference/mean": 0.0042495976124579705, "step": 4770, "step_time": 7.964652934538511, "student/entropy": 0.12749640084803104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029166668467223645, "completions/max_length": 495.5, "completions/max_terminated_length": 463.46666666666664, "completions/mean_length": 193.05111948649088, "completions/mean_terminated_length": 183.62896118164062, "completions/min_length": 49.63333333333333, "completions/min_terminated_length": 49.63333333333333, "entropy": 0.13271870849033196, "epoch": 0.18228078836440967, "eval/gt_acc_batch": 0.7855555653572083, "frac_reward_zero_std": 1.0, "grad_norm": 0.2192717343568802, "kd/policy_loss": 0.008937719250873973, "kd/rkl_advantage_mean": 0.6457948190470536, "kd/rkl_advantage_p95": 4.806369322538376, "kd/rkl_advantage_std": 2.207584433754285, "kd/ssd_loss": 0.0, "learning_rate": 8.177571867998328e-07, "loss": 0.03575087785720825, "num_tokens": 157014637.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555534362793, "rewards/gt_logging_reward/std": 0.4032997796932856, "sampling/importance_sampling_ratio/max": 1.9682828823725382, "sampling/importance_sampling_ratio/mean": 0.9949515104293823, "sampling/importance_sampling_ratio/min": 0.3712858721613884, "sampling/sampling_logp_difference/max": 0.4728123962879181, "sampling/sampling_logp_difference/mean": 0.004397773587455352, "step": 4800, "step_time": 8.16429967811564, "student/entropy": 0.13271870849033196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01944444558272759, "completions/max_length": 482.56666666666666, "completions/max_terminated_length": 448.7, "completions/mean_length": 186.10223236083985, "completions/mean_terminated_length": 179.69716135660806, "completions/min_length": 48.8, "completions/min_terminated_length": 48.8, "entropy": 0.12820262523988882, "epoch": 0.18342004329168723, "eval/gt_acc_batch": 0.7880555709203084, "frac_reward_zero_std": 1.0, "grad_norm": 0.24675236642360687, "kd/policy_loss": 0.008679888884459312, "kd/rkl_advantage_mean": 0.6667423952370882, "kd/rkl_advantage_p95": 4.81252782146136, "kd/rkl_advantage_std": 2.1807959894339244, "kd/ssd_loss": 0.0, "learning_rate": 8.166179318725553e-07, "loss": 0.03471955458323161, "num_tokens": 157979365.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.788055557012558, "rewards/gt_logging_reward/std": 0.40147166748841606, "sampling/importance_sampling_ratio/max": 1.9819732308387756, "sampling/importance_sampling_ratio/mean": 0.9982752879460652, "sampling/importance_sampling_ratio/min": 0.3733826662103335, "sampling/sampling_logp_difference/max": 0.44332319299379985, "sampling/sampling_logp_difference/mean": 0.004285374369161824, "step": 4830, "step_time": 7.907448230842904, "student/entropy": 0.12820262523988882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02750000124797225, "completions/max_length": 491.3666666666667, "completions/max_terminated_length": 447.1, "completions/mean_length": 193.73084564208983, "completions/mean_terminated_length": 184.77484792073568, "completions/min_length": 54.7, "completions/min_terminated_length": 54.7, "entropy": 0.13906484823673965, "epoch": 0.1845592982189648, "eval/gt_acc_batch": 0.7638888895511627, "frac_reward_zero_std": 1.0, "grad_norm": 0.32641205191612244, "kd/policy_loss": 0.009824709431268275, "kd/rkl_advantage_mean": 0.5822771724623939, "kd/rkl_advantage_p95": 4.74753347436587, "kd/rkl_advantage_std": 2.206132439772288, "kd/ssd_loss": 0.0, "learning_rate": 8.154786769452778e-07, "loss": 0.03929883639017741, "num_tokens": 158980868.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7638888895511627, "rewards/gt_logging_reward/std": 0.413290203611056, "sampling/importance_sampling_ratio/max": 2.044093358516693, "sampling/importance_sampling_ratio/mean": 0.9971038361390432, "sampling/importance_sampling_ratio/min": 0.35653166522582375, "sampling/sampling_logp_difference/max": 0.4385534107685089, "sampling/sampling_logp_difference/mean": 0.004586326191201806, "step": 4860, "step_time": 8.134320569430322, "student/entropy": 0.13906484823673965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445775200924, "completions/max_length": 499.8333333333333, "completions/max_terminated_length": 465.06666666666666, "completions/mean_length": 192.41556549072266, "completions/mean_terminated_length": 185.13451588948567, "completions/min_length": 53.63333333333333, "completions/min_terminated_length": 53.63333333333333, "entropy": 0.12990116843332847, "epoch": 0.18569855314624237, "eval/gt_acc_batch": 0.7800000190734864, "frac_reward_zero_std": 1.0, "grad_norm": 0.2687862813472748, "kd/policy_loss": 0.009216914876985054, "kd/rkl_advantage_mean": 0.6565036421467084, "kd/rkl_advantage_p95": 4.883481299877166, "kd/rkl_advantage_std": 2.2266299734512964, "kd/ssd_loss": 0.0, "learning_rate": 8.143394220180002e-07, "loss": 0.03686766624450684, "num_tokens": 159981644.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7800000011920929, "rewards/gt_logging_reward/std": 0.4075392936666807, "sampling/importance_sampling_ratio/max": 1.9859662214914957, "sampling/importance_sampling_ratio/mean": 1.0052743216355642, "sampling/importance_sampling_ratio/min": 0.394437434275945, "sampling/sampling_logp_difference/max": 0.44280216892560326, "sampling/sampling_logp_difference/mean": 0.0043404339347034694, "step": 4890, "step_time": 8.200424632627982, "student/entropy": 0.12990116843332847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01694444545234243, "completions/max_length": 480.96666666666664, "completions/max_terminated_length": 452.8666666666667, "completions/mean_length": 185.49445343017578, "completions/mean_terminated_length": 179.85411580403647, "completions/min_length": 45.06666666666667, "completions/min_terminated_length": 45.06666666666667, "entropy": 0.13092610612511635, "epoch": 0.1868378080735199, "eval/gt_acc_batch": 0.7972222447395325, "frac_reward_zero_std": 1.0, "grad_norm": 0.29051917791366577, "kd/policy_loss": 0.009051901889809718, "kd/rkl_advantage_mean": 0.6398770049369583, "kd/rkl_advantage_p95": 4.769393529494604, "kd/rkl_advantage_std": 2.1895749241113664, "kd/ssd_loss": 0.0, "learning_rate": 8.132001670907226e-07, "loss": 0.03620761235555013, "num_tokens": 160962136.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7972222189108531, "rewards/gt_logging_reward/std": 0.3965260634819667, "sampling/importance_sampling_ratio/max": 1.9661730527877808, "sampling/importance_sampling_ratio/mean": 0.9977195878823598, "sampling/importance_sampling_ratio/min": 0.42481893400351206, "sampling/sampling_logp_difference/max": 0.4476340413093567, "sampling/sampling_logp_difference/mean": 0.004376055602915585, "step": 4920, "step_time": 8.137886932783294, "student/entropy": 0.13092610612511635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01638889005407691, "completions/max_length": 483.5, "completions/max_terminated_length": 463.03333333333336, "completions/mean_length": 186.76278737386068, "completions/mean_terminated_length": 181.33468424479167, "completions/min_length": 52.03333333333333, "completions/min_terminated_length": 52.03333333333333, "entropy": 0.12739101592451335, "epoch": 0.18797706300079747, "eval/gt_acc_batch": 0.8158333539962769, "frac_reward_zero_std": 1.0, "grad_norm": 0.3056957721710205, "kd/policy_loss": 0.008308556980531042, "kd/rkl_advantage_mean": 0.5943945596305032, "kd/rkl_advantage_p95": 4.767317426204682, "kd/rkl_advantage_std": 2.2104207346836726, "kd/ssd_loss": 0.0, "learning_rate": 8.120609121634451e-07, "loss": 0.033234230677286786, "num_tokens": 161942434.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8158333400885264, "rewards/gt_logging_reward/std": 0.37926527559757234, "sampling/importance_sampling_ratio/max": 1.8978254556655885, "sampling/importance_sampling_ratio/mean": 0.9990658899148305, "sampling/importance_sampling_ratio/min": 0.4368948757648468, "sampling/sampling_logp_difference/max": 0.3956498364607493, "sampling/sampling_logp_difference/mean": 0.004294725328994294, "step": 4950, "step_time": 8.144437074377977, "student/entropy": 0.12739101592451335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556658655404, "completions/max_length": 476.1666666666667, "completions/max_terminated_length": 434.8333333333333, "completions/mean_length": 187.70223185221354, "completions/mean_terminated_length": 180.94827372233073, "completions/min_length": 54.63333333333333, "completions/min_terminated_length": 54.63333333333333, "entropy": 0.1277453636129697, "epoch": 0.18911631792807504, "eval/gt_acc_batch": 0.8047222435474396, "frac_reward_zero_std": 1.0, "grad_norm": 0.23080161213874817, "kd/policy_loss": 0.008772418657705809, "kd/rkl_advantage_mean": 0.6119360719031344, "kd/rkl_advantage_p95": 4.730212795734405, "kd/rkl_advantage_std": 2.1742894967397053, "kd/ssd_loss": 0.0, "learning_rate": 8.109216572361674e-07, "loss": 0.03508967955907186, "num_tokens": 162913658.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222197055817, "rewards/gt_logging_reward/std": 0.384920242925485, "sampling/importance_sampling_ratio/max": 2.0114851593971252, "sampling/importance_sampling_ratio/mean": 1.0040387193361917, "sampling/importance_sampling_ratio/min": 0.44306305249532063, "sampling/sampling_logp_difference/max": 0.45012377897898354, "sampling/sampling_logp_difference/mean": 0.004320841197234889, "step": 4980, "step_time": 7.891175898998821, "student/entropy": 0.1277453636129697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334326744078, "completions/max_length": 481.4, "completions/max_terminated_length": 440.6666666666667, "completions/mean_length": 187.43695475260418, "completions/mean_terminated_length": 180.63911997477214, "completions/min_length": 52.6, "completions/min_terminated_length": 52.6, "entropy": 0.1292789799471696, "epoch": 0.1902555728553526, "eval/gt_acc_batch": 0.801944476366043, "frac_reward_zero_std": 1.0, "grad_norm": 0.29492053389549255, "kd/policy_loss": 0.008603862760355695, "kd/rkl_advantage_mean": 0.6031553730756666, "kd/rkl_advantage_p95": 4.743026771148046, "kd/rkl_advantage_std": 2.1745767345031104, "kd/ssd_loss": 0.0, "learning_rate": 8.097824023088899e-07, "loss": 0.034415451685587566, "num_tokens": 163908775.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444425900777, "rewards/gt_logging_reward/std": 0.38749160220225654, "sampling/importance_sampling_ratio/max": 1.940081258614858, "sampling/importance_sampling_ratio/mean": 1.0053059120972951, "sampling/importance_sampling_ratio/min": 0.4202765787641207, "sampling/sampling_logp_difference/max": 0.4033089299996694, "sampling/sampling_logp_difference/mean": 0.004302563704550266, "step": 5010, "step_time": 8.305452114871393, "student/entropy": 0.1292789799471696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779282381136, "completions/max_length": 490.5, "completions/max_terminated_length": 447.0, "completions/mean_length": 187.22278645833333, "completions/mean_terminated_length": 178.78634440104167, "completions/min_length": 45.4, "completions/min_terminated_length": 45.4, "entropy": 0.12703756699338556, "epoch": 0.19139482778263017, "eval/gt_acc_batch": 0.7916666865348816, "frac_reward_zero_std": 1.0, "grad_norm": 0.2517487406730652, "kd/policy_loss": 0.008873986780721073, "kd/rkl_advantage_mean": 0.627296318858862, "kd/rkl_advantage_p95": 4.721908416350683, "kd/rkl_advantage_std": 2.168072532614072, "kd/ssd_loss": 0.0, "learning_rate": 8.086431473816125e-07, "loss": 0.035495952765146895, "num_tokens": 164883609.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7916666686534881, "rewards/gt_logging_reward/std": 0.398375674088796, "sampling/importance_sampling_ratio/max": 1.880922516187032, "sampling/importance_sampling_ratio/mean": 1.001639034350713, "sampling/importance_sampling_ratio/min": 0.4364289810260137, "sampling/sampling_logp_difference/max": 0.3999603629112244, "sampling/sampling_logp_difference/mean": 0.004204002728996178, "step": 5040, "step_time": 8.080870116921142, "student/entropy": 0.12703756699338556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778866390386, "completions/max_length": 477.3333333333333, "completions/max_terminated_length": 448.4, "completions/mean_length": 189.91112060546874, "completions/mean_terminated_length": 183.48850911458334, "completions/min_length": 53.266666666666666, "completions/min_terminated_length": 53.266666666666666, "entropy": 0.12819868040581545, "epoch": 0.1925340827099077, "eval/gt_acc_batch": 0.8019444644451141, "frac_reward_zero_std": 1.0, "grad_norm": 0.301169753074646, "kd/policy_loss": 0.008471824407267074, "kd/rkl_advantage_mean": 0.6075117065571248, "kd/rkl_advantage_p95": 4.729077784220378, "kd/rkl_advantage_std": 2.1599731942017875, "kd/ssd_loss": 0.0, "learning_rate": 8.075038924543348e-07, "loss": 0.0338873028755188, "num_tokens": 165878561.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444386164347, "rewards/gt_logging_reward/std": 0.39135581254959106, "sampling/importance_sampling_ratio/max": 1.8973191181818645, "sampling/importance_sampling_ratio/mean": 0.9952382663885753, "sampling/importance_sampling_ratio/min": 0.41198920061190925, "sampling/sampling_logp_difference/max": 0.4686159054438273, "sampling/sampling_logp_difference/mean": 0.004321890999563038, "step": 5070, "step_time": 8.086159905372188, "student/entropy": 0.12819868040581545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223381201427, "completions/max_length": 488.73333333333335, "completions/max_terminated_length": 447.3, "completions/mean_length": 185.92445373535156, "completions/mean_terminated_length": 178.7709701538086, "completions/min_length": 58.3, "completions/min_terminated_length": 58.3, "entropy": 0.12574077397584915, "epoch": 0.19367333763718528, "eval/gt_acc_batch": 0.7988889058430989, "frac_reward_zero_std": 1.0, "grad_norm": 0.397256076335907, "kd/policy_loss": 0.00864977237652056, "kd/rkl_advantage_mean": 0.6163639348388339, "kd/rkl_advantage_p95": 4.708701378107071, "kd/rkl_advantage_std": 2.175132308403651, "kd/ssd_loss": 0.0, "learning_rate": 8.063646375270573e-07, "loss": 0.03459908962249756, "num_tokens": 166852305.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7988888879617055, "rewards/gt_logging_reward/std": 0.3913717826207479, "sampling/importance_sampling_ratio/max": 2.023026971022288, "sampling/importance_sampling_ratio/mean": 1.0017088691393534, "sampling/importance_sampling_ratio/min": 0.3961103657881419, "sampling/sampling_logp_difference/max": 0.4183955013751984, "sampling/sampling_logp_difference/mean": 0.004254672094248235, "step": 5100, "step_time": 8.027666551379177, "student/entropy": 0.12574077397584915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0200000012293458, "completions/max_length": 487.46666666666664, "completions/max_terminated_length": 466.3666666666667, "completions/mean_length": 187.02306569417317, "completions/mean_terminated_length": 180.5190668741862, "completions/min_length": 51.56666666666667, "completions/min_terminated_length": 51.56666666666667, "entropy": 0.1283335148667296, "epoch": 0.19481259256446284, "eval/gt_acc_batch": 0.7955555776755016, "frac_reward_zero_std": 1.0, "grad_norm": 0.2991281747817993, "kd/policy_loss": 0.008480795009139305, "kd/rkl_advantage_mean": 0.6773272602508466, "kd/rkl_advantage_p95": 4.959339034557343, "kd/rkl_advantage_std": 2.241229225198428, "kd/ssd_loss": 0.0, "learning_rate": 8.052253825997797e-07, "loss": 0.0339231808980306, "num_tokens": 167825572.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555558204651, "rewards/gt_logging_reward/std": 0.3988847315311432, "sampling/importance_sampling_ratio/max": 1.8942663153012593, "sampling/importance_sampling_ratio/mean": 0.9986085593700409, "sampling/importance_sampling_ratio/min": 0.42663000027338666, "sampling/sampling_logp_difference/max": 0.4068620602289836, "sampling/sampling_logp_difference/mean": 0.004280555450047056, "step": 5130, "step_time": 8.023011575367612, "student/entropy": 0.1283335148667296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.035000002042700845, "completions/max_length": 486.3666666666667, "completions/max_terminated_length": 455.96666666666664, "completions/mean_length": 196.36223246256512, "completions/mean_terminated_length": 185.0490976969401, "completions/min_length": 53.3, "completions/min_terminated_length": 53.3, "entropy": 0.13094600594292086, "epoch": 0.1959518474917404, "eval/gt_acc_batch": 0.7791666766007741, "frac_reward_zero_std": 1.0, "grad_norm": 0.20581738650798798, "kd/policy_loss": 0.008756941882893443, "kd/rkl_advantage_mean": 0.6540660596452653, "kd/rkl_advantage_p95": 4.8508529424667355, "kd/rkl_advantage_std": 2.200080918272336, "kd/ssd_loss": 0.0, "learning_rate": 8.040861276725022e-07, "loss": 0.03502777417500814, "num_tokens": 168839564.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7791666666666667, "rewards/gt_logging_reward/std": 0.4110870490471522, "sampling/importance_sampling_ratio/max": 2.022777565320333, "sampling/importance_sampling_ratio/mean": 1.0010766545931498, "sampling/importance_sampling_ratio/min": 0.3326531062523524, "sampling/sampling_logp_difference/max": 0.4323797384897868, "sampling/sampling_logp_difference/mean": 0.004378063588713606, "step": 5160, "step_time": 8.149499145289884, "student/entropy": 0.13094600594292086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02356321978029506, "completions/max_length": 501.7931034482759, "completions/max_terminated_length": 457.0689655172414, "completions/mean_length": 191.34253666318696, "completions/mean_terminated_length": 183.60046597184805, "completions/min_length": 56.06896551724138, "completions/min_terminated_length": 56.06896551724138, "entropy": 0.13048019484970078, "epoch": 0.19709110241901795, "eval/gt_acc_batch": 0.7916666885902142, "frac_reward_zero_std": 1.0, "grad_norm": 0.3537169396877289, "kd/policy_loss": 0.008676694882712487, "kd/rkl_advantage_mean": 0.7002256462543175, "kd/rkl_advantage_p95": 4.935907851005423, "kd/rkl_advantage_std": 2.2219893387679397, "kd/ssd_loss": 0.0, "learning_rate": 8.029468727452245e-07, "loss": 0.03354988892873128, "num_tokens": 169796484.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7916666639262232, "rewards/gt_logging_reward/std": 0.39545962163086595, "sampling/importance_sampling_ratio/max": 1.9990886532027146, "sampling/importance_sampling_ratio/mean": 0.9966847896575928, "sampling/importance_sampling_ratio/min": 0.42183478690426923, "sampling/sampling_logp_difference/max": 0.3994228141061191, "sampling/sampling_logp_difference/mean": 0.004324385205861823, "step": 5190, "step_time": 7.840925910071625, "student/entropy": 0.13048019484970078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223530213037, "completions/max_length": 487.76666666666665, "completions/max_terminated_length": 459.43333333333334, "completions/mean_length": 185.7936223347982, "completions/mean_terminated_length": 179.30196584065754, "completions/min_length": 54.233333333333334, "completions/min_terminated_length": 54.233333333333334, "entropy": 0.1265678278480967, "epoch": 0.19823035734629552, "eval/gt_acc_batch": 0.8041666785875956, "frac_reward_zero_std": 1.0, "grad_norm": 0.22883690893650055, "kd/policy_loss": 0.008932129476064195, "kd/rkl_advantage_mean": 0.6151348318671808, "kd/rkl_advantage_p95": 4.764021501938502, "kd/rkl_advantage_std": 2.20256006916364, "kd/ssd_loss": 0.0, "learning_rate": 8.01807617817947e-07, "loss": 0.035728514194488525, "num_tokens": 170770701.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666666666667, "rewards/gt_logging_reward/std": 0.3862100834647814, "sampling/importance_sampling_ratio/max": 1.9690973877906799, "sampling/importance_sampling_ratio/mean": 1.006190800666809, "sampling/importance_sampling_ratio/min": 0.4551168272892634, "sampling/sampling_logp_difference/max": 0.43445527950922647, "sampling/sampling_logp_difference/mean": 0.004255938918019334, "step": 5220, "step_time": 8.000387248342546, "student/entropy": 0.1265678278480967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0222222234432896, "completions/max_length": 485.6666666666667, "completions/max_terminated_length": 446.1, "completions/mean_length": 187.16528778076173, "completions/mean_terminated_length": 179.80814361572266, "completions/min_length": 54.766666666666666, "completions/min_terminated_length": 54.766666666666666, "entropy": 0.12948254483441513, "epoch": 0.19936961227357308, "eval/gt_acc_batch": 0.7894444684187572, "frac_reward_zero_std": 1.0, "grad_norm": 0.3559996485710144, "kd/policy_loss": 0.009033587885399659, "kd/rkl_advantage_mean": 0.5340243935410399, "kd/rkl_advantage_p95": 4.595965071519216, "kd/rkl_advantage_std": 2.1638169447580973, "kd/ssd_loss": 0.0, "learning_rate": 8.006683628906695e-07, "loss": 0.03613435029983521, "num_tokens": 171764464.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7894444445768992, "rewards/gt_logging_reward/std": 0.4005243529876073, "sampling/importance_sampling_ratio/max": 2.0004639585812884, "sampling/importance_sampling_ratio/mean": 1.0050809939702352, "sampling/importance_sampling_ratio/min": 0.42858825822671254, "sampling/sampling_logp_difference/max": 0.3965487778186798, "sampling/sampling_logp_difference/mean": 0.004340576683171093, "step": 5250, "step_time": 8.222769333243681, "student/entropy": 0.12948254483441513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112305273613, "completions/max_length": 475.76666666666665, "completions/max_terminated_length": 437.6666666666667, "completions/mean_length": 190.09334462483724, "completions/mean_terminated_length": 183.29903767903645, "completions/min_length": 56.43333333333333, "completions/min_terminated_length": 56.43333333333333, "entropy": 0.12669192937513193, "epoch": 0.20050886720085065, "eval/gt_acc_batch": 0.7938889125982921, "frac_reward_zero_std": 1.0, "grad_norm": 0.23240606486797333, "kd/policy_loss": 0.00856164278035673, "kd/rkl_advantage_mean": 0.6027971886253606, "kd/rkl_advantage_p95": 4.661067821582159, "kd/rkl_advantage_std": 2.156881257891655, "kd/ssd_loss": 0.0, "learning_rate": 7.995291079633919e-07, "loss": 0.03424656788508097, "num_tokens": 172752912.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7938888927300771, "rewards/gt_logging_reward/std": 0.3966137280066808, "sampling/importance_sampling_ratio/max": 1.9265421311060587, "sampling/importance_sampling_ratio/mean": 0.9947320242722829, "sampling/importance_sampling_ratio/min": 0.42185368786255517, "sampling/sampling_logp_difference/max": 0.45608983337879183, "sampling/sampling_logp_difference/mean": 0.004258148170386751, "step": 5280, "step_time": 8.025129339339522, "student/entropy": 0.12669192937513193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02888889064391454, "completions/max_length": 476.3, "completions/max_terminated_length": 430.8, "completions/mean_length": 190.5291758219401, "completions/mean_terminated_length": 181.04527842203777, "completions/min_length": 50.36666666666667, "completions/min_terminated_length": 50.36666666666667, "entropy": 0.1415137327586611, "epoch": 0.20164812212812822, "eval/gt_acc_batch": 0.7747222403685252, "frac_reward_zero_std": 1.0, "grad_norm": 0.3147912323474884, "kd/policy_loss": 0.00929154625434118, "kd/rkl_advantage_mean": 0.5417258228641003, "kd/rkl_advantage_p95": 4.54262808561325, "kd/rkl_advantage_std": 2.1196047266324363, "kd/ssd_loss": 0.0, "learning_rate": 7.983898530361144e-07, "loss": 0.03716618220011393, "num_tokens": 173742713.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7747222244739532, "rewards/gt_logging_reward/std": 0.4078964153925578, "sampling/importance_sampling_ratio/max": 2.061245242754618, "sampling/importance_sampling_ratio/mean": 1.0026904304822286, "sampling/importance_sampling_ratio/min": 0.40884735584259035, "sampling/sampling_logp_difference/max": 0.4097396651903788, "sampling/sampling_logp_difference/mean": 0.0044444727788989745, "step": 5310, "step_time": 7.993606386139678, "student/entropy": 0.1415137327586611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028611112448076407, "completions/max_length": 477.56666666666666, "completions/max_terminated_length": 449.7, "completions/mean_length": 190.17056477864583, "completions/mean_terminated_length": 180.9695805867513, "completions/min_length": 46.0, "completions/min_terminated_length": 46.0, "entropy": 0.12938817447672288, "epoch": 0.20278737705540575, "eval/gt_acc_batch": 0.7919444640477499, "frac_reward_zero_std": 1.0, "grad_norm": 0.3003069758415222, "kd/policy_loss": 0.008795870268174136, "kd/rkl_advantage_mean": 0.6206797004677356, "kd/rkl_advantage_p95": 4.788314775625865, "kd/rkl_advantage_std": 2.1878727465867995, "kd/ssd_loss": 0.0, "learning_rate": 7.972505981088368e-07, "loss": 0.03518348534901937, "num_tokens": 174727943.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7919444461663564, "rewards/gt_logging_reward/std": 0.39309439311424893, "sampling/importance_sampling_ratio/max": 1.8762022852897644, "sampling/importance_sampling_ratio/mean": 0.9934730947017669, "sampling/importance_sampling_ratio/min": 0.43314648866653443, "sampling/sampling_logp_difference/max": 0.42374132871627807, "sampling/sampling_logp_difference/mean": 0.004241814968797068, "step": 5340, "step_time": 8.006405598910836, "student/entropy": 0.12938817447672288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029444446414709093, "completions/max_length": 501.3333333333333, "completions/max_terminated_length": 465.4, "completions/mean_length": 192.84139862060547, "completions/mean_terminated_length": 183.28834838867186, "completions/min_length": 58.56666666666667, "completions/min_terminated_length": 58.56666666666667, "entropy": 0.13499051611870527, "epoch": 0.20392663198268332, "eval/gt_acc_batch": 0.7866666913032532, "frac_reward_zero_std": 1.0, "grad_norm": 0.260213166475296, "kd/policy_loss": 0.00901132765187261, "kd/rkl_advantage_mean": 0.7174080243334174, "kd/rkl_advantage_p95": 4.978605473041535, "kd/rkl_advantage_std": 2.242840287089348, "kd/ssd_loss": 0.0, "learning_rate": 7.961113431815592e-07, "loss": 0.03604531288146973, "num_tokens": 175732116.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666634877523, "rewards/gt_logging_reward/std": 0.40042660335699715, "sampling/importance_sampling_ratio/max": 1.976708217461904, "sampling/importance_sampling_ratio/mean": 0.9894313891728719, "sampling/importance_sampling_ratio/min": 0.3614824761946996, "sampling/sampling_logp_difference/max": 0.4407424529393514, "sampling/sampling_logp_difference/mean": 0.004449075429389874, "step": 5370, "step_time": 8.251546183294462, "student/entropy": 0.13499051611870527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02972222389653325, "completions/max_length": 489.0, "completions/max_terminated_length": 454.56666666666666, "completions/mean_length": 195.61112009684246, "completions/mean_terminated_length": 186.0920415242513, "completions/min_length": 49.86666666666667, "completions/min_terminated_length": 49.86666666666667, "entropy": 0.13337129422773916, "epoch": 0.2050658869099609, "eval/gt_acc_batch": 0.7741666932900747, "frac_reward_zero_std": 1.0, "grad_norm": 0.32319706678390503, "kd/policy_loss": 0.009309611310406278, "kd/rkl_advantage_mean": 0.6194044188033634, "kd/rkl_advantage_p95": 4.796221750974655, "kd/rkl_advantage_std": 2.1968612293402354, "kd/ssd_loss": 0.0, "learning_rate": 7.949720882542816e-07, "loss": 0.037238446871439616, "num_tokens": 176745684.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7741666694482168, "rewards/gt_logging_reward/std": 0.4075356274843216, "sampling/importance_sampling_ratio/max": 2.051927149295807, "sampling/importance_sampling_ratio/mean": 1.001786035299301, "sampling/importance_sampling_ratio/min": 0.38589755396048225, "sampling/sampling_logp_difference/max": 0.4028500219186147, "sampling/sampling_logp_difference/mean": 0.004370854290512701, "step": 5400, "step_time": 8.194908946795234, "student/entropy": 0.13337129422773916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021839081727225203, "completions/max_length": 469.41379310344826, "completions/max_terminated_length": 428.0, "completions/mean_length": 180.7502978094693, "completions/mean_terminated_length": 173.53646166571255, "completions/min_length": 53.48275862068966, "completions/min_terminated_length": 53.48275862068966, "entropy": 0.12902775766520663, "epoch": 0.20620514183723845, "eval/gt_acc_batch": 0.8198276038827568, "frac_reward_zero_std": 1.0, "grad_norm": 0.2562881410121918, "kd/policy_loss": 0.008586089999880642, "kd/rkl_advantage_mean": 0.49612416630482364, "kd/rkl_advantage_p95": 4.4460516366465335, "kd/rkl_advantage_std": 2.103765595575859, "kd/ssd_loss": 0.0, "learning_rate": 7.938328333270042e-07, "loss": 0.03319954872131348, "num_tokens": 177663631.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8198275853847635, "rewards/gt_logging_reward/std": 0.3731264240782836, "sampling/importance_sampling_ratio/max": 1.9136873976937656, "sampling/importance_sampling_ratio/mean": 0.9994074928349462, "sampling/importance_sampling_ratio/min": 0.4185659263668389, "sampling/sampling_logp_difference/max": 0.41368394679036635, "sampling/sampling_logp_difference/mean": 0.004347394204473701, "step": 5430, "step_time": 7.587093648842226, "student/entropy": 0.12902775766520663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667945683002, "completions/max_length": 472.6333333333333, "completions/max_terminated_length": 439.9, "completions/mean_length": 182.5577860514323, "completions/mean_terminated_length": 176.2287582397461, "completions/min_length": 48.53333333333333, "completions/min_terminated_length": 48.53333333333333, "entropy": 0.125232717829446, "epoch": 0.20734439676451602, "eval/gt_acc_batch": 0.8066666920979818, "frac_reward_zero_std": 1.0, "grad_norm": 0.2500506043434143, "kd/policy_loss": 0.008496791384338091, "kd/rkl_advantage_mean": 0.6938993069032828, "kd/rkl_advantage_p95": 4.873514552911122, "kd/rkl_advantage_std": 2.2173260579506557, "kd/ssd_loss": 0.0, "learning_rate": 7.926935783997265e-07, "loss": 0.03398716847101847, "num_tokens": 178628487.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8066666642824809, "rewards/gt_logging_reward/std": 0.3899006962776184, "sampling/importance_sampling_ratio/max": 1.9564496914545695, "sampling/importance_sampling_ratio/mean": 0.9964917778968811, "sampling/importance_sampling_ratio/min": 0.435832021633784, "sampling/sampling_logp_difference/max": 0.47459025382995607, "sampling/sampling_logp_difference/mean": 0.00423008860864987, "step": 5460, "step_time": 8.107880292429279, "student/entropy": 0.125232717829446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01888888996715347, "completions/max_length": 457.1333333333333, "completions/max_terminated_length": 422.26666666666665, "completions/mean_length": 179.6688975016276, "completions/mean_terminated_length": 173.43395538330077, "completions/min_length": 55.36666666666667, "completions/min_terminated_length": 55.36666666666667, "entropy": 0.12709176043669382, "epoch": 0.20848365169179356, "eval/gt_acc_batch": 0.8152777989705403, "frac_reward_zero_std": 1.0, "grad_norm": 0.2315036505460739, "kd/policy_loss": 0.00822673201134118, "kd/rkl_advantage_mean": 0.5791924027338003, "kd/rkl_advantage_p95": 4.521941963831583, "kd/rkl_advantage_std": 2.088363617658615, "kd/ssd_loss": 0.0, "learning_rate": 7.91554323472449e-07, "loss": 0.03290692567825317, "num_tokens": 179567119.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8152777791023255, "rewards/gt_logging_reward/std": 0.3801032821337382, "sampling/importance_sampling_ratio/max": 1.8747817079226177, "sampling/importance_sampling_ratio/mean": 0.9970693548520406, "sampling/importance_sampling_ratio/min": 0.438566625614961, "sampling/sampling_logp_difference/max": 0.42173524498939513, "sampling/sampling_logp_difference/mean": 0.004261413309723139, "step": 5490, "step_time": 7.836766444030218, "student/entropy": 0.12709176043669382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890078912177, "completions/max_length": 499.4, "completions/max_terminated_length": 445.73333333333335, "completions/mean_length": 182.31889750162762, "completions/mean_terminated_length": 173.25070393880208, "completions/min_length": 49.5, "completions/min_terminated_length": 49.5, "entropy": 0.13152502570301294, "epoch": 0.20962290661907113, "eval/gt_acc_batch": 0.7872222522894542, "frac_reward_zero_std": 1.0, "grad_norm": 0.34442585706710815, "kd/policy_loss": 0.00889303203051289, "kd/rkl_advantage_mean": 0.64286714073581, "kd/rkl_advantage_p95": 4.745522250731786, "kd/rkl_advantage_std": 2.208394727110863, "kd/ssd_loss": 0.0, "learning_rate": 7.904150685451714e-07, "loss": 0.03557213147481283, "num_tokens": 180528259.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7872222205003102, "rewards/gt_logging_reward/std": 0.40022837817668916, "sampling/importance_sampling_ratio/max": 1.8933269262313843, "sampling/importance_sampling_ratio/mean": 0.9930647174517314, "sampling/importance_sampling_ratio/min": 0.41604195858041443, "sampling/sampling_logp_difference/max": 0.46777871052424114, "sampling/sampling_logp_difference/mean": 0.0044169434113428, "step": 5520, "step_time": 8.185300577261176, "student/entropy": 0.13152502570301294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02138889003545046, "completions/max_length": 481.5, "completions/max_terminated_length": 447.2, "completions/mean_length": 185.97834218343098, "completions/mean_terminated_length": 178.99661610921223, "completions/min_length": 49.3, "completions/min_terminated_length": 49.3, "entropy": 0.13055832802007597, "epoch": 0.2107621615463487, "eval/gt_acc_batch": 0.7947222431500752, "frac_reward_zero_std": 1.0, "grad_norm": 0.24751830101013184, "kd/policy_loss": 0.008780611907908072, "kd/rkl_advantage_mean": 0.6928253055550158, "kd/rkl_advantage_p95": 4.90375883380572, "kd/rkl_advantage_std": 2.2180306990941365, "kd/ssd_loss": 0.0, "learning_rate": 7.892758136178938e-07, "loss": 0.035122446219126385, "num_tokens": 181499733.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222252686819, "rewards/gt_logging_reward/std": 0.3969564184546471, "sampling/importance_sampling_ratio/max": 1.946396783987681, "sampling/importance_sampling_ratio/mean": 0.9970524112383524, "sampling/importance_sampling_ratio/min": 0.4360778411229452, "sampling/sampling_logp_difference/max": 0.4017364263534546, "sampling/sampling_logp_difference/mean": 0.0043715935898944736, "step": 5550, "step_time": 7.931471060790742, "student/entropy": 0.13055832802007597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01750000109896064, "completions/max_length": 473.0, "completions/max_terminated_length": 448.3, "completions/mean_length": 191.0466771443685, "completions/mean_terminated_length": 185.58940633138022, "completions/min_length": 55.46666666666667, "completions/min_terminated_length": 55.46666666666667, "entropy": 0.1250668091699481, "epoch": 0.21190141647362626, "eval/gt_acc_batch": 0.7822222530841827, "frac_reward_zero_std": 1.0, "grad_norm": 0.23923687636852264, "kd/policy_loss": 0.008436628081835807, "kd/rkl_advantage_mean": 0.5996271646115929, "kd/rkl_advantage_p95": 4.706129682064057, "kd/rkl_advantage_std": 2.1609189033508303, "kd/ssd_loss": 0.0, "learning_rate": 7.881365586906163e-07, "loss": 0.03374650875727336, "num_tokens": 182489789.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7822222272555034, "rewards/gt_logging_reward/std": 0.40455165406068166, "sampling/importance_sampling_ratio/max": 1.8589762568473815, "sampling/importance_sampling_ratio/mean": 0.9921265800793966, "sampling/importance_sampling_ratio/min": 0.44579907357692716, "sampling/sampling_logp_difference/max": 0.4165138840675354, "sampling/sampling_logp_difference/mean": 0.004173792052703599, "step": 5580, "step_time": 7.886463295835226, "student/entropy": 0.1250668091699481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223629554113, "completions/max_length": 497.23333333333335, "completions/max_terminated_length": 459.03333333333336, "completions/mean_length": 189.0438980102539, "completions/mean_terminated_length": 181.76290893554688, "completions/min_length": 51.56666666666667, "completions/min_terminated_length": 51.56666666666667, "entropy": 0.12730340926597516, "epoch": 0.2130406714009038, "eval/gt_acc_batch": 0.7813889046510061, "frac_reward_zero_std": 1.0, "grad_norm": 0.2746838331222534, "kd/policy_loss": 0.008674928439237799, "kd/rkl_advantage_mean": 0.5897966586713058, "kd/rkl_advantage_p95": 4.702760471900304, "kd/rkl_advantage_std": 2.1689340581496555, "kd/ssd_loss": 0.0, "learning_rate": 7.869973037633387e-07, "loss": 0.03469971418380737, "num_tokens": 183479027.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7813888887564341, "rewards/gt_logging_reward/std": 0.40876571337382, "sampling/importance_sampling_ratio/max": 1.8567088921864827, "sampling/importance_sampling_ratio/mean": 0.9960138281186421, "sampling/importance_sampling_ratio/min": 0.3820172796646754, "sampling/sampling_logp_difference/max": 0.3804553061723709, "sampling/sampling_logp_difference/mean": 0.0042133899948870145, "step": 5610, "step_time": 8.163071076389558, "student/entropy": 0.12730340926597516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667920847735, "completions/max_length": 488.9, "completions/max_terminated_length": 453.9, "completions/mean_length": 193.83473205566406, "completions/mean_terminated_length": 187.0249008178711, "completions/min_length": 51.53333333333333, "completions/min_terminated_length": 51.53333333333333, "entropy": 0.12789043970406055, "epoch": 0.21417992632818136, "eval/gt_acc_batch": 0.7900000214576721, "frac_reward_zero_std": 1.0, "grad_norm": 0.34008148312568665, "kd/policy_loss": 0.008629230262401203, "kd/rkl_advantage_mean": 0.6330489767715335, "kd/rkl_advantage_p95": 4.772487719853719, "kd/rkl_advantage_std": 2.178160043557485, "kd/ssd_loss": 0.0, "learning_rate": 7.858580488360613e-07, "loss": 0.03451692263285319, "num_tokens": 184496928.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7899999996026357, "rewards/gt_logging_reward/std": 0.39965747197469076, "sampling/importance_sampling_ratio/max": 1.9363914529482524, "sampling/importance_sampling_ratio/mean": 0.9969874779383342, "sampling/importance_sampling_ratio/min": 0.39826582024494805, "sampling/sampling_logp_difference/max": 0.45875527858734133, "sampling/sampling_logp_difference/mean": 0.004241201366918783, "step": 5640, "step_time": 8.381786244517814, "student/entropy": 0.12789043970406055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03083333515872558, "completions/max_length": 503.5, "completions/max_terminated_length": 459.1666666666667, "completions/mean_length": 197.1408447265625, "completions/mean_terminated_length": 187.2719477335612, "completions/min_length": 59.3, "completions/min_terminated_length": 59.3, "entropy": 0.13283644306162992, "epoch": 0.21531918125545893, "eval/gt_acc_batch": 0.7733333528041839, "frac_reward_zero_std": 1.0, "grad_norm": 0.188642218708992, "kd/policy_loss": 0.009595150810976822, "kd/rkl_advantage_mean": 0.679622879313926, "kd/rkl_advantage_p95": 4.931788782278697, "kd/rkl_advantage_std": 2.232171208659808, "kd/ssd_loss": 0.0, "learning_rate": 7.847187939087836e-07, "loss": 0.03838060696919759, "num_tokens": 185515003.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7733333329359691, "rewards/gt_logging_reward/std": 0.41041067739327747, "sampling/importance_sampling_ratio/max": 2.1348331610361737, "sampling/importance_sampling_ratio/mean": 1.0055423299471538, "sampling/importance_sampling_ratio/min": 0.3753095120191574, "sampling/sampling_logp_difference/max": 0.3968024333318075, "sampling/sampling_logp_difference/mean": 0.004346788814291358, "step": 5670, "step_time": 8.326860562207488, "student/entropy": 0.13283644306162992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001390775046, "completions/max_length": 486.8666666666667, "completions/max_terminated_length": 459.53333333333336, "completions/mean_length": 191.12223205566406, "completions/mean_terminated_length": 183.7868682861328, "completions/min_length": 53.86666666666667, "completions/min_terminated_length": 53.86666666666667, "entropy": 0.12788576477517685, "epoch": 0.2164584361827365, "eval/gt_acc_batch": 0.7952777862548828, "frac_reward_zero_std": 1.0, "grad_norm": 0.28096887469291687, "kd/policy_loss": 0.008937225226933758, "kd/rkl_advantage_mean": 0.6630268151561419, "kd/rkl_advantage_p95": 4.852271602551142, "kd/rkl_advantage_std": 2.2115610977013906, "kd/ssd_loss": 0.0, "learning_rate": 7.835795389815061e-07, "loss": 0.035748895009358725, "num_tokens": 186529203.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777743339539, "rewards/gt_logging_reward/std": 0.3970170249541601, "sampling/importance_sampling_ratio/max": 1.9355895876884461, "sampling/importance_sampling_ratio/mean": 1.001184888680776, "sampling/importance_sampling_ratio/min": 0.37919475038846334, "sampling/sampling_logp_difference/max": 0.42830678820610046, "sampling/sampling_logp_difference/mean": 0.0042645724800725775, "step": 5700, "step_time": 8.304077066869164, "student/entropy": 0.12788576477517685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890221714973, "completions/max_length": 492.93333333333334, "completions/max_terminated_length": 443.6333333333333, "completions/mean_length": 189.16112060546874, "completions/mean_terminated_length": 182.12786966959635, "completions/min_length": 57.1, "completions/min_terminated_length": 57.1, "entropy": 0.12831549756228924, "epoch": 0.21759769111001406, "eval/gt_acc_batch": 0.8127777953942616, "frac_reward_zero_std": 1.0, "grad_norm": 0.30303895473480225, "kd/policy_loss": 0.008849725604522973, "kd/rkl_advantage_mean": 0.7632906638396283, "kd/rkl_advantage_p95": 4.994319760799408, "kd/rkl_advantage_std": 2.2230107605457308, "kd/ssd_loss": 0.0, "learning_rate": 7.824402840542285e-07, "loss": 0.0353989044825236, "num_tokens": 187511343.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8127777755260468, "rewards/gt_logging_reward/std": 0.3808879022796949, "sampling/importance_sampling_ratio/max": 2.060555680592855, "sampling/importance_sampling_ratio/mean": 1.0089763919512431, "sampling/importance_sampling_ratio/min": 0.45285008996725085, "sampling/sampling_logp_difference/max": 0.4202309330304464, "sampling/sampling_logp_difference/mean": 0.00427888873188446, "step": 5730, "step_time": 8.076755245414097, "student/entropy": 0.12831549756228924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03000000144044558, "completions/max_length": 490.1, "completions/max_terminated_length": 466.76666666666665, "completions/mean_length": 193.43584289550782, "completions/mean_terminated_length": 183.68018900553386, "completions/min_length": 42.06666666666667, "completions/min_terminated_length": 42.06666666666667, "entropy": 0.13391668715824684, "epoch": 0.2187369460372916, "eval/gt_acc_batch": 0.7730555792649587, "frac_reward_zero_std": 1.0, "grad_norm": 0.3678651750087738, "kd/policy_loss": 0.00943789459997788, "kd/rkl_advantage_mean": 0.6817300123473008, "kd/rkl_advantage_p95": 4.9047264417012535, "kd/rkl_advantage_std": 2.2196541875600815, "kd/ssd_loss": 0.0, "learning_rate": 7.813010291269509e-07, "loss": 0.03775157928466797, "num_tokens": 188514200.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7730555574099223, "rewards/gt_logging_reward/std": 0.4085520620147387, "sampling/importance_sampling_ratio/max": 2.041526945432027, "sampling/importance_sampling_ratio/mean": 1.0028298477331796, "sampling/importance_sampling_ratio/min": 0.4376339763402939, "sampling/sampling_logp_difference/max": 0.41535300215085347, "sampling/sampling_logp_difference/mean": 0.004383208137005568, "step": 5760, "step_time": 8.222226228266178, "student/entropy": 0.13391668715824684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556491017341, "completions/max_length": 478.1, "completions/max_terminated_length": 442.56666666666666, "completions/mean_length": 181.13251037597655, "completions/mean_terminated_length": 175.98905436197916, "completions/min_length": 48.36666666666667, "completions/min_terminated_length": 48.36666666666667, "entropy": 0.12858205481121937, "epoch": 0.21987620096456917, "eval/gt_acc_batch": 0.8066666881243388, "frac_reward_zero_std": 1.0, "grad_norm": 0.3343258202075958, "kd/policy_loss": 0.008838289661798625, "kd/rkl_advantage_mean": 0.6625545490843554, "kd/rkl_advantage_p95": 4.888026799758276, "kd/rkl_advantage_std": 2.2402245422204334, "kd/ssd_loss": 0.0, "learning_rate": 7.801617741996733e-07, "loss": 0.03535315990447998, "num_tokens": 189465613.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8066666642824809, "rewards/gt_logging_reward/std": 0.3908297528823217, "sampling/importance_sampling_ratio/max": 1.9224931875864664, "sampling/importance_sampling_ratio/mean": 1.0016495029131571, "sampling/importance_sampling_ratio/min": 0.44221571187178293, "sampling/sampling_logp_difference/max": 0.4424039125442505, "sampling/sampling_logp_difference/mean": 0.004322101408615708, "step": 5790, "step_time": 7.943245146784466, "student/entropy": 0.12858205481121937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.012777778475234905, "completions/max_length": 476.1666666666667, "completions/max_terminated_length": 431.46666666666664, "completions/mean_length": 183.53695424397787, "completions/mean_terminated_length": 179.37187652587892, "completions/min_length": 54.233333333333334, "completions/min_terminated_length": 54.233333333333334, "entropy": 0.12296071797609329, "epoch": 0.22101545589184673, "eval/gt_acc_batch": 0.8019444664319356, "frac_reward_zero_std": 1.0, "grad_norm": 0.21569356322288513, "kd/policy_loss": 0.00799797362803171, "kd/rkl_advantage_mean": 0.6836033448576927, "kd/rkl_advantage_p95": 4.846912217140198, "kd/rkl_advantage_std": 2.173480122288068, "kd/ssd_loss": 0.0, "learning_rate": 7.790225192723958e-07, "loss": 0.0319918950398763, "num_tokens": 190417418.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444465637207, "rewards/gt_logging_reward/std": 0.3880488167206446, "sampling/importance_sampling_ratio/max": 1.8490684946378073, "sampling/importance_sampling_ratio/mean": 1.0028865834077199, "sampling/importance_sampling_ratio/min": 0.4574317678809166, "sampling/sampling_logp_difference/max": 0.3523656706015269, "sampling/sampling_logp_difference/mean": 0.0040835080901160834, "step": 5820, "step_time": 7.809561456561399, "student/entropy": 0.12296071797609329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0244444456572334, "completions/max_length": 489.2, "completions/max_terminated_length": 448.03333333333336, "completions/mean_length": 191.64945576985676, "completions/mean_terminated_length": 183.8726557413737, "completions/min_length": 52.766666666666666, "completions/min_terminated_length": 52.766666666666666, "entropy": 0.13544726489732664, "epoch": 0.2221547108191243, "eval/gt_acc_batch": 0.7725000242392223, "frac_reward_zero_std": 1.0, "grad_norm": 0.28815704584121704, "kd/policy_loss": 0.009313079774923001, "kd/rkl_advantage_mean": 0.6865894771491488, "kd/rkl_advantage_p95": 4.964809554815292, "kd/rkl_advantage_std": 2.238858007391294, "kd/ssd_loss": 0.0, "learning_rate": 7.778832643451183e-07, "loss": 0.03725232283274333, "num_tokens": 191413148.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7724999984105428, "rewards/gt_logging_reward/std": 0.4087403694788615, "sampling/importance_sampling_ratio/max": 1.8854342063268026, "sampling/importance_sampling_ratio/mean": 0.9945890764395396, "sampling/importance_sampling_ratio/min": 0.43529290954271954, "sampling/sampling_logp_difference/max": 0.3980265339215597, "sampling/sampling_logp_difference/mean": 0.004457079316489398, "step": 5850, "step_time": 8.036277526799434, "student/entropy": 0.13544726489732664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890221714973, "completions/max_length": 475.3, "completions/max_terminated_length": 431.7, "completions/mean_length": 183.9769536336263, "completions/mean_terminated_length": 176.9757044474284, "completions/min_length": 44.2, "completions/min_terminated_length": 44.2, "entropy": 0.12993071439365547, "epoch": 0.22329396574640184, "eval/gt_acc_batch": 0.8066666781902313, "frac_reward_zero_std": 1.0, "grad_norm": 0.29278483986854553, "kd/policy_loss": 0.008625036644904565, "kd/rkl_advantage_mean": 0.6887329362643262, "kd/rkl_advantage_p95": 4.913207626342773, "kd/rkl_advantage_std": 2.2223395725091297, "kd/ssd_loss": 0.0, "learning_rate": 7.767440094178407e-07, "loss": 0.03450014591217041, "num_tokens": 192378105.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8066666662693024, "rewards/gt_logging_reward/std": 0.3845016111930211, "sampling/importance_sampling_ratio/max": 1.9385174711545308, "sampling/importance_sampling_ratio/mean": 0.9991305847962697, "sampling/importance_sampling_ratio/min": 0.452227528889974, "sampling/sampling_logp_difference/max": 0.40471582214037577, "sampling/sampling_logp_difference/mean": 0.0043562284205108884, "step": 5880, "step_time": 7.974808844792036, "student/entropy": 0.12993071439365547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778804302215, "completions/max_length": 500.96666666666664, "completions/max_terminated_length": 451.6333333333333, "completions/mean_length": 186.37000986735026, "completions/mean_terminated_length": 179.6831319173177, "completions/min_length": 48.13333333333333, "completions/min_terminated_length": 48.13333333333333, "entropy": 0.12896088728060326, "epoch": 0.2244332206736794, "eval/gt_acc_batch": 0.798333364725113, "frac_reward_zero_std": 1.0, "grad_norm": 0.24819177389144897, "kd/policy_loss": 0.008702729817014188, "kd/rkl_advantage_mean": 0.6213313197096189, "kd/rkl_advantage_p95": 4.733808664480845, "kd/rkl_advantage_std": 2.162604829668999, "kd/ssd_loss": 0.0, "learning_rate": 7.756047544905632e-07, "loss": 0.03481091658274333, "num_tokens": 193353013.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.798333328962326, "rewards/gt_logging_reward/std": 0.39396425187587736, "sampling/importance_sampling_ratio/max": 2.0117841323216754, "sampling/importance_sampling_ratio/mean": 1.00590673883756, "sampling/importance_sampling_ratio/min": 0.421308138469855, "sampling/sampling_logp_difference/max": 0.40876758893330895, "sampling/sampling_logp_difference/mean": 0.004307455491895477, "step": 5910, "step_time": 8.192492198028292, "student/entropy": 0.12896088728060326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556882172822, "completions/max_length": 476.23333333333335, "completions/max_terminated_length": 447.4, "completions/mean_length": 188.5438985188802, "completions/mean_terminated_length": 181.27578786214193, "completions/min_length": 50.56666666666667, "completions/min_terminated_length": 50.56666666666667, "entropy": 0.13853238094598055, "epoch": 0.22557247560095697, "eval/gt_acc_batch": 0.7897222419579824, "frac_reward_zero_std": 1.0, "grad_norm": 0.27892500162124634, "kd/policy_loss": 0.009205534724363437, "kd/rkl_advantage_mean": 0.6743422707853218, "kd/rkl_advantage_p95": 4.869641643762589, "kd/rkl_advantage_std": 2.205375549197197, "kd/ssd_loss": 0.0, "learning_rate": 7.744654995632855e-07, "loss": 0.03682214021682739, "num_tokens": 194333467.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.789722220102946, "rewards/gt_logging_reward/std": 0.39648331999778746, "sampling/importance_sampling_ratio/max": 1.878988258043925, "sampling/importance_sampling_ratio/mean": 0.9979678650697072, "sampling/importance_sampling_ratio/min": 0.4188748821616173, "sampling/sampling_logp_difference/max": 0.39626409610112506, "sampling/sampling_logp_difference/mean": 0.004361696417133013, "step": 5940, "step_time": 7.935669683668918, "student/entropy": 0.13853238094598055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223822027444, "completions/max_length": 483.46666666666664, "completions/max_terminated_length": 445.93333333333334, "completions/mean_length": 185.52500864664714, "completions/mean_terminated_length": 177.2699737548828, "completions/min_length": 52.9, "completions/min_terminated_length": 52.9, "entropy": 0.12607061869154373, "epoch": 0.22671173052823454, "eval/gt_acc_batch": 0.8116666833559673, "frac_reward_zero_std": 1.0, "grad_norm": 0.28922736644744873, "kd/policy_loss": 0.008378829122132931, "kd/rkl_advantage_mean": 0.6698283930541947, "kd/rkl_advantage_p95": 4.85495662689209, "kd/rkl_advantage_std": 2.2240504691998164, "kd/ssd_loss": 0.0, "learning_rate": 7.73326244636008e-07, "loss": 0.033515310287475585, "num_tokens": 195296557.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8116666654745738, "rewards/gt_logging_reward/std": 0.3806858465075493, "sampling/importance_sampling_ratio/max": 1.942570964495341, "sampling/importance_sampling_ratio/mean": 0.9995030244191487, "sampling/importance_sampling_ratio/min": 0.4062091747919718, "sampling/sampling_logp_difference/max": 0.4503404418627421, "sampling/sampling_logp_difference/mean": 0.004235495355290672, "step": 5970, "step_time": 7.9241529256648695, "student/entropy": 0.12607061869154373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001235554616, "completions/max_length": 498.3333333333333, "completions/max_terminated_length": 455.6333333333333, "completions/mean_length": 189.9730651855469, "completions/mean_terminated_length": 182.57928161621095, "completions/min_length": 52.03333333333333, "completions/min_terminated_length": 52.03333333333333, "entropy": 0.13104532758394877, "epoch": 0.2278509854555121, "eval/gt_acc_batch": 0.7958333452542623, "frac_reward_zero_std": 1.0, "grad_norm": 0.22975988686084747, "kd/policy_loss": 0.009143940704719473, "kd/rkl_advantage_mean": 0.6065754485627015, "kd/rkl_advantage_p95": 4.670005939404169, "kd/rkl_advantage_std": 2.151283989350001, "kd/ssd_loss": 0.0, "learning_rate": 7.721869897087304e-07, "loss": 0.03657575845718384, "num_tokens": 196284268.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7958333373069764, "rewards/gt_logging_reward/std": 0.3970336039861043, "sampling/importance_sampling_ratio/max": 1.9158796389897665, "sampling/importance_sampling_ratio/mean": 1.0000604828198751, "sampling/importance_sampling_ratio/min": 0.4204437827070554, "sampling/sampling_logp_difference/max": 0.353840716679891, "sampling/sampling_logp_difference/mean": 0.0043151885891954105, "step": 6000, "step_time": 8.087329691345804, "student/entropy": 0.13104532758394877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444558893641, "completions/max_length": 504.6666666666667, "completions/max_terminated_length": 467.23333333333335, "completions/mean_length": 190.24278666178387, "completions/mean_terminated_length": 183.09772186279298, "completions/min_length": 48.46666666666667, "completions/min_terminated_length": 48.46666666666667, "entropy": 0.13260588670770326, "epoch": 0.22899024038278964, "eval/gt_acc_batch": 0.7736111402511596, "frac_reward_zero_std": 1.0, "grad_norm": 0.27430352568626404, "kd/policy_loss": 0.009140055797373255, "kd/rkl_advantage_mean": 0.63308949538817, "kd/rkl_advantage_p95": 4.786375486850739, "kd/rkl_advantage_std": 2.1824997772773105, "kd/ssd_loss": 0.0, "learning_rate": 7.71047734781453e-07, "loss": 0.036560225486755374, "num_tokens": 197274390.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7736111104488372, "rewards/gt_logging_reward/std": 0.4075977742671967, "sampling/importance_sampling_ratio/max": 1.9368695259094237, "sampling/importance_sampling_ratio/mean": 0.9889886975288391, "sampling/importance_sampling_ratio/min": 0.3951011781891187, "sampling/sampling_logp_difference/max": 0.44110994736353554, "sampling/sampling_logp_difference/mean": 0.004443427136478325, "step": 6030, "step_time": 8.315528792398982, "student/entropy": 0.13260588670770326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444596146544, "completions/max_length": 476.2, "completions/max_terminated_length": 454.73333333333335, "completions/mean_length": 189.7800099690755, "completions/mean_terminated_length": 182.65061899820964, "completions/min_length": 59.166666666666664, "completions/min_terminated_length": 59.166666666666664, "entropy": 0.12737384450932343, "epoch": 0.2301294953100672, "eval/gt_acc_batch": 0.7997222383817036, "frac_reward_zero_std": 1.0, "grad_norm": 0.23629285395145416, "kd/policy_loss": 0.008441779058193788, "kd/rkl_advantage_mean": 0.6419032241528233, "kd/rkl_advantage_p95": 4.866001093387604, "kd/rkl_advantage_std": 2.2196168065071107, "kd/ssd_loss": 0.0, "learning_rate": 7.699084798541753e-07, "loss": 0.03376712004343669, "num_tokens": 198259958.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7997222264607747, "rewards/gt_logging_reward/std": 0.3867082769672076, "sampling/importance_sampling_ratio/max": 1.882184370358785, "sampling/importance_sampling_ratio/mean": 1.0009948054949442, "sampling/importance_sampling_ratio/min": 0.40929896434148155, "sampling/sampling_logp_difference/max": 0.43364791870117186, "sampling/sampling_logp_difference/mean": 0.004253823085067173, "step": 6060, "step_time": 7.995410609742006, "student/entropy": 0.12737384450932343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890103747448, "completions/max_length": 494.26666666666665, "completions/max_terminated_length": 441.06666666666666, "completions/mean_length": 191.42084248860678, "completions/mean_terminated_length": 183.6046900431315, "completions/min_length": 48.96666666666667, "completions/min_terminated_length": 48.96666666666667, "entropy": 0.13534168048451345, "epoch": 0.23126875023734478, "eval/gt_acc_batch": 0.7702777981758118, "frac_reward_zero_std": 1.0, "grad_norm": 0.3110235631465912, "kd/policy_loss": 0.009326867169390123, "kd/rkl_advantage_mean": 0.655918958162268, "kd/rkl_advantage_p95": 4.913002063830693, "kd/rkl_advantage_std": 2.2106439491113026, "kd/ssd_loss": 0.0, "learning_rate": 7.687692249268978e-07, "loss": 0.03730746905008952, "num_tokens": 199251641.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7702777743339538, "rewards/gt_logging_reward/std": 0.4103247116009394, "sampling/importance_sampling_ratio/max": 2.0024213910102846, "sampling/importance_sampling_ratio/mean": 0.9991628030935923, "sampling/importance_sampling_ratio/min": 0.3916111042102178, "sampling/sampling_logp_difference/max": 0.40241243243217467, "sampling/sampling_logp_difference/mean": 0.0044058912356073655, "step": 6090, "step_time": 8.227268930318921, "student/entropy": 0.13534168048451345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01611111213763555, "completions/max_length": 491.3666666666667, "completions/max_terminated_length": 452.76666666666665, "completions/mean_length": 183.8480656941732, "completions/mean_terminated_length": 178.61597951253256, "completions/min_length": 55.46666666666667, "completions/min_terminated_length": 55.46666666666667, "entropy": 0.13214289546012878, "epoch": 0.23240800516462234, "eval/gt_acc_batch": 0.809166685740153, "frac_reward_zero_std": 1.0, "grad_norm": 0.2709805369377136, "kd/policy_loss": 0.008674674958456308, "kd/rkl_advantage_mean": 0.6511059725036223, "kd/rkl_advantage_p95": 4.827355154355367, "kd/rkl_advantage_std": 2.1967110057671864, "kd/ssd_loss": 0.0, "learning_rate": 7.676299699996203e-07, "loss": 0.03469870090484619, "num_tokens": 200209886.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8091666658719381, "rewards/gt_logging_reward/std": 0.38665656248728436, "sampling/importance_sampling_ratio/max": 1.8723712046941121, "sampling/importance_sampling_ratio/mean": 0.9903748591740926, "sampling/importance_sampling_ratio/min": 0.42082366247971853, "sampling/sampling_logp_difference/max": 0.3971408446629842, "sampling/sampling_logp_difference/mean": 0.00438405170571059, "step": 6120, "step_time": 8.095425741401657, "student/entropy": 0.13214289546012878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001111378273, "completions/max_length": 488.3333333333333, "completions/max_terminated_length": 457.26666666666665, "completions/mean_length": 191.24473215738934, "completions/mean_terminated_length": 184.0130599975586, "completions/min_length": 56.53333333333333, "completions/min_terminated_length": 56.53333333333333, "entropy": 0.12994815253963074, "epoch": 0.2335472600918999, "eval/gt_acc_batch": 0.795000014702479, "frac_reward_zero_std": 1.0, "grad_norm": 0.34392327070236206, "kd/policy_loss": 0.009036428155377507, "kd/rkl_advantage_mean": 0.610159932806467, "kd/rkl_advantage_p95": 4.792581123113632, "kd/rkl_advantage_std": 2.176404374837875, "kd/ssd_loss": 0.0, "learning_rate": 7.664907150723426e-07, "loss": 0.036145710945129396, "num_tokens": 201197231.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7950000007947285, "rewards/gt_logging_reward/std": 0.3875947813193003, "sampling/importance_sampling_ratio/max": 1.9073782364527385, "sampling/importance_sampling_ratio/mean": 1.0088681022326151, "sampling/importance_sampling_ratio/min": 0.45198328197002413, "sampling/sampling_logp_difference/max": 0.41112184325853984, "sampling/sampling_logp_difference/mean": 0.004254262164855997, "step": 6150, "step_time": 8.101859429955947, "student/entropy": 0.12994815253963074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028611112479120494, "completions/max_length": 488.43333333333334, "completions/max_terminated_length": 433.2, "completions/mean_length": 186.45889841715496, "completions/mean_terminated_length": 177.04283040364584, "completions/min_length": 56.266666666666666, "completions/min_terminated_length": 56.266666666666666, "entropy": 0.12880239176253477, "epoch": 0.23468651501917745, "eval/gt_acc_batch": 0.7927778005599976, "frac_reward_zero_std": 1.0, "grad_norm": 0.32142725586891174, "kd/policy_loss": 0.008768509357469156, "kd/rkl_advantage_mean": 0.688782718560348, "kd/rkl_advantage_p95": 4.932652097940445, "kd/rkl_advantage_std": 2.247202789783478, "kd/ssd_loss": 0.0, "learning_rate": 7.653514601450651e-07, "loss": 0.03507403930028279, "num_tokens": 202176435.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7927777767181396, "rewards/gt_logging_reward/std": 0.40062175691127777, "sampling/importance_sampling_ratio/max": 1.8828734040260315, "sampling/importance_sampling_ratio/mean": 0.9986003637313843, "sampling/importance_sampling_ratio/min": 0.4192064791917801, "sampling/sampling_logp_difference/max": 0.4846411069234212, "sampling/sampling_logp_difference/mean": 0.0042101180413737895, "step": 6180, "step_time": 8.13547533043505, "student/entropy": 0.12880239176253477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01472222302109003, "completions/max_length": 463.3, "completions/max_terminated_length": 441.43333333333334, "completions/mean_length": 175.86139780680338, "completions/mean_terminated_length": 171.03020884195965, "completions/min_length": 59.2, "completions/min_terminated_length": 59.2, "entropy": 0.1299978608265519, "epoch": 0.23582576994645502, "eval/gt_acc_batch": 0.8186111251513163, "frac_reward_zero_std": 1.0, "grad_norm": 0.3235873878002167, "kd/policy_loss": 0.008322320242101947, "kd/rkl_advantage_mean": 0.6483541022947369, "kd/rkl_advantage_p95": 4.797852172454198, "kd/rkl_advantage_std": 2.194501131772995, "kd/ssd_loss": 0.0, "learning_rate": 7.642122052177875e-07, "loss": 0.03328927954037984, "num_tokens": 203106048.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8186111132303874, "rewards/gt_logging_reward/std": 0.37935285915931066, "sampling/importance_sampling_ratio/max": 1.8870131333669027, "sampling/importance_sampling_ratio/mean": 0.9963464637597402, "sampling/importance_sampling_ratio/min": 0.41821932395299277, "sampling/sampling_logp_difference/max": 0.42629339297612506, "sampling/sampling_logp_difference/mean": 0.004366864954742293, "step": 6210, "step_time": 7.833092168206349, "student/entropy": 0.1299978608265519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445619980494, "completions/max_length": 491.8333333333333, "completions/max_terminated_length": 453.3, "completions/mean_length": 186.61028747558595, "completions/mean_terminated_length": 179.38998006184895, "completions/min_length": 54.53333333333333, "completions/min_terminated_length": 54.53333333333333, "entropy": 0.12331887781620025, "epoch": 0.23696502487373258, "eval/gt_acc_batch": 0.8130555768807729, "frac_reward_zero_std": 1.0, "grad_norm": 0.27219873666763306, "kd/policy_loss": 0.008462951225616659, "kd/rkl_advantage_mean": 0.7024667259926597, "kd/rkl_advantage_p95": 4.9172854125499725, "kd/rkl_advantage_std": 2.2145619869232176, "kd/ssd_loss": 0.0, "learning_rate": 7.6307295029051e-07, "loss": 0.03385180632273356, "num_tokens": 204082253.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8130555589993794, "rewards/gt_logging_reward/std": 0.38183150192101795, "sampling/importance_sampling_ratio/max": 1.8632219115893045, "sampling/importance_sampling_ratio/mean": 0.9992580831050872, "sampling/importance_sampling_ratio/min": 0.44758213063081104, "sampling/sampling_logp_difference/max": 0.4689243157704671, "sampling/sampling_logp_difference/mean": 0.004147490758138399, "step": 6240, "step_time": 8.21339652123473, "student/entropy": 0.12331887781620025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016111112075547378, "completions/max_length": 476.0, "completions/max_terminated_length": 441.46666666666664, "completions/mean_length": 185.91917775472004, "completions/mean_terminated_length": 180.76614634195963, "completions/min_length": 52.46666666666667, "completions/min_terminated_length": 52.46666666666667, "entropy": 0.13043399980912607, "epoch": 0.23810427980101015, "eval/gt_acc_batch": 0.8119444807370504, "frac_reward_zero_std": 1.0, "grad_norm": 0.23726686835289001, "kd/policy_loss": 0.008217162999790162, "kd/rkl_advantage_mean": 0.5797906673513353, "kd/rkl_advantage_p95": 4.708313755194346, "kd/rkl_advantage_std": 2.1769098550081254, "kd/ssd_loss": 0.0, "learning_rate": 7.619336953632324e-07, "loss": 0.03286864558855693, "num_tokens": 205063010.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8119444449742635, "rewards/gt_logging_reward/std": 0.38343019286791485, "sampling/importance_sampling_ratio/max": 1.845543372631073, "sampling/importance_sampling_ratio/mean": 0.9952125469843547, "sampling/importance_sampling_ratio/min": 0.4258480389912923, "sampling/sampling_logp_difference/max": 0.4127208431561788, "sampling/sampling_logp_difference/mean": 0.004199273030584057, "step": 6270, "step_time": 8.093902048631572, "student/entropy": 0.13043399980912607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030555556993931532, "completions/max_length": 475.6, "completions/max_terminated_length": 446.9, "completions/mean_length": 190.7813975016276, "completions/mean_terminated_length": 180.85533294677734, "completions/min_length": 49.9, "completions/min_terminated_length": 49.9, "entropy": 0.12496696536739667, "epoch": 0.2392435347282877, "eval/gt_acc_batch": 0.8097222407658895, "frac_reward_zero_std": 1.0, "grad_norm": 0.24982069432735443, "kd/policy_loss": 0.008748501294758172, "kd/rkl_advantage_mean": 0.6741078318251917, "kd/rkl_advantage_p95": 4.787641861041387, "kd/rkl_advantage_std": 2.17306227684021, "kd/ssd_loss": 0.0, "learning_rate": 7.607944404359549e-07, "loss": 0.03499400615692139, "num_tokens": 206057623.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.809722222884496, "rewards/gt_logging_reward/std": 0.3808087254563967, "sampling/importance_sampling_ratio/max": 2.011062236626943, "sampling/importance_sampling_ratio/mean": 1.0031454384326934, "sampling/importance_sampling_ratio/min": 0.4314197281996409, "sampling/sampling_logp_difference/max": 0.4187386751174927, "sampling/sampling_logp_difference/mean": 0.004139232930416862, "step": 6300, "step_time": 8.099489710200578, "student/entropy": 0.12496696536739667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779381722212, "completions/max_length": 498.56666666666666, "completions/max_terminated_length": 463.6666666666667, "completions/mean_length": 192.26584218343098, "completions/mean_terminated_length": 183.15040486653646, "completions/min_length": 48.96666666666667, "completions/min_terminated_length": 48.96666666666667, "entropy": 0.12913793232291937, "epoch": 0.24038278965556525, "eval/gt_acc_batch": 0.8002778013547261, "frac_reward_zero_std": 1.0, "grad_norm": 0.3103610575199127, "kd/policy_loss": 0.008718625331918398, "kd/rkl_advantage_mean": 0.7567344908912976, "kd/rkl_advantage_p95": 5.039508686463038, "kd/rkl_advantage_std": 2.2230162034432093, "kd/ssd_loss": 0.0, "learning_rate": 7.596551855086772e-07, "loss": 0.03487450281778971, "num_tokens": 207049772.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8002777755260467, "rewards/gt_logging_reward/std": 0.3912219415108363, "sampling/importance_sampling_ratio/max": 1.9777502139409384, "sampling/importance_sampling_ratio/mean": 0.995680965979894, "sampling/importance_sampling_ratio/min": 0.4151444842418035, "sampling/sampling_logp_difference/max": 0.4296450783809026, "sampling/sampling_logp_difference/mean": 0.004319851961918175, "step": 6330, "step_time": 8.138093703538955, "student/entropy": 0.12913793232291937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556720743575, "completions/max_length": 485.9, "completions/max_terminated_length": 451.6666666666667, "completions/mean_length": 186.33167572021483, "completions/mean_terminated_length": 179.57020416259766, "completions/min_length": 52.733333333333334, "completions/min_terminated_length": 52.733333333333334, "entropy": 0.1327652420848608, "epoch": 0.24152204458284282, "eval/gt_acc_batch": 0.783611132701238, "frac_reward_zero_std": 1.0, "grad_norm": 0.28265801072120667, "kd/policy_loss": 0.008860272904469942, "kd/rkl_advantage_mean": 0.6699662565719336, "kd/rkl_advantage_p95": 4.84623194138209, "kd/rkl_advantage_std": 2.212257832288742, "kd/ssd_loss": 0.0, "learning_rate": 7.585159305813997e-07, "loss": 0.035441092650095624, "num_tokens": 208023062.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7836111128330231, "rewards/gt_logging_reward/std": 0.4034515793124835, "sampling/importance_sampling_ratio/max": 1.9859682321548462, "sampling/importance_sampling_ratio/mean": 1.001838900645574, "sampling/importance_sampling_ratio/min": 0.40199041962623594, "sampling/sampling_logp_difference/max": 0.42122057477633157, "sampling/sampling_logp_difference/mean": 0.004427524105024835, "step": 6360, "step_time": 8.023761424177792, "student/entropy": 0.1327652420848608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556640028954, "completions/max_length": 480.1, "completions/max_terminated_length": 452.03333333333336, "completions/mean_length": 188.52806549072267, "completions/mean_terminated_length": 180.02882486979166, "completions/min_length": 62.733333333333334, "completions/min_terminated_length": 62.733333333333334, "entropy": 0.13002879321575164, "epoch": 0.2426612995101204, "eval/gt_acc_batch": 0.7816666881243388, "frac_reward_zero_std": 1.0, "grad_norm": 0.2970639765262604, "kd/policy_loss": 0.009217195356419931, "kd/rkl_advantage_mean": 0.7006503107647101, "kd/rkl_advantage_p95": 4.91127490401268, "kd/rkl_advantage_std": 2.235979250073433, "kd/ssd_loss": 0.0, "learning_rate": 7.573766756541222e-07, "loss": 0.036868778864542644, "num_tokens": 208999827.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7816666662693024, "rewards/gt_logging_reward/std": 0.40385886033376056, "sampling/importance_sampling_ratio/max": 2.0565109650293985, "sampling/importance_sampling_ratio/mean": 1.0089419225851695, "sampling/importance_sampling_ratio/min": 0.43466459810733793, "sampling/sampling_logp_difference/max": 0.4010078251361847, "sampling/sampling_logp_difference/mean": 0.004328290419653058, "step": 6390, "step_time": 7.967229126777966, "student/entropy": 0.13002879321575164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944445818662642, "completions/max_length": 491.56666666666666, "completions/max_terminated_length": 458.8, "completions/mean_length": 185.486675008138, "completions/mean_terminated_length": 176.42356872558594, "completions/min_length": 49.166666666666664, "completions/min_terminated_length": 49.166666666666664, "entropy": 0.13192854622999828, "epoch": 0.24380055443739795, "eval/gt_acc_batch": 0.7997222443421682, "frac_reward_zero_std": 1.0, "grad_norm": 0.24751074612140656, "kd/policy_loss": 0.008718036629337197, "kd/rkl_advantage_mean": 0.6847877079310517, "kd/rkl_advantage_p95": 4.862166025241216, "kd/rkl_advantage_std": 2.2022369503974915, "kd/ssd_loss": 0.0, "learning_rate": 7.562374207268446e-07, "loss": 0.034872158368428545, "num_tokens": 209971371.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7997222205003103, "rewards/gt_logging_reward/std": 0.3935115804274877, "sampling/importance_sampling_ratio/max": 1.9359491189320883, "sampling/importance_sampling_ratio/mean": 0.9943469723065694, "sampling/importance_sampling_ratio/min": 0.40121693710486095, "sampling/sampling_logp_difference/max": 0.4213083803653717, "sampling/sampling_logp_difference/mean": 0.004418986073384683, "step": 6420, "step_time": 8.150758177247674, "student/entropy": 0.13192854622999828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112336317697, "completions/max_length": 473.73333333333335, "completions/max_terminated_length": 448.4, "completions/mean_length": 183.6369524637858, "completions/mean_terminated_length": 176.74135818481446, "completions/min_length": 53.46666666666667, "completions/min_terminated_length": 53.46666666666667, "entropy": 0.1297277480363846, "epoch": 0.2449398093646755, "eval/gt_acc_batch": 0.8111111263434092, "frac_reward_zero_std": 1.0, "grad_norm": 0.3232152760028839, "kd/policy_loss": 0.008284555396918828, "kd/rkl_advantage_mean": 0.5611398845522975, "kd/rkl_advantage_p95": 4.650587461392084, "kd/rkl_advantage_std": 2.177451287706693, "kd/ssd_loss": 0.0, "learning_rate": 7.550981657995671e-07, "loss": 0.03313822348912557, "num_tokens": 210930864.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8111111164093018, "rewards/gt_logging_reward/std": 0.3862161229054133, "sampling/importance_sampling_ratio/max": 1.8320292313893636, "sampling/importance_sampling_ratio/mean": 0.9973168015480042, "sampling/importance_sampling_ratio/min": 0.4025011857350667, "sampling/sampling_logp_difference/max": 0.39460267424583434, "sampling/sampling_logp_difference/mean": 0.004344195515538256, "step": 6450, "step_time": 7.838175813570463, "student/entropy": 0.1297277480363846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02416666814436515, "completions/max_length": 490.73333333333335, "completions/max_terminated_length": 456.76666666666665, "completions/mean_length": 188.2527877807617, "completions/mean_terminated_length": 180.5466049194336, "completions/min_length": 55.1, "completions/min_terminated_length": 55.1, "entropy": 0.13442746407041947, "epoch": 0.24607906429195306, "eval/gt_acc_batch": 0.7913889050483703, "frac_reward_zero_std": 1.0, "grad_norm": 0.24111120402812958, "kd/policy_loss": 0.009167794570870077, "kd/rkl_advantage_mean": 0.7579849595824878, "kd/rkl_advantage_p95": 5.011103264490763, "kd/rkl_advantage_std": 2.2465117196242015, "kd/ssd_loss": 0.0, "learning_rate": 7.539589108722895e-07, "loss": 0.036671189467112224, "num_tokens": 211910646.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7913888812065124, "rewards/gt_logging_reward/std": 0.3950616712371508, "sampling/importance_sampling_ratio/max": 1.85566113392512, "sampling/importance_sampling_ratio/mean": 0.996570869286855, "sampling/importance_sampling_ratio/min": 0.4182213837901751, "sampling/sampling_logp_difference/max": 0.47024984161059064, "sampling/sampling_logp_difference/mean": 0.004407738917507231, "step": 6480, "step_time": 8.058678201582127, "student/entropy": 0.13442746407041947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030000001409401497, "completions/max_length": 479.1333333333333, "completions/max_terminated_length": 446.46666666666664, "completions/mean_length": 186.81112009684244, "completions/mean_terminated_length": 177.01209309895833, "completions/min_length": 56.666666666666664, "completions/min_terminated_length": 56.666666666666664, "entropy": 0.1317155930524071, "epoch": 0.24721831921923063, "eval/gt_acc_batch": 0.7902777989705404, "frac_reward_zero_std": 1.0, "grad_norm": 0.25881513953208923, "kd/policy_loss": 0.009148378877822931, "kd/rkl_advantage_mean": 0.6807209809931616, "kd/rkl_advantage_p95": 4.806604232390722, "kd/rkl_advantage_std": 2.1716851701339084, "kd/ssd_loss": 0.0, "learning_rate": 7.52819655945012e-07, "loss": 0.03659351666768392, "num_tokens": 212880982.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7902777830759684, "rewards/gt_logging_reward/std": 0.3958704407016436, "sampling/importance_sampling_ratio/max": 1.938192371527354, "sampling/importance_sampling_ratio/mean": 0.9952379405498505, "sampling/importance_sampling_ratio/min": 0.4255646526813507, "sampling/sampling_logp_difference/max": 0.4084997693697611, "sampling/sampling_logp_difference/mean": 0.004432754870504141, "step": 6510, "step_time": 7.9764029064021695, "student/entropy": 0.1317155930524071 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334581305584, "completions/max_length": 481.46666666666664, "completions/max_terminated_length": 444.8, "completions/mean_length": 190.84639841715494, "completions/mean_terminated_length": 183.44420420328777, "completions/min_length": 58.233333333333334, "completions/min_terminated_length": 58.233333333333334, "entropy": 0.1307209754983584, "epoch": 0.2483575741465082, "eval/gt_acc_batch": 0.7791666984558105, "frac_reward_zero_std": 1.0, "grad_norm": 0.2319793403148651, "kd/policy_loss": 0.009327594250983869, "kd/rkl_advantage_mean": 0.6250596908542018, "kd/rkl_advantage_p95": 4.798074014981588, "kd/rkl_advantage_std": 2.1958048224449156, "kd/ssd_loss": 0.0, "learning_rate": 7.516804010177343e-07, "loss": 0.03731037775675456, "num_tokens": 213879413.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7791666686534882, "rewards/gt_logging_reward/std": 0.4056950479745865, "sampling/importance_sampling_ratio/max": 1.9356088439623516, "sampling/importance_sampling_ratio/mean": 1.0063813308874765, "sampling/importance_sampling_ratio/min": 0.44139294723669686, "sampling/sampling_logp_difference/max": 0.40160841941833497, "sampling/sampling_logp_difference/mean": 0.0042986598253871005, "step": 6540, "step_time": 8.149581079778727, "student/entropy": 0.1307209754983584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556751787663, "completions/max_length": 468.3, "completions/max_terminated_length": 435.1, "completions/mean_length": 185.22778778076173, "completions/mean_terminated_length": 178.52667592366535, "completions/min_length": 55.06666666666667, "completions/min_terminated_length": 55.06666666666667, "entropy": 0.13345299810171127, "epoch": 0.24949682907378573, "eval/gt_acc_batch": 0.7919444600741069, "frac_reward_zero_std": 1.0, "grad_norm": 0.2400812804698944, "kd/policy_loss": 0.009333005623193457, "kd/rkl_advantage_mean": 0.5901699352854242, "kd/rkl_advantage_p95": 4.581990871826807, "kd/rkl_advantage_std": 2.115486776828766, "kd/ssd_loss": 0.0, "learning_rate": 7.505411460904568e-07, "loss": 0.037332022190093996, "num_tokens": 214848073.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7919444461663564, "rewards/gt_logging_reward/std": 0.3982202430566152, "sampling/importance_sampling_ratio/max": 2.0125489314397176, "sampling/importance_sampling_ratio/mean": 1.0062717696030934, "sampling/importance_sampling_ratio/min": 0.4366688181956609, "sampling/sampling_logp_difference/max": 0.39994831879933673, "sampling/sampling_logp_difference/mean": 0.004463640429700415, "step": 6570, "step_time": 7.946685146885769, "student/entropy": 0.13345299810171127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556919425727, "completions/max_length": 473.2, "completions/max_terminated_length": 451.06666666666666, "completions/mean_length": 193.67862243652343, "completions/mean_terminated_length": 185.421001180013, "completions/min_length": 59.1, "completions/min_terminated_length": 59.1, "entropy": 0.12893886485447487, "epoch": 0.2506360840010633, "eval/gt_acc_batch": 0.7733333647251129, "frac_reward_zero_std": 1.0, "grad_norm": 0.2646295130252838, "kd/policy_loss": 0.008955428580520675, "kd/rkl_advantage_mean": 0.6700256830023136, "kd/rkl_advantage_p95": 4.750998731454214, "kd/rkl_advantage_std": 2.14154637157917, "kd/ssd_loss": 0.0, "learning_rate": 7.494018911631792e-07, "loss": 0.03582171599070231, "num_tokens": 215849692.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7733333329359691, "rewards/gt_logging_reward/std": 0.4087906301021576, "sampling/importance_sampling_ratio/max": 1.9598654786745706, "sampling/importance_sampling_ratio/mean": 0.9995945533116658, "sampling/importance_sampling_ratio/min": 0.42593573182821276, "sampling/sampling_logp_difference/max": 0.4021790583928426, "sampling/sampling_logp_difference/mean": 0.004294938625146945, "step": 6600, "step_time": 8.025439435092267, "student/entropy": 0.12893886485447487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667920847735, "completions/max_length": 484.06666666666666, "completions/max_terminated_length": 441.3666666666667, "completions/mean_length": 184.8466766357422, "completions/mean_terminated_length": 177.56533915201823, "completions/min_length": 53.266666666666666, "completions/min_terminated_length": 53.266666666666666, "entropy": 0.12881126534193754, "epoch": 0.25177533892834086, "eval/gt_acc_batch": 0.7897222340106964, "frac_reward_zero_std": 1.0, "grad_norm": 0.3228004276752472, "kd/policy_loss": 0.0086019679515933, "kd/rkl_advantage_mean": 0.6352578810949733, "kd/rkl_advantage_p95": 4.7387493332227075, "kd/rkl_advantage_std": 2.17497730354468, "kd/ssd_loss": 0.0, "learning_rate": 7.482626362359017e-07, "loss": 0.03440786997477214, "num_tokens": 216806052.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7897222220897675, "rewards/gt_logging_reward/std": 0.40149479707082114, "sampling/importance_sampling_ratio/max": 1.9305628577868144, "sampling/importance_sampling_ratio/mean": 0.9963873545328776, "sampling/importance_sampling_ratio/min": 0.4073688745498657, "sampling/sampling_logp_difference/max": 0.4689078430334727, "sampling/sampling_logp_difference/mean": 0.004301401708895962, "step": 6630, "step_time": 7.864458196129029, "student/entropy": 0.12881126534193754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014166667684912681, "completions/max_length": 478.0, "completions/max_terminated_length": 455.6666666666667, "completions/mean_length": 182.60973205566407, "completions/mean_terminated_length": 177.8578130086263, "completions/min_length": 52.06666666666667, "completions/min_terminated_length": 52.06666666666667, "entropy": 0.12573809946576756, "epoch": 0.25291459385561843, "eval/gt_acc_batch": 0.8016666869322459, "frac_reward_zero_std": 1.0, "grad_norm": 0.2257617563009262, "kd/policy_loss": 0.008368931417741502, "kd/rkl_advantage_mean": 0.7087296484038234, "kd/rkl_advantage_p95": 4.909722928206126, "kd/rkl_advantage_std": 2.220593562722206, "kd/ssd_loss": 0.0, "learning_rate": 7.471233813086242e-07, "loss": 0.03347572485605876, "num_tokens": 217762599.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.801666667064031, "rewards/gt_logging_reward/std": 0.38974318106969197, "sampling/importance_sampling_ratio/max": 1.8878875891367595, "sampling/importance_sampling_ratio/mean": 0.998793379465739, "sampling/importance_sampling_ratio/min": 0.4301188935836156, "sampling/sampling_logp_difference/max": 0.41356134017308555, "sampling/sampling_logp_difference/mean": 0.004266316164284944, "step": 6660, "step_time": 7.945062089591132, "student/entropy": 0.12573809946576756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0188888899050653, "completions/max_length": 481.2, "completions/max_terminated_length": 450.7, "completions/mean_length": 181.994731648763, "completions/mean_terminated_length": 175.68707733154298, "completions/min_length": 60.43333333333333, "completions/min_terminated_length": 60.43333333333333, "entropy": 0.12240205487857263, "epoch": 0.254053848782896, "eval/gt_acc_batch": 0.820555571715037, "frac_reward_zero_std": 1.0, "grad_norm": 0.2635529339313507, "kd/policy_loss": 0.00791559810944212, "kd/rkl_advantage_mean": 0.737532316086193, "kd/rkl_advantage_p95": 4.912347207466762, "kd/rkl_advantage_std": 2.195744394262632, "kd/ssd_loss": 0.0, "learning_rate": 7.459841263813466e-07, "loss": 0.03166239658991496, "num_tokens": 218715980.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8205555578072866, "rewards/gt_logging_reward/std": 0.3758873715996742, "sampling/importance_sampling_ratio/max": 1.9192961613337198, "sampling/importance_sampling_ratio/mean": 1.0019152363141377, "sampling/importance_sampling_ratio/min": 0.439955398440361, "sampling/sampling_logp_difference/max": 0.41455527742703757, "sampling/sampling_logp_difference/mean": 0.004177936585620046, "step": 6690, "step_time": 7.9371159586861415, "student/entropy": 0.12240205487857263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667827715477, "completions/max_length": 486.6666666666667, "completions/max_terminated_length": 449.73333333333335, "completions/mean_length": 189.77334340413412, "completions/mean_terminated_length": 182.6819859822591, "completions/min_length": 61.166666666666664, "completions/min_terminated_length": 61.166666666666664, "entropy": 0.11928754728287458, "epoch": 0.25519310371017356, "eval/gt_acc_batch": 0.8069444537162781, "frac_reward_zero_std": 1.0, "grad_norm": 0.34236061573028564, "kd/policy_loss": 0.008116432700383787, "kd/rkl_advantage_mean": 0.661946140229702, "kd/rkl_advantage_p95": 4.835807512203853, "kd/rkl_advantage_std": 2.178380723794301, "kd/ssd_loss": 0.0, "learning_rate": 7.44844871454069e-07, "loss": 0.03246572812398275, "num_tokens": 219699204.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8069444457689922, "rewards/gt_logging_reward/std": 0.3882801706592242, "sampling/importance_sampling_ratio/max": 1.8033640702565512, "sampling/importance_sampling_ratio/mean": 1.0015222907066346, "sampling/importance_sampling_ratio/min": 0.4643369361758232, "sampling/sampling_logp_difference/max": 0.3749047636985779, "sampling/sampling_logp_difference/mean": 0.004006231296807527, "step": 6720, "step_time": 8.023343049207082, "student/entropy": 0.11928754728287458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02722222382823626, "completions/max_length": 484.3333333333333, "completions/max_terminated_length": 452.8, "completions/mean_length": 192.28112335205077, "completions/mean_terminated_length": 183.23143056233724, "completions/min_length": 62.06666666666667, "completions/min_terminated_length": 62.06666666666667, "entropy": 0.1325053472071886, "epoch": 0.25633235863745113, "eval/gt_acc_batch": 0.7963889102141063, "frac_reward_zero_std": 1.0, "grad_norm": 0.2888845205307007, "kd/policy_loss": 0.009105391229968517, "kd/rkl_advantage_mean": 0.6889169408008456, "kd/rkl_advantage_p95": 4.872095646460851, "kd/rkl_advantage_std": 2.1957727124293647, "kd/ssd_loss": 0.0, "learning_rate": 7.437056165267914e-07, "loss": 0.03642156521479289, "num_tokens": 220694480.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888903458913, "rewards/gt_logging_reward/std": 0.3958563576141993, "sampling/importance_sampling_ratio/max": 1.9580243508021036, "sampling/importance_sampling_ratio/mean": 1.0023346523443857, "sampling/importance_sampling_ratio/min": 0.4149521400531133, "sampling/sampling_logp_difference/max": 0.41691152652104696, "sampling/sampling_logp_difference/mean": 0.004414762471181651, "step": 6750, "step_time": 7.930047904727204, "student/entropy": 0.1325053472071886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889060666164, "completions/max_length": 499.3666666666667, "completions/max_terminated_length": 469.43333333333334, "completions/mean_length": 191.4836217244466, "completions/mean_terminated_length": 182.9739242553711, "completions/min_length": 59.13333333333333, "completions/min_terminated_length": 59.13333333333333, "entropy": 0.12661677629997334, "epoch": 0.25747161356472864, "eval/gt_acc_batch": 0.783055579662323, "frac_reward_zero_std": 1.0, "grad_norm": 0.2637076675891876, "kd/policy_loss": 0.009063981263898313, "kd/rkl_advantage_mean": 0.6177466770478834, "kd/rkl_advantage_p95": 4.769889424244563, "kd/rkl_advantage_std": 2.1949843148390453, "kd/ssd_loss": 0.0, "learning_rate": 7.425663615995139e-07, "loss": 0.03625592788060506, "num_tokens": 221680421.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7830555498600006, "rewards/gt_logging_reward/std": 0.40643223822116853, "sampling/importance_sampling_ratio/max": 2.007291229565938, "sampling/importance_sampling_ratio/mean": 0.9946927607059479, "sampling/importance_sampling_ratio/min": 0.43872424215078354, "sampling/sampling_logp_difference/max": 0.423018886645635, "sampling/sampling_logp_difference/mean": 0.00422237473540008, "step": 6780, "step_time": 8.102001418848522, "student/entropy": 0.12661677629997334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030555557552725078, "completions/max_length": 492.76666666666665, "completions/max_terminated_length": 457.03333333333336, "completions/mean_length": 195.664453125, "completions/mean_terminated_length": 185.8007044474284, "completions/min_length": 53.2, "completions/min_terminated_length": 53.2, "entropy": 0.13151656972865264, "epoch": 0.2586108684920062, "eval/gt_acc_batch": 0.7702777942021688, "frac_reward_zero_std": 1.0, "grad_norm": 0.2575989067554474, "kd/policy_loss": 0.009302911534905433, "kd/rkl_advantage_mean": 0.6222567290067673, "kd/rkl_advantage_p95": 4.763335400819779, "kd/rkl_advantage_std": 2.174221494793892, "kd/ssd_loss": 0.0, "learning_rate": 7.414271066722363e-07, "loss": 0.037211652596791586, "num_tokens": 222701549.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7702777763207753, "rewards/gt_logging_reward/std": 0.41113004287083943, "sampling/importance_sampling_ratio/max": 1.975506556034088, "sampling/importance_sampling_ratio/mean": 0.998224671681722, "sampling/importance_sampling_ratio/min": 0.4338963061571121, "sampling/sampling_logp_difference/max": 0.39469812711079916, "sampling/sampling_logp_difference/mean": 0.0043824572348967195, "step": 6810, "step_time": 8.274163667031099, "student/entropy": 0.13151656972865264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02722222385928035, "completions/max_length": 493.3333333333333, "completions/max_terminated_length": 457.6, "completions/mean_length": 192.0694549560547, "completions/mean_terminated_length": 183.44447224934896, "completions/min_length": 54.4, "completions/min_terminated_length": 54.4, "entropy": 0.12972833973666031, "epoch": 0.2597501234192838, "eval/gt_acc_batch": 0.7844444692134858, "frac_reward_zero_std": 1.0, "grad_norm": 0.31347623467445374, "kd/policy_loss": 0.008824968349654228, "kd/rkl_advantage_mean": 0.7010617584921419, "kd/rkl_advantage_p95": 4.939160501956939, "kd/rkl_advantage_std": 2.2384482741355898, "kd/ssd_loss": 0.0, "learning_rate": 7.402878517449588e-07, "loss": 0.03529987335205078, "num_tokens": 223695503.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7844444453716278, "rewards/gt_logging_reward/std": 0.40485833187898, "sampling/importance_sampling_ratio/max": 1.9926565011342368, "sampling/importance_sampling_ratio/mean": 1.0009188214937845, "sampling/importance_sampling_ratio/min": 0.4117788036664327, "sampling/sampling_logp_difference/max": 0.40339021682739257, "sampling/sampling_logp_difference/mean": 0.004281430559543272, "step": 6840, "step_time": 8.092743775981944, "student/entropy": 0.12972833973666031 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779220292965, "completions/max_length": 480.1333333333333, "completions/max_terminated_length": 436.73333333333335, "completions/mean_length": 188.49195353190103, "completions/mean_terminated_length": 180.44965311686198, "completions/min_length": 58.8, "completions/min_terminated_length": 58.8, "entropy": 0.12720373688886563, "epoch": 0.26088937834656134, "eval/gt_acc_batch": 0.7763888994852702, "frac_reward_zero_std": 1.0, "grad_norm": 0.27323204278945923, "kd/policy_loss": 0.008373104158090427, "kd/rkl_advantage_mean": 0.7390439737277726, "kd/rkl_advantage_p95": 4.948255254824956, "kd/rkl_advantage_std": 2.2000600516796114, "kd/ssd_loss": 0.0, "learning_rate": 7.391485968176812e-07, "loss": 0.03349241813023885, "num_tokens": 224672466.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7763888875643412, "rewards/gt_logging_reward/std": 0.4053095986445745, "sampling/importance_sampling_ratio/max": 1.832367436091105, "sampling/importance_sampling_ratio/mean": 0.9945196191469828, "sampling/importance_sampling_ratio/min": 0.411879796286424, "sampling/sampling_logp_difference/max": 0.39355329672495526, "sampling/sampling_logp_difference/mean": 0.00426455324050039, "step": 6870, "step_time": 8.098365814813102, "student/entropy": 0.12720373688886563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445874541997, "completions/max_length": 488.23333333333335, "completions/max_terminated_length": 443.93333333333334, "completions/mean_length": 195.02278900146484, "completions/mean_terminated_length": 187.10968780517578, "completions/min_length": 55.4, "completions/min_terminated_length": 55.4, "entropy": 0.1266275266185403, "epoch": 0.2620286332738389, "eval/gt_acc_batch": 0.7902777830759684, "frac_reward_zero_std": 1.0, "grad_norm": 0.28177309036254883, "kd/policy_loss": 0.009025950568805759, "kd/rkl_advantage_mean": 0.6612976046899954, "kd/rkl_advantage_p95": 4.786916029453278, "kd/rkl_advantage_std": 2.1466736018657686, "kd/ssd_loss": 0.0, "learning_rate": 7.380093418904036e-07, "loss": 0.03610380490620931, "num_tokens": 225682740.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7902777830759684, "rewards/gt_logging_reward/std": 0.4004335562388102, "sampling/importance_sampling_ratio/max": 2.1212918559710183, "sampling/importance_sampling_ratio/mean": 1.0057276030381521, "sampling/importance_sampling_ratio/min": 0.4316197156906128, "sampling/sampling_logp_difference/max": 0.3992898722489675, "sampling/sampling_logp_difference/mean": 0.004189158207736909, "step": 6900, "step_time": 8.199730098616177, "student/entropy": 0.1266275266185403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556658655404, "completions/max_length": 489.76666666666665, "completions/max_terminated_length": 454.6666666666667, "completions/mean_length": 189.16528727213543, "completions/mean_terminated_length": 182.4216049194336, "completions/min_length": 52.03333333333333, "completions/min_terminated_length": 52.03333333333333, "entropy": 0.12821080069988966, "epoch": 0.2631678882011165, "eval/gt_acc_batch": 0.811388905843099, "frac_reward_zero_std": 1.0, "grad_norm": 0.2238578200340271, "kd/policy_loss": 0.008730334241408854, "kd/rkl_advantage_mean": 0.663798309272776, "kd/rkl_advantage_p95": 4.841734168926875, "kd/rkl_advantage_std": 2.2082900096972784, "kd/ssd_loss": 0.0, "learning_rate": 7.368700869631261e-07, "loss": 0.0349213441212972, "num_tokens": 226673655.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8113888919353485, "rewards/gt_logging_reward/std": 0.387039249142011, "sampling/importance_sampling_ratio/max": 2.1290335416793824, "sampling/importance_sampling_ratio/mean": 0.9974997460842132, "sampling/importance_sampling_ratio/min": 0.43055165658394495, "sampling/sampling_logp_difference/max": 0.40321135719617207, "sampling/sampling_logp_difference/mean": 0.0042367249572028715, "step": 6930, "step_time": 8.189786452997941, "student/entropy": 0.12821080069988966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.031666668690741064, "completions/max_length": 494.56666666666666, "completions/max_terminated_length": 462.1, "completions/mean_length": 192.49667714436848, "completions/mean_terminated_length": 182.14723052978516, "completions/min_length": 49.13333333333333, "completions/min_terminated_length": 49.13333333333333, "entropy": 0.13307219793399175, "epoch": 0.26430714312839404, "eval/gt_acc_batch": 0.7766666909058889, "frac_reward_zero_std": 1.0, "grad_norm": 0.2867799699306488, "kd/policy_loss": 0.009403167595155536, "kd/rkl_advantage_mean": 0.6886448294545213, "kd/rkl_advantage_p95": 4.91659289598465, "kd/rkl_advantage_std": 2.213735107580821, "kd/ssd_loss": 0.0, "learning_rate": 7.357308320358485e-07, "loss": 0.037612672646840414, "num_tokens": 227667539.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.776666663090388, "rewards/gt_logging_reward/std": 0.40818662544091544, "sampling/importance_sampling_ratio/max": 1.926716697216034, "sampling/importance_sampling_ratio/mean": 1.0026012301445006, "sampling/importance_sampling_ratio/min": 0.4004194806019465, "sampling/sampling_logp_difference/max": 0.42244641383488973, "sampling/sampling_logp_difference/mean": 0.004397509875707329, "step": 6960, "step_time": 8.1596174281983, "student/entropy": 0.13307219793399175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02305555685112874, "completions/max_length": 492.43333333333334, "completions/max_terminated_length": 441.8, "completions/mean_length": 188.94306640625, "completions/mean_terminated_length": 181.394411722819, "completions/min_length": 53.833333333333336, "completions/min_terminated_length": 53.833333333333336, "entropy": 0.13091566003859043, "epoch": 0.2654463980556716, "eval/gt_acc_batch": 0.7855555812517802, "frac_reward_zero_std": 1.0, "grad_norm": 0.2603132128715515, "kd/policy_loss": 0.009042247613736738, "kd/rkl_advantage_mean": 0.7262087893749898, "kd/rkl_advantage_p95": 5.0299200514952345, "kd/rkl_advantage_std": 2.2588981946309405, "kd/ssd_loss": 0.0, "learning_rate": 7.34591577108571e-07, "loss": 0.036168996493021646, "num_tokens": 228661230.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555514494578, "rewards/gt_logging_reward/std": 0.40349901616573336, "sampling/importance_sampling_ratio/max": 1.995185093084971, "sampling/importance_sampling_ratio/mean": 1.006988956530889, "sampling/importance_sampling_ratio/min": 0.45723100900650027, "sampling/sampling_logp_difference/max": 0.3838215818007787, "sampling/sampling_logp_difference/mean": 0.00437279900846382, "step": 6990, "step_time": 8.180050293639457, "student/entropy": 0.13091566003859043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556813875834, "completions/max_length": 495.03333333333336, "completions/max_terminated_length": 457.56666666666666, "completions/mean_length": 190.94223225911458, "completions/mean_terminated_length": 184.3092030843099, "completions/min_length": 53.36666666666667, "completions/min_terminated_length": 53.36666666666667, "entropy": 0.1255717379351457, "epoch": 0.2665856529829492, "eval/gt_acc_batch": 0.7977778077125549, "frac_reward_zero_std": 1.0, "grad_norm": 0.20469558238983154, "kd/policy_loss": 0.00831234985186408, "kd/rkl_advantage_mean": 0.7257567259172598, "kd/rkl_advantage_p95": 4.922374834616979, "kd/rkl_advantage_std": 2.196360992391904, "kd/ssd_loss": 0.0, "learning_rate": 7.334523221812934e-07, "loss": 0.03324940005938212, "num_tokens": 229645862.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.797777779897054, "rewards/gt_logging_reward/std": 0.39365256975094476, "sampling/importance_sampling_ratio/max": 1.8745338598887125, "sampling/importance_sampling_ratio/mean": 0.9993794580300649, "sampling/importance_sampling_ratio/min": 0.4187064816554387, "sampling/sampling_logp_difference/max": 0.39551650285720824, "sampling/sampling_logp_difference/mean": 0.004188337169277172, "step": 7020, "step_time": 8.080893895879854, "student/entropy": 0.1255717379351457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556857337555, "completions/max_length": 482.03333333333336, "completions/max_terminated_length": 458.56666666666666, "completions/mean_length": 188.92889760335285, "completions/mean_terminated_length": 180.48760274251302, "completions/min_length": 60.86666666666667, "completions/min_terminated_length": 60.86666666666667, "entropy": 0.1270712294926246, "epoch": 0.26772490791022674, "eval/gt_acc_batch": 0.8113889177640279, "frac_reward_zero_std": 1.0, "grad_norm": 0.2673044800758362, "kd/policy_loss": 0.008358221322608491, "kd/rkl_advantage_mean": 0.7365761488676071, "kd/rkl_advantage_p95": 4.984905964136123, "kd/rkl_advantage_std": 2.2323709189891816, "kd/ssd_loss": 0.0, "learning_rate": 7.323130672540159e-07, "loss": 0.03343288898468018, "num_tokens": 230627654.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8113888839880625, "rewards/gt_logging_reward/std": 0.38493500848611195, "sampling/importance_sampling_ratio/max": 1.9668651938438415, "sampling/importance_sampling_ratio/mean": 0.9977368811766306, "sampling/importance_sampling_ratio/min": 0.4035266732176145, "sampling/sampling_logp_difference/max": 0.40533265670140584, "sampling/sampling_logp_difference/mean": 0.004242446009690563, "step": 7050, "step_time": 8.021556702989619, "student/entropy": 0.1270712294926246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03111111291994651, "completions/max_length": 484.26666666666665, "completions/max_terminated_length": 452.3, "completions/mean_length": 191.30334345499674, "completions/mean_terminated_length": 181.0300615946452, "completions/min_length": 51.166666666666664, "completions/min_terminated_length": 51.166666666666664, "entropy": 0.12857031021267176, "epoch": 0.26886416283750425, "eval/gt_acc_batch": 0.8055555721124014, "frac_reward_zero_std": 1.0, "grad_norm": 0.3902371823787689, "kd/policy_loss": 0.008661734559185182, "kd/rkl_advantage_mean": 0.7123775197193026, "kd/rkl_advantage_p95": 4.882406133413315, "kd/rkl_advantage_std": 2.2138723770777387, "kd/ssd_loss": 0.0, "learning_rate": 7.311738123267383e-07, "loss": 0.034646936257680255, "num_tokens": 231618258.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8055555522441864, "rewards/gt_logging_reward/std": 0.39114587704340614, "sampling/importance_sampling_ratio/max": 1.9993839820226034, "sampling/importance_sampling_ratio/mean": 0.9966413577397665, "sampling/importance_sampling_ratio/min": 0.41114200750986735, "sampling/sampling_logp_difference/max": 0.45211298068364464, "sampling/sampling_logp_difference/mean": 0.0042783253903811175, "step": 7080, "step_time": 8.08023926265693, "student/entropy": 0.12857031021267176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055557049810886, "completions/max_length": 491.06666666666666, "completions/max_terminated_length": 447.26666666666665, "completions/mean_length": 193.54806569417318, "completions/mean_terminated_length": 184.5179463704427, "completions/min_length": 50.56666666666667, "completions/min_terminated_length": 50.56666666666667, "entropy": 0.13106240443885325, "epoch": 0.2700034177647818, "eval/gt_acc_batch": 0.7863889137903849, "frac_reward_zero_std": 1.0, "grad_norm": 0.2919648587703705, "kd/policy_loss": 0.008959093854840224, "kd/rkl_advantage_mean": 0.6709450084716082, "kd/rkl_advantage_p95": 4.825467518965403, "kd/rkl_advantage_std": 2.185019408663114, "kd/ssd_loss": 0.0, "learning_rate": 7.300345573994607e-07, "loss": 0.035836378733317055, "num_tokens": 232627087.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7863888919353486, "rewards/gt_logging_reward/std": 0.4013235032558441, "sampling/importance_sampling_ratio/max": 1.9113041043281556, "sampling/importance_sampling_ratio/mean": 0.9942569812138875, "sampling/importance_sampling_ratio/min": 0.41415459165970486, "sampling/sampling_logp_difference/max": 0.3969536532958349, "sampling/sampling_logp_difference/mean": 0.004340172636633118, "step": 7110, "step_time": 8.28920640903137, "student/entropy": 0.13106240443885325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001452863218, "completions/max_length": 491.1, "completions/max_terminated_length": 464.6666666666667, "completions/mean_length": 186.26639912923176, "completions/mean_terminated_length": 178.84396107991537, "completions/min_length": 53.833333333333336, "completions/min_terminated_length": 53.833333333333336, "entropy": 0.12979006450623273, "epoch": 0.2711426726920594, "eval/gt_acc_batch": 0.785833348830541, "frac_reward_zero_std": 1.0, "grad_norm": 0.35547754168510437, "kd/policy_loss": 0.009000564616871997, "kd/rkl_advantage_mean": 0.6805640195806821, "kd/rkl_advantage_p95": 4.892049342393875, "kd/rkl_advantage_std": 2.230516183376312, "kd/ssd_loss": 0.0, "learning_rate": 7.288953024721831e-07, "loss": 0.03600225448608398, "num_tokens": 233603910.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.785833336909612, "rewards/gt_logging_reward/std": 0.4035194824139277, "sampling/importance_sampling_ratio/max": 1.9517660895983378, "sampling/importance_sampling_ratio/mean": 0.9995789388815562, "sampling/importance_sampling_ratio/min": 0.4216391801834106, "sampling/sampling_logp_difference/max": 0.3953616182009379, "sampling/sampling_logp_difference/mean": 0.004260512984668215, "step": 7140, "step_time": 8.109515184594784, "student/entropy": 0.12979006450623273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001459072035, "completions/max_length": 485.6666666666667, "completions/max_terminated_length": 433.9, "completions/mean_length": 183.98306528727213, "completions/mean_terminated_length": 175.77496999104818, "completions/min_length": 50.53333333333333, "completions/min_terminated_length": 50.53333333333333, "entropy": 0.12913555689156056, "epoch": 0.27228192761933695, "eval/gt_acc_batch": 0.8030555744965872, "frac_reward_zero_std": 1.0, "grad_norm": 0.320027619600296, "kd/policy_loss": 0.008827545577272151, "kd/rkl_advantage_mean": 0.650126864258588, "kd/rkl_advantage_p95": 4.842671497662862, "kd/rkl_advantage_std": 2.216081448396047, "kd/ssd_loss": 0.0, "learning_rate": 7.277560475449056e-07, "loss": 0.03531018098195394, "num_tokens": 234575753.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555566151937, "rewards/gt_logging_reward/std": 0.3885568579037984, "sampling/importance_sampling_ratio/max": 1.9812754273414612, "sampling/importance_sampling_ratio/mean": 0.9992644190788269, "sampling/importance_sampling_ratio/min": 0.4359300305445989, "sampling/sampling_logp_difference/max": 0.374162882566452, "sampling/sampling_logp_difference/mean": 0.0043146274906272685, "step": 7170, "step_time": 8.13958897848691, "student/entropy": 0.12913555689156056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888889842977128, "completions/max_length": 482.7, "completions/max_terminated_length": 451.56666666666666, "completions/mean_length": 184.91528828938803, "completions/mean_terminated_length": 178.75211130777996, "completions/min_length": 53.3, "completions/min_terminated_length": 53.3, "entropy": 0.13244923390448093, "epoch": 0.2734211825466145, "eval/gt_acc_batch": 0.8022222479184469, "frac_reward_zero_std": 1.0, "grad_norm": 0.31756749749183655, "kd/policy_loss": 0.00868869288630473, "kd/rkl_advantage_mean": 0.6775181104894727, "kd/rkl_advantage_p95": 4.850653626521429, "kd/rkl_advantage_std": 2.2005867153406142, "kd/ssd_loss": 0.0, "learning_rate": 7.266167926176281e-07, "loss": 0.03475476900736491, "num_tokens": 235543216.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.802222220102946, "rewards/gt_logging_reward/std": 0.3916413774092992, "sampling/importance_sampling_ratio/max": 1.8890231649080913, "sampling/importance_sampling_ratio/mean": 0.9985008060932159, "sampling/importance_sampling_ratio/min": 0.42834109465281167, "sampling/sampling_logp_difference/max": 0.3872853934764862, "sampling/sampling_logp_difference/mean": 0.004358966369181871, "step": 7200, "step_time": 8.033701635780744, "student/entropy": 0.13244923390448093 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0336111129882435, "completions/max_length": 493.3, "completions/max_terminated_length": 446.46666666666664, "completions/mean_length": 191.88806610107423, "completions/mean_terminated_length": 180.7874959309896, "completions/min_length": 51.86666666666667, "completions/min_terminated_length": 51.86666666666667, "entropy": 0.13195215879629055, "epoch": 0.2745604374738921, "eval/gt_acc_batch": 0.7922222534815471, "frac_reward_zero_std": 1.0, "grad_norm": 0.2795392870903015, "kd/policy_loss": 0.008633831812767312, "kd/rkl_advantage_mean": 0.5973191520587231, "kd/rkl_advantage_p95": 4.7056278149286905, "kd/rkl_advantage_std": 2.1815122971932093, "kd/ssd_loss": 0.0, "learning_rate": 7.254775376903505e-07, "loss": 0.03453532854715983, "num_tokens": 236554029.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222256660462, "rewards/gt_logging_reward/std": 0.3966766794522603, "sampling/importance_sampling_ratio/max": 1.8418158928553263, "sampling/importance_sampling_ratio/mean": 0.9981395463148753, "sampling/importance_sampling_ratio/min": 0.4349077920118968, "sampling/sampling_logp_difference/max": 0.41477184891700747, "sampling/sampling_logp_difference/mean": 0.004349384177476168, "step": 7230, "step_time": 8.328828496458785, "student/entropy": 0.13195215879629055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001260389883, "completions/max_length": 496.03333333333336, "completions/max_terminated_length": 454.93333333333334, "completions/mean_length": 193.3747319539388, "completions/mean_terminated_length": 186.90281473795574, "completions/min_length": 60.93333333333333, "completions/min_terminated_length": 60.93333333333333, "entropy": 0.12653256058692933, "epoch": 0.27569969240116965, "eval/gt_acc_batch": 0.820555579662323, "frac_reward_zero_std": 1.0, "grad_norm": 0.29712221026420593, "kd/policy_loss": 0.00855220912101989, "kd/rkl_advantage_mean": 0.667461157652239, "kd/rkl_advantage_p95": 4.884639322757721, "kd/rkl_advantage_std": 2.2042930165926617, "kd/ssd_loss": 0.0, "learning_rate": 7.24338282763073e-07, "loss": 0.034208842118581134, "num_tokens": 237548714.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.820555551846822, "rewards/gt_logging_reward/std": 0.3738074233134588, "sampling/importance_sampling_ratio/max": 1.9909626205762228, "sampling/importance_sampling_ratio/mean": 1.0061950266361237, "sampling/importance_sampling_ratio/min": 0.423416376610597, "sampling/sampling_logp_difference/max": 0.4035428563753764, "sampling/sampling_logp_difference/mean": 0.004203322754862408, "step": 7260, "step_time": 8.156133976695127, "student/entropy": 0.12653256058692933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112491538127, "completions/max_length": 489.8, "completions/max_terminated_length": 450.03333333333336, "completions/mean_length": 187.83167877197266, "completions/mean_terminated_length": 181.01654205322265, "completions/min_length": 54.333333333333336, "completions/min_terminated_length": 54.333333333333336, "entropy": 0.12847090512514114, "epoch": 0.2768389473284472, "eval/gt_acc_batch": 0.806944461663564, "frac_reward_zero_std": 1.0, "grad_norm": 0.22150930762290955, "kd/policy_loss": 0.008667389988355959, "kd/rkl_advantage_mean": 0.7316719402869543, "kd/rkl_advantage_p95": 5.000080126523971, "kd/rkl_advantage_std": 2.232233554124832, "kd/ssd_loss": 0.0, "learning_rate": 7.231990278357953e-07, "loss": 0.03466955820719401, "num_tokens": 238526476.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8069444457689922, "rewards/gt_logging_reward/std": 0.3811695019404093, "sampling/importance_sampling_ratio/max": 1.8845135569572449, "sampling/importance_sampling_ratio/mean": 1.0062776704629262, "sampling/importance_sampling_ratio/min": 0.4106526712576548, "sampling/sampling_logp_difference/max": 0.37148555914560955, "sampling/sampling_logp_difference/mean": 0.004256860058133801, "step": 7290, "step_time": 8.128393263334875, "student/entropy": 0.12847090512514114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01888888996715347, "completions/max_length": 493.93333333333334, "completions/max_terminated_length": 454.1, "completions/mean_length": 182.88167572021484, "completions/mean_terminated_length": 176.5390833536784, "completions/min_length": 51.63333333333333, "completions/min_terminated_length": 51.63333333333333, "entropy": 0.1317394000167648, "epoch": 0.2779782022557248, "eval/gt_acc_batch": 0.7908333738644918, "frac_reward_zero_std": 1.0, "grad_norm": 0.21553803980350494, "kd/policy_loss": 0.009112790285144001, "kd/rkl_advantage_mean": 0.62514662258327, "kd/rkl_advantage_p95": 4.819916617870331, "kd/rkl_advantage_std": 2.2130232016245523, "kd/ssd_loss": 0.0, "learning_rate": 7.220597729085178e-07, "loss": 0.036451164881388345, "num_tokens": 239494826.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7908333321412404, "rewards/gt_logging_reward/std": 0.3985697249571482, "sampling/importance_sampling_ratio/max": 1.8780595302581786, "sampling/importance_sampling_ratio/mean": 1.0039331833521525, "sampling/importance_sampling_ratio/min": 0.4182099297642708, "sampling/sampling_logp_difference/max": 0.4154897650082906, "sampling/sampling_logp_difference/mean": 0.004377705844429632, "step": 7320, "step_time": 8.279885052916748, "student/entropy": 0.1317394000167648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778829137485, "completions/max_length": 473.43333333333334, "completions/max_terminated_length": 441.3, "completions/mean_length": 189.6658442179362, "completions/mean_terminated_length": 184.02222290039063, "completions/min_length": 58.96666666666667, "completions/min_terminated_length": 58.96666666666667, "entropy": 0.11890160012990236, "epoch": 0.2791174571830023, "eval/gt_acc_batch": 0.8130555729071299, "frac_reward_zero_std": 1.0, "grad_norm": 0.2616545259952545, "kd/policy_loss": 0.007866497751092539, "kd/rkl_advantage_mean": 0.6495922372986873, "kd/rkl_advantage_p95": 4.831064222256343, "kd/rkl_advantage_std": 2.193978323539098, "kd/ssd_loss": 0.0, "learning_rate": 7.209205179812402e-07, "loss": 0.03146599133809407, "num_tokens": 240479431.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8130555550257365, "rewards/gt_logging_reward/std": 0.3771941527724266, "sampling/importance_sampling_ratio/max": 1.9455264488855997, "sampling/importance_sampling_ratio/mean": 0.9973666826883952, "sampling/importance_sampling_ratio/min": 0.4294523626565933, "sampling/sampling_logp_difference/max": 0.39302099744478863, "sampling/sampling_logp_difference/mean": 0.003974258697902163, "step": 7350, "step_time": 7.888823867736694, "student/entropy": 0.11890160012990236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334773778915, "completions/max_length": 488.3, "completions/max_terminated_length": 459.1333333333333, "completions/mean_length": 188.53306477864584, "completions/mean_terminated_length": 180.14781850179037, "completions/min_length": 60.2, "completions/min_terminated_length": 60.2, "entropy": 0.13104477574427922, "epoch": 0.28025671211027986, "eval/gt_acc_batch": 0.7866666972637176, "frac_reward_zero_std": 1.0, "grad_norm": 0.2555178701877594, "kd/policy_loss": 0.00885378066644383, "kd/rkl_advantage_mean": 0.7146113731898367, "kd/rkl_advantage_p95": 4.894355609019597, "kd/rkl_advantage_std": 2.187800592184067, "kd/ssd_loss": 0.0, "learning_rate": 7.197812630539626e-07, "loss": 0.035415124893188474, "num_tokens": 241457710.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666674613952, "rewards/gt_logging_reward/std": 0.4010653465986252, "sampling/importance_sampling_ratio/max": 1.9251043200492859, "sampling/importance_sampling_ratio/mean": 0.9978241821130117, "sampling/importance_sampling_ratio/min": 0.44129207531611125, "sampling/sampling_logp_difference/max": 0.42639047304789224, "sampling/sampling_logp_difference/mean": 0.0043221672686437765, "step": 7380, "step_time": 8.088440095757445, "student/entropy": 0.13104477574427922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018611112081756195, "completions/max_length": 474.9, "completions/max_terminated_length": 437.6333333333333, "completions/mean_length": 187.0247319539388, "completions/mean_terminated_length": 180.97616678873698, "completions/min_length": 57.666666666666664, "completions/min_terminated_length": 57.666666666666664, "entropy": 0.12927573726822933, "epoch": 0.2813959670375574, "eval/gt_acc_batch": 0.7902777969837189, "frac_reward_zero_std": 1.0, "grad_norm": 0.3095906972885132, "kd/policy_loss": 0.00896775655176801, "kd/rkl_advantage_mean": 0.6248621855241557, "kd/rkl_advantage_p95": 4.730440032482147, "kd/rkl_advantage_std": 2.170497782031695, "kd/ssd_loss": 0.0, "learning_rate": 7.186420081266851e-07, "loss": 0.035871028900146484, "num_tokens": 242433759.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7902777791023254, "rewards/gt_logging_reward/std": 0.399994333088398, "sampling/importance_sampling_ratio/max": 1.9782690167427064, "sampling/importance_sampling_ratio/mean": 1.00160311460495, "sampling/importance_sampling_ratio/min": 0.41860113143920896, "sampling/sampling_logp_difference/max": 0.3959237615267436, "sampling/sampling_logp_difference/mean": 0.0043514217172438896, "step": 7410, "step_time": 7.998041561160547, "student/entropy": 0.12927573726822933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223604718842, "completions/max_length": 495.73333333333335, "completions/max_terminated_length": 460.23333333333335, "completions/mean_length": 188.46945343017578, "completions/mean_terminated_length": 180.50446217854818, "completions/min_length": 52.86666666666667, "completions/min_terminated_length": 52.86666666666667, "entropy": 0.12449201655884584, "epoch": 0.282535221964835, "eval/gt_acc_batch": 0.8194444676240286, "frac_reward_zero_std": 1.0, "grad_norm": 0.24058017134666443, "kd/policy_loss": 0.008094085634608443, "kd/rkl_advantage_mean": 0.6648565145209432, "kd/rkl_advantage_p95": 4.889782045284907, "kd/rkl_advantage_std": 2.2213917315006255, "kd/ssd_loss": 0.0, "learning_rate": 7.175027531994076e-07, "loss": 0.03237634499867757, "num_tokens": 243418561.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8194444398085277, "rewards/gt_logging_reward/std": 0.37773456076780954, "sampling/importance_sampling_ratio/max": 1.8634984135627746, "sampling/importance_sampling_ratio/mean": 0.9950605074564616, "sampling/importance_sampling_ratio/min": 0.39722345074017845, "sampling/sampling_logp_difference/max": 0.36180466413497925, "sampling/sampling_logp_difference/mean": 0.004165480192750692, "step": 7440, "step_time": 8.13850879422777, "student/entropy": 0.12449201655884584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223188728093, "completions/max_length": 480.3666666666667, "completions/max_terminated_length": 452.6666666666667, "completions/mean_length": 181.80084330240885, "completions/mean_terminated_length": 175.26335754394532, "completions/min_length": 53.8, "completions/min_terminated_length": 53.8, "entropy": 0.1290100621059537, "epoch": 0.28367447689211256, "eval/gt_acc_batch": 0.8011111239592235, "frac_reward_zero_std": 1.0, "grad_norm": 0.30264878273010254, "kd/policy_loss": 0.008407128961213555, "kd/rkl_advantage_mean": 0.6937856125334899, "kd/rkl_advantage_p95": 4.926431512832641, "kd/rkl_advantage_std": 2.2306670884291333, "kd/ssd_loss": 0.0, "learning_rate": 7.1636349827213e-07, "loss": 0.03362851937611898, "num_tokens": 244379356.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8011111140251159, "rewards/gt_logging_reward/std": 0.3908787359793981, "sampling/importance_sampling_ratio/max": 1.8494520982106526, "sampling/importance_sampling_ratio/mean": 0.9922216852506002, "sampling/importance_sampling_ratio/min": 0.4032910118500392, "sampling/sampling_logp_difference/max": 0.40983497301737465, "sampling/sampling_logp_difference/mean": 0.004308179983248313, "step": 7470, "step_time": 8.049019245733508, "student/entropy": 0.1290100621059537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01954023083993073, "completions/max_length": 482.7586206896552, "completions/max_terminated_length": 441.6896551724138, "completions/mean_length": 187.14397404111665, "completions/mean_terminated_length": 180.80667429956895, "completions/min_length": 54.172413793103445, "completions/min_terminated_length": 54.172413793103445, "entropy": 0.12974398114301008, "epoch": 0.2848137318193901, "eval/gt_acc_batch": 0.8112069162829169, "frac_reward_zero_std": 1.0, "grad_norm": 0.25777268409729004, "kd/policy_loss": 0.008724484102928946, "kd/rkl_advantage_mean": 0.6844629948509151, "kd/rkl_advantage_p95": 4.8789235394576504, "kd/rkl_advantage_std": 2.2019814293960045, "kd/ssd_loss": 0.0, "learning_rate": 7.152242433448524e-07, "loss": 0.033734679222106934, "num_tokens": 245314345.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8112068957295911, "rewards/gt_logging_reward/std": 0.37981374073645163, "sampling/importance_sampling_ratio/max": 1.9253342069428543, "sampling/importance_sampling_ratio/mean": 0.9967258367045172, "sampling/importance_sampling_ratio/min": 0.4212525199199545, "sampling/sampling_logp_difference/max": 0.4117835862883206, "sampling/sampling_logp_difference/mean": 0.004299978926328236, "step": 7500, "step_time": 7.802094062033575, "student/entropy": 0.12974398114301008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445775200924, "completions/max_length": 477.6333333333333, "completions/max_terminated_length": 441.6333333333333, "completions/mean_length": 191.60501047770182, "completions/mean_terminated_length": 184.62628326416015, "completions/min_length": 52.266666666666666, "completions/min_terminated_length": 52.266666666666666, "entropy": 0.12551198303699493, "epoch": 0.2859529867466677, "eval/gt_acc_batch": 0.7858333508173625, "frac_reward_zero_std": 1.0, "grad_norm": 0.32289934158325195, "kd/policy_loss": 0.00851394320682933, "kd/rkl_advantage_mean": 0.6326868486280243, "kd/rkl_advantage_p95": 4.793203415473302, "kd/rkl_advantage_std": 2.175092860062917, "kd/ssd_loss": 0.0, "learning_rate": 7.140849884175749e-07, "loss": 0.03405577739079793, "num_tokens": 246300931.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7858333349227905, "rewards/gt_logging_reward/std": 0.40166600545247394, "sampling/importance_sampling_ratio/max": 1.902752161026001, "sampling/importance_sampling_ratio/mean": 0.9973043084144593, "sampling/importance_sampling_ratio/min": 0.4102367252111435, "sampling/sampling_logp_difference/max": 0.37177858750025433, "sampling/sampling_logp_difference/mean": 0.0041945129089678325, "step": 7530, "step_time": 8.027995697952186, "student/entropy": 0.12551198303699493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02666666836788257, "completions/max_length": 500.4, "completions/max_terminated_length": 456.96666666666664, "completions/mean_length": 192.5469548543294, "completions/mean_terminated_length": 183.86858723958332, "completions/min_length": 47.0, "completions/min_terminated_length": 47.0, "entropy": 0.13432998632391294, "epoch": 0.28709224167394526, "eval/gt_acc_batch": 0.7869444568951924, "frac_reward_zero_std": 1.0, "grad_norm": 0.2551116347312927, "kd/policy_loss": 0.009300643752794713, "kd/rkl_advantage_mean": 0.6909839316251843, "kd/rkl_advantage_p95": 4.9054478724797566, "kd/rkl_advantage_std": 2.2254164030154544, "kd/ssd_loss": 0.0, "learning_rate": 7.129457334902973e-07, "loss": 0.0372025728225708, "num_tokens": 247299316.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444449742635, "rewards/gt_logging_reward/std": 0.4019670362273852, "sampling/importance_sampling_ratio/max": 2.0223362684249877, "sampling/importance_sampling_ratio/mean": 1.0082279165585837, "sampling/importance_sampling_ratio/min": 0.4176556602120399, "sampling/sampling_logp_difference/max": 0.4476656913757324, "sampling/sampling_logp_difference/mean": 0.004463448965301116, "step": 7560, "step_time": 8.372763120798238, "student/entropy": 0.13432998632391294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02694444600492716, "completions/max_length": 489.8, "completions/max_terminated_length": 442.1, "completions/mean_length": 194.866677347819, "completions/mean_terminated_length": 186.37992350260416, "completions/min_length": 56.733333333333334, "completions/min_terminated_length": 56.733333333333334, "entropy": 0.13449308909475804, "epoch": 0.2882314966012228, "eval/gt_acc_batch": 0.771666693687439, "frac_reward_zero_std": 1.0, "grad_norm": 0.2850792706012726, "kd/policy_loss": 0.009342372687145446, "kd/rkl_advantage_mean": 0.6919974890848001, "kd/rkl_advantage_p95": 4.904853465159734, "kd/rkl_advantage_std": 2.1925798912843066, "kd/ssd_loss": 0.0, "learning_rate": 7.118064785630198e-07, "loss": 0.03736949364344279, "num_tokens": 248305508.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7716666618982951, "rewards/gt_logging_reward/std": 0.41171499093373615, "sampling/importance_sampling_ratio/max": 1.8823186755180359, "sampling/importance_sampling_ratio/mean": 0.9978046496709188, "sampling/importance_sampling_ratio/min": 0.3852042456467946, "sampling/sampling_logp_difference/max": 0.36237057447433474, "sampling/sampling_logp_difference/mean": 0.004371510089064638, "step": 7590, "step_time": 8.20140072212865, "student/entropy": 0.13449308909475804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223437080782, "completions/max_length": 485.9, "completions/max_terminated_length": 456.93333333333334, "completions/mean_length": 181.93584442138672, "completions/mean_terminated_length": 175.43326873779296, "completions/min_length": 56.56666666666667, "completions/min_terminated_length": 56.56666666666667, "entropy": 0.13278786235799392, "epoch": 0.28937075152850034, "eval/gt_acc_batch": 0.7930555661519368, "frac_reward_zero_std": 1.0, "grad_norm": 0.2813803255558014, "kd/policy_loss": 0.00862608606306215, "kd/rkl_advantage_mean": 0.722017059638165, "kd/rkl_advantage_p95": 4.981088976065318, "kd/rkl_advantage_std": 2.2586400995651883, "kd/ssd_loss": 0.0, "learning_rate": 7.106672236357422e-07, "loss": 0.0345043420791626, "num_tokens": 249252445.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555601914724, "rewards/gt_logging_reward/std": 0.3994779924551646, "sampling/importance_sampling_ratio/max": 1.8742060701052348, "sampling/importance_sampling_ratio/mean": 0.9938172340393067, "sampling/importance_sampling_ratio/min": 0.4252035493652026, "sampling/sampling_logp_difference/max": 0.4132633924484253, "sampling/sampling_logp_difference/mean": 0.0044174450837696595, "step": 7620, "step_time": 7.9955654951899, "student/entropy": 0.13278786235799392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223784774543, "completions/max_length": 475.8666666666667, "completions/max_terminated_length": 452.06666666666666, "completions/mean_length": 188.2452875773112, "completions/mean_terminated_length": 180.88937428792318, "completions/min_length": 51.46666666666667, "completions/min_terminated_length": 51.46666666666667, "entropy": 0.12492223164687553, "epoch": 0.2905100064557779, "eval/gt_acc_batch": 0.8138889094193776, "frac_reward_zero_std": 1.0, "grad_norm": 0.1973373293876648, "kd/policy_loss": 0.008381914508451398, "kd/rkl_advantage_mean": 0.676210269642373, "kd/rkl_advantage_p95": 4.913306049505869, "kd/rkl_advantage_std": 2.2185465345780053, "kd/ssd_loss": 0.0, "learning_rate": 7.095279687084647e-07, "loss": 0.033527652422587075, "num_tokens": 250241840.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8138888895511627, "rewards/gt_logging_reward/std": 0.38042057305574417, "sampling/importance_sampling_ratio/max": 1.9463424563407898, "sampling/importance_sampling_ratio/mean": 0.9944198906421662, "sampling/importance_sampling_ratio/min": 0.42315922379493714, "sampling/sampling_logp_difference/max": 0.37017068862915037, "sampling/sampling_logp_difference/mean": 0.004117939559121927, "step": 7650, "step_time": 8.127120964054484, "student/entropy": 0.12492223164687553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333469927311, "completions/max_length": 494.7, "completions/max_terminated_length": 464.96666666666664, "completions/mean_length": 193.02806599934897, "completions/mean_terminated_length": 186.34808197021485, "completions/min_length": 55.36666666666667, "completions/min_terminated_length": 55.36666666666667, "entropy": 0.12791847232729198, "epoch": 0.29164926138305547, "eval/gt_acc_batch": 0.7844444731871287, "frac_reward_zero_std": 1.0, "grad_norm": 0.30657410621643066, "kd/policy_loss": 0.008794536468728135, "kd/rkl_advantage_mean": 0.6878189840043585, "kd/rkl_advantage_p95": 4.888978544871012, "kd/rkl_advantage_std": 2.1933288226524987, "kd/ssd_loss": 0.0, "learning_rate": 7.08388713781187e-07, "loss": 0.035178144772847496, "num_tokens": 251248117.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7844444453716278, "rewards/gt_logging_reward/std": 0.40195291737715405, "sampling/importance_sampling_ratio/max": 1.9462915937105814, "sampling/importance_sampling_ratio/mean": 1.0002789874871572, "sampling/importance_sampling_ratio/min": 0.4279101143280665, "sampling/sampling_logp_difference/max": 0.3927750507990519, "sampling/sampling_logp_difference/mean": 0.0041761129628866914, "step": 7680, "step_time": 8.354513695770098, "student/entropy": 0.12791847232729198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556596567232, "completions/max_length": 495.2, "completions/max_terminated_length": 463.1333333333333, "completions/mean_length": 185.58306477864582, "completions/mean_terminated_length": 178.80243225097655, "completions/min_length": 53.7, "completions/min_terminated_length": 53.7, "entropy": 0.13234983968238037, "epoch": 0.29278851631033304, "eval/gt_acc_batch": 0.8133333563804627, "frac_reward_zero_std": 1.0, "grad_norm": 0.1916031539440155, "kd/policy_loss": 0.008344037287558118, "kd/rkl_advantage_mean": 0.719113462670551, "kd/rkl_advantage_p95": 4.960255457957586, "kd/rkl_advantage_std": 2.2320378263791403, "kd/ssd_loss": 0.0, "learning_rate": 7.072494588539095e-07, "loss": 0.033376153310139975, "num_tokens": 252210184.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8133333325386047, "rewards/gt_logging_reward/std": 0.38271885812282563, "sampling/importance_sampling_ratio/max": 1.9597482244173685, "sampling/importance_sampling_ratio/mean": 0.9950135747591654, "sampling/importance_sampling_ratio/min": 0.4136969188849131, "sampling/sampling_logp_difference/max": 0.3874598741531372, "sampling/sampling_logp_difference/mean": 0.004173777477505306, "step": 7710, "step_time": 8.03531077800629, "student/entropy": 0.13234983968238037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777940655748, "completions/max_length": 484.06666666666666, "completions/max_terminated_length": 452.53333333333336, "completions/mean_length": 189.4819544474284, "completions/mean_terminated_length": 181.40251922607422, "completions/min_length": 47.46666666666667, "completions/min_terminated_length": 47.46666666666667, "entropy": 0.13221267834305764, "epoch": 0.2939277712376106, "eval/gt_acc_batch": 0.7791666805744171, "frac_reward_zero_std": 1.0, "grad_norm": 0.2826479375362396, "kd/policy_loss": 0.009280706817905108, "kd/rkl_advantage_mean": 0.6660173285752535, "kd/rkl_advantage_p95": 4.80612416168054, "kd/rkl_advantage_std": 2.183683686455091, "kd/ssd_loss": 0.0, "learning_rate": 7.061102039266319e-07, "loss": 0.037122829755147295, "num_tokens": 253202255.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7791666666666667, "rewards/gt_logging_reward/std": 0.4074968273440997, "sampling/importance_sampling_ratio/max": 1.9393693486849466, "sampling/importance_sampling_ratio/mean": 1.0003304700056712, "sampling/importance_sampling_ratio/min": 0.4589948852856954, "sampling/sampling_logp_difference/max": 0.3769507606824239, "sampling/sampling_logp_difference/mean": 0.004351368996625146, "step": 7740, "step_time": 8.202671994284417, "student/entropy": 0.13221267834305764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778717378776, "completions/max_length": 477.0, "completions/max_terminated_length": 447.76666666666665, "completions/mean_length": 186.90445556640626, "completions/mean_terminated_length": 179.49115702311198, "completions/min_length": 50.06666666666667, "completions/min_terminated_length": 50.06666666666667, "entropy": 0.12832566568007073, "epoch": 0.29506702616488817, "eval/gt_acc_batch": 0.8000000238418579, "frac_reward_zero_std": 1.0, "grad_norm": 0.26327311992645264, "kd/policy_loss": 0.0085234900994692, "kd/rkl_advantage_mean": 0.7063379841235776, "kd/rkl_advantage_p95": 4.975066602230072, "kd/rkl_advantage_std": 2.2228442390759784, "kd/ssd_loss": 0.0, "learning_rate": 7.049709489993543e-07, "loss": 0.034093964099884036, "num_tokens": 254184943.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7999999980131786, "rewards/gt_logging_reward/std": 0.38735744506120684, "sampling/importance_sampling_ratio/max": 1.9168116807937623, "sampling/importance_sampling_ratio/mean": 0.9992313047250112, "sampling/importance_sampling_ratio/min": 0.40390019367138547, "sampling/sampling_logp_difference/max": 0.4154171387354533, "sampling/sampling_logp_difference/mean": 0.004274530002536873, "step": 7770, "step_time": 8.147790236837075, "student/entropy": 0.12832566568007073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112367361785, "completions/max_length": 471.23333333333335, "completions/max_terminated_length": 446.6333333333333, "completions/mean_length": 189.66389872233074, "completions/mean_terminated_length": 182.97745513916016, "completions/min_length": 52.733333333333334, "completions/min_terminated_length": 52.733333333333334, "entropy": 0.12946879186977941, "epoch": 0.29620628109216574, "eval/gt_acc_batch": 0.7977777977784475, "frac_reward_zero_std": 1.0, "grad_norm": 0.29174333810806274, "kd/policy_loss": 0.008949945903926467, "kd/rkl_advantage_mean": 0.5659873115151034, "kd/rkl_advantage_p95": 4.618185230096182, "kd/rkl_advantage_std": 2.1430855482816695, "kd/ssd_loss": 0.0, "learning_rate": 7.038316940720769e-07, "loss": 0.03579978942871094, "num_tokens": 255173189.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7977777779102325, "rewards/gt_logging_reward/std": 0.39027885695298514, "sampling/importance_sampling_ratio/max": 1.884654426574707, "sampling/importance_sampling_ratio/mean": 0.9992243687311808, "sampling/importance_sampling_ratio/min": 0.4084655463695526, "sampling/sampling_logp_difference/max": 0.47365596691767375, "sampling/sampling_logp_difference/mean": 0.004216033150441945, "step": 7800, "step_time": 8.037247585625543, "student/entropy": 0.12946879186977941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779381722212, "completions/max_length": 491.43333333333334, "completions/max_terminated_length": 441.8, "completions/mean_length": 190.92001088460287, "completions/mean_terminated_length": 181.79068145751953, "completions/min_length": 57.333333333333336, "completions/min_terminated_length": 57.333333333333336, "entropy": 0.12996432402481636, "epoch": 0.2973455360194433, "eval/gt_acc_batch": 0.7794444600741068, "frac_reward_zero_std": 1.0, "grad_norm": 0.28397703170776367, "kd/policy_loss": 0.008921714971074834, "kd/rkl_advantage_mean": 0.6956824646020929, "kd/rkl_advantage_p95": 4.90191034078598, "kd/rkl_advantage_std": 2.2564422369003294, "kd/ssd_loss": 0.0, "learning_rate": 7.026924391447993e-07, "loss": 0.03568686644236247, "num_tokens": 256159637.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.779444444179535, "rewards/gt_logging_reward/std": 0.40902405778566997, "sampling/importance_sampling_ratio/max": 2.0407096266746523, "sampling/importance_sampling_ratio/mean": 1.0010959188143411, "sampling/importance_sampling_ratio/min": 0.42104661067326865, "sampling/sampling_logp_difference/max": 0.41013426780700685, "sampling/sampling_logp_difference/mean": 0.0042616487558310235, "step": 7830, "step_time": 8.06226682367657, "student/entropy": 0.12996432402481636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667945683002, "completions/max_length": 480.6333333333333, "completions/max_terminated_length": 448.43333333333334, "completions/mean_length": 184.67528788248697, "completions/mean_terminated_length": 178.30347239176433, "completions/min_length": 48.9, "completions/min_terminated_length": 48.9, "entropy": 0.12672165986150502, "epoch": 0.29848479094672087, "eval/gt_acc_batch": 0.8019444723924001, "frac_reward_zero_std": 1.0, "grad_norm": 0.3114898204803467, "kd/policy_loss": 0.008144811327413966, "kd/rkl_advantage_mean": 0.7411577735717098, "kd/rkl_advantage_p95": 4.976274504264196, "kd/rkl_advantage_std": 2.2099788675705594, "kd/ssd_loss": 0.0, "learning_rate": 7.015531842175218e-07, "loss": 0.032579243183135986, "num_tokens": 257125636.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444425900777, "rewards/gt_logging_reward/std": 0.3912709911664327, "sampling/importance_sampling_ratio/max": 1.948303226629893, "sampling/importance_sampling_ratio/mean": 0.9968331297238667, "sampling/importance_sampling_ratio/min": 0.42082385619481405, "sampling/sampling_logp_difference/max": 0.40383689999580386, "sampling/sampling_logp_difference/mean": 0.004239057330414653, "step": 7860, "step_time": 7.923556204411822, "student/entropy": 0.12672165986150502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013611111727853616, "completions/max_length": 484.3666666666667, "completions/max_terminated_length": 455.6, "completions/mean_length": 187.53667653401692, "completions/mean_terminated_length": 183.1054453531901, "completions/min_length": 57.233333333333334, "completions/min_terminated_length": 57.233333333333334, "entropy": 0.12915795811762412, "epoch": 0.2996240458739984, "eval/gt_acc_batch": 0.7911111275355022, "frac_reward_zero_std": 1.0, "grad_norm": 0.27148157358169556, "kd/policy_loss": 0.009231099726942677, "kd/rkl_advantage_mean": 0.7701348431486016, "kd/rkl_advantage_p95": 5.098598366975784, "kd/rkl_advantage_std": 2.2579104880491894, "kd/ssd_loss": 0.0, "learning_rate": 7.004139292902441e-07, "loss": 0.03692439397176107, "num_tokens": 258096856.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7911111056804657, "rewards/gt_logging_reward/std": 0.3998505746324857, "sampling/importance_sampling_ratio/max": 2.0587182799975077, "sampling/importance_sampling_ratio/mean": 1.000919508934021, "sampling/importance_sampling_ratio/min": 0.44793639580408734, "sampling/sampling_logp_difference/max": 0.3850701113541921, "sampling/sampling_logp_difference/mean": 0.004254226906535526, "step": 7890, "step_time": 8.084433331685917, "student/entropy": 0.12915795811762412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112280438343, "completions/max_length": 499.46666666666664, "completions/max_terminated_length": 451.6333333333333, "completions/mean_length": 188.5541763305664, "completions/mean_terminated_length": 180.78846740722656, "completions/min_length": 52.266666666666666, "completions/min_terminated_length": 52.266666666666666, "entropy": 0.12820197536299627, "epoch": 0.30076330080127595, "eval/gt_acc_batch": 0.8072222272555033, "frac_reward_zero_std": 1.0, "grad_norm": 0.29439711570739746, "kd/policy_loss": 0.008424010098678991, "kd/rkl_advantage_mean": 0.7260900138023619, "kd/rkl_advantage_p95": 4.9656450311342875, "kd/rkl_advantage_std": 2.2236838678518933, "kd/ssd_loss": 0.0, "learning_rate": 6.992746743629666e-07, "loss": 0.03369603951772054, "num_tokens": 259068715.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222232818603, "rewards/gt_logging_reward/std": 0.385924868285656, "sampling/importance_sampling_ratio/max": 1.9858551740646362, "sampling/importance_sampling_ratio/mean": 1.000299878915151, "sampling/importance_sampling_ratio/min": 0.4306361754735311, "sampling/sampling_logp_difference/max": 0.3604607343673706, "sampling/sampling_logp_difference/mean": 0.004226866651636859, "step": 7920, "step_time": 8.095093808676271, "student/entropy": 0.12820197536299627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028611112696429093, "completions/max_length": 489.53333333333336, "completions/max_terminated_length": 454.93333333333334, "completions/mean_length": 189.22445526123047, "completions/mean_terminated_length": 179.97029520670574, "completions/min_length": 54.13333333333333, "completions/min_terminated_length": 54.13333333333333, "entropy": 0.12671863163510957, "epoch": 0.3019025557285535, "eval/gt_acc_batch": 0.7836111426353455, "frac_reward_zero_std": 1.0, "grad_norm": 0.2785177528858185, "kd/policy_loss": 0.008541210826175908, "kd/rkl_advantage_mean": 0.7461035484292855, "kd/rkl_advantage_p95": 5.028793956836065, "kd/rkl_advantage_std": 2.2635088751713437, "kd/ssd_loss": 0.0, "learning_rate": 6.98135419435689e-07, "loss": 0.034164841969807944, "num_tokens": 260060531.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7836111148198446, "rewards/gt_logging_reward/std": 0.40433193842569987, "sampling/importance_sampling_ratio/max": 1.9323698242505392, "sampling/importance_sampling_ratio/mean": 0.9928284029165904, "sampling/importance_sampling_ratio/min": 0.38991453299919765, "sampling/sampling_logp_difference/max": 0.4393004635969798, "sampling/sampling_logp_difference/mean": 0.004181418009102345, "step": 7950, "step_time": 8.178795074737476, "student/entropy": 0.12671863163510957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890414188305, "completions/max_length": 496.03333333333336, "completions/max_terminated_length": 455.8, "completions/mean_length": 187.12501017252603, "completions/mean_terminated_length": 179.17007904052736, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "entropy": 0.13045757524669171, "epoch": 0.3030418106558311, "eval/gt_acc_batch": 0.7916666785875957, "frac_reward_zero_std": 1.0, "grad_norm": 0.3191779851913452, "kd/policy_loss": 0.008630218651766578, "kd/rkl_advantage_mean": 0.7810533531631033, "kd/rkl_advantage_p95": 5.025513519843419, "kd/rkl_advantage_std": 2.240301220615705, "kd/ssd_loss": 0.0, "learning_rate": 6.969961645084114e-07, "loss": 0.03452087640762329, "num_tokens": 261035469.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7916666686534881, "rewards/gt_logging_reward/std": 0.3992473170161247, "sampling/importance_sampling_ratio/max": 1.8990725914637248, "sampling/importance_sampling_ratio/mean": 0.9935619751612346, "sampling/importance_sampling_ratio/min": 0.4321083347002665, "sampling/sampling_logp_difference/max": 0.3817866325378418, "sampling/sampling_logp_difference/mean": 0.004284323728643358, "step": 7980, "step_time": 8.140066279924941, "student/entropy": 0.13045757524669171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667765627305, "completions/max_length": 471.6, "completions/max_terminated_length": 439.8, "completions/mean_length": 193.52639821370443, "completions/mean_terminated_length": 186.6849802652995, "completions/min_length": 56.8, "completions/min_terminated_length": 56.8, "entropy": 0.127671993834277, "epoch": 0.30418106558310865, "eval/gt_acc_batch": 0.7986111402511596, "frac_reward_zero_std": 1.0, "grad_norm": 0.3080930709838867, "kd/policy_loss": 0.008873973111622035, "kd/rkl_advantage_mean": 0.6301079919716964, "kd/rkl_advantage_p95": 4.7425407310326895, "kd/rkl_advantage_std": 2.172306857506434, "kd/ssd_loss": 0.0, "learning_rate": 6.958569095811339e-07, "loss": 0.03549589316050212, "num_tokens": 262037884.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111084620158, "rewards/gt_logging_reward/std": 0.39281623462835946, "sampling/importance_sampling_ratio/max": 2.023660441239675, "sampling/importance_sampling_ratio/mean": 0.9990261316299438, "sampling/importance_sampling_ratio/min": 0.42471478978792826, "sampling/sampling_logp_difference/max": 0.3664470911026001, "sampling/sampling_logp_difference/mean": 0.004173655831255019, "step": 8010, "step_time": 8.005879651029439, "student/entropy": 0.127671993834277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334773778915, "completions/max_length": 494.6, "completions/max_terminated_length": 463.0, "completions/mean_length": 190.5358434041341, "completions/mean_terminated_length": 182.1780990600586, "completions/min_length": 51.9, "completions/min_terminated_length": 51.9, "entropy": 0.13477400820702315, "epoch": 0.3053203205103862, "eval/gt_acc_batch": 0.7725000242392223, "frac_reward_zero_std": 1.0, "grad_norm": 0.2903474271297455, "kd/policy_loss": 0.00920351585179257, "kd/rkl_advantage_mean": 0.7555451489984989, "kd/rkl_advantage_p95": 5.020200232664744, "kd/rkl_advantage_std": 2.278938070933024, "kd/ssd_loss": 0.0, "learning_rate": 6.947176546538564e-07, "loss": 0.03681406577428182, "num_tokens": 263041437.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7725000023841858, "rewards/gt_logging_reward/std": 0.41078338225682576, "sampling/importance_sampling_ratio/max": 1.9809740344683329, "sampling/importance_sampling_ratio/mean": 0.996174430847168, "sampling/importance_sampling_ratio/min": 0.40766132523616155, "sampling/sampling_logp_difference/max": 0.4593360821406047, "sampling/sampling_logp_difference/mean": 0.004428394588952263, "step": 8040, "step_time": 8.51191537710062, "student/entropy": 0.13477400820702315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02305555697530508, "completions/max_length": 495.5, "completions/max_terminated_length": 455.3666666666667, "completions/mean_length": 193.46334177652994, "completions/mean_terminated_length": 185.95989227294922, "completions/min_length": 59.93333333333333, "completions/min_terminated_length": 59.93333333333333, "entropy": 0.12734902128577233, "epoch": 0.3064595754376638, "eval/gt_acc_batch": 0.7911111374696096, "frac_reward_zero_std": 1.0, "grad_norm": 0.22734984755516052, "kd/policy_loss": 0.008530391209448377, "kd/rkl_advantage_mean": 0.6589152462780475, "kd/rkl_advantage_p95": 4.838178829352061, "kd/rkl_advantage_std": 2.1820967386166257, "kd/ssd_loss": 0.0, "learning_rate": 6.935783997265788e-07, "loss": 0.03412156899770101, "num_tokens": 264034881.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7911111156145731, "rewards/gt_logging_reward/std": 0.3981965839862823, "sampling/importance_sampling_ratio/max": 1.9726892232894897, "sampling/importance_sampling_ratio/mean": 0.9993841032187144, "sampling/importance_sampling_ratio/min": 0.45649208426475524, "sampling/sampling_logp_difference/max": 0.41073160568873085, "sampling/sampling_logp_difference/mean": 0.004234331636689603, "step": 8070, "step_time": 8.1781967502359, "student/entropy": 0.12734902128577233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02388889013479153, "completions/max_length": 483.06666666666666, "completions/max_terminated_length": 456.6, "completions/mean_length": 192.9100102742513, "completions/mean_terminated_length": 185.313818359375, "completions/min_length": 52.233333333333334, "completions/min_terminated_length": 52.233333333333334, "entropy": 0.13224350636204082, "epoch": 0.30759883036494134, "eval/gt_acc_batch": 0.766111147403717, "frac_reward_zero_std": 1.0, "grad_norm": 0.253036230802536, "kd/policy_loss": 0.009226441546343268, "kd/rkl_advantage_mean": 0.7030738583455484, "kd/rkl_advantage_p95": 4.915995558102925, "kd/rkl_advantage_std": 2.183627490202586, "kd/ssd_loss": 0.0, "learning_rate": 6.924391447993012e-07, "loss": 0.036905765533447266, "num_tokens": 265032901.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7661111136277516, "rewards/gt_logging_reward/std": 0.416210866967837, "sampling/importance_sampling_ratio/max": 2.024342075983683, "sampling/importance_sampling_ratio/mean": 1.0014472345511118, "sampling/importance_sampling_ratio/min": 0.3877014418443044, "sampling/sampling_logp_difference/max": 0.4013754228750865, "sampling/sampling_logp_difference/mean": 0.004328109960382184, "step": 8100, "step_time": 8.255580194457433, "student/entropy": 0.13224350636204082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028611112603296837, "completions/max_length": 498.96666666666664, "completions/max_terminated_length": 464.6333333333333, "completions/mean_length": 191.4966781616211, "completions/mean_terminated_length": 182.22972310384114, "completions/min_length": 56.766666666666666, "completions/min_terminated_length": 56.766666666666666, "entropy": 0.13312652961661417, "epoch": 0.3087380852922189, "eval/gt_acc_batch": 0.7669444719950358, "frac_reward_zero_std": 1.0, "grad_norm": 0.2479233592748642, "kd/policy_loss": 0.0094405393251994, "kd/rkl_advantage_mean": 0.7439281607667605, "kd/rkl_advantage_p95": 5.002383544047674, "kd/rkl_advantage_std": 2.2426028827826183, "kd/ssd_loss": 0.0, "learning_rate": 6.912998898720237e-07, "loss": 0.037762161095937094, "num_tokens": 266022913.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7669444421927134, "rewards/gt_logging_reward/std": 0.41500459412733715, "sampling/importance_sampling_ratio/max": 1.9273448864618936, "sampling/importance_sampling_ratio/mean": 0.9959462583065033, "sampling/importance_sampling_ratio/min": 0.40525284508864085, "sampling/sampling_logp_difference/max": 0.43919384280840557, "sampling/sampling_logp_difference/mean": 0.004366057319566607, "step": 8130, "step_time": 8.405102116540851, "student/entropy": 0.13312652961661417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026666668088485796, "completions/max_length": 498.1666666666667, "completions/max_terminated_length": 461.03333333333336, "completions/mean_length": 191.2344538370768, "completions/mean_terminated_length": 182.50704905192057, "completions/min_length": 51.233333333333334, "completions/min_terminated_length": 51.233333333333334, "entropy": 0.13743435287227232, "epoch": 0.3098773402194964, "eval/gt_acc_batch": 0.7830555617809296, "frac_reward_zero_std": 1.0, "grad_norm": 0.2827863097190857, "kd/policy_loss": 0.009014608097883563, "kd/rkl_advantage_mean": 0.7831722415750846, "kd/rkl_advantage_p95": 5.051604572931925, "kd/rkl_advantage_std": 2.241032110651334, "kd/ssd_loss": 0.0, "learning_rate": 6.90160634944746e-07, "loss": 0.0360584298769633, "num_tokens": 267009469.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.783055555820465, "rewards/gt_logging_reward/std": 0.4067437688509623, "sampling/importance_sampling_ratio/max": 1.912741482257843, "sampling/importance_sampling_ratio/mean": 0.9999624331792195, "sampling/importance_sampling_ratio/min": 0.4051282371083895, "sampling/sampling_logp_difference/max": 0.3907126565774282, "sampling/sampling_logp_difference/mean": 0.0043076858234902225, "step": 8160, "step_time": 8.202666431777955, "student/entropy": 0.13743435287227232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001297642788, "completions/max_length": 493.6333333333333, "completions/max_terminated_length": 450.9, "completions/mean_length": 189.31334482828777, "completions/mean_terminated_length": 181.97473551432293, "completions/min_length": 54.666666666666664, "completions/min_terminated_length": 54.666666666666664, "entropy": 0.1276719604929288, "epoch": 0.311016595146774, "eval/gt_acc_batch": 0.7936111311117808, "frac_reward_zero_std": 1.0, "grad_norm": 0.2598976790904999, "kd/policy_loss": 0.00841502429684624, "kd/rkl_advantage_mean": 0.6778800476963321, "kd/rkl_advantage_p95": 4.9318534553051, "kd/rkl_advantage_std": 2.234111000100772, "kd/ssd_loss": 0.0, "learning_rate": 6.890213800174686e-07, "loss": 0.03366009394327799, "num_tokens": 268002389.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111052831014, "rewards/gt_logging_reward/std": 0.396973118185997, "sampling/importance_sampling_ratio/max": 1.9005735874176026, "sampling/importance_sampling_ratio/mean": 0.9972155114014943, "sampling/importance_sampling_ratio/min": 0.45818411409854887, "sampling/sampling_logp_difference/max": 0.39282062848409016, "sampling/sampling_logp_difference/mean": 0.004195075559740265, "step": 8190, "step_time": 8.307744815022064, "student/entropy": 0.1276719604929288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055556956678628, "completions/max_length": 505.3333333333333, "completions/max_terminated_length": 459.23333333333335, "completions/mean_length": 193.1975102742513, "completions/mean_terminated_length": 184.11436716715494, "completions/min_length": 61.4, "completions/min_terminated_length": 61.4, "entropy": 0.12783271130174398, "epoch": 0.31215585007405156, "eval/gt_acc_batch": 0.7933333535989125, "frac_reward_zero_std": 1.0, "grad_norm": 0.2338627725839615, "kd/policy_loss": 0.008694928539140771, "kd/rkl_advantage_mean": 0.7189088577792669, "kd/rkl_advantage_p95": 4.94693039059639, "kd/rkl_advantage_std": 2.213687347372373, "kd/ssd_loss": 0.0, "learning_rate": 6.87882125090191e-07, "loss": 0.034779715538024905, "num_tokens": 269014724.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7933333317438761, "rewards/gt_logging_reward/std": 0.3931067168712616, "sampling/importance_sampling_ratio/max": 1.988636883099874, "sampling/importance_sampling_ratio/mean": 1.0064445932706196, "sampling/importance_sampling_ratio/min": 0.39764509797872355, "sampling/sampling_logp_difference/max": 0.5656359553337097, "sampling/sampling_logp_difference/mean": 0.004217798379249871, "step": 8220, "step_time": 8.490134261389418, "student/entropy": 0.12783271130174398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223449498416, "completions/max_length": 478.6, "completions/max_terminated_length": 444.26666666666665, "completions/mean_length": 189.33028666178384, "completions/mean_terminated_length": 181.47399495442707, "completions/min_length": 60.53333333333333, "completions/min_terminated_length": 60.53333333333333, "entropy": 0.1246159682671229, "epoch": 0.3132951050013291, "eval/gt_acc_batch": 0.7883333543936412, "frac_reward_zero_std": 1.0, "grad_norm": 0.30631735920906067, "kd/policy_loss": 0.008417430040814603, "kd/rkl_advantage_mean": 0.6260349647918095, "kd/rkl_advantage_p95": 4.71088000535965, "kd/rkl_advantage_std": 2.151095740000407, "kd/ssd_loss": 0.0, "learning_rate": 6.867428701629134e-07, "loss": 0.033669718106587726, "num_tokens": 269994065.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7883333365122477, "rewards/gt_logging_reward/std": 0.3969871600468953, "sampling/importance_sampling_ratio/max": 1.9124387184778848, "sampling/importance_sampling_ratio/mean": 1.001412703593572, "sampling/importance_sampling_ratio/min": 0.44709332287311554, "sampling/sampling_logp_difference/max": 0.4296522696812948, "sampling/sampling_logp_difference/mean": 0.004147849534638226, "step": 8250, "step_time": 8.130510303401389, "student/entropy": 0.1246159682671229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778996775546, "completions/max_length": 482.4, "completions/max_terminated_length": 436.8666666666667, "completions/mean_length": 186.02223205566406, "completions/mean_terminated_length": 178.4942189534505, "completions/min_length": 44.93333333333333, "completions/min_terminated_length": 44.93333333333333, "entropy": 0.13258436204244692, "epoch": 0.3144343599286067, "eval/gt_acc_batch": 0.7988889078299205, "frac_reward_zero_std": 1.0, "grad_norm": 0.2419353723526001, "kd/policy_loss": 0.008726734506975238, "kd/rkl_advantage_mean": 0.7162716191262006, "kd/rkl_advantage_p95": 4.933476175864538, "kd/rkl_advantage_std": 2.2069155822197595, "kd/ssd_loss": 0.0, "learning_rate": 6.856036152356358e-07, "loss": 0.034906939665476484, "num_tokens": 270963393.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.79888889392217, "rewards/gt_logging_reward/std": 0.3957697411378225, "sampling/importance_sampling_ratio/max": 2.057222068309784, "sampling/importance_sampling_ratio/mean": 0.9967463751633961, "sampling/importance_sampling_ratio/min": 0.41508087317148845, "sampling/sampling_logp_difference/max": 0.39457394580046334, "sampling/sampling_logp_difference/mean": 0.004321820506205162, "step": 8280, "step_time": 8.14594660057531, "student/entropy": 0.13258436204244692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029166668467223645, "completions/max_length": 501.23333333333335, "completions/max_terminated_length": 455.26666666666665, "completions/mean_length": 190.74556579589844, "completions/mean_terminated_length": 181.19400126139323, "completions/min_length": 48.2, "completions/min_terminated_length": 48.2, "entropy": 0.13169103444864352, "epoch": 0.31557361485588425, "eval/gt_acc_batch": 0.7838889042536418, "frac_reward_zero_std": 1.0, "grad_norm": 0.2878350019454956, "kd/policy_loss": 0.008935515367193147, "kd/rkl_advantage_mean": 0.7181084876259168, "kd/rkl_advantage_p95": 4.957957132657369, "kd/rkl_advantage_std": 2.2619640678167343, "kd/ssd_loss": 0.0, "learning_rate": 6.844643603083583e-07, "loss": 0.03574206034342448, "num_tokens": 271951997.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7838888883590698, "rewards/gt_logging_reward/std": 0.40466095209121705, "sampling/importance_sampling_ratio/max": 1.9629389286041259, "sampling/importance_sampling_ratio/mean": 0.9942821621894836, "sampling/importance_sampling_ratio/min": 0.41048687100410464, "sampling/sampling_logp_difference/max": 0.36530070304870604, "sampling/sampling_logp_difference/mean": 0.004328181517, "step": 8310, "step_time": 8.415632698615081, "student/entropy": 0.13169103444864352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016111112168679633, "completions/max_length": 471.56666666666666, "completions/max_terminated_length": 443.03333333333336, "completions/mean_length": 182.46834259033204, "completions/mean_terminated_length": 177.42541961669923, "completions/min_length": 50.93333333333333, "completions/min_terminated_length": 50.93333333333333, "entropy": 0.1259428852548202, "epoch": 0.3167128697831618, "eval/gt_acc_batch": 0.8177778005599976, "frac_reward_zero_std": 1.0, "grad_norm": 0.27428439259529114, "kd/policy_loss": 0.00825489557852658, "kd/rkl_advantage_mean": 0.6506200798631956, "kd/rkl_advantage_p95": 4.795858526229859, "kd/rkl_advantage_std": 2.182651687661807, "kd/ssd_loss": 0.0, "learning_rate": 6.833251053810808e-07, "loss": 0.03301958640416463, "num_tokens": 272910931.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8177777767181397, "rewards/gt_logging_reward/std": 0.37430750479300817, "sampling/importance_sampling_ratio/max": 1.9026568015416463, "sampling/importance_sampling_ratio/mean": 0.9962528447310129, "sampling/importance_sampling_ratio/min": 0.45954305281241736, "sampling/sampling_logp_difference/max": 0.37396459182103475, "sampling/sampling_logp_difference/mean": 0.0041381387660900755, "step": 8340, "step_time": 8.14997990312598, "student/entropy": 0.1259428852548202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668020188808, "completions/max_length": 501.26666666666665, "completions/max_terminated_length": 457.73333333333335, "completions/mean_length": 188.49000854492186, "completions/mean_terminated_length": 180.44159138997395, "completions/min_length": 60.333333333333336, "completions/min_terminated_length": 60.333333333333336, "entropy": 0.12785292336096366, "epoch": 0.3178521247104394, "eval/gt_acc_batch": 0.8016666869322459, "frac_reward_zero_std": 1.0, "grad_norm": 0.1784721463918686, "kd/policy_loss": 0.00863059035424764, "kd/rkl_advantage_mean": 0.6254451589658856, "kd/rkl_advantage_p95": 4.760281370083491, "kd/rkl_advantage_std": 2.1916444500287375, "kd/ssd_loss": 0.0, "learning_rate": 6.821858504538031e-07, "loss": 0.034522358576456705, "num_tokens": 273889695.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8016666690508525, "rewards/gt_logging_reward/std": 0.3935589780410131, "sampling/importance_sampling_ratio/max": 1.9925379077593486, "sampling/importance_sampling_ratio/mean": 0.997292673587799, "sampling/importance_sampling_ratio/min": 0.4320015569527944, "sampling/sampling_logp_difference/max": 0.38558813333511355, "sampling/sampling_logp_difference/mean": 0.0041940762506177025, "step": 8370, "step_time": 8.192613052867818, "student/entropy": 0.12785292336096366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333482965827, "completions/max_length": 462.03333333333336, "completions/max_terminated_length": 428.6333333333333, "completions/mean_length": 178.7136210123698, "completions/mean_terminated_length": 170.9363260904948, "completions/min_length": 50.666666666666664, "completions/min_terminated_length": 50.666666666666664, "entropy": 0.12815146911889314, "epoch": 0.31899137963771695, "eval/gt_acc_batch": 0.8200000206629435, "frac_reward_zero_std": 1.0, "grad_norm": 0.30205821990966797, "kd/policy_loss": 0.00842558623990044, "kd/rkl_advantage_mean": 0.6699591166650255, "kd/rkl_advantage_p95": 4.9575323839982355, "kd/rkl_advantage_std": 2.2477012405792873, "kd/ssd_loss": 0.0, "learning_rate": 6.810465955265257e-07, "loss": 0.03370234966278076, "num_tokens": 274848720.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.82000000278155, "rewards/gt_logging_reward/std": 0.37882622281710304, "sampling/importance_sampling_ratio/max": 1.8505788127581277, "sampling/importance_sampling_ratio/mean": 0.9951942483584086, "sampling/importance_sampling_ratio/min": 0.43621045102675754, "sampling/sampling_logp_difference/max": 0.4792909324169159, "sampling/sampling_logp_difference/mean": 0.004283950167397658, "step": 8400, "step_time": 8.25512477181231, "student/entropy": 0.12815146911889314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02000000113621354, "completions/max_length": 488.8666666666667, "completions/max_terminated_length": 444.43333333333334, "completions/mean_length": 185.8372319539388, "completions/mean_terminated_length": 179.2657435099284, "completions/min_length": 55.96666666666667, "completions/min_terminated_length": 55.96666666666667, "entropy": 0.12987772847215334, "epoch": 0.3201306345649945, "eval/gt_acc_batch": 0.798333348830541, "frac_reward_zero_std": 1.0, "grad_norm": 0.3064788579940796, "kd/policy_loss": 0.008578827158392718, "kd/rkl_advantage_mean": 0.6666143011301756, "kd/rkl_advantage_p95": 4.855276914437612, "kd/rkl_advantage_std": 2.1976909418900807, "kd/ssd_loss": 0.0, "learning_rate": 6.799073405992481e-07, "loss": 0.034315311908721925, "num_tokens": 275818270.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7983333349227906, "rewards/gt_logging_reward/std": 0.39602545102437336, "sampling/importance_sampling_ratio/max": 1.9320125261942545, "sampling/importance_sampling_ratio/mean": 0.9922675212224324, "sampling/importance_sampling_ratio/min": 0.39782090733448666, "sampling/sampling_logp_difference/max": 0.38894654512405397, "sampling/sampling_logp_difference/mean": 0.004279561868558327, "step": 8430, "step_time": 8.026568830866987, "student/entropy": 0.12987772847215334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223641971747, "completions/max_length": 489.46666666666664, "completions/max_terminated_length": 455.46666666666664, "completions/mean_length": 194.21278737386066, "completions/mean_terminated_length": 185.42237904866536, "completions/min_length": 59.86666666666667, "completions/min_terminated_length": 59.86666666666667, "entropy": 0.1275697650387883, "epoch": 0.32126988949227203, "eval/gt_acc_batch": 0.7855555733044942, "frac_reward_zero_std": 1.0, "grad_norm": 0.2833205759525299, "kd/policy_loss": 0.00856663526695532, "kd/rkl_advantage_mean": 0.652627089060843, "kd/rkl_advantage_p95": 4.738772612810135, "kd/rkl_advantage_std": 2.128523947795232, "kd/ssd_loss": 0.0, "learning_rate": 6.787680856719705e-07, "loss": 0.03426653544108073, "num_tokens": 276813228.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555514494578, "rewards/gt_logging_reward/std": 0.4048611670732498, "sampling/importance_sampling_ratio/max": 1.9869576573371888, "sampling/importance_sampling_ratio/mean": 0.99285413424174, "sampling/importance_sampling_ratio/min": 0.3901004955172539, "sampling/sampling_logp_difference/max": 0.388815446694692, "sampling/sampling_logp_difference/mean": 0.0041831432453667125, "step": 8460, "step_time": 8.019526541062321, "student/entropy": 0.1275697650387883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026666668119529884, "completions/max_length": 466.6666666666667, "completions/max_terminated_length": 439.3666666666667, "completions/mean_length": 182.00362040201824, "completions/mean_terminated_length": 173.09803822835286, "completions/min_length": 53.43333333333333, "completions/min_terminated_length": 53.43333333333333, "entropy": 0.12936283126473427, "epoch": 0.3224091444195496, "eval/gt_acc_batch": 0.8183333476384481, "frac_reward_zero_std": 1.0, "grad_norm": 0.2698460817337036, "kd/policy_loss": 0.008577545986433203, "kd/rkl_advantage_mean": 0.6268694808522317, "kd/rkl_advantage_p95": 4.743229977289835, "kd/rkl_advantage_std": 2.172880221406619, "kd/ssd_loss": 0.0, "learning_rate": 6.776288307446929e-07, "loss": 0.03431018590927124, "num_tokens": 277770489.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8183333337306976, "rewards/gt_logging_reward/std": 0.37817918409903845, "sampling/importance_sampling_ratio/max": 1.9058789451917013, "sampling/importance_sampling_ratio/mean": 0.9927881062030792, "sampling/importance_sampling_ratio/min": 0.44554262856642407, "sampling/sampling_logp_difference/max": 0.4148217936356862, "sampling/sampling_logp_difference/mean": 0.004285782909331222, "step": 8490, "step_time": 7.914119776672063, "student/entropy": 0.12936283126473427 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223511586587, "completions/max_length": 501.93333333333334, "completions/max_terminated_length": 468.3666666666667, "completions/mean_length": 192.94862111409506, "completions/mean_terminated_length": 184.9907480875651, "completions/min_length": 53.63333333333333, "completions/min_terminated_length": 53.63333333333333, "entropy": 0.12570547225574652, "epoch": 0.32354839934682716, "eval/gt_acc_batch": 0.7875000139077505, "frac_reward_zero_std": 1.0, "grad_norm": 0.24622979760169983, "kd/policy_loss": 0.00862590159716395, "kd/rkl_advantage_mean": 0.7666273268638178, "kd/rkl_advantage_p95": 5.038853277762731, "kd/rkl_advantage_std": 2.236299588282903, "kd/ssd_loss": 0.0, "learning_rate": 6.764895758174154e-07, "loss": 0.03450361092885335, "num_tokens": 278771800.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.787500003973643, "rewards/gt_logging_reward/std": 0.40542584160963696, "sampling/importance_sampling_ratio/max": 1.978078297773997, "sampling/importance_sampling_ratio/mean": 1.0015506168206534, "sampling/importance_sampling_ratio/min": 0.4456667939821879, "sampling/sampling_logp_difference/max": 0.36251864234606423, "sampling/sampling_logp_difference/mean": 0.00413245406622688, "step": 8520, "step_time": 8.29042280811506, "student/entropy": 0.12570547225574652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444558893641, "completions/max_length": 482.93333333333334, "completions/max_terminated_length": 444.9, "completions/mean_length": 187.05084279378255, "completions/mean_terminated_length": 179.85850830078124, "completions/min_length": 49.733333333333334, "completions/min_terminated_length": 49.733333333333334, "entropy": 0.12555604142447313, "epoch": 0.32468765427410473, "eval/gt_acc_batch": 0.7886111319065094, "frac_reward_zero_std": 1.0, "grad_norm": 0.2899806797504425, "kd/policy_loss": 0.008494836480046312, "kd/rkl_advantage_mean": 0.710896366473753, "kd/rkl_advantage_p95": 4.920590837796529, "kd/rkl_advantage_std": 2.2340145309766135, "kd/ssd_loss": 0.0, "learning_rate": 6.753503208901377e-07, "loss": 0.03397934436798096, "num_tokens": 279745247.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7886111080646515, "rewards/gt_logging_reward/std": 0.4017337441444397, "sampling/importance_sampling_ratio/max": 1.8342518091201783, "sampling/importance_sampling_ratio/mean": 0.9968911170959472, "sampling/importance_sampling_ratio/min": 0.4346864109237989, "sampling/sampling_logp_difference/max": 0.3759603301684062, "sampling/sampling_logp_difference/mean": 0.004122713720425964, "step": 8550, "step_time": 8.048140358917104, "student/entropy": 0.12555604142447313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500001285225154, "completions/max_length": 475.3, "completions/max_terminated_length": 444.43333333333334, "completions/mean_length": 181.6952880859375, "completions/mean_terminated_length": 175.9866185506185, "completions/min_length": 46.96666666666667, "completions/min_terminated_length": 46.96666666666667, "entropy": 0.1266918442522486, "epoch": 0.3258269092013823, "eval/gt_acc_batch": 0.8202778061230978, "frac_reward_zero_std": 1.0, "grad_norm": 0.2735084891319275, "kd/policy_loss": 0.00836192957746486, "kd/rkl_advantage_mean": 0.6579931124734382, "kd/rkl_advantage_p95": 4.769901090860367, "kd/rkl_advantage_std": 2.19208063185215, "kd/ssd_loss": 0.0, "learning_rate": 6.742110659628603e-07, "loss": 0.03344771862030029, "num_tokens": 280705534.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8202777783075968, "rewards/gt_logging_reward/std": 0.37307341198126476, "sampling/importance_sampling_ratio/max": 1.9243560274442038, "sampling/importance_sampling_ratio/mean": 1.002505115667979, "sampling/importance_sampling_ratio/min": 0.43364440302054086, "sampling/sampling_logp_difference/max": 0.4582654396692912, "sampling/sampling_logp_difference/mean": 0.0041228556695083775, "step": 8580, "step_time": 7.901649216030879, "student/entropy": 0.1266918442522486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668051232895, "completions/max_length": 500.7, "completions/max_terminated_length": 460.56666666666666, "completions/mean_length": 188.99028727213542, "completions/mean_terminated_length": 181.12115681966145, "completions/min_length": 53.733333333333334, "completions/min_terminated_length": 53.733333333333334, "entropy": 0.1317533024897178, "epoch": 0.32696616412865986, "eval/gt_acc_batch": 0.7955555776755016, "frac_reward_zero_std": 1.0, "grad_norm": 0.34527796506881714, "kd/policy_loss": 0.008734636439476163, "kd/rkl_advantage_mean": 0.6765692373427252, "kd/rkl_advantage_p95": 4.828026390075683, "kd/rkl_advantage_std": 2.2019779562950133, "kd/ssd_loss": 0.0, "learning_rate": 6.730718110355828e-07, "loss": 0.034938542048136394, "num_tokens": 281693611.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555578072866, "rewards/gt_logging_reward/std": 0.39107387165228524, "sampling/importance_sampling_ratio/max": 2.0069327394167584, "sampling/importance_sampling_ratio/mean": 1.0024435599644979, "sampling/importance_sampling_ratio/min": 0.3937139679988225, "sampling/sampling_logp_difference/max": 0.39501031835873923, "sampling/sampling_logp_difference/mean": 0.00435396859732767, "step": 8610, "step_time": 8.335681490010272, "student/entropy": 0.1317533024897178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001396983863, "completions/max_length": 483.23333333333335, "completions/max_terminated_length": 437.3666666666667, "completions/mean_length": 184.03306528727214, "completions/mean_terminated_length": 175.75660909016926, "completions/min_length": 53.46666666666667, "completions/min_terminated_length": 53.46666666666667, "entropy": 0.12758857781688374, "epoch": 0.32810541905593743, "eval/gt_acc_batch": 0.8061111251513163, "frac_reward_zero_std": 1.0, "grad_norm": 0.31180712580680847, "kd/policy_loss": 0.008596957839714984, "kd/rkl_advantage_mean": 0.6594313100601236, "kd/rkl_advantage_p95": 4.750219132502874, "kd/rkl_advantage_std": 2.1639523377021153, "kd/ssd_loss": 0.0, "learning_rate": 6.719325561083051e-07, "loss": 0.03438783089319865, "num_tokens": 282650266.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8061111132303874, "rewards/gt_logging_reward/std": 0.38705718914667764, "sampling/importance_sampling_ratio/max": 1.9615538040796916, "sampling/importance_sampling_ratio/mean": 1.0011113663514455, "sampling/importance_sampling_ratio/min": 0.4335600634415944, "sampling/sampling_logp_difference/max": 0.42324379086494446, "sampling/sampling_logp_difference/mean": 0.004143522493541241, "step": 8640, "step_time": 7.904211141654135, "student/entropy": 0.12758857781688374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01465517327446362, "completions/max_length": 475.7241379310345, "completions/max_terminated_length": 443.2413793103448, "completions/mean_length": 185.0603537723936, "completions/mean_terminated_length": 180.2525661073882, "completions/min_length": 56.206896551724135, "completions/min_terminated_length": 56.206896551724135, "entropy": 0.12215300488831668, "epoch": 0.329244673983215, "eval/gt_acc_batch": 0.815517261110503, "frac_reward_zero_std": 1.0, "grad_norm": 0.2989735007286072, "kd/policy_loss": 0.008225711560326403, "kd/rkl_advantage_mean": 0.6570834801773188, "kd/rkl_advantage_p95": 4.815799877561372, "kd/rkl_advantage_std": 2.185256608601274, "kd/ssd_loss": 0.0, "learning_rate": 6.707933011810276e-07, "loss": 0.03180609146753947, "num_tokens": 283587804.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8155172364465122, "rewards/gt_logging_reward/std": 0.38034029202214603, "sampling/importance_sampling_ratio/max": 1.9751177697346127, "sampling/importance_sampling_ratio/mean": 1.0028353682879745, "sampling/importance_sampling_ratio/min": 0.44089852427614146, "sampling/sampling_logp_difference/max": 0.3634093177729639, "sampling/sampling_logp_difference/mean": 0.004027233980917211, "step": 8670, "step_time": 7.733067771679877, "student/entropy": 0.12215300488831668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016944445421298346, "completions/max_length": 488.1333333333333, "completions/max_terminated_length": 446.8, "completions/mean_length": 185.5886210123698, "completions/mean_terminated_length": 179.977663675944, "completions/min_length": 53.3, "completions/min_terminated_length": 53.3, "entropy": 0.12636293284595013, "epoch": 0.33038392891049256, "eval/gt_acc_batch": 0.8047222395737966, "frac_reward_zero_std": 1.0, "grad_norm": 0.28756144642829895, "kd/policy_loss": 0.007992266789854815, "kd/rkl_advantage_mean": 0.7485401768547794, "kd/rkl_advantage_p95": 5.063990251223246, "kd/rkl_advantage_std": 2.2321324408054353, "kd/ssd_loss": 0.0, "learning_rate": 6.6965404625375e-07, "loss": 0.03196906646092733, "num_tokens": 284554987.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222177187602, "rewards/gt_logging_reward/std": 0.38999810020128883, "sampling/importance_sampling_ratio/max": 1.8561342517534891, "sampling/importance_sampling_ratio/mean": 0.9944567700227102, "sampling/importance_sampling_ratio/min": 0.4003780464331309, "sampling/sampling_logp_difference/max": 0.3593200186888377, "sampling/sampling_logp_difference/mean": 0.004189534190421303, "step": 8700, "step_time": 8.081669338210485, "student/entropy": 0.12636293284595013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778965731462, "completions/max_length": 483.8666666666667, "completions/max_terminated_length": 446.7, "completions/mean_length": 189.24501037597656, "completions/mean_terminated_length": 181.92221425374348, "completions/min_length": 62.2, "completions/min_terminated_length": 62.2, "entropy": 0.12661002191404502, "epoch": 0.3315231838377701, "eval/gt_acc_batch": 0.7969444731871287, "frac_reward_zero_std": 1.0, "grad_norm": 0.33669647574424744, "kd/policy_loss": 0.008446681421870987, "kd/rkl_advantage_mean": 0.6703882876162728, "kd/rkl_advantage_p95": 4.841607226928075, "kd/rkl_advantage_std": 2.1675153464078902, "kd/ssd_loss": 0.0, "learning_rate": 6.685147913264724e-07, "loss": 0.03378672997156779, "num_tokens": 285539877.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444473584493, "rewards/gt_logging_reward/std": 0.39350444823503494, "sampling/importance_sampling_ratio/max": 1.8871628085772196, "sampling/importance_sampling_ratio/mean": 1.0006260097026825, "sampling/importance_sampling_ratio/min": 0.440630312760671, "sampling/sampling_logp_difference/max": 0.341574572523435, "sampling/sampling_logp_difference/mean": 0.004156845008643965, "step": 8730, "step_time": 8.06389281974795, "student/entropy": 0.12661002191404502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013333334183941285, "completions/max_length": 456.1666666666667, "completions/max_terminated_length": 415.96666666666664, "completions/mean_length": 177.18889973958332, "completions/mean_terminated_length": 172.8654551188151, "completions/min_length": 58.8, "completions/min_terminated_length": 58.8, "entropy": 0.12926998740683, "epoch": 0.33266243876504764, "eval/gt_acc_batch": 0.8202778021494548, "frac_reward_zero_std": 1.0, "grad_norm": 0.2306824028491974, "kd/policy_loss": 0.008299718197667972, "kd/rkl_advantage_mean": 0.5694848079505997, "kd/rkl_advantage_p95": 4.630170973141988, "kd/rkl_advantage_std": 2.1634669651587806, "kd/ssd_loss": 0.0, "learning_rate": 6.673755363991948e-07, "loss": 0.03319887121518453, "num_tokens": 286475157.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8202777723471324, "rewards/gt_logging_reward/std": 0.3751339892546336, "sampling/importance_sampling_ratio/max": 1.8881141583124796, "sampling/importance_sampling_ratio/mean": 0.99896293481191, "sampling/importance_sampling_ratio/min": 0.4474107414484024, "sampling/sampling_logp_difference/max": 0.35313992500305175, "sampling/sampling_logp_difference/mean": 0.004293617179306845, "step": 8760, "step_time": 7.851646132945704, "student/entropy": 0.12926998740683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223641971747, "completions/max_length": 482.23333333333335, "completions/max_terminated_length": 451.1666666666667, "completions/mean_length": 188.8858444213867, "completions/mean_terminated_length": 180.0051254272461, "completions/min_length": 56.266666666666666, "completions/min_terminated_length": 56.266666666666666, "entropy": 0.12959283174326022, "epoch": 0.3338016936923252, "eval/gt_acc_batch": 0.7905555764834086, "frac_reward_zero_std": 1.0, "grad_norm": 0.2782401442527771, "kd/policy_loss": 0.00859983594273217, "kd/rkl_advantage_mean": 0.6542566049844026, "kd/rkl_advantage_p95": 4.800155123074849, "kd/rkl_advantage_std": 2.1831458896398543, "kd/ssd_loss": 0.0, "learning_rate": 6.662362814719174e-07, "loss": 0.03439934253692627, "num_tokens": 287456706.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7905555526415508, "rewards/gt_logging_reward/std": 0.39903866350650785, "sampling/importance_sampling_ratio/max": 2.022561824321747, "sampling/importance_sampling_ratio/mean": 0.9967817405859629, "sampling/importance_sampling_ratio/min": 0.3830740670363108, "sampling/sampling_logp_difference/max": 0.45671876668930056, "sampling/sampling_logp_difference/mean": 0.004218928104576965, "step": 8790, "step_time": 8.139063032270254, "student/entropy": 0.12959283174326022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02416666829958558, "completions/max_length": 480.8333333333333, "completions/max_terminated_length": 447.3333333333333, "completions/mean_length": 189.6797317504883, "completions/mean_terminated_length": 181.63480072021486, "completions/min_length": 62.06666666666667, "completions/min_terminated_length": 62.06666666666667, "entropy": 0.12951488538334768, "epoch": 0.3349409486196028, "eval/gt_acc_batch": 0.7869444668293, "frac_reward_zero_std": 1.0, "grad_norm": 0.3576202392578125, "kd/policy_loss": 0.008692857139976695, "kd/rkl_advantage_mean": 0.7121804726426489, "kd/rkl_advantage_p95": 4.885150104761124, "kd/rkl_advantage_std": 2.1809640884399415, "kd/ssd_loss": 0.0, "learning_rate": 6.650970265446398e-07, "loss": 0.034771426518758135, "num_tokens": 288441561.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444449742635, "rewards/gt_logging_reward/std": 0.39771625846624375, "sampling/importance_sampling_ratio/max": 1.886244567235311, "sampling/importance_sampling_ratio/mean": 0.9938910722732544, "sampling/importance_sampling_ratio/min": 0.4463439474503199, "sampling/sampling_logp_difference/max": 0.37362276315689086, "sampling/sampling_logp_difference/mean": 0.004255933687090874, "step": 8820, "step_time": 8.231736096103365, "student/entropy": 0.12951488538334768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03000000159566601, "completions/max_length": 502.5, "completions/max_terminated_length": 455.06666666666666, "completions/mean_length": 192.93750966389973, "completions/mean_terminated_length": 183.137717183431, "completions/min_length": 52.0, "completions/min_terminated_length": 52.0, "entropy": 0.12689653125902017, "epoch": 0.33608020354688034, "eval/gt_acc_batch": 0.7866666932900747, "frac_reward_zero_std": 1.0, "grad_norm": 0.22728019952774048, "kd/policy_loss": 0.008711807429790497, "kd/rkl_advantage_mean": 0.7152943244514366, "kd/rkl_advantage_p95": 4.9652286489804585, "kd/rkl_advantage_std": 2.230519098043442, "kd/ssd_loss": 0.0, "learning_rate": 6.639577716173622e-07, "loss": 0.034847227732340495, "num_tokens": 289446464.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666674613952, "rewards/gt_logging_reward/std": 0.3998132675886154, "sampling/importance_sampling_ratio/max": 1.9588260610898336, "sampling/importance_sampling_ratio/mean": 0.9983718554178874, "sampling/importance_sampling_ratio/min": 0.42282937467098236, "sampling/sampling_logp_difference/max": 0.3648748795191447, "sampling/sampling_logp_difference/mean": 0.004169857882273694, "step": 8850, "step_time": 8.323017490162359, "student/entropy": 0.12689653125902017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890389353038, "completions/max_length": 484.3666666666667, "completions/max_terminated_length": 455.56666666666666, "completions/mean_length": 191.95612131754558, "completions/mean_terminated_length": 183.4303009033203, "completions/min_length": 59.266666666666666, "completions/min_terminated_length": 59.266666666666666, "entropy": 0.1319762883707881, "epoch": 0.3372194584741579, "eval/gt_acc_batch": 0.7763889173666636, "frac_reward_zero_std": 1.0, "grad_norm": 0.3590720295906067, "kd/policy_loss": 0.008954666772236426, "kd/rkl_advantage_mean": 0.7488782301545143, "kd/rkl_advantage_p95": 4.980200443665186, "kd/rkl_advantage_std": 2.2150216897328696, "kd/ssd_loss": 0.0, "learning_rate": 6.628185166900847e-07, "loss": 0.03581867218017578, "num_tokens": 290440786.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7763888855775197, "rewards/gt_logging_reward/std": 0.4092646916707357, "sampling/importance_sampling_ratio/max": 1.897650178273519, "sampling/importance_sampling_ratio/mean": 0.9989227732022603, "sampling/importance_sampling_ratio/min": 0.40374892999728523, "sampling/sampling_logp_difference/max": 0.3965316285689672, "sampling/sampling_logp_difference/mean": 0.004295167629607022, "step": 8880, "step_time": 8.243943615832055, "student/entropy": 0.1319762883707881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445713112753, "completions/max_length": 488.06666666666666, "completions/max_terminated_length": 459.8666666666667, "completions/mean_length": 190.010009765625, "completions/mean_terminated_length": 182.87838541666667, "completions/min_length": 56.46666666666667, "completions/min_terminated_length": 56.46666666666667, "entropy": 0.1271217186624805, "epoch": 0.3383587134014355, "eval/gt_acc_batch": 0.8075000226497651, "frac_reward_zero_std": 1.0, "grad_norm": 0.2668098509311676, "kd/policy_loss": 0.008471001511982953, "kd/rkl_advantage_mean": 0.6084519537786642, "kd/rkl_advantage_p95": 4.685510146617889, "kd/rkl_advantage_std": 2.159306467572848, "kd/ssd_loss": 0.0, "learning_rate": 6.616792617628071e-07, "loss": 0.0338840126991272, "num_tokens": 291428710.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8075000027815501, "rewards/gt_logging_reward/std": 0.3905651867389679, "sampling/importance_sampling_ratio/max": 2.0461269418398538, "sampling/importance_sampling_ratio/mean": 1.003346081574758, "sampling/importance_sampling_ratio/min": 0.45734696586926776, "sampling/sampling_logp_difference/max": 0.4023272136847178, "sampling/sampling_logp_difference/mean": 0.0041735503744954865, "step": 8910, "step_time": 8.177752814996833, "student/entropy": 0.1271217186624805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001390775046, "completions/max_length": 482.76666666666665, "completions/max_terminated_length": 445.6333333333333, "completions/mean_length": 185.96417643229168, "completions/mean_terminated_length": 178.544921875, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "entropy": 0.13381755575537682, "epoch": 0.33949796832871304, "eval/gt_acc_batch": 0.8008333563804626, "frac_reward_zero_std": 1.0, "grad_norm": 0.3079426884651184, "kd/policy_loss": 0.009124745948550601, "kd/rkl_advantage_mean": 0.6921505567928156, "kd/rkl_advantage_p95": 4.874881263573965, "kd/rkl_advantage_std": 2.2090082238117854, "kd/ssd_loss": 0.0, "learning_rate": 6.605400068355295e-07, "loss": 0.03649898370107015, "num_tokens": 292392669.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333265781402, "rewards/gt_logging_reward/std": 0.39618745843569436, "sampling/importance_sampling_ratio/max": 1.921631133556366, "sampling/importance_sampling_ratio/mean": 1.0023756702740987, "sampling/importance_sampling_ratio/min": 0.40795307358105976, "sampling/sampling_logp_difference/max": 0.4085219661394755, "sampling/sampling_logp_difference/mean": 0.004373661599432429, "step": 8940, "step_time": 8.169943514233456, "student/entropy": 0.13381755575537682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01500000103066365, "completions/max_length": 468.2, "completions/max_terminated_length": 439.3666666666667, "completions/mean_length": 186.9555643717448, "completions/mean_terminated_length": 182.2327082316081, "completions/min_length": 50.766666666666666, "completions/min_terminated_length": 50.766666666666666, "entropy": 0.1293987731138865, "epoch": 0.3406372232559906, "eval/gt_acc_batch": 0.8072222491105397, "frac_reward_zero_std": 1.0, "grad_norm": 0.23846212029457092, "kd/policy_loss": 0.00873971525967742, "kd/rkl_advantage_mean": 0.6573196187304954, "kd/rkl_advantage_p95": 4.785433608293533, "kd/rkl_advantage_std": 2.156586045026779, "kd/ssd_loss": 0.0, "learning_rate": 6.59400751908252e-07, "loss": 0.03495886325836182, "num_tokens": 293362421.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222262620926, "rewards/gt_logging_reward/std": 0.38333106438318887, "sampling/importance_sampling_ratio/max": 1.934975000222524, "sampling/importance_sampling_ratio/mean": 1.000257913271586, "sampling/importance_sampling_ratio/min": 0.4318603148063024, "sampling/sampling_logp_difference/max": 0.4488362669944763, "sampling/sampling_logp_difference/mean": 0.004218335457456608, "step": 8970, "step_time": 7.994321068314215, "student/entropy": 0.1293987731138865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667765627305, "completions/max_length": 479.3333333333333, "completions/max_terminated_length": 427.1666666666667, "completions/mean_length": 185.44945373535157, "completions/mean_terminated_length": 178.3060267130534, "completions/min_length": 52.56666666666667, "completions/min_terminated_length": 52.56666666666667, "entropy": 0.13491631646951038, "epoch": 0.3417764781832681, "eval/gt_acc_batch": 0.8197222471237182, "frac_reward_zero_std": 1.0, "grad_norm": 0.46091240644454956, "kd/policy_loss": 0.008325056741402175, "kd/rkl_advantage_mean": 0.6438771753261486, "kd/rkl_advantage_p95": 4.855372665325801, "kd/rkl_advantage_std": 2.1953291376431783, "kd/ssd_loss": 0.0, "learning_rate": 6.582614969809745e-07, "loss": 0.033300224939982095, "num_tokens": 294331399.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8197222212950389, "rewards/gt_logging_reward/std": 0.3737477863828341, "sampling/importance_sampling_ratio/max": 1.9432935078938802, "sampling/importance_sampling_ratio/mean": 1.0001396576563517, "sampling/importance_sampling_ratio/min": 0.45445378025372823, "sampling/sampling_logp_difference/max": 0.4072263975938161, "sampling/sampling_logp_difference/mean": 0.004199628330146273, "step": 9000, "step_time": 8.10061858277768, "student/entropy": 0.13491631646951038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778953313828, "completions/max_length": 482.23333333333335, "completions/max_terminated_length": 428.53333333333336, "completions/mean_length": 189.89362131754558, "completions/mean_terminated_length": 184.15562896728517, "completions/min_length": 52.63333333333333, "completions/min_terminated_length": 52.63333333333333, "entropy": 0.1254212811589241, "epoch": 0.3429157331105457, "eval/gt_acc_batch": 0.7838889221350352, "frac_reward_zero_std": 1.0, "grad_norm": 0.3086441457271576, "kd/policy_loss": 0.008148523980829244, "kd/rkl_advantage_mean": 0.6280489933085239, "kd/rkl_advantage_p95": 4.83648251692454, "kd/rkl_advantage_std": 2.2041015048821766, "kd/ssd_loss": 0.0, "learning_rate": 6.571222420536968e-07, "loss": 0.032594090700149535, "num_tokens": 295327144.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7838888903458913, "rewards/gt_logging_reward/std": 0.40477747122446694, "sampling/importance_sampling_ratio/max": 1.8843031843503317, "sampling/importance_sampling_ratio/mean": 0.9921175003051758, "sampling/importance_sampling_ratio/min": 0.44654210408528644, "sampling/sampling_logp_difference/max": 0.4032543659210205, "sampling/sampling_logp_difference/mean": 0.00413291829948624, "step": 9030, "step_time": 8.363437591737602, "student/entropy": 0.1254212811589241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01861111239219705, "completions/max_length": 491.3333333333333, "completions/max_terminated_length": 456.6, "completions/mean_length": 183.68417561848958, "completions/mean_terminated_length": 177.5930658976237, "completions/min_length": 47.03333333333333, "completions/min_terminated_length": 47.03333333333333, "entropy": 0.13182379926244417, "epoch": 0.34405498803782325, "eval/gt_acc_batch": 0.8011111358801524, "frac_reward_zero_std": 1.0, "grad_norm": 0.2587132155895233, "kd/policy_loss": 0.009006756231732046, "kd/rkl_advantage_mean": 0.6711975915997754, "kd/rkl_advantage_p95": 4.796699341138204, "kd/rkl_advantage_std": 2.188013333082199, "kd/ssd_loss": 0.0, "learning_rate": 6.559829871264193e-07, "loss": 0.036027026176452634, "num_tokens": 296299791.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8011111080646515, "rewards/gt_logging_reward/std": 0.38936307430267336, "sampling/importance_sampling_ratio/max": 1.887475355466207, "sampling/importance_sampling_ratio/mean": 1.0012987474600474, "sampling/importance_sampling_ratio/min": 0.4040193140506744, "sampling/sampling_logp_difference/max": 0.38825971881548565, "sampling/sampling_logp_difference/mean": 0.0043161343472699325, "step": 9060, "step_time": 8.320210897736251, "student/entropy": 0.13182379926244417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01916666766628623, "completions/max_length": 475.46666666666664, "completions/max_terminated_length": 444.56666666666666, "completions/mean_length": 184.8322326660156, "completions/mean_terminated_length": 178.61439005533853, "completions/min_length": 52.3, "completions/min_terminated_length": 52.3, "entropy": 0.13092005519817274, "epoch": 0.3451942429651008, "eval/gt_acc_batch": 0.80055557290713, "frac_reward_zero_std": 1.0, "grad_norm": 0.2776658535003662, "kd/policy_loss": 0.00885129823582247, "kd/rkl_advantage_mean": 0.6356165125655632, "kd/rkl_advantage_p95": 4.761682732899984, "kd/rkl_advantage_std": 2.183447990814845, "kd/ssd_loss": 0.0, "learning_rate": 6.548437321991417e-07, "loss": 0.035405194759368895, "num_tokens": 297274795.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.800555557012558, "rewards/gt_logging_reward/std": 0.3911796083052953, "sampling/importance_sampling_ratio/max": 1.9102790236473084, "sampling/importance_sampling_ratio/mean": 1.0014369825522105, "sampling/importance_sampling_ratio/min": 0.4164133980870247, "sampling/sampling_logp_difference/max": 0.37316996455192564, "sampling/sampling_logp_difference/mean": 0.0042652916784087815, "step": 9090, "step_time": 8.245698942181965, "student/entropy": 0.13092005519817274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668051232895, "completions/max_length": 499.3, "completions/max_terminated_length": 443.26666666666665, "completions/mean_length": 188.71556498209637, "completions/mean_terminated_length": 180.861865234375, "completions/min_length": 55.266666666666666, "completions/min_terminated_length": 55.266666666666666, "entropy": 0.12890561819076538, "epoch": 0.3463334978923784, "eval/gt_acc_batch": 0.8041666865348815, "frac_reward_zero_std": 1.0, "grad_norm": 0.287801593542099, "kd/policy_loss": 0.008728590194368734, "kd/rkl_advantage_mean": 0.6650229601189495, "kd/rkl_advantage_p95": 4.870977316300074, "kd/rkl_advantage_std": 2.1969819257656735, "kd/ssd_loss": 0.0, "learning_rate": 6.537044772718641e-07, "loss": 0.03491436243057251, "num_tokens": 298259963.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666686534882, "rewards/gt_logging_reward/std": 0.3914580777287483, "sampling/importance_sampling_ratio/max": 1.8678146282831827, "sampling/importance_sampling_ratio/mean": 0.9956495920817058, "sampling/importance_sampling_ratio/min": 0.43996299505233766, "sampling/sampling_logp_difference/max": 0.37470380067825315, "sampling/sampling_logp_difference/mean": 0.004261499332884947, "step": 9120, "step_time": 8.391516423415547, "student/entropy": 0.12890561819076538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334798614182, "completions/max_length": 484.76666666666665, "completions/max_terminated_length": 462.46666666666664, "completions/mean_length": 188.17501017252604, "completions/mean_terminated_length": 180.4718022664388, "completions/min_length": 52.06666666666667, "completions/min_terminated_length": 52.06666666666667, "entropy": 0.13524126249055068, "epoch": 0.34747275281965595, "eval/gt_acc_batch": 0.7822222451368968, "frac_reward_zero_std": 1.0, "grad_norm": 0.3709876537322998, "kd/policy_loss": 0.00920374914809751, "kd/rkl_advantage_mean": 0.76872334900933, "kd/rkl_advantage_p95": 5.057345676422119, "kd/rkl_advantage_std": 2.2645326435565947, "kd/ssd_loss": 0.0, "learning_rate": 6.525652223445866e-07, "loss": 0.03681499163309733, "num_tokens": 299236313.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7822222232818603, "rewards/gt_logging_reward/std": 0.40362928559382755, "sampling/importance_sampling_ratio/max": 1.8849472085634866, "sampling/importance_sampling_ratio/mean": 0.9995686014493307, "sampling/importance_sampling_ratio/min": 0.4413154751062393, "sampling/sampling_logp_difference/max": 0.3650614023208618, "sampling/sampling_logp_difference/mean": 0.004365714422116677, "step": 9150, "step_time": 8.18719573849424, "student/entropy": 0.13524126249055068 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333423982064, "completions/max_length": 485.03333333333336, "completions/max_terminated_length": 454.6, "completions/mean_length": 185.55973256429036, "completions/mean_terminated_length": 177.80599110921224, "completions/min_length": 52.56666666666667, "completions/min_terminated_length": 52.56666666666667, "entropy": 0.1345903476079305, "epoch": 0.3486120077469335, "eval/gt_acc_batch": 0.7961111227671306, "frac_reward_zero_std": 1.0, "grad_norm": 0.31358328461647034, "kd/policy_loss": 0.008836147404508665, "kd/rkl_advantage_mean": 0.6459995245716225, "kd/rkl_advantage_p95": 4.792645688851675, "kd/rkl_advantage_std": 2.2186480422814685, "kd/ssd_loss": 0.0, "learning_rate": 6.514259674173091e-07, "loss": 0.035344592730204266, "num_tokens": 300201248.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111148198445, "rewards/gt_logging_reward/std": 0.3925876274704933, "sampling/importance_sampling_ratio/max": 1.9299854358037314, "sampling/importance_sampling_ratio/mean": 0.9989327768484751, "sampling/importance_sampling_ratio/min": 0.41303906639417015, "sampling/sampling_logp_difference/max": 0.37383585969607036, "sampling/sampling_logp_difference/mean": 0.004358282700801889, "step": 9180, "step_time": 8.219485089858063, "student/entropy": 0.1345903476079305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556789040567, "completions/max_length": 494.1333333333333, "completions/max_terminated_length": 459.6333333333333, "completions/mean_length": 184.59334208170574, "completions/mean_terminated_length": 177.01032155354818, "completions/min_length": 48.166666666666664, "completions/min_terminated_length": 48.166666666666664, "entropy": 0.1298273166641593, "epoch": 0.3497512626742111, "eval/gt_acc_batch": 0.7894444584846496, "frac_reward_zero_std": 1.0, "grad_norm": 0.2816925048828125, "kd/policy_loss": 0.008748850511619822, "kd/rkl_advantage_mean": 0.6325677173600222, "kd/rkl_advantage_p95": 4.716289190451304, "kd/rkl_advantage_std": 2.1376844227313994, "kd/ssd_loss": 0.0, "learning_rate": 6.502867124900316e-07, "loss": 0.034995404879252116, "num_tokens": 301165712.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7894444445768992, "rewards/gt_logging_reward/std": 0.3980559875567754, "sampling/importance_sampling_ratio/max": 1.9008180816968283, "sampling/importance_sampling_ratio/mean": 1.0034698784351348, "sampling/importance_sampling_ratio/min": 0.41833346436421076, "sampling/sampling_logp_difference/max": 0.36410555243492126, "sampling/sampling_logp_difference/mean": 0.004256947905135652, "step": 9210, "step_time": 8.329688590369187, "student/entropy": 0.1298273166641593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890004406372, "completions/max_length": 468.26666666666665, "completions/max_terminated_length": 428.8333333333333, "completions/mean_length": 186.89139760335286, "completions/mean_terminated_length": 179.79624837239584, "completions/min_length": 49.1, "completions/min_terminated_length": 49.1, "entropy": 0.12749013031522433, "epoch": 0.35089051760148865, "eval/gt_acc_batch": 0.8008333444595337, "frac_reward_zero_std": 1.0, "grad_norm": 0.31007659435272217, "kd/policy_loss": 0.008355037652654574, "kd/rkl_advantage_mean": 0.657732292295744, "kd/rkl_advantage_p95": 4.776065043608347, "kd/rkl_advantage_std": 2.1498479783535003, "kd/ssd_loss": 0.0, "learning_rate": 6.491474575627539e-07, "loss": 0.03342014948527018, "num_tokens": 302146049.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333345254263, "rewards/gt_logging_reward/std": 0.39380994339783987, "sampling/importance_sampling_ratio/max": 1.8663376291592917, "sampling/importance_sampling_ratio/mean": 1.000386267900467, "sampling/importance_sampling_ratio/min": 0.41665874073902764, "sampling/sampling_logp_difference/max": 0.35977547566095985, "sampling/sampling_logp_difference/mean": 0.004130745042736332, "step": 9240, "step_time": 8.197469069669022, "student/entropy": 0.12749013031522433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166667989144724, "completions/max_length": 475.3666666666667, "completions/max_terminated_length": 447.06666666666666, "completions/mean_length": 185.19056549072266, "completions/mean_terminated_length": 177.26368357340496, "completions/min_length": 53.7, "completions/min_terminated_length": 53.7, "entropy": 0.133661797704796, "epoch": 0.35202977252876616, "eval/gt_acc_batch": 0.7777778009573618, "frac_reward_zero_std": 1.0, "grad_norm": 0.2714308202266693, "kd/policy_loss": 0.009325351705774665, "kd/rkl_advantage_mean": 0.6939681196119636, "kd/rkl_advantage_p95": 4.942061366637548, "kd/rkl_advantage_std": 2.2540754874547324, "kd/ssd_loss": 0.0, "learning_rate": 6.480082026354764e-07, "loss": 0.03730141321818034, "num_tokens": 303112191.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.777777777115504, "rewards/gt_logging_reward/std": 0.4081997801860174, "sampling/importance_sampling_ratio/max": 1.9407877643903098, "sampling/importance_sampling_ratio/mean": 1.0079192340373992, "sampling/importance_sampling_ratio/min": 0.44201667110125226, "sampling/sampling_logp_difference/max": 0.35082286794980366, "sampling/sampling_logp_difference/mean": 0.004375100694596767, "step": 9270, "step_time": 8.121871920869065, "student/entropy": 0.133661797704796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333445712924, "completions/max_length": 478.6333333333333, "completions/max_terminated_length": 438.46666666666664, "completions/mean_length": 189.10139872233074, "completions/mean_terminated_length": 181.68348236083983, "completions/min_length": 59.4, "completions/min_terminated_length": 59.4, "entropy": 0.13047293461859227, "epoch": 0.3531690274560437, "eval/gt_acc_batch": 0.8055555760860443, "frac_reward_zero_std": 1.0, "grad_norm": 0.24611951410770416, "kd/policy_loss": 0.00853938504587859, "kd/rkl_advantage_mean": 0.6977147083060118, "kd/rkl_advantage_p95": 4.8433394948641455, "kd/rkl_advantage_std": 2.1831346998612084, "kd/ssd_loss": 0.0, "learning_rate": 6.468689477081988e-07, "loss": 0.034157538414001466, "num_tokens": 304098236.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8055555502573649, "rewards/gt_logging_reward/std": 0.38850513895352684, "sampling/importance_sampling_ratio/max": 1.8928034106890361, "sampling/importance_sampling_ratio/mean": 0.9941564639409383, "sampling/importance_sampling_ratio/min": 0.4235199640194575, "sampling/sampling_logp_difference/max": 0.35629377563794457, "sampling/sampling_logp_difference/mean": 0.004163155130421122, "step": 9300, "step_time": 8.10758480411411, "student/entropy": 0.13047293461859227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02722222392136852, "completions/max_length": 475.6666666666667, "completions/max_terminated_length": 446.43333333333334, "completions/mean_length": 189.8466766357422, "completions/mean_terminated_length": 180.94353332519532, "completions/min_length": 55.06666666666667, "completions/min_terminated_length": 55.06666666666667, "entropy": 0.12814508496473234, "epoch": 0.3543082823833213, "eval/gt_acc_batch": 0.7775000214576722, "frac_reward_zero_std": 1.0, "grad_norm": 0.3473929166793823, "kd/policy_loss": 0.008719173081529637, "kd/rkl_advantage_mean": 0.6683686429013809, "kd/rkl_advantage_p95": 4.826034231980642, "kd/rkl_advantage_std": 2.1841811150312425, "kd/ssd_loss": 0.0, "learning_rate": 6.457296927809212e-07, "loss": 0.034876692295074466, "num_tokens": 305087636.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7775000035762787, "rewards/gt_logging_reward/std": 0.40620450526475904, "sampling/importance_sampling_ratio/max": 1.912766933441162, "sampling/importance_sampling_ratio/mean": 1.0001560906569162, "sampling/importance_sampling_ratio/min": 0.4233213598529498, "sampling/sampling_logp_difference/max": 0.41106130679448444, "sampling/sampling_logp_difference/mean": 0.0042153894901275635, "step": 9330, "step_time": 8.281981782704436, "student/entropy": 0.12814508496473234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0269444456944863, "completions/max_length": 497.3333333333333, "completions/max_terminated_length": 466.46666666666664, "completions/mean_length": 189.01250915527345, "completions/mean_terminated_length": 180.22821960449218, "completions/min_length": 54.6, "completions/min_terminated_length": 54.6, "entropy": 0.13169871295491856, "epoch": 0.35544753731059886, "eval/gt_acc_batch": 0.795555579662323, "frac_reward_zero_std": 1.0, "grad_norm": 0.38635683059692383, "kd/policy_loss": 0.009149243445911755, "kd/rkl_advantage_mean": 0.7161016612003247, "kd/rkl_advantage_p95": 4.974159638086955, "kd/rkl_advantage_std": 2.2399903506040575, "kd/ssd_loss": 0.0, "learning_rate": 6.445904378536436e-07, "loss": 0.036596973737080894, "num_tokens": 306079145.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555558204651, "rewards/gt_logging_reward/std": 0.3988635162512461, "sampling/importance_sampling_ratio/max": 1.9010874191919962, "sampling/importance_sampling_ratio/mean": 0.999707160393397, "sampling/importance_sampling_ratio/min": 0.38848243554433187, "sampling/sampling_logp_difference/max": 0.3742696662743886, "sampling/sampling_logp_difference/mean": 0.004331180298080047, "step": 9360, "step_time": 8.500752118125092, "student/entropy": 0.13169871295491856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001428027947, "completions/max_length": 493.96666666666664, "completions/max_terminated_length": 462.8666666666667, "completions/mean_length": 194.90528767903646, "completions/mean_terminated_length": 186.95475209554036, "completions/min_length": 55.2, "completions/min_terminated_length": 55.2, "entropy": 0.12660903359452882, "epoch": 0.3565867922378764, "eval/gt_acc_batch": 0.7916666825612386, "frac_reward_zero_std": 1.0, "grad_norm": 0.3097769021987915, "kd/policy_loss": 0.008473035041242838, "kd/rkl_advantage_mean": 0.5819644222036005, "kd/rkl_advantage_p95": 4.695436841249466, "kd/rkl_advantage_std": 2.1663837254047396, "kd/ssd_loss": 0.0, "learning_rate": 6.434511829263662e-07, "loss": 0.03389214674631755, "num_tokens": 307085340.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7916666646798451, "rewards/gt_logging_reward/std": 0.3986374964316686, "sampling/importance_sampling_ratio/max": 1.937364363670349, "sampling/importance_sampling_ratio/mean": 0.9974105854829153, "sampling/importance_sampling_ratio/min": 0.42100464502970375, "sampling/sampling_logp_difference/max": 0.38091036876042683, "sampling/sampling_logp_difference/mean": 0.004104954570842286, "step": 9390, "step_time": 8.426375470974017, "student/entropy": 0.12660903359452882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166667989144724, "completions/max_length": 493.4, "completions/max_terminated_length": 454.0, "completions/mean_length": 187.19473215738932, "completions/mean_terminated_length": 179.3586451212565, "completions/min_length": 54.8, "completions/min_terminated_length": 54.8, "entropy": 0.1332708147043983, "epoch": 0.357726047165154, "eval/gt_acc_batch": 0.8019444564978282, "frac_reward_zero_std": 1.0, "grad_norm": 0.27597299218177795, "kd/policy_loss": 0.008820178626532045, "kd/rkl_advantage_mean": 0.7215134064046045, "kd/rkl_advantage_p95": 4.935378134250641, "kd/rkl_advantage_std": 2.221444809436798, "kd/ssd_loss": 0.0, "learning_rate": 6.423119279990886e-07, "loss": 0.035280712445576984, "num_tokens": 308065897.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444406032562, "rewards/gt_logging_reward/std": 0.3845115855336189, "sampling/importance_sampling_ratio/max": 1.9273186564445495, "sampling/importance_sampling_ratio/mean": 0.9955881555875142, "sampling/importance_sampling_ratio/min": 0.38406532605489097, "sampling/sampling_logp_difference/max": 0.411193044980367, "sampling/sampling_logp_difference/mean": 0.004280142361919085, "step": 9420, "step_time": 8.406993264394503, "student/entropy": 0.1332708147043983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02583333505317569, "completions/max_length": 484.53333333333336, "completions/max_terminated_length": 428.8666666666667, "completions/mean_length": 183.7336201985677, "completions/mean_terminated_length": 175.10066680908204, "completions/min_length": 52.96666666666667, "completions/min_terminated_length": 52.96666666666667, "entropy": 0.1259868914571901, "epoch": 0.35886530209243156, "eval/gt_acc_batch": 0.8047222415606181, "frac_reward_zero_std": 1.0, "grad_norm": 0.23642845451831818, "kd/policy_loss": 0.008446283609373495, "kd/rkl_advantage_mean": 0.6746477019662659, "kd/rkl_advantage_p95": 4.860642115275065, "kd/rkl_advantage_std": 2.1904714703559875, "kd/ssd_loss": 0.0, "learning_rate": 6.41172673071811e-07, "loss": 0.033785136540730794, "num_tokens": 309036106.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222256660461, "rewards/gt_logging_reward/std": 0.388948118686676, "sampling/importance_sampling_ratio/max": 1.8903741240501404, "sampling/importance_sampling_ratio/mean": 0.9937155286471049, "sampling/importance_sampling_ratio/min": 0.4160972754160563, "sampling/sampling_logp_difference/max": 0.3876836061477661, "sampling/sampling_logp_difference/mean": 0.004130588060555359, "step": 9450, "step_time": 8.398102629549491, "student/entropy": 0.1259868914571901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028333334904164075, "completions/max_length": 483.8666666666667, "completions/max_terminated_length": 445.46666666666664, "completions/mean_length": 189.77528686523436, "completions/mean_terminated_length": 180.5810979207357, "completions/min_length": 58.5, "completions/min_terminated_length": 58.5, "entropy": 0.13478964579602082, "epoch": 0.3600045570197091, "eval/gt_acc_batch": 0.7866666853427887, "frac_reward_zero_std": 1.0, "grad_norm": 0.31807762384414673, "kd/policy_loss": 0.009065636868278186, "kd/rkl_advantage_mean": 0.7391211165736119, "kd/rkl_advantage_p95": 4.886943592627843, "kd/rkl_advantage_std": 2.1811495810747146, "kd/ssd_loss": 0.0, "learning_rate": 6.400334181445335e-07, "loss": 0.03626254399617513, "num_tokens": 310020593.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666694482167, "rewards/gt_logging_reward/std": 0.4022507021824519, "sampling/importance_sampling_ratio/max": 1.8695498029390971, "sampling/importance_sampling_ratio/mean": 0.9980003078778584, "sampling/importance_sampling_ratio/min": 0.4240652432044347, "sampling/sampling_logp_difference/max": 0.37015681465466815, "sampling/sampling_logp_difference/mean": 0.004379474340627591, "step": 9480, "step_time": 8.245613798611641, "student/entropy": 0.13478964579602082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0233333347675701, "completions/max_length": 500.26666666666665, "completions/max_terminated_length": 458.43333333333334, "completions/mean_length": 186.6872319539388, "completions/mean_terminated_length": 179.145166015625, "completions/min_length": 60.53333333333333, "completions/min_terminated_length": 60.53333333333333, "entropy": 0.12981611688931782, "epoch": 0.3611438119469867, "eval/gt_acc_batch": 0.7866666773955028, "frac_reward_zero_std": 1.0, "grad_norm": 0.26110631227493286, "kd/policy_loss": 0.008487485640216619, "kd/rkl_advantage_mean": 0.8014332528536519, "kd/rkl_advantage_p95": 5.02556734085083, "kd/rkl_advantage_std": 2.2485829989115396, "kd/ssd_loss": 0.0, "learning_rate": 6.388941632172558e-07, "loss": 0.03394994338353475, "num_tokens": 310996747.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666674613952, "rewards/gt_logging_reward/std": 0.40047278106212614, "sampling/importance_sampling_ratio/max": 1.9332031965255738, "sampling/importance_sampling_ratio/mean": 0.993144420782725, "sampling/importance_sampling_ratio/min": 0.44345046480496725, "sampling/sampling_logp_difference/max": 0.3874913175900777, "sampling/sampling_logp_difference/mean": 0.0042376039394487934, "step": 9510, "step_time": 8.332020094249552, "student/entropy": 0.12981611688931782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01861111233010888, "completions/max_length": 462.8, "completions/max_terminated_length": 431.53333333333336, "completions/mean_length": 183.13140004475912, "completions/mean_terminated_length": 177.01922912597655, "completions/min_length": 51.9, "completions/min_terminated_length": 51.9, "entropy": 0.13161723017692567, "epoch": 0.3622830668742642, "eval/gt_acc_batch": 0.7855555772781372, "frac_reward_zero_std": 1.0, "grad_norm": 0.2264113575220108, "kd/policy_loss": 0.009053950652014465, "kd/rkl_advantage_mean": 0.7328890686544279, "kd/rkl_advantage_p95": 4.945240958531698, "kd/rkl_advantage_std": 2.206369627515475, "kd/ssd_loss": 0.0, "learning_rate": 6.377549082899783e-07, "loss": 0.036215806007385255, "num_tokens": 311966428.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555574099223, "rewards/gt_logging_reward/std": 0.40231440563996634, "sampling/importance_sampling_ratio/max": 1.9194711168607077, "sampling/importance_sampling_ratio/mean": 1.0002658228079477, "sampling/importance_sampling_ratio/min": 0.43808894554773964, "sampling/sampling_logp_difference/max": 0.3976186235745748, "sampling/sampling_logp_difference/mean": 0.0042328283889219165, "step": 9540, "step_time": 8.350950199237559, "student/entropy": 0.13161723017692567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888889998197555, "completions/max_length": 480.0, "completions/max_terminated_length": 448.6333333333333, "completions/mean_length": 187.38362147013348, "completions/mean_terminated_length": 181.3318967183431, "completions/min_length": 49.56666666666667, "completions/min_terminated_length": 49.56666666666667, "entropy": 0.12906674252202113, "epoch": 0.36342232180154177, "eval/gt_acc_batch": 0.8025000135103861, "frac_reward_zero_std": 1.0, "grad_norm": 0.21750858426094055, "kd/policy_loss": 0.008429230359615758, "kd/rkl_advantage_mean": 0.7529655702412128, "kd/rkl_advantage_p95": 4.9720430235068, "kd/rkl_advantage_std": 2.2233044395844144, "kd/ssd_loss": 0.0, "learning_rate": 6.366156533627008e-07, "loss": 0.033716924985249835, "num_tokens": 312940321.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8025000015894572, "rewards/gt_logging_reward/std": 0.38901411394278207, "sampling/importance_sampling_ratio/max": 1.8935015002886455, "sampling/importance_sampling_ratio/mean": 0.9970864832401276, "sampling/importance_sampling_ratio/min": 0.41628940800825753, "sampling/sampling_logp_difference/max": 0.4016576488812765, "sampling/sampling_logp_difference/mean": 0.004203814050803582, "step": 9570, "step_time": 8.164428117126226, "student/entropy": 0.12906674252202113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444596146544, "completions/max_length": 482.3333333333333, "completions/max_terminated_length": 450.43333333333334, "completions/mean_length": 187.108620707194, "completions/mean_terminated_length": 179.8906682332357, "completions/min_length": 56.733333333333334, "completions/min_terminated_length": 56.733333333333334, "entropy": 0.1274147034312288, "epoch": 0.36456157672881934, "eval/gt_acc_batch": 0.8100000242392222, "frac_reward_zero_std": 1.0, "grad_norm": 0.2903934717178345, "kd/policy_loss": 0.008450969201900686, "kd/rkl_advantage_mean": 0.6795555861666799, "kd/rkl_advantage_p95": 4.913840295871099, "kd/rkl_advantage_std": 2.206829917430878, "kd/ssd_loss": 0.0, "learning_rate": 6.354763984354232e-07, "loss": 0.03380387624104818, "num_tokens": 313918104.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8099999944368999, "rewards/gt_logging_reward/std": 0.38297681162754693, "sampling/importance_sampling_ratio/max": 1.9153131127357483, "sampling/importance_sampling_ratio/mean": 0.9934942960739136, "sampling/importance_sampling_ratio/min": 0.4420156409343084, "sampling/sampling_logp_difference/max": 0.4360279440879822, "sampling/sampling_logp_difference/mean": 0.004174494370818138, "step": 9600, "step_time": 8.206924693771482, "student/entropy": 0.1274147034312288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02222222350537777, "completions/max_length": 491.4, "completions/max_terminated_length": 443.23333333333335, "completions/mean_length": 187.79056447347006, "completions/mean_terminated_length": 180.577294921875, "completions/min_length": 54.6, "completions/min_terminated_length": 54.6, "entropy": 0.13150816597044468, "epoch": 0.3657008316560969, "eval/gt_acc_batch": 0.8022222419579824, "frac_reward_zero_std": 1.0, "grad_norm": 0.4527110755443573, "kd/policy_loss": 0.008835546396827947, "kd/rkl_advantage_mean": 0.6466934797198822, "kd/rkl_advantage_p95": 4.84856506784757, "kd/rkl_advantage_std": 2.212957883874575, "kd/ssd_loss": 0.0, "learning_rate": 6.343371435081456e-07, "loss": 0.03534218470255534, "num_tokens": 314887486.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8022222240765889, "rewards/gt_logging_reward/std": 0.3909742956360181, "sampling/importance_sampling_ratio/max": 1.9889745712280273, "sampling/importance_sampling_ratio/mean": 0.9998695154984792, "sampling/importance_sampling_ratio/min": 0.40135740439097084, "sampling/sampling_logp_difference/max": 0.3845667362213135, "sampling/sampling_logp_difference/mean": 0.004193580981033544, "step": 9630, "step_time": 8.22805117207269, "student/entropy": 0.13150816597044468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02555555704360207, "completions/max_length": 486.6, "completions/max_terminated_length": 449.8666666666667, "completions/mean_length": 195.80223236083984, "completions/mean_terminated_length": 187.84988454182943, "completions/min_length": 49.166666666666664, "completions/min_terminated_length": 49.166666666666664, "entropy": 0.12793788177271684, "epoch": 0.36684008658337447, "eval/gt_acc_batch": 0.769444465637207, "frac_reward_zero_std": 1.0, "grad_norm": 0.27880409359931946, "kd/policy_loss": 0.008468073880067095, "kd/rkl_advantage_mean": 0.7700247859427084, "kd/rkl_advantage_p95": 4.972950311501821, "kd/rkl_advantage_std": 2.202912465731303, "kd/ssd_loss": 0.0, "learning_rate": 6.331978885808681e-07, "loss": 0.03387229442596436, "num_tokens": 315892030.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7694444398085276, "rewards/gt_logging_reward/std": 0.41239723761876423, "sampling/importance_sampling_ratio/max": 1.9893386761347454, "sampling/importance_sampling_ratio/mean": 0.9970775326093038, "sampling/importance_sampling_ratio/min": 0.4341450870037079, "sampling/sampling_logp_difference/max": 0.38411622246106464, "sampling/sampling_logp_difference/mean": 0.004134661556842427, "step": 9660, "step_time": 8.255183985363692, "student/entropy": 0.12793788177271684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02388889032105605, "completions/max_length": 472.06666666666666, "completions/max_terminated_length": 441.6666666666667, "completions/mean_length": 186.61195576985676, "completions/mean_terminated_length": 178.65365651448568, "completions/min_length": 49.5, "completions/min_terminated_length": 49.5, "entropy": 0.13046905342489482, "epoch": 0.36797934151065204, "eval/gt_acc_batch": 0.7930555681387583, "frac_reward_zero_std": 1.0, "grad_norm": 0.27260613441467285, "kd/policy_loss": 0.008860151091357692, "kd/rkl_advantage_mean": 0.6591877078947922, "kd/rkl_advantage_p95": 4.845076235135396, "kd/rkl_advantage_std": 2.2051877230405807, "kd/ssd_loss": 0.0, "learning_rate": 6.320586336535906e-07, "loss": 0.03544060389200846, "num_tokens": 316859953.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555522441864, "rewards/gt_logging_reward/std": 0.3932260269920031, "sampling/importance_sampling_ratio/max": 1.8667312383651733, "sampling/importance_sampling_ratio/mean": 0.9995024303595225, "sampling/importance_sampling_ratio/min": 0.42264144817988075, "sampling/sampling_logp_difference/max": 0.3771143893400828, "sampling/sampling_logp_difference/mean": 0.004256996512413025, "step": 9690, "step_time": 8.02468842509358, "student/entropy": 0.13046905342489482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02250000135973096, "completions/max_length": 488.5, "completions/max_terminated_length": 463.23333333333335, "completions/mean_length": 191.95445353190104, "completions/mean_terminated_length": 184.623882039388, "completions/min_length": 48.6, "completions/min_terminated_length": 48.6, "entropy": 0.12544294483959675, "epoch": 0.3691185964379296, "eval/gt_acc_batch": 0.7952777981758118, "frac_reward_zero_std": 1.0, "grad_norm": 0.3248385488986969, "kd/policy_loss": 0.008660057093948126, "kd/rkl_advantage_mean": 0.684173948628207, "kd/rkl_advantage_p95": 4.957046578327815, "kd/rkl_advantage_std": 2.2364496260881426, "kd/ssd_loss": 0.0, "learning_rate": 6.309193787263129e-07, "loss": 0.034640232721964516, "num_tokens": 317857757.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777723471324, "rewards/gt_logging_reward/std": 0.39913299183050793, "sampling/importance_sampling_ratio/max": 1.9571431477864583, "sampling/importance_sampling_ratio/mean": 0.9994183123111725, "sampling/importance_sampling_ratio/min": 0.43334222535292305, "sampling/sampling_logp_difference/max": 0.37234841187795004, "sampling/sampling_logp_difference/mean": 0.004089299278954665, "step": 9720, "step_time": 8.404127637762576, "student/entropy": 0.12544294483959675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667641450962, "completions/max_length": 479.56666666666666, "completions/max_terminated_length": 447.6, "completions/mean_length": 182.15695495605468, "completions/mean_terminated_length": 174.95840199788412, "completions/min_length": 56.3, "completions/min_terminated_length": 56.3, "entropy": 0.13400048421074948, "epoch": 0.37025785136520717, "eval/gt_acc_batch": 0.8094444751739502, "frac_reward_zero_std": 1.0, "grad_norm": 0.25068312883377075, "kd/policy_loss": 0.008765360026154667, "kd/rkl_advantage_mean": 0.7229058289900422, "kd/rkl_advantage_p95": 4.968716514110565, "kd/rkl_advantage_std": 2.2319500784079236, "kd/ssd_loss": 0.0, "learning_rate": 6.297801237990355e-07, "loss": 0.03506143887837728, "num_tokens": 318814434.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8094444453716279, "rewards/gt_logging_reward/std": 0.3846282680829366, "sampling/importance_sampling_ratio/max": 1.8931588570276896, "sampling/importance_sampling_ratio/mean": 0.999732498327891, "sampling/importance_sampling_ratio/min": 0.4198591609795888, "sampling/sampling_logp_difference/max": 0.37176328102747597, "sampling/sampling_logp_difference/mean": 0.00431956029497087, "step": 9750, "step_time": 8.143168282861977, "student/entropy": 0.13400048421074948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444558893641, "completions/max_length": 481.03333333333336, "completions/max_terminated_length": 435.9, "completions/mean_length": 184.41778564453125, "completions/mean_terminated_length": 177.20860290527344, "completions/min_length": 50.666666666666664, "completions/min_terminated_length": 50.666666666666664, "entropy": 0.12405492809290687, "epoch": 0.37139710629248474, "eval/gt_acc_batch": 0.8091666877269745, "frac_reward_zero_std": 1.0, "grad_norm": 0.26208576560020447, "kd/policy_loss": 0.008014675714851668, "kd/rkl_advantage_mean": 0.7431144814938306, "kd/rkl_advantage_p95": 4.996762901544571, "kd/rkl_advantage_std": 2.2202061623334886, "kd/ssd_loss": 0.0, "learning_rate": 6.286408688717579e-07, "loss": 0.03205870787302653, "num_tokens": 319774714.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.809166669845581, "rewards/gt_logging_reward/std": 0.38235337684551873, "sampling/importance_sampling_ratio/max": 1.9204445997873942, "sampling/importance_sampling_ratio/mean": 1.00126656293869, "sampling/importance_sampling_ratio/min": 0.45836518704891205, "sampling/sampling_logp_difference/max": 0.37431111335754397, "sampling/sampling_logp_difference/mean": 0.00403997964070489, "step": 9780, "step_time": 8.110478018115584, "student/entropy": 0.12405492809290687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01611111219972372, "completions/max_length": 458.3666666666667, "completions/max_terminated_length": 420.93333333333334, "completions/mean_length": 182.8052874247233, "completions/mean_terminated_length": 177.46178665161133, "completions/min_length": 56.7, "completions/min_terminated_length": 56.7, "entropy": 0.13145444554587205, "epoch": 0.3725363612197623, "eval/gt_acc_batch": 0.8150000254313151, "frac_reward_zero_std": 1.0, "grad_norm": 0.3131003975868225, "kd/policy_loss": 0.008640028615870203, "kd/rkl_advantage_mean": 0.6463978946985056, "kd/rkl_advantage_p95": 4.7554896434148155, "kd/rkl_advantage_std": 2.1571982463200885, "kd/ssd_loss": 0.0, "learning_rate": 6.275016139444803e-07, "loss": 0.034560108184814455, "num_tokens": 320736037.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8149999976158142, "rewards/gt_logging_reward/std": 0.3770480225483576, "sampling/importance_sampling_ratio/max": 1.8777260899543762, "sampling/importance_sampling_ratio/mean": 0.9948222021261851, "sampling/importance_sampling_ratio/min": 0.47452540894349415, "sampling/sampling_logp_difference/max": 0.3661513090133667, "sampling/sampling_logp_difference/mean": 0.004219039025095602, "step": 9810, "step_time": 7.889701761169515, "student/entropy": 0.13145444554587205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030555557397504648, "completions/max_length": 502.1, "completions/max_terminated_length": 453.03333333333336, "completions/mean_length": 196.26028747558593, "completions/mean_terminated_length": 186.37083536783854, "completions/min_length": 49.36666666666667, "completions/min_terminated_length": 49.36666666666667, "entropy": 0.13344640346864858, "epoch": 0.3736756161470398, "eval/gt_acc_batch": 0.7783333480358123, "frac_reward_zero_std": 1.0, "grad_norm": 0.23301827907562256, "kd/policy_loss": 0.008788678174217543, "kd/rkl_advantage_mean": 0.7809964195514719, "kd/rkl_advantage_p95": 5.111612468957901, "kd/rkl_advantage_std": 2.247116985917091, "kd/ssd_loss": 0.0, "learning_rate": 6.263623590172027e-07, "loss": 0.03515472014745077, "num_tokens": 321749894.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7783333321412405, "rewards/gt_logging_reward/std": 0.407735471924146, "sampling/importance_sampling_ratio/max": 1.9431749860445657, "sampling/importance_sampling_ratio/mean": 0.9951545377572377, "sampling/importance_sampling_ratio/min": 0.4263415902853012, "sampling/sampling_logp_difference/max": 0.35411285360654193, "sampling/sampling_logp_difference/mean": 0.004295963848320147, "step": 9840, "step_time": 8.266102849540767, "student/entropy": 0.13344640346864858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334426085154, "completions/max_length": 491.23333333333335, "completions/max_terminated_length": 452.1333333333333, "completions/mean_length": 192.54362030029296, "completions/mean_terminated_length": 185.11483103434244, "completions/min_length": 53.03333333333333, "completions/min_terminated_length": 53.03333333333333, "entropy": 0.1281501216813922, "epoch": 0.3748148710743174, "eval/gt_acc_batch": 0.772777799765269, "frac_reward_zero_std": 1.0, "grad_norm": 0.32173681259155273, "kd/policy_loss": 0.00838592100966101, "kd/rkl_advantage_mean": 0.7101572992901007, "kd/rkl_advantage_p95": 4.934435325860977, "kd/rkl_advantage_std": 2.2035057326157887, "kd/ssd_loss": 0.0, "learning_rate": 6.252231040899252e-07, "loss": 0.033543686072031655, "num_tokens": 322742627.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7727777779102325, "rewards/gt_logging_reward/std": 0.40975583692391715, "sampling/importance_sampling_ratio/max": 1.9260729908943177, "sampling/importance_sampling_ratio/mean": 0.996148532629013, "sampling/importance_sampling_ratio/min": 0.4186852087577184, "sampling/sampling_logp_difference/max": 0.38078646461168925, "sampling/sampling_logp_difference/mean": 0.004149565170519054, "step": 9870, "step_time": 8.282067395692382, "student/entropy": 0.1281501216813922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334798614182, "completions/max_length": 500.1333333333333, "completions/max_terminated_length": 458.4, "completions/mean_length": 194.48861999511718, "completions/mean_terminated_length": 187.01885325113932, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.13451248041043679, "epoch": 0.37595412600159495, "eval/gt_acc_batch": 0.7738889078299205, "frac_reward_zero_std": 1.0, "grad_norm": 0.31938645243644714, "kd/policy_loss": 0.00912278784865824, "kd/rkl_advantage_mean": 0.6920032396912574, "kd/rkl_advantage_p95": 4.877023730675379, "kd/rkl_advantage_std": 2.20730052391688, "kd/ssd_loss": 0.0, "learning_rate": 6.240838491626475e-07, "loss": 0.03649114767710368, "num_tokens": 323742474.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7738888879617055, "rewards/gt_logging_reward/std": 0.4118642409642537, "sampling/importance_sampling_ratio/max": 2.007888146241506, "sampling/importance_sampling_ratio/mean": 0.9968001782894135, "sampling/importance_sampling_ratio/min": 0.39600727905829747, "sampling/sampling_logp_difference/max": 0.4670242210229238, "sampling/sampling_logp_difference/mean": 0.004281714372336864, "step": 9900, "step_time": 8.376614935928956, "student/entropy": 0.13451248041043679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016111111951371035, "completions/max_length": 474.73333333333335, "completions/max_terminated_length": 440.1333333333333, "completions/mean_length": 185.07084147135416, "completions/mean_terminated_length": 179.7723592122396, "completions/min_length": 49.46666666666667, "completions/min_terminated_length": 49.46666666666667, "entropy": 0.1310834500938654, "epoch": 0.3770933809288725, "eval/gt_acc_batch": 0.7922222475210826, "frac_reward_zero_std": 1.0, "grad_norm": 0.2839756906032562, "kd/policy_loss": 0.008929014718160034, "kd/rkl_advantage_mean": 0.7055610371753573, "kd/rkl_advantage_p95": 4.846905070543289, "kd/rkl_advantage_std": 2.183122459053993, "kd/ssd_loss": 0.0, "learning_rate": 6.2294459423537e-07, "loss": 0.03571605682373047, "num_tokens": 324709353.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222216924032, "rewards/gt_logging_reward/std": 0.40013818244139354, "sampling/importance_sampling_ratio/max": 1.906126085917155, "sampling/importance_sampling_ratio/mean": 0.9985102017720541, "sampling/importance_sampling_ratio/min": 0.4232762138048808, "sampling/sampling_logp_difference/max": 0.3585298418998718, "sampling/sampling_logp_difference/mean": 0.004237664025276899, "step": 9930, "step_time": 8.154406209317191, "student/entropy": 0.1310834500938654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030277779325842857, "completions/max_length": 486.3666666666667, "completions/max_terminated_length": 444.26666666666665, "completions/mean_length": 193.0711212158203, "completions/mean_terminated_length": 183.22935587565104, "completions/min_length": 46.233333333333334, "completions/min_terminated_length": 46.233333333333334, "entropy": 0.13445791322737932, "epoch": 0.3782326358561501, "eval/gt_acc_batch": 0.7888889114061991, "frac_reward_zero_std": 1.0, "grad_norm": 0.3098655343055725, "kd/policy_loss": 0.009096783812856302, "kd/rkl_advantage_mean": 0.7015199278694733, "kd/rkl_advantage_p95": 4.867603911956151, "kd/rkl_advantage_std": 2.196913417180379, "kd/ssd_loss": 0.0, "learning_rate": 6.218053393080925e-07, "loss": 0.03638713359832764, "num_tokens": 325709385.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7888888855775197, "rewards/gt_logging_reward/std": 0.40138972202936807, "sampling/importance_sampling_ratio/max": 1.9480564792950947, "sampling/importance_sampling_ratio/mean": 0.9978017389774323, "sampling/importance_sampling_ratio/min": 0.4050105080008507, "sampling/sampling_logp_difference/max": 0.36892584959665936, "sampling/sampling_logp_difference/mean": 0.00428771705677112, "step": 9960, "step_time": 8.212174342293292, "student/entropy": 0.13445791322737932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778996775546, "completions/max_length": 489.06666666666666, "completions/max_terminated_length": 459.06666666666666, "completions/mean_length": 192.6358434041341, "completions/mean_terminated_length": 185.0674840291341, "completions/min_length": 54.7, "completions/min_terminated_length": 54.7, "entropy": 0.12343849763274192, "epoch": 0.37937189078342765, "eval/gt_acc_batch": 0.7947222371896108, "frac_reward_zero_std": 1.0, "grad_norm": 0.31597113609313965, "kd/policy_loss": 0.008433795580640436, "kd/rkl_advantage_mean": 0.7190398591104895, "kd/rkl_advantage_p95": 4.927952102820078, "kd/rkl_advantage_std": 2.2250452746947604, "kd/ssd_loss": 0.0, "learning_rate": 6.206660843808149e-07, "loss": 0.03373518387476603, "num_tokens": 326710082.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222212950389, "rewards/gt_logging_reward/std": 0.3931838974356651, "sampling/importance_sampling_ratio/max": 1.940947941939036, "sampling/importance_sampling_ratio/mean": 0.9977596402168274, "sampling/importance_sampling_ratio/min": 0.4043866559863091, "sampling/sampling_logp_difference/max": 0.39347648521264394, "sampling/sampling_logp_difference/mean": 0.003971061048408349, "step": 9990, "step_time": 8.216579726350028, "student/entropy": 0.12343849763274192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779282381136, "completions/max_length": 479.76666666666665, "completions/max_terminated_length": 453.0, "completions/mean_length": 185.3175084431966, "completions/mean_terminated_length": 176.9407704671224, "completions/min_length": 53.6, "completions/min_terminated_length": 53.6, "entropy": 0.12489491999149323, "epoch": 0.3805111457107052, "eval/gt_acc_batch": 0.7969444672266642, "frac_reward_zero_std": 1.0, "grad_norm": 0.2321690022945404, "kd/policy_loss": 0.008339020662242546, "kd/rkl_advantage_mean": 0.7345149738093217, "kd/rkl_advantage_p95": 4.980962312221527, "kd/rkl_advantage_std": 2.2169794837633767, "kd/ssd_loss": 0.0, "learning_rate": 6.195268294535374e-07, "loss": 0.033356086413065596, "num_tokens": 327684137.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444433848063, "rewards/gt_logging_reward/std": 0.3933699478705724, "sampling/importance_sampling_ratio/max": 1.8897527694702148, "sampling/importance_sampling_ratio/mean": 1.0035442729791006, "sampling/importance_sampling_ratio/min": 0.4623557021220525, "sampling/sampling_logp_difference/max": 0.34225456714630126, "sampling/sampling_logp_difference/mean": 0.004014853178523481, "step": 10020, "step_time": 8.270922718724856, "student/entropy": 0.12489491999149323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223455707233, "completions/max_length": 492.8, "completions/max_terminated_length": 446.26666666666665, "completions/mean_length": 191.29806569417318, "completions/mean_terminated_length": 182.66209513346354, "completions/min_length": 53.13333333333333, "completions/min_terminated_length": 53.13333333333333, "entropy": 0.1242199797804157, "epoch": 0.3816504006379828, "eval/gt_acc_batch": 0.8055555760860443, "frac_reward_zero_std": 1.0, "grad_norm": 0.25575730204582214, "kd/policy_loss": 0.00811742366834854, "kd/rkl_advantage_mean": 0.7123264652987321, "kd/rkl_advantage_p95": 4.924403454860052, "kd/rkl_advantage_std": 2.1964634944995245, "kd/ssd_loss": 0.0, "learning_rate": 6.183875745262598e-07, "loss": 0.03246969779332479, "num_tokens": 328669850.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8055555522441864, "rewards/gt_logging_reward/std": 0.38956208725770314, "sampling/importance_sampling_ratio/max": 1.9209080497423807, "sampling/importance_sampling_ratio/mean": 1.0025264024734497, "sampling/importance_sampling_ratio/min": 0.4312830130259196, "sampling/sampling_logp_difference/max": 0.4031056781609853, "sampling/sampling_logp_difference/mean": 0.004052581319895884, "step": 10050, "step_time": 8.19740539599831, "student/entropy": 0.1242199797804157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556522061427, "completions/max_length": 477.7, "completions/max_terminated_length": 441.43333333333334, "completions/mean_length": 179.40973307291668, "completions/mean_terminated_length": 174.1570930480957, "completions/min_length": 52.4, "completions/min_terminated_length": 52.4, "entropy": 0.12652920155475536, "epoch": 0.38278965556526034, "eval/gt_acc_batch": 0.8147222379843394, "frac_reward_zero_std": 1.0, "grad_norm": 0.23832790553569794, "kd/policy_loss": 0.008377545200831567, "kd/rkl_advantage_mean": 0.6914118158786248, "kd/rkl_advantage_p95": 4.8867443362871805, "kd/rkl_advantage_std": 2.2129261285066604, "kd/ssd_loss": 0.0, "learning_rate": 6.172483195989822e-07, "loss": 0.03351018031438192, "num_tokens": 329611149.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8147222240765889, "rewards/gt_logging_reward/std": 0.38156190812587737, "sampling/importance_sampling_ratio/max": 1.930700163046519, "sampling/importance_sampling_ratio/mean": 0.998485134045283, "sampling/importance_sampling_ratio/min": 0.4519537885983785, "sampling/sampling_logp_difference/max": 0.3678448021411896, "sampling/sampling_logp_difference/mean": 0.0041094814582417405, "step": 10080, "step_time": 7.987613461337363, "student/entropy": 0.12652920155475536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015000000906487307, "completions/max_length": 477.46666666666664, "completions/max_terminated_length": 448.4, "completions/mean_length": 190.323898824056, "completions/mean_terminated_length": 185.529665629069, "completions/min_length": 62.766666666666666, "completions/min_terminated_length": 62.766666666666666, "entropy": 0.12530008647590876, "epoch": 0.38392891049253786, "eval/gt_acc_batch": 0.8169444620609283, "frac_reward_zero_std": 1.0, "grad_norm": 0.3986154794692993, "kd/policy_loss": 0.008167816283336531, "kd/rkl_advantage_mean": 0.5976809051508705, "kd/rkl_advantage_p95": 4.688899372021357, "kd/rkl_advantage_std": 2.1312288880348205, "kd/ssd_loss": 0.0, "learning_rate": 6.161090646717046e-07, "loss": 0.03267126679420471, "num_tokens": 330602107.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8169444461663564, "rewards/gt_logging_reward/std": 0.38069140712420146, "sampling/importance_sampling_ratio/max": 1.9561619758605957, "sampling/importance_sampling_ratio/mean": 0.9918095648288727, "sampling/importance_sampling_ratio/min": 0.42063508729139965, "sampling/sampling_logp_difference/max": 0.398466153939565, "sampling/sampling_logp_difference/mean": 0.0040523367002606395, "step": 10110, "step_time": 8.233962846104987, "student/entropy": 0.12530008647590876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01722222308938702, "completions/max_length": 472.1333333333333, "completions/max_terminated_length": 445.8666666666667, "completions/mean_length": 185.90528767903646, "completions/mean_terminated_length": 180.39632771809895, "completions/min_length": 47.2, "completions/min_terminated_length": 47.2, "entropy": 0.12697908735523622, "epoch": 0.3850681654198154, "eval/gt_acc_batch": 0.7952778081099192, "frac_reward_zero_std": 1.0, "grad_norm": 0.2450903058052063, "kd/policy_loss": 0.00861721879725034, "kd/rkl_advantage_mean": 0.7205763199439389, "kd/rkl_advantage_p95": 4.880520806709925, "kd/rkl_advantage_std": 2.181394581993421, "kd/ssd_loss": 0.0, "learning_rate": 6.149698097444272e-07, "loss": 0.03446887334187825, "num_tokens": 331576494.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777763207753, "rewards/gt_logging_reward/std": 0.39568873941898347, "sampling/importance_sampling_ratio/max": 1.9337709625562032, "sampling/importance_sampling_ratio/mean": 1.0059771756331126, "sampling/importance_sampling_ratio/min": 0.45979283303022384, "sampling/sampling_logp_difference/max": 0.34328846335411073, "sampling/sampling_logp_difference/mean": 0.0041526952292770146, "step": 10140, "step_time": 8.267443561627685, "student/entropy": 0.12697908735523622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02555555695046981, "completions/max_length": 486.46666666666664, "completions/max_terminated_length": 444.23333333333335, "completions/mean_length": 187.49445546468098, "completions/mean_terminated_length": 179.03060760498047, "completions/min_length": 51.53333333333333, "completions/min_terminated_length": 51.53333333333333, "entropy": 0.12802983671426774, "epoch": 0.386207420347093, "eval/gt_acc_batch": 0.7969444553057353, "frac_reward_zero_std": 1.0, "grad_norm": 0.31947869062423706, "kd/policy_loss": 0.00867235321784392, "kd/rkl_advantage_mean": 0.6759752164284388, "kd/rkl_advantage_p95": 4.906816260019938, "kd/rkl_advantage_std": 2.2060528606176377, "kd/ssd_loss": 0.0, "learning_rate": 6.138305548171496e-07, "loss": 0.03468941847483317, "num_tokens": 332557042.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444453716278, "rewards/gt_logging_reward/std": 0.39634432395299274, "sampling/importance_sampling_ratio/max": 1.878971564769745, "sampling/importance_sampling_ratio/mean": 1.001742168267568, "sampling/importance_sampling_ratio/min": 0.38673013548056284, "sampling/sampling_logp_difference/max": 0.37305970589319865, "sampling/sampling_logp_difference/mean": 0.0041400702747826775, "step": 10170, "step_time": 8.324218877331198, "student/entropy": 0.12802983671426774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500001223136982, "completions/max_length": 482.8666666666667, "completions/max_terminated_length": 448.96666666666664, "completions/mean_length": 187.5069544474284, "completions/mean_terminated_length": 181.83792826334636, "completions/min_length": 57.06666666666667, "completions/min_terminated_length": 57.06666666666667, "entropy": 0.12697829399257898, "epoch": 0.38734667527437056, "eval/gt_acc_batch": 0.804166688521703, "frac_reward_zero_std": 1.0, "grad_norm": 0.2994507849216461, "kd/policy_loss": 0.008319904703724509, "kd/rkl_advantage_mean": 0.7224339076628288, "kd/rkl_advantage_p95": 4.9260926425457, "kd/rkl_advantage_std": 2.199181161324183, "kd/ssd_loss": 0.0, "learning_rate": 6.12691299889872e-07, "loss": 0.03327962358792623, "num_tokens": 333534491.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666666666667, "rewards/gt_logging_reward/std": 0.3889688104391098, "sampling/importance_sampling_ratio/max": 1.9056219140688577, "sampling/importance_sampling_ratio/mean": 0.9997601052125294, "sampling/importance_sampling_ratio/min": 0.42182309925556183, "sampling/sampling_logp_difference/max": 0.36165018479029337, "sampling/sampling_logp_difference/mean": 0.00411458876915276, "step": 10200, "step_time": 8.131652444709713, "student/entropy": 0.12697829399257898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556826293468, "completions/max_length": 492.1333333333333, "completions/max_terminated_length": 453.43333333333334, "completions/mean_length": 195.04001057942708, "completions/mean_terminated_length": 186.84317220052083, "completions/min_length": 59.666666666666664, "completions/min_terminated_length": 59.666666666666664, "entropy": 0.12655156745264928, "epoch": 0.3884859302016481, "eval/gt_acc_batch": 0.7855555792649587, "frac_reward_zero_std": 1.0, "grad_norm": 0.2493259608745575, "kd/policy_loss": 0.008427743144178141, "kd/rkl_advantage_mean": 0.6948126898457606, "kd/rkl_advantage_p95": 4.9485011061032615, "kd/rkl_advantage_std": 2.2344351450602216, "kd/ssd_loss": 0.0, "learning_rate": 6.115520449625944e-07, "loss": 0.03371097246805827, "num_tokens": 334541339.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7855555633703868, "rewards/gt_logging_reward/std": 0.4039842625459035, "sampling/importance_sampling_ratio/max": 1.8600865443547567, "sampling/importance_sampling_ratio/mean": 0.9938223004341126, "sampling/importance_sampling_ratio/min": 0.4558413833379745, "sampling/sampling_logp_difference/max": 0.3778587241967519, "sampling/sampling_logp_difference/mean": 0.0040210429656629765, "step": 10230, "step_time": 8.250182303999706, "student/entropy": 0.12655156745264928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03472222428147991, "completions/max_length": 482.73333333333335, "completions/max_terminated_length": 444.73333333333335, "completions/mean_length": 191.9763982137044, "completions/mean_terminated_length": 180.6290481567383, "completions/min_length": 54.43333333333333, "completions/min_terminated_length": 54.43333333333333, "entropy": 0.12902700199435155, "epoch": 0.3896251851289257, "eval/gt_acc_batch": 0.7744444688161214, "frac_reward_zero_std": 1.0, "grad_norm": 0.2781360447406769, "kd/policy_loss": 0.008976664431005095, "kd/rkl_advantage_mean": 0.7903159651129196, "kd/rkl_advantage_p95": 5.0526085575421655, "kd/rkl_advantage_std": 2.244813864429792, "kd/ssd_loss": 0.0, "learning_rate": 6.104127900353169e-07, "loss": 0.035906656583150225, "num_tokens": 335546830.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.774444446961085, "rewards/gt_logging_reward/std": 0.40676483511924744, "sampling/importance_sampling_ratio/max": 1.9644323348999024, "sampling/importance_sampling_ratio/mean": 1.0102351943651835, "sampling/importance_sampling_ratio/min": 0.44939912259578707, "sampling/sampling_logp_difference/max": 0.3913691242535909, "sampling/sampling_logp_difference/mean": 0.004154381869981686, "step": 10260, "step_time": 8.354230338754132, "student/entropy": 0.12902700199435155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334705481927, "completions/max_length": 489.8333333333333, "completions/max_terminated_length": 455.8333333333333, "completions/mean_length": 188.3997319539388, "completions/mean_terminated_length": 180.69031982421876, "completions/min_length": 52.8, "completions/min_terminated_length": 52.8, "entropy": 0.12628398487965267, "epoch": 0.39076444005620325, "eval/gt_acc_batch": 0.7872222542762757, "frac_reward_zero_std": 1.0, "grad_norm": 0.26478201150894165, "kd/policy_loss": 0.008441130966336157, "kd/rkl_advantage_mean": 0.7879686736812194, "kd/rkl_advantage_p95": 5.058080005645752, "kd/rkl_advantage_std": 2.245703478654226, "kd/ssd_loss": 0.0, "learning_rate": 6.092735351080393e-07, "loss": 0.03376452922821045, "num_tokens": 336530941.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7872222224871318, "rewards/gt_logging_reward/std": 0.40412157972653706, "sampling/importance_sampling_ratio/max": 1.9847240130106607, "sampling/importance_sampling_ratio/mean": 0.9955563644568125, "sampling/importance_sampling_ratio/min": 0.40349572400252026, "sampling/sampling_logp_difference/max": 0.3871663471062978, "sampling/sampling_logp_difference/mean": 0.004089852864854038, "step": 10290, "step_time": 8.294336703182974, "student/entropy": 0.12628398487965267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223759939273, "completions/max_length": 483.8333333333333, "completions/max_terminated_length": 448.03333333333336, "completions/mean_length": 189.35139923095704, "completions/mean_terminated_length": 181.5300715128581, "completions/min_length": 57.1, "completions/min_terminated_length": 57.1, "entropy": 0.1261238032951951, "epoch": 0.3919036949834808, "eval/gt_acc_batch": 0.7994444648424784, "frac_reward_zero_std": 1.0, "grad_norm": 0.4802449643611908, "kd/policy_loss": 0.008257585397223011, "kd/rkl_advantage_mean": 0.6362483878697579, "kd/rkl_advantage_p95": 4.8008205711841585, "kd/rkl_advantage_std": 2.1885229219992954, "kd/ssd_loss": 0.0, "learning_rate": 6.081342801807617e-07, "loss": 0.033030347029368086, "num_tokens": 337513766.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.799444442987442, "rewards/gt_logging_reward/std": 0.3903897196054459, "sampling/importance_sampling_ratio/max": 1.8780599673589071, "sampling/importance_sampling_ratio/mean": 0.9952785432338714, "sampling/importance_sampling_ratio/min": 0.3958033618827661, "sampling/sampling_logp_difference/max": 0.5941510299841563, "sampling/sampling_logp_difference/mean": 0.004130534168022375, "step": 10320, "step_time": 8.143811307494373, "student/entropy": 0.1261238032951951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890476276476, "completions/max_length": 468.43333333333334, "completions/max_terminated_length": 433.26666666666665, "completions/mean_length": 187.5100102742513, "completions/mean_terminated_length": 179.8037338256836, "completions/min_length": 57.1, "completions/min_terminated_length": 57.1, "entropy": 0.13294159881770612, "epoch": 0.3930429499107584, "eval/gt_acc_batch": 0.7988889018694559, "frac_reward_zero_std": 1.0, "grad_norm": 0.2596026360988617, "kd/policy_loss": 0.009348773337357367, "kd/rkl_advantage_mean": 0.6829862282727845, "kd/rkl_advantage_p95": 4.806303509076437, "kd/rkl_advantage_std": 2.182272347807884, "kd/ssd_loss": 0.0, "learning_rate": 6.069950252534843e-07, "loss": 0.03739509185155233, "num_tokens": 338486130.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.798888885974884, "rewards/gt_logging_reward/std": 0.38875184108813604, "sampling/importance_sampling_ratio/max": 2.041721733411153, "sampling/importance_sampling_ratio/mean": 0.9983010629812876, "sampling/importance_sampling_ratio/min": 0.39211993118127186, "sampling/sampling_logp_difference/max": 0.4021190007527669, "sampling/sampling_logp_difference/mean": 0.0042422649761041, "step": 10350, "step_time": 7.90422823264574, "student/entropy": 0.13294159881770612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334866911174, "completions/max_length": 491.96666666666664, "completions/max_terminated_length": 452.9, "completions/mean_length": 196.6063990275065, "completions/mean_terminated_length": 188.25635681152343, "completions/min_length": 54.53333333333333, "completions/min_terminated_length": 54.53333333333333, "entropy": 0.12412393266956011, "epoch": 0.3941822048380359, "eval/gt_acc_batch": 0.7980555772781373, "frac_reward_zero_std": 1.0, "grad_norm": 0.299937903881073, "kd/policy_loss": 0.008200121785436446, "kd/rkl_advantage_mean": 0.7375374026286106, "kd/rkl_advantage_p95": 4.981474326054255, "kd/rkl_advantage_std": 2.219531876842181, "kd/ssd_loss": 0.0, "learning_rate": 6.058557703262066e-07, "loss": 0.032800483703613284, "num_tokens": 339495505.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7980555574099223, "rewards/gt_logging_reward/std": 0.39063335458437604, "sampling/importance_sampling_ratio/max": 1.8451335668563842, "sampling/importance_sampling_ratio/mean": 1.000986115137736, "sampling/importance_sampling_ratio/min": 0.40850267906983695, "sampling/sampling_logp_difference/max": 0.3902754247188568, "sampling/sampling_logp_difference/mean": 0.004027543659321964, "step": 10380, "step_time": 8.236227757619538, "student/entropy": 0.12412393266956011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02888889058182637, "completions/max_length": 498.1333333333333, "completions/max_terminated_length": 453.56666666666666, "completions/mean_length": 193.67084299723308, "completions/mean_terminated_length": 184.268679300944, "completions/min_length": 55.8, "completions/min_terminated_length": 55.8, "entropy": 0.12769989718993505, "epoch": 0.39532145976531347, "eval/gt_acc_batch": 0.7922222356001536, "frac_reward_zero_std": 1.0, "grad_norm": 0.2254512906074524, "kd/policy_loss": 0.008641013136366383, "kd/rkl_advantage_mean": 0.7566630424155544, "kd/rkl_advantage_p95": 5.049440878629684, "kd/rkl_advantage_std": 2.2569124668836595, "kd/ssd_loss": 0.0, "learning_rate": 6.047165153989291e-07, "loss": 0.03456405798594157, "num_tokens": 340499280.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222256660462, "rewards/gt_logging_reward/std": 0.39891810715198517, "sampling/importance_sampling_ratio/max": 1.9558592240015666, "sampling/importance_sampling_ratio/mean": 0.9976052006085714, "sampling/importance_sampling_ratio/min": 0.43653740907708805, "sampling/sampling_logp_difference/max": 0.33924348950386046, "sampling/sampling_logp_difference/mean": 0.004150114712926248, "step": 10410, "step_time": 8.418215082407308, "student/entropy": 0.12769989718993505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019444445644815764, "completions/max_length": 484.1666666666667, "completions/max_terminated_length": 447.6333333333333, "completions/mean_length": 193.13167826334634, "completions/mean_terminated_length": 186.9132512410482, "completions/min_length": 58.36666666666667, "completions/min_terminated_length": 58.36666666666667, "entropy": 0.13044219029446444, "epoch": 0.39646071469259103, "eval/gt_acc_batch": 0.7852777977784474, "frac_reward_zero_std": 1.0, "grad_norm": 0.260187029838562, "kd/policy_loss": 0.008849827916128561, "kd/rkl_advantage_mean": 0.7837284609675408, "kd/rkl_advantage_p95": 5.084891470273336, "kd/rkl_advantage_std": 2.237830387552579, "kd/ssd_loss": 0.0, "learning_rate": 6.035772604716515e-07, "loss": 0.03539931376775106, "num_tokens": 341505858.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7852777739365896, "rewards/gt_logging_reward/std": 0.40534499088923137, "sampling/importance_sampling_ratio/max": 1.9373135050137837, "sampling/importance_sampling_ratio/mean": 1.0003085672855376, "sampling/importance_sampling_ratio/min": 0.4221147398153941, "sampling/sampling_logp_difference/max": 0.3763553977012634, "sampling/sampling_logp_difference/mean": 0.004201953198450307, "step": 10440, "step_time": 8.218302009592298, "student/entropy": 0.13044219029446444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779505898555, "completions/max_length": 495.3333333333333, "completions/max_terminated_length": 455.8333333333333, "completions/mean_length": 188.92084299723308, "completions/mean_terminated_length": 179.72516784667968, "completions/min_length": 46.13333333333333, "completions/min_terminated_length": 46.13333333333333, "entropy": 0.12828426553557318, "epoch": 0.3975999696198686, "eval/gt_acc_batch": 0.8025000294049581, "frac_reward_zero_std": 1.0, "grad_norm": 0.3363610506057739, "kd/policy_loss": 0.008382120749835546, "kd/rkl_advantage_mean": 0.7918566590795915, "kd/rkl_advantage_p95": 5.148655641078949, "kd/rkl_advantage_std": 2.2626444419225056, "kd/ssd_loss": 0.0, "learning_rate": 6.024380055443739e-07, "loss": 0.03352848688761393, "num_tokens": 342500141.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8024999976158143, "rewards/gt_logging_reward/std": 0.38850411723057426, "sampling/importance_sampling_ratio/max": 1.8759971817334493, "sampling/importance_sampling_ratio/mean": 0.994582058986028, "sampling/importance_sampling_ratio/min": 0.41367944578329724, "sampling/sampling_logp_difference/max": 0.37943705519040427, "sampling/sampling_logp_difference/mean": 0.004206137157355746, "step": 10470, "step_time": 8.471915021845295, "student/entropy": 0.12828426553557318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667734583218, "completions/max_length": 479.8, "completions/max_terminated_length": 446.03333333333336, "completions/mean_length": 187.4488983154297, "completions/mean_terminated_length": 180.25680643717448, "completions/min_length": 58.766666666666666, "completions/min_terminated_length": 58.766666666666666, "entropy": 0.12673202647517126, "epoch": 0.39873922454714616, "eval/gt_acc_batch": 0.8125000278155009, "frac_reward_zero_std": 1.0, "grad_norm": 0.25298401713371277, "kd/policy_loss": 0.008066516858525575, "kd/rkl_advantage_mean": 0.7252732593566179, "kd/rkl_advantage_p95": 4.921247969071071, "kd/rkl_advantage_std": 2.2072092284758886, "kd/ssd_loss": 0.0, "learning_rate": 6.012987506170963e-07, "loss": 0.03226606647173564, "num_tokens": 343476861.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8125000079472859, "rewards/gt_logging_reward/std": 0.3833318630854289, "sampling/importance_sampling_ratio/max": 1.935193125406901, "sampling/importance_sampling_ratio/mean": 0.9993380347887675, "sampling/importance_sampling_ratio/min": 0.42956757446130117, "sampling/sampling_logp_difference/max": 0.38223926424980165, "sampling/sampling_logp_difference/mean": 0.00408207622046272, "step": 10500, "step_time": 8.151610345432225, "student/entropy": 0.12673202647517126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890041659276, "completions/max_length": 497.1333333333333, "completions/max_terminated_length": 459.43333333333334, "completions/mean_length": 193.55278828938802, "completions/mean_terminated_length": 185.99297993977865, "completions/min_length": 58.733333333333334, "completions/min_terminated_length": 58.733333333333334, "entropy": 0.12207784838974475, "epoch": 0.39987847947442373, "eval/gt_acc_batch": 0.8055555820465088, "frac_reward_zero_std": 1.0, "grad_norm": 0.2624225318431854, "kd/policy_loss": 0.008701883456281697, "kd/rkl_advantage_mean": 0.6952139757573604, "kd/rkl_advantage_p95": 4.951360183954239, "kd/rkl_advantage_std": 2.238209065794945, "kd/ssd_loss": 0.0, "learning_rate": 6.001594956898188e-07, "loss": 0.03480753898620605, "num_tokens": 344494907.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8055555502573649, "rewards/gt_logging_reward/std": 0.3899929761886597, "sampling/importance_sampling_ratio/max": 1.897826651732127, "sampling/importance_sampling_ratio/mean": 0.9988108396530151, "sampling/importance_sampling_ratio/min": 0.444990145166715, "sampling/sampling_logp_difference/max": 0.353996217250824, "sampling/sampling_logp_difference/mean": 0.003978785833654305, "step": 10530, "step_time": 8.241240713705094, "student/entropy": 0.12207784838974475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016944445421298346, "completions/max_length": 477.6, "completions/max_terminated_length": 443.23333333333335, "completions/mean_length": 183.74778645833334, "completions/mean_terminated_length": 178.13280080159504, "completions/min_length": 52.4, "completions/min_terminated_length": 52.4, "entropy": 0.12364342007786036, "epoch": 0.4010177344017013, "eval/gt_acc_batch": 0.7994444747765859, "frac_reward_zero_std": 1.0, "grad_norm": 0.2695307731628418, "kd/policy_loss": 0.00850772473301428, "kd/rkl_advantage_mean": 0.8033940463637312, "kd/rkl_advantage_p95": 5.1964630762736, "kd/rkl_advantage_std": 2.2766631583372754, "kd/ssd_loss": 0.0, "learning_rate": 5.990202407625414e-07, "loss": 0.0340308944384257, "num_tokens": 345466167.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.799444439013799, "rewards/gt_logging_reward/std": 0.3941947663823763, "sampling/importance_sampling_ratio/max": 1.95120876232783, "sampling/importance_sampling_ratio/mean": 0.9986362914244334, "sampling/importance_sampling_ratio/min": 0.4449803670247396, "sampling/sampling_logp_difference/max": 0.3559811015923818, "sampling/sampling_logp_difference/mean": 0.004017813402848939, "step": 10560, "step_time": 7.899725150633215, "student/entropy": 0.12364342007786036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334742734828, "completions/max_length": 498.73333333333335, "completions/max_terminated_length": 454.7, "completions/mean_length": 191.54278869628905, "completions/mean_terminated_length": 183.08368835449218, "completions/min_length": 58.13333333333333, "completions/min_terminated_length": 58.13333333333333, "entropy": 0.12120778858661652, "epoch": 0.40215698932897886, "eval/gt_acc_batch": 0.8055555721124014, "frac_reward_zero_std": 1.0, "grad_norm": 0.2632123529911041, "kd/policy_loss": 0.008499011216918007, "kd/rkl_advantage_mean": 0.8394763531784216, "kd/rkl_advantage_p95": 5.201586111386617, "kd/rkl_advantage_std": 2.2697713593641917, "kd/ssd_loss": 0.0, "learning_rate": 5.978809858352637e-07, "loss": 0.03399604558944702, "num_tokens": 346463369.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8055555522441864, "rewards/gt_logging_reward/std": 0.39117661317189534, "sampling/importance_sampling_ratio/max": 1.9139292081197103, "sampling/importance_sampling_ratio/mean": 0.9941296319166819, "sampling/importance_sampling_ratio/min": 0.43435709476470946, "sampling/sampling_logp_difference/max": 0.3893790364265442, "sampling/sampling_logp_difference/mean": 0.003904475651991864, "step": 10590, "step_time": 7.9608642486666215, "student/entropy": 0.12120778858661652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777918924888, "completions/max_length": 490.6333333333333, "completions/max_terminated_length": 449.8666666666667, "completions/mean_length": 187.89362030029298, "completions/mean_terminated_length": 179.5716807047526, "completions/min_length": 59.53333333333333, "completions/min_terminated_length": 59.53333333333333, "entropy": 0.11787495004634062, "epoch": 0.40329624425625643, "eval/gt_acc_batch": 0.8155555685361227, "frac_reward_zero_std": 1.0, "grad_norm": 0.23196366429328918, "kd/policy_loss": 0.00792016582369494, "kd/rkl_advantage_mean": 0.7885824842999379, "kd/rkl_advantage_p95": 5.133045214414596, "kd/rkl_advantage_std": 2.2537056396404904, "kd/ssd_loss": 0.0, "learning_rate": 5.967417309079862e-07, "loss": 0.03168066740036011, "num_tokens": 347450946.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8155555526415507, "rewards/gt_logging_reward/std": 0.38325621088345846, "sampling/importance_sampling_ratio/max": 1.851384131113688, "sampling/importance_sampling_ratio/mean": 0.9941918273766835, "sampling/importance_sampling_ratio/min": 0.43764520188172656, "sampling/sampling_logp_difference/max": 0.3574194570382436, "sampling/sampling_logp_difference/mean": 0.0038523649408792457, "step": 10620, "step_time": 7.895587878096558, "student/entropy": 0.11787495004634062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0211111121190091, "completions/max_length": 486.3, "completions/max_terminated_length": 436.26666666666665, "completions/mean_length": 180.9225112915039, "completions/mean_terminated_length": 173.75101064046223, "completions/min_length": 55.03333333333333, "completions/min_terminated_length": 55.03333333333333, "entropy": 0.12748178448528052, "epoch": 0.40443549918353394, "eval/gt_acc_batch": 0.7977777938048045, "frac_reward_zero_std": 1.0, "grad_norm": 0.31685635447502136, "kd/policy_loss": 0.00867722161541072, "kd/rkl_advantage_mean": 0.8010423277815183, "kd/rkl_advantage_p95": 5.068478218714396, "kd/rkl_advantage_std": 2.2548077086607616, "kd/ssd_loss": 0.0, "learning_rate": 5.956024759807086e-07, "loss": 0.0347088893254598, "num_tokens": 348391091.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.797777775923411, "rewards/gt_logging_reward/std": 0.39485761324564617, "sampling/importance_sampling_ratio/max": 1.865190533796946, "sampling/importance_sampling_ratio/mean": 0.9953875561555227, "sampling/importance_sampling_ratio/min": 0.4089632083972295, "sampling/sampling_logp_difference/max": 0.4063802202542623, "sampling/sampling_logp_difference/mean": 0.004106242287283142, "step": 10650, "step_time": 7.649718028967133, "student/entropy": 0.12748178448528052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112435658773, "completions/max_length": 500.8, "completions/max_terminated_length": 462.43333333333334, "completions/mean_length": 190.5261220296224, "completions/mean_terminated_length": 182.68347727457683, "completions/min_length": 54.86666666666667, "completions/min_terminated_length": 54.86666666666667, "entropy": 0.12614295004556578, "epoch": 0.4055747541108115, "eval/gt_acc_batch": 0.7866666952768961, "frac_reward_zero_std": 1.0, "grad_norm": 0.2542363107204437, "kd/policy_loss": 0.008874607998101662, "kd/rkl_advantage_mean": 0.8650987508396307, "kd/rkl_advantage_p95": 5.276357622941335, "kd/rkl_advantage_std": 2.2966013381878536, "kd/ssd_loss": 0.0, "learning_rate": 5.94463221053431e-07, "loss": 0.035498428344726565, "num_tokens": 349378937.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666575272878, "rewards/gt_logging_reward/std": 0.40179196894168856, "sampling/importance_sampling_ratio/max": 1.9289963046709697, "sampling/importance_sampling_ratio/mean": 0.9995276510715485, "sampling/importance_sampling_ratio/min": 0.455810709297657, "sampling/sampling_logp_difference/max": 0.38887224594751996, "sampling/sampling_logp_difference/mean": 0.004029872027846674, "step": 10680, "step_time": 7.942622533199028, "student/entropy": 0.12614295004556578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02472222385307153, "completions/max_length": 472.76666666666665, "completions/max_terminated_length": 436.53333333333336, "completions/mean_length": 186.0519536336263, "completions/mean_terminated_length": 178.06328684488932, "completions/min_length": 54.166666666666664, "completions/min_terminated_length": 54.166666666666664, "entropy": 0.12194116935133933, "epoch": 0.4067140090380891, "eval/gt_acc_batch": 0.8063889046510061, "frac_reward_zero_std": 1.0, "grad_norm": 0.5285079479217529, "kd/policy_loss": 0.008404361054999754, "kd/rkl_advantage_mean": 0.750483313932394, "kd/rkl_advantage_p95": 4.960639995336533, "kd/rkl_advantage_std": 2.2089276144901913, "kd/ssd_loss": 0.0, "learning_rate": 5.933239661261534e-07, "loss": 0.03361744483311971, "num_tokens": 350351924.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8063888867696126, "rewards/gt_logging_reward/std": 0.3829671124617259, "sampling/importance_sampling_ratio/max": 2.0044728914896646, "sampling/importance_sampling_ratio/mean": 1.003524480263392, "sampling/importance_sampling_ratio/min": 0.47735979755719504, "sampling/sampling_logp_difference/max": 0.349388853708903, "sampling/sampling_logp_difference/mean": 0.003918835047322015, "step": 10710, "step_time": 7.777415638530511, "student/entropy": 0.12194116935133933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02816092142642572, "completions/max_length": 505.0344827586207, "completions/max_terminated_length": 452.8965517241379, "completions/mean_length": 187.26064168995825, "completions/mean_terminated_length": 177.90686193005791, "completions/min_length": 57.03448275862069, "completions/min_terminated_length": 57.03448275862069, "entropy": 0.12918645650919142, "epoch": 0.40785326396536664, "eval/gt_acc_batch": 0.790517256177705, "frac_reward_zero_std": 1.0, "grad_norm": 0.38703030347824097, "kd/policy_loss": 0.009054176336764518, "kd/rkl_advantage_mean": 0.8436282056158987, "kd/rkl_advantage_p95": 5.169765945138602, "kd/rkl_advantage_std": 2.2698032033854516, "kd/ssd_loss": 0.0, "learning_rate": 5.92184711198876e-07, "loss": 0.03500947952270508, "num_tokens": 351300695.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7905172479563746, "rewards/gt_logging_reward/std": 0.4029906587354068, "sampling/importance_sampling_ratio/max": 1.9759930290024856, "sampling/importance_sampling_ratio/mean": 1.00705441113176, "sampling/importance_sampling_ratio/min": 0.43564349104618205, "sampling/sampling_logp_difference/max": 0.3534643444521674, "sampling/sampling_logp_difference/mean": 0.004125198453729008, "step": 10740, "step_time": 7.7762239007007645, "student/entropy": 0.12918645650919142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166667989144724, "completions/max_length": 486.06666666666666, "completions/max_terminated_length": 437.2, "completions/mean_length": 185.3975102742513, "completions/mean_terminated_length": 177.42224629720053, "completions/min_length": 49.3, "completions/min_terminated_length": 49.3, "entropy": 0.125607464214166, "epoch": 0.4089925188926442, "eval/gt_acc_batch": 0.7930555760860443, "frac_reward_zero_std": 1.0, "grad_norm": 0.33320990204811096, "kd/policy_loss": 0.00856847232595707, "kd/rkl_advantage_mean": 0.7880529242567718, "kd/rkl_advantage_p95": 5.120797036091487, "kd/rkl_advantage_std": 2.2659881502389907, "kd/ssd_loss": 0.0, "learning_rate": 5.910454562715983e-07, "loss": 0.03427388668060303, "num_tokens": 352261886.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555542310079, "rewards/gt_logging_reward/std": 0.3989254966378212, "sampling/importance_sampling_ratio/max": 1.9457431435585022, "sampling/importance_sampling_ratio/mean": 0.9940714100996654, "sampling/importance_sampling_ratio/min": 0.3950306514898936, "sampling/sampling_logp_difference/max": 0.3592452963193258, "sampling/sampling_logp_difference/mean": 0.004083046449037889, "step": 10770, "step_time": 7.764885103199534, "student/entropy": 0.125607464214166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445936630168, "completions/max_length": 481.56666666666666, "completions/max_terminated_length": 439.26666666666665, "completions/mean_length": 193.4327885945638, "completions/mean_terminated_length": 185.64923400878905, "completions/min_length": 63.13333333333333, "completions/min_terminated_length": 63.13333333333333, "entropy": 0.12752354020873705, "epoch": 0.4101317738199218, "eval/gt_acc_batch": 0.8077777961889903, "frac_reward_zero_std": 1.0, "grad_norm": 0.2653803527355194, "kd/policy_loss": 0.008474512726146107, "kd/rkl_advantage_mean": 0.7582023064916332, "kd/rkl_advantage_p95": 4.966840287049611, "kd/rkl_advantage_std": 2.2277325659990312, "kd/ssd_loss": 0.0, "learning_rate": 5.899062013443208e-07, "loss": 0.033898051579793295, "num_tokens": 353264036.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8077777822812399, "rewards/gt_logging_reward/std": 0.3846756408611933, "sampling/importance_sampling_ratio/max": 1.9891152262687684, "sampling/importance_sampling_ratio/mean": 0.993836905558904, "sampling/importance_sampling_ratio/min": 0.41395831455787024, "sampling/sampling_logp_difference/max": 0.3352519909540812, "sampling/sampling_logp_difference/mean": 0.004051566830215355, "step": 10800, "step_time": 7.886331143337399, "student/entropy": 0.12752354020873705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223343948523, "completions/max_length": 480.8666666666667, "completions/max_terminated_length": 435.76666666666665, "completions/mean_length": 188.80306498209634, "completions/mean_terminated_length": 182.4061731974284, "completions/min_length": 52.766666666666666, "completions/min_terminated_length": 52.766666666666666, "entropy": 0.12422056545813878, "epoch": 0.41127102874719934, "eval/gt_acc_batch": 0.7963889102141063, "frac_reward_zero_std": 1.0, "grad_norm": 0.3042498528957367, "kd/policy_loss": 0.00866517920124655, "kd/rkl_advantage_mean": 0.7662867549263562, "kd/rkl_advantage_p95": 5.042865946888924, "kd/rkl_advantage_std": 2.236620851357778, "kd/ssd_loss": 0.0, "learning_rate": 5.887669464170433e-07, "loss": 0.03466071685155233, "num_tokens": 354249543.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888923327128, "rewards/gt_logging_reward/std": 0.39596930146217346, "sampling/importance_sampling_ratio/max": 1.8969589710235595, "sampling/importance_sampling_ratio/mean": 0.9995357116063436, "sampling/importance_sampling_ratio/min": 0.44987736344337464, "sampling/sampling_logp_difference/max": 0.3585905780394872, "sampling/sampling_logp_difference/mean": 0.003967347244421641, "step": 10830, "step_time": 7.764017306226985, "student/entropy": 0.12422056545813878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500000912696122, "completions/max_length": 466.23333333333335, "completions/max_terminated_length": 437.03333333333336, "completions/mean_length": 186.15945536295573, "completions/mean_terminated_length": 180.540407816569, "completions/min_length": 52.9, "completions/min_terminated_length": 52.9, "entropy": 0.12542476101468006, "epoch": 0.4124102836744769, "eval/gt_acc_batch": 0.8008333524068196, "frac_reward_zero_std": 1.0, "grad_norm": 0.21073341369628906, "kd/policy_loss": 0.008808878952792535, "kd/rkl_advantage_mean": 0.7740348021189372, "kd/rkl_advantage_p95": 4.9942957878112795, "kd/rkl_advantage_std": 2.2034556527932483, "kd/ssd_loss": 0.0, "learning_rate": 5.876276914897656e-07, "loss": 0.03523551225662232, "num_tokens": 355216237.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333384990692, "rewards/gt_logging_reward/std": 0.3917003264029821, "sampling/importance_sampling_ratio/max": 2.0196523865063987, "sampling/importance_sampling_ratio/mean": 1.007257310549418, "sampling/importance_sampling_ratio/min": 0.4608916292587916, "sampling/sampling_logp_difference/max": 0.35771257877349855, "sampling/sampling_logp_difference/mean": 0.004021588790540894, "step": 10860, "step_time": 7.641711621867338, "student/entropy": 0.12542476101468006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02722222382823626, "completions/max_length": 486.3666666666667, "completions/max_terminated_length": 449.6333333333333, "completions/mean_length": 191.60278778076173, "completions/mean_terminated_length": 182.71871999104818, "completions/min_length": 60.766666666666666, "completions/min_terminated_length": 60.766666666666666, "entropy": 0.12420201965918144, "epoch": 0.4135495386017545, "eval/gt_acc_batch": 0.8077777981758117, "frac_reward_zero_std": 1.0, "grad_norm": 0.4471946656703949, "kd/policy_loss": 0.008470389169330399, "kd/rkl_advantage_mean": 0.8011665217578411, "kd/rkl_advantage_p95": 5.090583336353302, "kd/rkl_advantage_std": 2.2563922186692555, "kd/ssd_loss": 0.0, "learning_rate": 5.864884365624881e-07, "loss": 0.033881564935048424, "num_tokens": 356214615.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8077777783075969, "rewards/gt_logging_reward/std": 0.38770818213621777, "sampling/importance_sampling_ratio/max": 1.90258998076121, "sampling/importance_sampling_ratio/mean": 1.0028245568275451, "sampling/importance_sampling_ratio/min": 0.42885444362958275, "sampling/sampling_logp_difference/max": 0.3720317800839742, "sampling/sampling_logp_difference/mean": 0.003941055620089173, "step": 10890, "step_time": 7.916933299636487, "student/entropy": 0.12420201965918144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028888890861223142, "completions/max_length": 497.6, "completions/max_terminated_length": 449.0, "completions/mean_length": 192.14834238688152, "completions/mean_terminated_length": 182.69435475667316, "completions/min_length": 60.3, "completions/min_terminated_length": 60.3, "entropy": 0.12500317922482887, "epoch": 0.41468879352903204, "eval/gt_acc_batch": 0.796666685740153, "frac_reward_zero_std": 1.0, "grad_norm": 0.2658860385417938, "kd/policy_loss": 0.008729055353129903, "kd/rkl_advantage_mean": 0.779737031335632, "kd/rkl_advantage_p95": 5.099222348133723, "kd/rkl_advantage_std": 2.256130571166674, "kd/ssd_loss": 0.0, "learning_rate": 5.853491816352105e-07, "loss": 0.0349162220954895, "num_tokens": 357221573.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7966666678587596, "rewards/gt_logging_reward/std": 0.3929822579026222, "sampling/importance_sampling_ratio/max": 1.895244308312734, "sampling/importance_sampling_ratio/mean": 0.9989401400089264, "sampling/importance_sampling_ratio/min": 0.4561264644066493, "sampling/sampling_logp_difference/max": 0.338992178440094, "sampling/sampling_logp_difference/mean": 0.003946910961531103, "step": 10920, "step_time": 8.230200215565855, "student/entropy": 0.12500317922482887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944446222235758, "completions/max_length": 488.43333333333334, "completions/max_terminated_length": 463.9, "completions/mean_length": 183.4452885945638, "completions/mean_terminated_length": 174.53964233398438, "completions/min_length": 55.03333333333333, "completions/min_terminated_length": 55.03333333333333, "entropy": 0.129709267988801, "epoch": 0.41582804845630955, "eval/gt_acc_batch": 0.8047222336133321, "frac_reward_zero_std": 1.0, "grad_norm": 0.21270230412483215, "kd/policy_loss": 0.008817604067735374, "kd/rkl_advantage_mean": 0.8292535634400944, "kd/rkl_advantage_p95": 5.188517554601034, "kd/rkl_advantage_std": 2.2989805420239766, "kd/ssd_loss": 0.0, "learning_rate": 5.84209926707933e-07, "loss": 0.0352704127629598, "num_tokens": 358178408.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222197055817, "rewards/gt_logging_reward/std": 0.3882374922434489, "sampling/importance_sampling_ratio/max": 1.8938692172368368, "sampling/importance_sampling_ratio/mean": 1.000975110133489, "sampling/importance_sampling_ratio/min": 0.44805833995342254, "sampling/sampling_logp_difference/max": 0.338328750928243, "sampling/sampling_logp_difference/mean": 0.004136632126756012, "step": 10950, "step_time": 7.780633494335052, "student/entropy": 0.129709267988801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223517795404, "completions/max_length": 489.03333333333336, "completions/max_terminated_length": 444.2, "completions/mean_length": 184.31556498209636, "completions/mean_terminated_length": 175.30699106852214, "completions/min_length": 44.7, "completions/min_terminated_length": 44.7, "entropy": 0.12932452528427044, "epoch": 0.4169673033835871, "eval/gt_acc_batch": 0.8050000250339509, "frac_reward_zero_std": 1.0, "grad_norm": 0.2591937482357025, "kd/policy_loss": 0.008842953960023199, "kd/rkl_advantage_mean": 0.8290664931914459, "kd/rkl_advantage_p95": 5.167586690187454, "kd/rkl_advantage_std": 2.2737001279989877, "kd/ssd_loss": 0.0, "learning_rate": 5.830706717806554e-07, "loss": 0.03537181218465169, "num_tokens": 359158608.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8049999952316285, "rewards/gt_logging_reward/std": 0.3874802549680074, "sampling/importance_sampling_ratio/max": 1.9606038252512614, "sampling/importance_sampling_ratio/mean": 0.9952649215857188, "sampling/importance_sampling_ratio/min": 0.44397147595882414, "sampling/sampling_logp_difference/max": 0.3612501780192057, "sampling/sampling_logp_difference/mean": 0.0040940668511514865, "step": 10980, "step_time": 8.248861019739222, "student/entropy": 0.12932452528427044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500001192092895, "completions/max_length": 488.73333333333335, "completions/max_terminated_length": 449.56666666666666, "completions/mean_length": 184.8725092569987, "completions/mean_terminated_length": 179.0597712198893, "completions/min_length": 50.2, "completions/min_terminated_length": 50.2, "entropy": 0.12394448798149824, "epoch": 0.4181065583108647, "eval/gt_acc_batch": 0.8172222435474396, "frac_reward_zero_std": 1.0, "grad_norm": 0.25163689255714417, "kd/policy_loss": 0.008220065449131652, "kd/rkl_advantage_mean": 0.7826869182288647, "kd/rkl_advantage_p95": 5.085982749859492, "kd/rkl_advantage_std": 2.2580818285544715, "kd/ssd_loss": 0.0, "learning_rate": 5.819314168533779e-07, "loss": 0.032880264520645144, "num_tokens": 360121261.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8172222197055816, "rewards/gt_logging_reward/std": 0.37853196561336516, "sampling/importance_sampling_ratio/max": 1.8593367298444112, "sampling/importance_sampling_ratio/mean": 1.0078315118948618, "sampling/importance_sampling_ratio/min": 0.4691827744245529, "sampling/sampling_logp_difference/max": 0.3082729816436768, "sampling/sampling_logp_difference/mean": 0.003924928815104067, "step": 11010, "step_time": 7.925345746831348, "student/entropy": 0.12394448798149824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02000000113621354, "completions/max_length": 477.5, "completions/max_terminated_length": 448.4, "completions/mean_length": 187.02528686523436, "completions/mean_terminated_length": 180.523823038737, "completions/min_length": 48.43333333333333, "completions/min_terminated_length": 48.43333333333333, "entropy": 0.12993011064827442, "epoch": 0.41924581323814225, "eval/gt_acc_batch": 0.7872222542762757, "frac_reward_zero_std": 1.0, "grad_norm": 0.24298010766506195, "kd/policy_loss": 0.009076951690561448, "kd/rkl_advantage_mean": 0.7553342174777451, "kd/rkl_advantage_p95": 5.012013920148214, "kd/rkl_advantage_std": 2.220237065354983, "kd/ssd_loss": 0.0, "learning_rate": 5.807921619261002e-07, "loss": 0.03630780776341756, "num_tokens": 361095336.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7872222224871318, "rewards/gt_logging_reward/std": 0.3995201165477435, "sampling/importance_sampling_ratio/max": 1.9528343240420023, "sampling/importance_sampling_ratio/mean": 0.9985416134198507, "sampling/importance_sampling_ratio/min": 0.433385768532753, "sampling/sampling_logp_difference/max": 0.36888420979181924, "sampling/sampling_logp_difference/mean": 0.004151241636524598, "step": 11040, "step_time": 7.904930816731454, "student/entropy": 0.12993011064827442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667790462572, "completions/max_length": 471.23333333333335, "completions/max_terminated_length": 432.46666666666664, "completions/mean_length": 184.54639739990233, "completions/mean_terminated_length": 178.18759104410807, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "entropy": 0.12880986283222834, "epoch": 0.4203850681654198, "eval/gt_acc_batch": 0.8008333563804626, "frac_reward_zero_std": 1.0, "grad_norm": 0.3085721731185913, "kd/policy_loss": 0.008803926119192814, "kd/rkl_advantage_mean": 0.6820449548463027, "kd/rkl_advantage_p95": 4.915043437480927, "kd/rkl_advantage_std": 2.1935487429300946, "kd/ssd_loss": 0.0, "learning_rate": 5.796529069988227e-07, "loss": 0.035215707619984944, "num_tokens": 362068111.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333305517833, "rewards/gt_logging_reward/std": 0.3921476120750109, "sampling/importance_sampling_ratio/max": 2.0264683683713276, "sampling/importance_sampling_ratio/mean": 1.0074478149414063, "sampling/importance_sampling_ratio/min": 0.4566768676042557, "sampling/sampling_logp_difference/max": 0.3410379747549693, "sampling/sampling_logp_difference/mean": 0.004086233008032044, "step": 11070, "step_time": 7.999714906234294, "student/entropy": 0.12880986283222834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444586833318, "completions/max_length": 478.26666666666665, "completions/max_terminated_length": 439.2, "completions/mean_length": 188.33306579589845, "completions/mean_terminated_length": 181.06346995035807, "completions/min_length": 57.833333333333336, "completions/min_terminated_length": 57.833333333333336, "entropy": 0.12791222501546146, "epoch": 0.4215243230926974, "eval/gt_acc_batch": 0.8047222415606181, "frac_reward_zero_std": 1.0, "grad_norm": 0.25002676248550415, "kd/policy_loss": 0.008760467489870886, "kd/rkl_advantage_mean": 0.7697031732027729, "kd/rkl_advantage_p95": 4.963197908798853, "kd/rkl_advantage_std": 2.2070022890965144, "kd/ssd_loss": 0.0, "learning_rate": 5.785136520715452e-07, "loss": 0.03504187266031901, "num_tokens": 363057374.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222256660461, "rewards/gt_logging_reward/std": 0.38869201838970185, "sampling/importance_sampling_ratio/max": 1.8413901646931967, "sampling/importance_sampling_ratio/mean": 0.9972897211710612, "sampling/importance_sampling_ratio/min": 0.4511428048213323, "sampling/sampling_logp_difference/max": 0.33121751944224037, "sampling/sampling_logp_difference/mean": 0.0040814243334655964, "step": 11100, "step_time": 8.089216584036087, "student/entropy": 0.12791222501546146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556857337555, "completions/max_length": 474.43333333333334, "completions/max_terminated_length": 436.1, "completions/mean_length": 188.72834116617838, "completions/mean_terminated_length": 180.58513488769532, "completions/min_length": 50.233333333333334, "completions/min_terminated_length": 50.233333333333334, "entropy": 0.11866806540638208, "epoch": 0.42266357801997495, "eval/gt_acc_batch": 0.8138889034589132, "frac_reward_zero_std": 1.0, "grad_norm": 0.2723023593425751, "kd/policy_loss": 0.00798421764629893, "kd/rkl_advantage_mean": 0.7725955202554663, "kd/rkl_advantage_p95": 5.09555814464887, "kd/rkl_advantage_std": 2.262621342142423, "kd/ssd_loss": 0.0, "learning_rate": 5.773743971442677e-07, "loss": 0.03193687399228414, "num_tokens": 364049372.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8138888875643412, "rewards/gt_logging_reward/std": 0.37864691615104673, "sampling/importance_sampling_ratio/max": 1.8349785168965658, "sampling/importance_sampling_ratio/mean": 0.9984891553719838, "sampling/importance_sampling_ratio/min": 0.47732709447542826, "sampling/sampling_logp_difference/max": 0.34683728714783985, "sampling/sampling_logp_difference/mean": 0.0037664261103297274, "step": 11130, "step_time": 8.02442361832994, "student/entropy": 0.11866806540638208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0247222236978511, "completions/max_length": 491.06666666666666, "completions/max_terminated_length": 445.06666666666666, "completions/mean_length": 186.99556528727214, "completions/mean_terminated_length": 178.92196197509764, "completions/min_length": 47.766666666666666, "completions/min_terminated_length": 47.766666666666666, "entropy": 0.13067898588875929, "epoch": 0.4238028329472525, "eval/gt_acc_batch": 0.797222242752711, "frac_reward_zero_std": 1.0, "grad_norm": 0.2978976368904114, "kd/policy_loss": 0.00865006383198003, "kd/rkl_advantage_mean": 0.7731514656295378, "kd/rkl_advantage_p95": 5.066205354531606, "kd/rkl_advantage_std": 2.251801476875941, "kd/ssd_loss": 0.0, "learning_rate": 5.762351422169901e-07, "loss": 0.03460024992624919, "num_tokens": 365031100.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7972222149372101, "rewards/gt_logging_reward/std": 0.39352360318104423, "sampling/importance_sampling_ratio/max": 1.9272628625233967, "sampling/importance_sampling_ratio/mean": 0.9933995525042216, "sampling/importance_sampling_ratio/min": 0.4079070970416069, "sampling/sampling_logp_difference/max": 0.32501941521962485, "sampling/sampling_logp_difference/mean": 0.00412472162861377, "step": 11160, "step_time": 8.166388187632158, "student/entropy": 0.13067898588875929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111281439662, "completions/max_length": 490.3666666666667, "completions/max_terminated_length": 451.7, "completions/mean_length": 193.73084309895833, "completions/mean_terminated_length": 185.43362630208333, "completions/min_length": 57.56666666666667, "completions/min_terminated_length": 57.56666666666667, "entropy": 0.13179865976174673, "epoch": 0.4249420878745301, "eval/gt_acc_batch": 0.7869444628556569, "frac_reward_zero_std": 1.0, "grad_norm": 0.3106021285057068, "kd/policy_loss": 0.009301108994986863, "kd/rkl_advantage_mean": 0.861307215380172, "kd/rkl_advantage_p95": 5.2087114453315735, "kd/rkl_advantage_std": 2.2753982186317443, "kd/ssd_loss": 0.0, "learning_rate": 5.750958872897125e-07, "loss": 0.037204432487487796, "num_tokens": 366035851.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444410006206, "rewards/gt_logging_reward/std": 0.4033940056959788, "sampling/importance_sampling_ratio/max": 2.037217450141907, "sampling/importance_sampling_ratio/mean": 1.0038142919540405, "sampling/importance_sampling_ratio/min": 0.4327598164478938, "sampling/sampling_logp_difference/max": 0.33052988052368165, "sampling/sampling_logp_difference/mean": 0.004164413960340122, "step": 11190, "step_time": 8.056227477396412, "student/entropy": 0.13179865976174673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112435658773, "completions/max_length": 483.6333333333333, "completions/max_terminated_length": 461.23333333333335, "completions/mean_length": 190.05250905354816, "completions/mean_terminated_length": 182.40032602945965, "completions/min_length": 49.53333333333333, "completions/min_terminated_length": 49.53333333333333, "entropy": 0.1266085082044204, "epoch": 0.4260813428018076, "eval/gt_acc_batch": 0.8019444684187571, "frac_reward_zero_std": 1.0, "grad_norm": 0.2158711552619934, "kd/policy_loss": 0.008484772534575313, "kd/rkl_advantage_mean": 0.8479551448641965, "kd/rkl_advantage_p95": 5.20724185705185, "kd/rkl_advantage_std": 2.2941170980532966, "kd/ssd_loss": 0.0, "learning_rate": 5.73956632362435e-07, "loss": 0.03393908739089966, "num_tokens": 367023832.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444485505421, "rewards/gt_logging_reward/std": 0.39036971429983774, "sampling/importance_sampling_ratio/max": 1.9822932243347169, "sampling/importance_sampling_ratio/mean": 0.9976930658022563, "sampling/importance_sampling_ratio/min": 0.46582283775011696, "sampling/sampling_logp_difference/max": 0.3527142037947973, "sampling/sampling_logp_difference/mean": 0.00399091235982875, "step": 11220, "step_time": 8.030181897330234, "student/entropy": 0.1266085082044204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667889803648, "completions/max_length": 465.46666666666664, "completions/max_terminated_length": 427.73333333333335, "completions/mean_length": 186.5983418782552, "completions/mean_terminated_length": 179.52516428629556, "completions/min_length": 62.43333333333333, "completions/min_terminated_length": 62.43333333333333, "entropy": 0.12533150172481936, "epoch": 0.42722059772908516, "eval/gt_acc_batch": 0.8100000301996867, "frac_reward_zero_std": 1.0, "grad_norm": 0.3758449852466583, "kd/policy_loss": 0.008399219021278744, "kd/rkl_advantage_mean": 0.7076372888560096, "kd/rkl_advantage_p95": 4.872074631849925, "kd/rkl_advantage_std": 2.1807295362154644, "kd/ssd_loss": 0.0, "learning_rate": 5.728173774351573e-07, "loss": 0.033596877257029215, "num_tokens": 367994506.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8100000023841858, "rewards/gt_logging_reward/std": 0.38448666483163835, "sampling/importance_sampling_ratio/max": 1.9318461855252584, "sampling/importance_sampling_ratio/mean": 0.999275267124176, "sampling/importance_sampling_ratio/min": 0.4572328085700671, "sampling/sampling_logp_difference/max": 0.322148597240448, "sampling/sampling_logp_difference/mean": 0.003928730318633219, "step": 11250, "step_time": 7.854440692966454, "student/entropy": 0.12533150172481936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001043081285, "completions/max_length": 480.23333333333335, "completions/max_terminated_length": 435.8, "completions/mean_length": 182.83000946044922, "completions/mean_terminated_length": 176.16265513102215, "completions/min_length": 49.46666666666667, "completions/min_terminated_length": 49.46666666666667, "entropy": 0.1275231911490361, "epoch": 0.4283598526563627, "eval/gt_acc_batch": 0.8069444795449575, "frac_reward_zero_std": 1.0, "grad_norm": 0.34125104546546936, "kd/policy_loss": 0.008516371071649095, "kd/rkl_advantage_mean": 0.7540580379466216, "kd/rkl_advantage_p95": 5.035788347323735, "kd/rkl_advantage_std": 2.2574027349551518, "kd/ssd_loss": 0.0, "learning_rate": 5.716781225078798e-07, "loss": 0.03406548500061035, "num_tokens": 368952294.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8069444417953491, "rewards/gt_logging_reward/std": 0.3903204739093781, "sampling/importance_sampling_ratio/max": 1.8893377741177877, "sampling/importance_sampling_ratio/mean": 1.0001621504624685, "sampling/importance_sampling_ratio/min": 0.4182889983057976, "sampling/sampling_logp_difference/max": 0.3155474017063777, "sampling/sampling_logp_difference/mean": 0.004037419753149152, "step": 11280, "step_time": 7.84328851623286, "student/entropy": 0.1275231911490361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112404614686, "completions/max_length": 480.23333333333335, "completions/max_terminated_length": 444.43333333333334, "completions/mean_length": 185.85501047770182, "completions/mean_terminated_length": 178.03470560709636, "completions/min_length": 54.0, "completions/min_terminated_length": 54.0, "entropy": 0.126002014738818, "epoch": 0.4294991075836403, "eval/gt_acc_batch": 0.795000022649765, "frac_reward_zero_std": 1.0, "grad_norm": 0.2704719305038452, "kd/policy_loss": 0.008470776128039386, "kd/rkl_advantage_mean": 0.8179263540854057, "kd/rkl_advantage_p95": 5.105910396575927, "kd/rkl_advantage_std": 2.238905473550161, "kd/ssd_loss": 0.0, "learning_rate": 5.705388675806022e-07, "loss": 0.03388310670852661, "num_tokens": 369927780.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7950000007947285, "rewards/gt_logging_reward/std": 0.39737036526203157, "sampling/importance_sampling_ratio/max": 1.9620817144711813, "sampling/importance_sampling_ratio/mean": 0.9985138674577078, "sampling/importance_sampling_ratio/min": 0.4493737662831942, "sampling/sampling_logp_difference/max": 0.36315704782803854, "sampling/sampling_logp_difference/mean": 0.004015575077695151, "step": 11310, "step_time": 8.089973823267792, "student/entropy": 0.126002014738818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0158333341901501, "completions/max_length": 481.76666666666665, "completions/max_terminated_length": 453.43333333333334, "completions/mean_length": 185.61945393880208, "completions/mean_terminated_length": 180.47264353434244, "completions/min_length": 52.8, "completions/min_terminated_length": 52.8, "entropy": 0.13106321015705666, "epoch": 0.43063836251091786, "eval/gt_acc_batch": 0.7986111422379811, "frac_reward_zero_std": 1.0, "grad_norm": 0.4565618634223938, "kd/policy_loss": 0.00892021637992002, "kd/rkl_advantage_mean": 0.7560482613742352, "kd/rkl_advantage_p95": 4.945070141553879, "kd/rkl_advantage_std": 2.19042622645696, "kd/ssd_loss": 0.0, "learning_rate": 5.693996126533247e-07, "loss": 0.03568086624145508, "num_tokens": 370895338.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111164093017, "rewards/gt_logging_reward/std": 0.39417697191238404, "sampling/importance_sampling_ratio/max": 1.9638185302416484, "sampling/importance_sampling_ratio/mean": 1.0016366561253867, "sampling/importance_sampling_ratio/min": 0.44007737189531326, "sampling/sampling_logp_difference/max": 0.3429566383361816, "sampling/sampling_logp_difference/mean": 0.004193193341294924, "step": 11340, "step_time": 7.953461444832889, "student/entropy": 0.13106321015705666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013333334121853113, "completions/max_length": 479.6333333333333, "completions/max_terminated_length": 454.43333333333334, "completions/mean_length": 184.0666768391927, "completions/mean_terminated_length": 179.66200002034506, "completions/min_length": 56.06666666666667, "completions/min_terminated_length": 56.06666666666667, "entropy": 0.12782959192991256, "epoch": 0.4317776174381954, "eval/gt_acc_batch": 0.8219444553057352, "frac_reward_zero_std": 1.0, "grad_norm": 0.2516724467277527, "kd/policy_loss": 0.008234480190246056, "kd/rkl_advantage_mean": 0.739900568438073, "kd/rkl_advantage_p95": 5.013181231419245, "kd/rkl_advantage_std": 2.237910489241282, "kd/ssd_loss": 0.0, "learning_rate": 5.682603577260472e-07, "loss": 0.032937920093536376, "num_tokens": 371865642.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8219444493452708, "rewards/gt_logging_reward/std": 0.37703707814216614, "sampling/importance_sampling_ratio/max": 1.9399539868036906, "sampling/importance_sampling_ratio/mean": 1.0008496026198068, "sampling/importance_sampling_ratio/min": 0.45469182829062144, "sampling/sampling_logp_difference/max": 0.32901699940363566, "sampling/sampling_logp_difference/mean": 0.003992986655794084, "step": 11370, "step_time": 8.11936890656604, "student/entropy": 0.12782959192991256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0255555571988225, "completions/max_length": 482.6333333333333, "completions/max_terminated_length": 447.5, "completions/mean_length": 193.7613993326823, "completions/mean_terminated_length": 185.53423207600912, "completions/min_length": 45.166666666666664, "completions/min_terminated_length": 45.166666666666664, "entropy": 0.12734760685513416, "epoch": 0.432916872365473, "eval/gt_acc_batch": 0.789722228050232, "frac_reward_zero_std": 1.0, "grad_norm": 0.34110063314437866, "kd/policy_loss": 0.008443815752010171, "kd/rkl_advantage_mean": 0.7463594195743402, "kd/rkl_advantage_p95": 5.032692964871725, "kd/rkl_advantage_std": 2.241734658678373, "kd/ssd_loss": 0.0, "learning_rate": 5.671211027987696e-07, "loss": 0.03377526601155599, "num_tokens": 372879711.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.789722224076589, "rewards/gt_logging_reward/std": 0.3992016464471817, "sampling/importance_sampling_ratio/max": 2.037269683678945, "sampling/importance_sampling_ratio/mean": 0.9982204020023346, "sampling/importance_sampling_ratio/min": 0.4168927232424418, "sampling/sampling_logp_difference/max": 0.35298062761624655, "sampling/sampling_logp_difference/mean": 0.004035789798945188, "step": 11400, "step_time": 8.074420509397168, "student/entropy": 0.12734760685513416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01805555643513799, "completions/max_length": 478.1333333333333, "completions/max_terminated_length": 444.73333333333335, "completions/mean_length": 185.20584309895833, "completions/mean_terminated_length": 179.3300313313802, "completions/min_length": 52.4, "completions/min_terminated_length": 52.4, "entropy": 0.1289910117785136, "epoch": 0.43405612729275056, "eval/gt_acc_batch": 0.8141666928927104, "frac_reward_zero_std": 1.0, "grad_norm": 0.3170662522315979, "kd/policy_loss": 0.008488527126610279, "kd/rkl_advantage_mean": 0.7495379378398259, "kd/rkl_advantage_p95": 5.026414149999619, "kd/rkl_advantage_std": 2.2278493573268254, "kd/ssd_loss": 0.0, "learning_rate": 5.65981847871492e-07, "loss": 0.03395411173502604, "num_tokens": 373859364.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8141666650772095, "rewards/gt_logging_reward/std": 0.3831405997276306, "sampling/importance_sampling_ratio/max": 1.8420002897580465, "sampling/importance_sampling_ratio/mean": 0.9978655020395915, "sampling/importance_sampling_ratio/min": 0.4287222623825073, "sampling/sampling_logp_difference/max": 0.37705453832944236, "sampling/sampling_logp_difference/mean": 0.004106219268093506, "step": 11430, "step_time": 8.100613432732159, "student/entropy": 0.1289910117785136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223424663145, "completions/max_length": 495.3, "completions/max_terminated_length": 457.56666666666666, "completions/mean_length": 188.74695383707683, "completions/mean_terminated_length": 179.63865458170574, "completions/min_length": 56.36666666666667, "completions/min_terminated_length": 56.36666666666667, "entropy": 0.1274974225088954, "epoch": 0.4351953822200281, "eval/gt_acc_batch": 0.8022222439448039, "frac_reward_zero_std": 1.0, "grad_norm": 0.2787800431251526, "kd/policy_loss": 0.008797077411630501, "kd/rkl_advantage_mean": 0.7305536361721655, "kd/rkl_advantage_p95": 4.981649388869603, "kd/rkl_advantage_std": 2.204659538467725, "kd/ssd_loss": 0.0, "learning_rate": 5.648425929442144e-07, "loss": 0.03518830935160319, "num_tokens": 374838765.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8022222260634104, "rewards/gt_logging_reward/std": 0.3934246381123861, "sampling/importance_sampling_ratio/max": 1.8553470571835835, "sampling/importance_sampling_ratio/mean": 0.9984675447146097, "sampling/importance_sampling_ratio/min": 0.44022238900264105, "sampling/sampling_logp_difference/max": 0.3477927188078562, "sampling/sampling_logp_difference/mean": 0.004018687549978495, "step": 11460, "step_time": 8.046144979037248, "student/entropy": 0.1274974225088954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.036388890879849596, "completions/max_length": 504.3666666666667, "completions/max_terminated_length": 453.8, "completions/mean_length": 194.79250895182292, "completions/mean_terminated_length": 183.3503189086914, "completions/min_length": 49.1, "completions/min_terminated_length": 49.1, "entropy": 0.13075141745309035, "epoch": 0.43633463714730564, "eval/gt_acc_batch": 0.7808333516120911, "frac_reward_zero_std": 1.0, "grad_norm": 0.19623211026191711, "kd/policy_loss": 0.008761590915188815, "kd/rkl_advantage_mean": 0.83263626024127, "kd/rkl_advantage_p95": 5.21355218688647, "kd/rkl_advantage_std": 2.289816269278526, "kd/ssd_loss": 0.0, "learning_rate": 5.637033380169369e-07, "loss": 0.035046366850535075, "num_tokens": 375849330.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7808333337306976, "rewards/gt_logging_reward/std": 0.40280557026465735, "sampling/importance_sampling_ratio/max": 1.8930689215660095, "sampling/importance_sampling_ratio/mean": 0.9942979296048482, "sampling/importance_sampling_ratio/min": 0.4450619806845983, "sampling/sampling_logp_difference/max": 0.3481112480163574, "sampling/sampling_logp_difference/mean": 0.004094026183399061, "step": 11490, "step_time": 8.260081543201037, "student/entropy": 0.13075141745309035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778996775546, "completions/max_length": 491.3, "completions/max_terminated_length": 447.46666666666664, "completions/mean_length": 190.40584208170574, "completions/mean_terminated_length": 182.98750864664714, "completions/min_length": 60.93333333333333, "completions/min_terminated_length": 60.93333333333333, "entropy": 0.12774408770104248, "epoch": 0.4374738920745832, "eval/gt_acc_batch": 0.7897222459316253, "frac_reward_zero_std": 1.0, "grad_norm": 0.24044856429100037, "kd/policy_loss": 0.0087102738112056, "kd/rkl_advantage_mean": 0.8949220889558395, "kd/rkl_advantage_p95": 5.332709223031998, "kd/rkl_advantage_std": 2.3312124073505402, "kd/ssd_loss": 0.0, "learning_rate": 5.625640830896594e-07, "loss": 0.034841092427571614, "num_tokens": 376832951.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.789722224076589, "rewards/gt_logging_reward/std": 0.39851535658041637, "sampling/importance_sampling_ratio/max": 1.9364305933316548, "sampling/importance_sampling_ratio/mean": 0.9997584740320842, "sampling/importance_sampling_ratio/min": 0.39720841497182846, "sampling/sampling_logp_difference/max": 0.31929536660512287, "sampling/sampling_logp_difference/mean": 0.004048191787054141, "step": 11520, "step_time": 7.933718773032402, "student/entropy": 0.12774408770104248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556726952395, "completions/max_length": 497.3666666666667, "completions/max_terminated_length": 446.1666666666667, "completions/mean_length": 188.77223205566406, "completions/mean_terminated_length": 181.24918416341146, "completions/min_length": 53.833333333333336, "completions/min_terminated_length": 53.833333333333336, "entropy": 0.12390445911635956, "epoch": 0.43861314700186077, "eval/gt_acc_batch": 0.8050000150998433, "frac_reward_zero_std": 1.0, "grad_norm": 0.2510381042957306, "kd/policy_loss": 0.008350844809319824, "kd/rkl_advantage_mean": 0.8598504309852918, "kd/rkl_advantage_p95": 5.234802307685216, "kd/rkl_advantage_std": 2.2683691531419754, "kd/ssd_loss": 0.0, "learning_rate": 5.614248281623818e-07, "loss": 0.03340338071187337, "num_tokens": 377816027.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8050000031789144, "rewards/gt_logging_reward/std": 0.3893147518237432, "sampling/importance_sampling_ratio/max": 1.9478103399276734, "sampling/importance_sampling_ratio/mean": 0.9985718846321106, "sampling/importance_sampling_ratio/min": 0.44102411915858586, "sampling/sampling_logp_difference/max": 0.3229081074396769, "sampling/sampling_logp_difference/mean": 0.003876647655852139, "step": 11550, "step_time": 7.96979540753479, "student/entropy": 0.12390445911635956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779344469307, "completions/max_length": 494.8, "completions/max_terminated_length": 472.56666666666666, "completions/mean_length": 192.90056355794272, "completions/mean_terminated_length": 184.84651947021484, "completions/min_length": 49.3, "completions/min_terminated_length": 49.3, "entropy": 0.12849274203181266, "epoch": 0.43975240192913834, "eval/gt_acc_batch": 0.7883333444595337, "frac_reward_zero_std": 1.0, "grad_norm": 0.22863176465034485, "kd/policy_loss": 0.008965088120506455, "kd/rkl_advantage_mean": 0.770871565490961, "kd/rkl_advantage_p95": 5.060108218590418, "kd/rkl_advantage_std": 2.224726704756419, "kd/ssd_loss": 0.0, "learning_rate": 5.602855732351042e-07, "loss": 0.035860343774159746, "num_tokens": 378817797.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7883333345254262, "rewards/gt_logging_reward/std": 0.4035507599512736, "sampling/importance_sampling_ratio/max": 2.0174263874689737, "sampling/importance_sampling_ratio/mean": 1.0029634833335876, "sampling/importance_sampling_ratio/min": 0.4231244067351023, "sampling/sampling_logp_difference/max": 0.33953218658765155, "sampling/sampling_logp_difference/mean": 0.003979000154261788, "step": 11580, "step_time": 8.05831890850095, "student/entropy": 0.12849274203181266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500000881652038, "completions/max_length": 490.2, "completions/max_terminated_length": 449.1333333333333, "completions/mean_length": 188.62723083496093, "completions/mean_terminated_length": 183.0042953491211, "completions/min_length": 58.8, "completions/min_terminated_length": 58.8, "entropy": 0.12196001832683881, "epoch": 0.4408916568564159, "eval/gt_acc_batch": 0.7969444712003072, "frac_reward_zero_std": 1.0, "grad_norm": 0.20473287999629974, "kd/policy_loss": 0.008157000772189348, "kd/rkl_advantage_mean": 0.8192717425525189, "kd/rkl_advantage_p95": 5.152626891930898, "kd/rkl_advantage_std": 2.267831738789876, "kd/ssd_loss": 0.0, "learning_rate": 5.591463183078267e-07, "loss": 0.032628007729848224, "num_tokens": 379794991.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444433848063, "rewards/gt_logging_reward/std": 0.39531231224536895, "sampling/importance_sampling_ratio/max": 2.0035086909929913, "sampling/importance_sampling_ratio/mean": 1.003898940483729, "sampling/importance_sampling_ratio/min": 0.46857076982657114, "sampling/sampling_logp_difference/max": 0.3511040290196737, "sampling/sampling_logp_difference/mean": 0.0038627872321133814, "step": 11610, "step_time": 7.8961040179992175, "student/entropy": 0.12196001832683881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944446160147587, "completions/max_length": 479.6666666666667, "completions/max_terminated_length": 449.96666666666664, "completions/mean_length": 190.4836217244466, "completions/mean_terminated_length": 181.6720001220703, "completions/min_length": 61.2, "completions/min_terminated_length": 61.2, "entropy": 0.1270018999154369, "epoch": 0.44203091178369347, "eval/gt_acc_batch": 0.7941667000452678, "frac_reward_zero_std": 1.0, "grad_norm": 0.33135125041007996, "kd/policy_loss": 0.008627993956906721, "kd/rkl_advantage_mean": 0.7327132009435445, "kd/rkl_advantage_p95": 4.954172877470652, "kd/rkl_advantage_std": 2.2100267390410107, "kd/ssd_loss": 0.0, "learning_rate": 5.580070633805491e-07, "loss": 0.03451197942097982, "num_tokens": 380790684.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7941666682561238, "rewards/gt_logging_reward/std": 0.39538770020008085, "sampling/importance_sampling_ratio/max": 1.9614120244979858, "sampling/importance_sampling_ratio/mean": 0.9972096582253774, "sampling/importance_sampling_ratio/min": 0.4698720335960388, "sampling/sampling_logp_difference/max": 0.3488775432109833, "sampling/sampling_logp_difference/mean": 0.003992912902807196, "step": 11640, "step_time": 8.053600064432734, "student/entropy": 0.1270018999154369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666801397999, "completions/max_length": 482.73333333333335, "completions/max_terminated_length": 448.76666666666665, "completions/mean_length": 187.06417694091797, "completions/mean_terminated_length": 179.96803080240886, "completions/min_length": 55.266666666666666, "completions/min_terminated_length": 55.266666666666666, "entropy": 0.12451817175994316, "epoch": 0.44317016671097104, "eval/gt_acc_batch": 0.8016666968663534, "frac_reward_zero_std": 1.0, "grad_norm": 0.30949196219444275, "kd/policy_loss": 0.00838309078826569, "kd/rkl_advantage_mean": 0.77105615294228, "kd/rkl_advantage_p95": 4.976109951734543, "kd/rkl_advantage_std": 2.210820542772611, "kd/ssd_loss": 0.0, "learning_rate": 5.568678084532715e-07, "loss": 0.033532365163167314, "num_tokens": 381769611.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8016666650772095, "rewards/gt_logging_reward/std": 0.38965435574452084, "sampling/importance_sampling_ratio/max": 1.9081045627593993, "sampling/importance_sampling_ratio/mean": 0.9979966700077056, "sampling/importance_sampling_ratio/min": 0.4625123590230942, "sampling/sampling_logp_difference/max": 0.3474987943967183, "sampling/sampling_logp_difference/mean": 0.003964844695292413, "step": 11670, "step_time": 7.898506471431756, "student/entropy": 0.12451817175994316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112150053183, "completions/max_length": 489.73333333333335, "completions/max_terminated_length": 452.1666666666667, "completions/mean_length": 189.44250996907553, "completions/mean_terminated_length": 182.7487782796224, "completions/min_length": 59.666666666666664, "completions/min_terminated_length": 59.666666666666664, "entropy": 0.1273148354763786, "epoch": 0.4443094216382486, "eval/gt_acc_batch": 0.7947222471237183, "frac_reward_zero_std": 1.0, "grad_norm": 0.38450661301612854, "kd/policy_loss": 0.008845442135740693, "kd/rkl_advantage_mean": 0.7522298318644365, "kd/rkl_advantage_p95": 4.995620630184809, "kd/rkl_advantage_std": 2.195026456316312, "kd/ssd_loss": 0.0, "learning_rate": 5.55728553525994e-07, "loss": 0.03538177013397217, "num_tokens": 382760892.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222153345744, "rewards/gt_logging_reward/std": 0.39516275425752007, "sampling/importance_sampling_ratio/max": 1.9572381258010865, "sampling/importance_sampling_ratio/mean": 1.0022704859574636, "sampling/importance_sampling_ratio/min": 0.42390644947687783, "sampling/sampling_logp_difference/max": 0.35675330956776935, "sampling/sampling_logp_difference/mean": 0.0040344038900608815, "step": 11700, "step_time": 8.070355656768758, "student/entropy": 0.1273148354763786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02888889048869411, "completions/max_length": 484.7, "completions/max_terminated_length": 447.3333333333333, "completions/mean_length": 191.41723175048827, "completions/mean_terminated_length": 182.01618092854818, "completions/min_length": 50.4, "completions/min_terminated_length": 50.4, "entropy": 0.13005729311456282, "epoch": 0.44544867656552617, "eval/gt_acc_batch": 0.7858333667119344, "frac_reward_zero_std": 1.0, "grad_norm": 0.37692791223526, "kd/policy_loss": 0.008497272369762261, "kd/rkl_advantage_mean": 0.8157773998255531, "kd/rkl_advantage_p95": 5.127555092175801, "kd/rkl_advantage_std": 2.2356165140867232, "kd/ssd_loss": 0.0, "learning_rate": 5.545892985987164e-07, "loss": 0.03398909171422323, "num_tokens": 383759482.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.785833328962326, "rewards/gt_logging_reward/std": 0.3994343648354212, "sampling/importance_sampling_ratio/max": 1.8703674793243408, "sampling/importance_sampling_ratio/mean": 0.9955096940199534, "sampling/importance_sampling_ratio/min": 0.4365615606307983, "sampling/sampling_logp_difference/max": 0.36432543595631917, "sampling/sampling_logp_difference/mean": 0.003969570777068536, "step": 11730, "step_time": 8.078351310167152, "student/entropy": 0.13005729311456282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778872599203, "completions/max_length": 493.3333333333333, "completions/max_terminated_length": 465.46666666666664, "completions/mean_length": 187.9247314453125, "completions/mean_terminated_length": 180.49365743001303, "completions/min_length": 50.5, "completions/min_terminated_length": 50.5, "entropy": 0.12723729057858388, "epoch": 0.4465879314928037, "eval/gt_acc_batch": 0.816111131509145, "frac_reward_zero_std": 1.0, "grad_norm": 0.32571759819984436, "kd/policy_loss": 0.008576208523785074, "kd/rkl_advantage_mean": 0.8299570726851623, "kd/rkl_advantage_p95": 5.217686629295349, "kd/rkl_advantage_std": 2.3087016830841702, "kd/ssd_loss": 0.0, "learning_rate": 5.534500436714389e-07, "loss": 0.03430484135945638, "num_tokens": 384740115.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8161111136277517, "rewards/gt_logging_reward/std": 0.3815735220909119, "sampling/importance_sampling_ratio/max": 1.9349141558011373, "sampling/importance_sampling_ratio/mean": 1.003368576367696, "sampling/importance_sampling_ratio/min": 0.4318300182620684, "sampling/sampling_logp_difference/max": 0.31116458773612976, "sampling/sampling_logp_difference/mean": 0.003987009970781704, "step": 11760, "step_time": 7.9829765179034435, "student/entropy": 0.12723729057858388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015000000906487307, "completions/max_length": 462.7, "completions/max_terminated_length": 433.3333333333333, "completions/mean_length": 184.37278671264647, "completions/mean_terminated_length": 179.48554865519205, "completions/min_length": 57.96666666666667, "completions/min_terminated_length": 57.96666666666667, "entropy": 0.11963666311154762, "epoch": 0.44772718642008125, "eval/gt_acc_batch": 0.8141666789849599, "frac_reward_zero_std": 1.0, "grad_norm": 0.23927870392799377, "kd/policy_loss": 0.007753877302942177, "kd/rkl_advantage_mean": 0.7611672976820653, "kd/rkl_advantage_p95": 5.006147203842799, "kd/rkl_advantage_std": 2.20835799574852, "kd/ssd_loss": 0.0, "learning_rate": 5.523107887441613e-07, "loss": 0.03101550738016764, "num_tokens": 385717305.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.814166667064031, "rewards/gt_logging_reward/std": 0.38008989989757536, "sampling/importance_sampling_ratio/max": 1.8327450275421142, "sampling/importance_sampling_ratio/mean": 0.9966122527917226, "sampling/importance_sampling_ratio/min": 0.4442432483037313, "sampling/sampling_logp_difference/max": 0.33312641779581703, "sampling/sampling_logp_difference/mean": 0.003806824532027046, "step": 11790, "step_time": 7.917652834999414, "student/entropy": 0.11963666311154762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779251337052, "completions/max_length": 499.8333333333333, "completions/max_terminated_length": 444.9, "completions/mean_length": 192.68306579589844, "completions/mean_terminated_length": 184.43591868082683, "completions/min_length": 58.86666666666667, "completions/min_terminated_length": 58.86666666666667, "entropy": 0.12605640826125938, "epoch": 0.4488664413473588, "eval/gt_acc_batch": 0.7997222522894541, "frac_reward_zero_std": 1.0, "grad_norm": 0.21553899347782135, "kd/policy_loss": 0.008403326947397241, "kd/rkl_advantage_mean": 0.7684778449436028, "kd/rkl_advantage_p95": 5.01877718369166, "kd/rkl_advantage_std": 2.24449217915535, "kd/ssd_loss": 0.0, "learning_rate": 5.511715338168837e-07, "loss": 0.03361331621805827, "num_tokens": 386714196.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7997222224871318, "rewards/gt_logging_reward/std": 0.38953013916810353, "sampling/importance_sampling_ratio/max": 1.9053038001060485, "sampling/importance_sampling_ratio/mean": 1.0007560352484386, "sampling/importance_sampling_ratio/min": 0.44970961660146713, "sampling/sampling_logp_difference/max": 0.3371387521425883, "sampling/sampling_logp_difference/mean": 0.003999390173703432, "step": 11820, "step_time": 7.98917300440177, "student/entropy": 0.12605640826125938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001210719346, "completions/max_length": 493.4, "completions/max_terminated_length": 452.9, "completions/mean_length": 194.28751118977866, "completions/mean_terminated_length": 186.20176340738934, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "entropy": 0.13216456808149815, "epoch": 0.4500056962746364, "eval/gt_acc_batch": 0.7719444513320923, "frac_reward_zero_std": 1.0, "grad_norm": 0.2585335671901703, "kd/policy_loss": 0.009133713343180716, "kd/rkl_advantage_mean": 0.7791572157293558, "kd/rkl_advantage_p95": 5.092261878649394, "kd/rkl_advantage_std": 2.253809830546379, "kd/ssd_loss": 0.0, "learning_rate": 5.500322788896061e-07, "loss": 0.03653485377629598, "num_tokens": 387719999.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7719444473584492, "rewards/gt_logging_reward/std": 0.4070851032932599, "sampling/importance_sampling_ratio/max": 1.8567302624384563, "sampling/importance_sampling_ratio/mean": 0.9980574309825897, "sampling/importance_sampling_ratio/min": 0.4460436473290125, "sampling/sampling_logp_difference/max": 0.3403144737084707, "sampling/sampling_logp_difference/mean": 0.004107474737490217, "step": 11850, "step_time": 8.086806227334213, "student/entropy": 0.13216456808149815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444446029762427, "completions/max_length": 494.3666666666667, "completions/max_terminated_length": 460.9, "completions/mean_length": 194.70334167480468, "completions/mean_terminated_length": 187.00848999023438, "completions/min_length": 55.3, "completions/min_terminated_length": 55.3, "entropy": 0.12698029465973376, "epoch": 0.45114495120191395, "eval/gt_acc_batch": 0.7947222491105398, "frac_reward_zero_std": 1.0, "grad_norm": 0.2579437792301178, "kd/policy_loss": 0.008462744660209864, "kd/rkl_advantage_mean": 0.7482010906562209, "kd/rkl_advantage_p95": 5.019400147596995, "kd/rkl_advantage_std": 2.2100096782048544, "kd/ssd_loss": 0.0, "learning_rate": 5.488930239623286e-07, "loss": 0.03385097980499267, "num_tokens": 388727891.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222272555033, "rewards/gt_logging_reward/std": 0.39679141690333686, "sampling/importance_sampling_ratio/max": 1.9057703097661336, "sampling/importance_sampling_ratio/mean": 1.0026654561360677, "sampling/importance_sampling_ratio/min": 0.4301087309916814, "sampling/sampling_logp_difference/max": 0.34376593430836994, "sampling/sampling_logp_difference/mean": 0.00401786375635614, "step": 11880, "step_time": 8.136607864434579, "student/entropy": 0.12698029465973376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0258333345875144, "completions/max_length": 487.56666666666666, "completions/max_terminated_length": 447.3, "completions/mean_length": 195.7136220296224, "completions/mean_terminated_length": 187.49509023030598, "completions/min_length": 52.766666666666666, "completions/min_terminated_length": 52.766666666666666, "entropy": 0.12829925219217936, "epoch": 0.4522842061291915, "eval/gt_acc_batch": 0.8052777965863546, "frac_reward_zero_std": 1.0, "grad_norm": 0.3354143798351288, "kd/policy_loss": 0.008491399263342221, "kd/rkl_advantage_mean": 0.7631049758289009, "kd/rkl_advantage_p95": 5.010220529635747, "kd/rkl_advantage_std": 2.2081862916549047, "kd/ssd_loss": 0.0, "learning_rate": 5.477537690350512e-07, "loss": 0.03396560351053874, "num_tokens": 389729844.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8052777806917827, "rewards/gt_logging_reward/std": 0.3923739194869995, "sampling/importance_sampling_ratio/max": 1.8995325922966004, "sampling/importance_sampling_ratio/mean": 1.0007681449254353, "sampling/importance_sampling_ratio/min": 0.44639915972948074, "sampling/sampling_logp_difference/max": 0.31044217745463054, "sampling/sampling_logp_difference/mean": 0.00396394773075978, "step": 11910, "step_time": 8.027570355097607, "student/entropy": 0.12829925219217936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944445973883072, "completions/max_length": 469.76666666666665, "completions/max_terminated_length": 448.4, "completions/mean_length": 189.8702870686849, "completions/mean_terminated_length": 181.0322463989258, "completions/min_length": 52.3, "completions/min_terminated_length": 52.3, "entropy": 0.12513675882170597, "epoch": 0.4534234610564691, "eval/gt_acc_batch": 0.7925000190734863, "frac_reward_zero_std": 1.0, "grad_norm": 0.23538579046726227, "kd/policy_loss": 0.008504804479889572, "kd/rkl_advantage_mean": 0.7597595894709229, "kd/rkl_advantage_p95": 4.992324103911717, "kd/rkl_advantage_std": 2.224011715253194, "kd/ssd_loss": 0.0, "learning_rate": 5.466145141077735e-07, "loss": 0.03401922384897868, "num_tokens": 390716529.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7924999972184499, "rewards/gt_logging_reward/std": 0.39806604087352754, "sampling/importance_sampling_ratio/max": 1.8765013376871744, "sampling/importance_sampling_ratio/mean": 1.0007257123788198, "sampling/importance_sampling_ratio/min": 0.4320206219951312, "sampling/sampling_logp_difference/max": 0.3308928906917572, "sampling/sampling_logp_difference/mean": 0.003912045410834253, "step": 11940, "step_time": 8.000353744334522, "student/entropy": 0.12513675882170597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03972222491477927, "completions/max_length": 492.3666666666667, "completions/max_terminated_length": 473.6, "completions/mean_length": 199.96056518554687, "completions/mean_terminated_length": 187.25412241617838, "completions/min_length": 49.86666666666667, "completions/min_terminated_length": 49.86666666666667, "entropy": 0.13511500060558318, "epoch": 0.45456271598374665, "eval/gt_acc_batch": 0.7816666901111603, "frac_reward_zero_std": 1.0, "grad_norm": 0.22071725130081177, "kd/policy_loss": 0.009446450124960393, "kd/rkl_advantage_mean": 0.7470880875363946, "kd/rkl_advantage_p95": 5.028647708892822, "kd/rkl_advantage_std": 2.2432570358117423, "kd/ssd_loss": 0.0, "learning_rate": 5.45475259180496e-07, "loss": 0.03778580824534098, "num_tokens": 391738907.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7816666662693024, "rewards/gt_logging_reward/std": 0.40555890202522277, "sampling/importance_sampling_ratio/max": 1.9463974237442017, "sampling/importance_sampling_ratio/mean": 0.9959320863087971, "sampling/importance_sampling_ratio/min": 0.4032391240199407, "sampling/sampling_logp_difference/max": 0.3672845125198364, "sampling/sampling_logp_difference/mean": 0.004156980368619164, "step": 11970, "step_time": 8.065991825701591, "student/entropy": 0.13511500060558318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555668969949, "completions/max_length": 472.03333333333336, "completions/max_terminated_length": 428.6, "completions/mean_length": 180.6336212158203, "completions/mean_terminated_length": 173.76263732910155, "completions/min_length": 51.733333333333334, "completions/min_terminated_length": 51.733333333333334, "entropy": 0.13032022460053364, "epoch": 0.4557019709110242, "eval/gt_acc_batch": 0.8186111489931742, "frac_reward_zero_std": 1.0, "grad_norm": 0.279405415058136, "kd/policy_loss": 0.008192786273624127, "kd/rkl_advantage_mean": 0.7063602925588687, "kd/rkl_advantage_p95": 4.922167044878006, "kd/rkl_advantage_std": 2.1944000830252963, "kd/ssd_loss": 0.0, "learning_rate": 5.443360042532184e-07, "loss": 0.032771148284276325, "num_tokens": 392691244.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8186111152172089, "rewards/gt_logging_reward/std": 0.3747957726319631, "sampling/importance_sampling_ratio/max": 1.7784202496210735, "sampling/importance_sampling_ratio/mean": 0.9927469909191131, "sampling/importance_sampling_ratio/min": 0.417162894209226, "sampling/sampling_logp_difference/max": 0.3315996050834656, "sampling/sampling_logp_difference/mean": 0.004049071320332586, "step": 12000, "step_time": 7.945789945797879, "student/entropy": 0.13032022460053364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016944445421298346, "completions/max_length": 486.96666666666664, "completions/max_terminated_length": 450.23333333333335, "completions/mean_length": 189.90945587158203, "completions/mean_terminated_length": 184.41861012776693, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "entropy": 0.12799301072955133, "epoch": 0.4568412258383017, "eval/gt_acc_batch": 0.8136111219724019, "frac_reward_zero_std": 1.0, "grad_norm": 0.2825597822666168, "kd/policy_loss": 0.008338462703007584, "kd/rkl_advantage_mean": 0.7807475455105305, "kd/rkl_advantage_p95": 5.135435656706492, "kd/rkl_advantage_std": 2.264342340826988, "kd/ssd_loss": 0.0, "learning_rate": 5.431967493259408e-07, "loss": 0.03335385322570801, "num_tokens": 393671838.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813611110051473, "rewards/gt_logging_reward/std": 0.3768211121360461, "sampling/importance_sampling_ratio/max": 1.9151448011398315, "sampling/importance_sampling_ratio/mean": 1.0074583450953165, "sampling/importance_sampling_ratio/min": 0.42339751273393633, "sampling/sampling_logp_difference/max": 0.3393979549407959, "sampling/sampling_logp_difference/mean": 0.004025807557627559, "step": 12030, "step_time": 7.904765253470396, "student/entropy": 0.12799301072955133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223629554113, "completions/max_length": 472.6, "completions/max_terminated_length": 439.9, "completions/mean_length": 188.8072311401367, "completions/mean_terminated_length": 181.54793345133464, "completions/min_length": 56.266666666666666, "completions/min_terminated_length": 56.266666666666666, "entropy": 0.13337998470912377, "epoch": 0.4579804807655793, "eval/gt_acc_batch": 0.7850000123182933, "frac_reward_zero_std": 1.0, "grad_norm": 0.31098082661628723, "kd/policy_loss": 0.008980945111640418, "kd/rkl_advantage_mean": 0.7959753692770998, "kd/rkl_advantage_p95": 5.065071958303451, "kd/rkl_advantage_std": 2.2123953054348626, "kd/ssd_loss": 0.0, "learning_rate": 5.420574943986632e-07, "loss": 0.03592378298441569, "num_tokens": 394653704.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7850000043710073, "rewards/gt_logging_reward/std": 0.40459794700145724, "sampling/importance_sampling_ratio/max": 1.9783270716667176, "sampling/importance_sampling_ratio/mean": 0.9989055732885996, "sampling/importance_sampling_ratio/min": 0.4522521351774534, "sampling/sampling_logp_difference/max": 0.30883174141248065, "sampling/sampling_logp_difference/mean": 0.004110450212222835, "step": 12060, "step_time": 7.930055189168585, "student/entropy": 0.13337998470912377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019444445427507163, "completions/max_length": 456.96666666666664, "completions/max_terminated_length": 426.8666666666667, "completions/mean_length": 182.3011220296224, "completions/mean_terminated_length": 175.9831278483073, "completions/min_length": 47.233333333333334, "completions/min_terminated_length": 47.233333333333334, "entropy": 0.1281306093558669, "epoch": 0.45911973569285686, "eval/gt_acc_batch": 0.8072222451368968, "frac_reward_zero_std": 1.0, "grad_norm": 0.2872403860092163, "kd/policy_loss": 0.008477109352437158, "kd/rkl_advantage_mean": 0.6978572747670114, "kd/rkl_advantage_p95": 4.835377901792526, "kd/rkl_advantage_std": 2.160737705230713, "kd/ssd_loss": 0.0, "learning_rate": 5.409182394713857e-07, "loss": 0.03390843868255615, "num_tokens": 395610044.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222193082174, "rewards/gt_logging_reward/std": 0.3904307097196579, "sampling/importance_sampling_ratio/max": 1.9422001123428345, "sampling/importance_sampling_ratio/mean": 1.0016786257425945, "sampling/importance_sampling_ratio/min": 0.45843269377946855, "sampling/sampling_logp_difference/max": 0.37298805713653566, "sampling/sampling_logp_difference/mean": 0.004057158568563561, "step": 12090, "step_time": 7.728222152529634, "student/entropy": 0.1281306093558669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028611112385988235, "completions/max_length": 489.1, "completions/max_terminated_length": 427.4, "completions/mean_length": 193.21195475260416, "completions/mean_terminated_length": 184.02374114990235, "completions/min_length": 52.233333333333334, "completions/min_terminated_length": 52.233333333333334, "entropy": 0.12577250252167385, "epoch": 0.4602589906201344, "eval/gt_acc_batch": 0.7969444652398427, "frac_reward_zero_std": 1.0, "grad_norm": 0.30789715051651, "kd/policy_loss": 0.00849053099906693, "kd/rkl_advantage_mean": 0.7787505216896534, "kd/rkl_advantage_p95": 5.1172090371449785, "kd/rkl_advantage_std": 2.2487810134887694, "kd/ssd_loss": 0.0, "learning_rate": 5.397789845441081e-07, "loss": 0.033962130546569824, "num_tokens": 396599823.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444473584493, "rewards/gt_logging_reward/std": 0.38750017384688057, "sampling/importance_sampling_ratio/max": 2.0049553592999776, "sampling/importance_sampling_ratio/mean": 1.0072013795375825, "sampling/importance_sampling_ratio/min": 0.4497729440530141, "sampling/sampling_logp_difference/max": 0.33171613017718, "sampling/sampling_logp_difference/mean": 0.003979091175521413, "step": 12120, "step_time": 8.030885695067505, "student/entropy": 0.12577250252167385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777912716071, "completions/max_length": 487.9, "completions/max_terminated_length": 453.93333333333334, "completions/mean_length": 182.65167795817058, "completions/mean_terminated_length": 174.12720947265626, "completions/min_length": 50.4, "completions/min_terminated_length": 50.4, "entropy": 0.12850449445346993, "epoch": 0.461398245547412, "eval/gt_acc_batch": 0.8016666869322459, "frac_reward_zero_std": 1.0, "grad_norm": 0.3051902651786804, "kd/policy_loss": 0.008476567268371583, "kd/rkl_advantage_mean": 0.8039091775504251, "kd/rkl_advantage_p95": 5.114813641707102, "kd/rkl_advantage_std": 2.290487782160441, "kd/ssd_loss": 0.0, "learning_rate": 5.386397296168306e-07, "loss": 0.033906273047129315, "num_tokens": 397555377.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8016666690508525, "rewards/gt_logging_reward/std": 0.3898630325992902, "sampling/importance_sampling_ratio/max": 1.925388475259145, "sampling/importance_sampling_ratio/mean": 1.0013880014419556, "sampling/importance_sampling_ratio/min": 0.43317556778589883, "sampling/sampling_logp_difference/max": 0.3799127479394277, "sampling/sampling_logp_difference/mean": 0.004026151386400064, "step": 12150, "step_time": 7.984927275465937, "student/entropy": 0.12850449445346993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01916666766628623, "completions/max_length": 483.26666666666665, "completions/max_terminated_length": 450.9, "completions/mean_length": 186.55723215738934, "completions/mean_terminated_length": 180.2538070678711, "completions/min_length": 54.833333333333336, "completions/min_terminated_length": 54.833333333333336, "entropy": 0.12956419413288434, "epoch": 0.46253750047468956, "eval/gt_acc_batch": 0.7958333591620127, "frac_reward_zero_std": 1.0, "grad_norm": 0.2549331486225128, "kd/policy_loss": 0.008816639733656, "kd/rkl_advantage_mean": 0.7689006454000871, "kd/rkl_advantage_p95": 5.07081171075503, "kd/rkl_advantage_std": 2.2311460425456366, "kd/ssd_loss": 0.0, "learning_rate": 5.375004746895531e-07, "loss": 0.03526655832926432, "num_tokens": 398530111.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7958333353201549, "rewards/gt_logging_reward/std": 0.3966590245564779, "sampling/importance_sampling_ratio/max": 1.9275898615519205, "sampling/importance_sampling_ratio/mean": 1.0009394566218057, "sampling/importance_sampling_ratio/min": 0.4812654346227646, "sampling/sampling_logp_difference/max": 0.3434873064359029, "sampling/sampling_logp_difference/mean": 0.004052965971641243, "step": 12180, "step_time": 7.907386727261959, "student/entropy": 0.12956419413288434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027500001713633537, "completions/max_length": 491.8333333333333, "completions/max_terminated_length": 460.2, "completions/mean_length": 193.01278635660807, "completions/mean_terminated_length": 184.16851552327475, "completions/min_length": 53.9, "completions/min_terminated_length": 53.9, "entropy": 0.12869466841220856, "epoch": 0.4636767554019671, "eval/gt_acc_batch": 0.7930555760860443, "frac_reward_zero_std": 1.0, "grad_norm": 0.4258292019367218, "kd/policy_loss": 0.008373634554057692, "kd/rkl_advantage_mean": 0.7480928803483645, "kd/rkl_advantage_p95": 5.043279594182968, "kd/rkl_advantage_std": 2.2217055052518844, "kd/ssd_loss": 0.0, "learning_rate": 5.363612197622754e-07, "loss": 0.03349453608194987, "num_tokens": 399532213.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555582046509, "rewards/gt_logging_reward/std": 0.39734552800655365, "sampling/importance_sampling_ratio/max": 1.8310639619827271, "sampling/importance_sampling_ratio/mean": 0.9924701273441314, "sampling/importance_sampling_ratio/min": 0.41165054539839424, "sampling/sampling_logp_difference/max": 0.36141204635302226, "sampling/sampling_logp_difference/mean": 0.0040165397571399804, "step": 12210, "step_time": 8.054497408536553, "student/entropy": 0.12869466841220856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02277777933826049, "completions/max_length": 493.0, "completions/max_terminated_length": 445.7, "completions/mean_length": 189.71639862060547, "completions/mean_terminated_length": 182.38717651367188, "completions/min_length": 52.733333333333334, "completions/min_terminated_length": 52.733333333333334, "entropy": 0.12905156395087639, "epoch": 0.4648160103292447, "eval/gt_acc_batch": 0.7861111442248027, "frac_reward_zero_std": 1.0, "grad_norm": 0.2826915681362152, "kd/policy_loss": 0.008705345463628571, "kd/rkl_advantage_mean": 0.7891308058674137, "kd/rkl_advantage_p95": 5.112093053261439, "kd/rkl_advantage_std": 2.278004095951716, "kd/ssd_loss": 0.0, "learning_rate": 5.352219648349979e-07, "loss": 0.03482138315836589, "num_tokens": 400520304.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111104488373, "rewards/gt_logging_reward/std": 0.3994013249874115, "sampling/importance_sampling_ratio/max": 1.9566477855046591, "sampling/importance_sampling_ratio/mean": 0.9952687402566274, "sampling/importance_sampling_ratio/min": 0.44381705621878303, "sampling/sampling_logp_difference/max": 0.3578454474608103, "sampling/sampling_logp_difference/mean": 0.003978842605526249, "step": 12240, "step_time": 7.93886410856503, "student/entropy": 0.12905156395087639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556944260994, "completions/max_length": 465.9, "completions/max_terminated_length": 439.0, "completions/mean_length": 184.73834330240885, "completions/mean_terminated_length": 177.3744333902995, "completions/min_length": 58.266666666666666, "completions/min_terminated_length": 58.266666666666666, "entropy": 0.1316041840861241, "epoch": 0.46595526525652226, "eval/gt_acc_batch": 0.8002778033415476, "frac_reward_zero_std": 1.0, "grad_norm": 0.28901320695877075, "kd/policy_loss": 0.008861186498931299, "kd/rkl_advantage_mean": 0.8377581735452017, "kd/rkl_advantage_p95": 5.188427786032359, "kd/rkl_advantage_std": 2.2697444836298626, "kd/ssd_loss": 0.0, "learning_rate": 5.340827099077203e-07, "loss": 0.035444752375284834, "num_tokens": 401492346.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8002777775128682, "rewards/gt_logging_reward/std": 0.3883898377418518, "sampling/importance_sampling_ratio/max": 1.9238938053448995, "sampling/importance_sampling_ratio/mean": 1.0013447026411693, "sampling/importance_sampling_ratio/min": 0.44839653968811033, "sampling/sampling_logp_difference/max": 0.35615657170613607, "sampling/sampling_logp_difference/mean": 0.0041023522227381665, "step": 12270, "step_time": 7.998448583833912, "student/entropy": 0.1316041840861241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02583333443229397, "completions/max_length": 491.9, "completions/max_terminated_length": 450.76666666666665, "completions/mean_length": 191.52584279378254, "completions/mean_terminated_length": 183.18075307210287, "completions/min_length": 51.6, "completions/min_terminated_length": 51.6, "entropy": 0.12544056779394547, "epoch": 0.4670945201837998, "eval/gt_acc_batch": 0.7936111330986023, "frac_reward_zero_std": 1.0, "grad_norm": 0.35356587171554565, "kd/policy_loss": 0.00853559344735307, "kd/rkl_advantage_mean": 0.7803453713655472, "kd/rkl_advantage_p95": 5.095985708634059, "kd/rkl_advantage_std": 2.2327536781628927, "kd/ssd_loss": 0.0, "learning_rate": 5.329434549804428e-07, "loss": 0.034142374992370605, "num_tokens": 402497455.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111152172088, "rewards/gt_logging_reward/std": 0.39681895474592843, "sampling/importance_sampling_ratio/max": 1.8700268546740213, "sampling/importance_sampling_ratio/mean": 1.001883480946223, "sampling/importance_sampling_ratio/min": 0.43674261420965194, "sampling/sampling_logp_difference/max": 0.34496406515439354, "sampling/sampling_logp_difference/mean": 0.003968942631036043, "step": 12300, "step_time": 8.146593546066045, "student/entropy": 0.12544056779394547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334742734828, "completions/max_length": 505.93333333333334, "completions/max_terminated_length": 461.4, "completions/mean_length": 190.9844538370768, "completions/mean_terminated_length": 182.51958211263022, "completions/min_length": 57.5, "completions/min_terminated_length": 57.5, "entropy": 0.13454602435231208, "epoch": 0.46823377511107733, "eval/gt_acc_batch": 0.8005555868148804, "frac_reward_zero_std": 1.0, "grad_norm": 0.28282517194747925, "kd/policy_loss": 0.008879554050508887, "kd/rkl_advantage_mean": 0.8585422556226452, "kd/rkl_advantage_p95": 5.18667913277944, "kd/rkl_advantage_std": 2.26518285771211, "kd/ssd_loss": 0.0, "learning_rate": 5.318042000531652e-07, "loss": 0.035518217086791995, "num_tokens": 403487479.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8005555550257365, "rewards/gt_logging_reward/std": 0.391682335237662, "sampling/importance_sampling_ratio/max": 1.9527653972307841, "sampling/importance_sampling_ratio/mean": 0.9971980492273966, "sampling/importance_sampling_ratio/min": 0.4198535069823265, "sampling/sampling_logp_difference/max": 0.37571562131245934, "sampling/sampling_logp_difference/mean": 0.004155640591246386, "step": 12330, "step_time": 8.262695367068712, "student/entropy": 0.13454602435231208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778953313828, "completions/max_length": 491.8666666666667, "completions/max_terminated_length": 447.03333333333336, "completions/mean_length": 182.05695292154948, "completions/mean_terminated_length": 176.0688496907552, "completions/min_length": 51.333333333333336, "completions/min_terminated_length": 51.333333333333336, "entropy": 0.12975815553218126, "epoch": 0.4693730300383549, "eval/gt_acc_batch": 0.7944444755713145, "frac_reward_zero_std": 1.0, "grad_norm": 0.3199203312397003, "kd/policy_loss": 0.00869534924859181, "kd/rkl_advantage_mean": 0.7870756124146283, "kd/rkl_advantage_p95": 5.114649401108424, "kd/rkl_advantage_std": 2.2828834851582847, "kd/ssd_loss": 0.0, "learning_rate": 5.306649451258877e-07, "loss": 0.03478139638900757, "num_tokens": 404446940.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7944444417953491, "rewards/gt_logging_reward/std": 0.3986867109934489, "sampling/importance_sampling_ratio/max": 1.8747958064079284, "sampling/importance_sampling_ratio/mean": 0.9929058909416199, "sampling/importance_sampling_ratio/min": 0.437885095179081, "sampling/sampling_logp_difference/max": 0.3231948455174764, "sampling/sampling_logp_difference/mean": 0.004070127957190076, "step": 12360, "step_time": 8.023203751935217, "student/entropy": 0.12975815553218126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030833334910372892, "completions/max_length": 494.73333333333335, "completions/max_terminated_length": 451.1666666666667, "completions/mean_length": 197.84945576985677, "completions/mean_terminated_length": 188.0748540242513, "completions/min_length": 51.56666666666667, "completions/min_terminated_length": 51.56666666666667, "entropy": 0.12908708558728296, "epoch": 0.47051228496563247, "eval/gt_acc_batch": 0.7930555701255798, "frac_reward_zero_std": 1.0, "grad_norm": 0.24288931488990784, "kd/policy_loss": 0.008531666034832597, "kd/rkl_advantage_mean": 0.8102629373470942, "kd/rkl_advantage_p95": 5.144208264350891, "kd/rkl_advantage_std": 2.2631691992282867, "kd/ssd_loss": 0.0, "learning_rate": 5.2952569019861e-07, "loss": 0.03412667115529378, "num_tokens": 405468046.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555522441864, "rewards/gt_logging_reward/std": 0.39825192590554553, "sampling/importance_sampling_ratio/max": 1.9421889185905457, "sampling/importance_sampling_ratio/mean": 1.0011794507503509, "sampling/importance_sampling_ratio/min": 0.45422680377960206, "sampling/sampling_logp_difference/max": 0.31996582746505736, "sampling/sampling_logp_difference/mean": 0.003985898375200729, "step": 12390, "step_time": 8.254475776099328, "student/entropy": 0.12908708558728296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02833333481103182, "completions/max_length": 486.6, "completions/max_terminated_length": 451.3666666666667, "completions/mean_length": 192.4555648803711, "completions/mean_terminated_length": 183.28630574544272, "completions/min_length": 56.36666666666667, "completions/min_terminated_length": 56.36666666666667, "entropy": 0.12820224712292352, "epoch": 0.47165153989291003, "eval/gt_acc_batch": 0.7944444636503856, "frac_reward_zero_std": 1.0, "grad_norm": 0.2593255937099457, "kd/policy_loss": 0.008674797390510018, "kd/rkl_advantage_mean": 0.8075697331999739, "kd/rkl_advantage_p95": 5.1253375271956125, "kd/rkl_advantage_std": 2.2392161319653194, "kd/ssd_loss": 0.0, "learning_rate": 5.283864352713325e-07, "loss": 0.034699193636576336, "num_tokens": 406455750.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7944444437821706, "rewards/gt_logging_reward/std": 0.39720159272352856, "sampling/importance_sampling_ratio/max": 1.976891847451528, "sampling/importance_sampling_ratio/mean": 0.9993873596191406, "sampling/importance_sampling_ratio/min": 0.42790596683820087, "sampling/sampling_logp_difference/max": 0.33373506863911945, "sampling/sampling_logp_difference/mean": 0.003946350538171828, "step": 12420, "step_time": 8.017524809301054, "student/entropy": 0.12820224712292352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112367361785, "completions/max_length": 491.3666666666667, "completions/max_terminated_length": 443.56666666666666, "completions/mean_length": 188.79195454915364, "completions/mean_terminated_length": 181.9593287150065, "completions/min_length": 57.2, "completions/min_terminated_length": 57.2, "entropy": 0.13498600851744413, "epoch": 0.4727907948201876, "eval/gt_acc_batch": 0.7972222487131755, "frac_reward_zero_std": 1.0, "grad_norm": 0.28816768527030945, "kd/policy_loss": 0.009159433022917558, "kd/rkl_advantage_mean": 0.8423999354243279, "kd/rkl_advantage_p95": 5.189157114426295, "kd/rkl_advantage_std": 2.2744650334119796, "kd/ssd_loss": 0.0, "learning_rate": 5.272471803440549e-07, "loss": 0.036637735366821286, "num_tokens": 407430553.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7972222189108531, "rewards/gt_logging_reward/std": 0.3963650673627853, "sampling/importance_sampling_ratio/max": 1.837975549697876, "sampling/importance_sampling_ratio/mean": 0.9944822669029236, "sampling/importance_sampling_ratio/min": 0.4234780485431353, "sampling/sampling_logp_difference/max": 0.35751566688219705, "sampling/sampling_logp_difference/mean": 0.004125487843217949, "step": 12450, "step_time": 8.069132091835975, "student/entropy": 0.13498600851744413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030833334972461064, "completions/max_length": 495.1, "completions/max_terminated_length": 452.1333333333333, "completions/mean_length": 188.45528920491537, "completions/mean_terminated_length": 178.19663848876954, "completions/min_length": 55.46666666666667, "completions/min_terminated_length": 55.46666666666667, "entropy": 0.13365290326376755, "epoch": 0.47393004974746517, "eval/gt_acc_batch": 0.7850000222524007, "frac_reward_zero_std": 1.0, "grad_norm": 0.25963732600212097, "kd/policy_loss": 0.008860667108092458, "kd/rkl_advantage_mean": 0.809007806951801, "kd/rkl_advantage_p95": 5.1819097936153415, "kd/rkl_advantage_std": 2.273602482676506, "kd/ssd_loss": 0.0, "learning_rate": 5.261079254167774e-07, "loss": 0.035442666212717695, "num_tokens": 408409088.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7849999984105428, "rewards/gt_logging_reward/std": 0.4033943961064021, "sampling/importance_sampling_ratio/max": 1.9409206748008727, "sampling/importance_sampling_ratio/mean": 0.9996331989765167, "sampling/importance_sampling_ratio/min": 0.4205568845073382, "sampling/sampling_logp_difference/max": 0.34457824428876244, "sampling/sampling_logp_difference/mean": 0.00412205553924044, "step": 12480, "step_time": 8.08000808700114, "student/entropy": 0.13365290326376755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666751727462, "completions/max_length": 491.0, "completions/max_terminated_length": 441.1, "completions/mean_length": 194.4136215209961, "completions/mean_terminated_length": 187.2980951944987, "completions/min_length": 51.63333333333333, "completions/min_terminated_length": 51.63333333333333, "entropy": 0.13033027419199547, "epoch": 0.47506930467474273, "eval/gt_acc_batch": 0.7777777969837188, "frac_reward_zero_std": 1.0, "grad_norm": 0.28601422905921936, "kd/policy_loss": 0.009291781892534346, "kd/rkl_advantage_mean": 0.8224023768678308, "kd/rkl_advantage_p95": 5.146378113826116, "kd/rkl_advantage_std": 2.2531438281138736, "kd/ssd_loss": 0.0, "learning_rate": 5.249686704894999e-07, "loss": 0.03716712792714437, "num_tokens": 409405441.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7777777791023255, "rewards/gt_logging_reward/std": 0.40889160335063934, "sampling/importance_sampling_ratio/max": 1.9376919468243916, "sampling/importance_sampling_ratio/mean": 1.0036546528339385, "sampling/importance_sampling_ratio/min": 0.39551488955815634, "sampling/sampling_logp_difference/max": 0.35029176870981854, "sampling/sampling_logp_difference/mean": 0.004034199534604947, "step": 12510, "step_time": 7.950618540866708, "student/entropy": 0.13033027419199547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556813875834, "completions/max_length": 483.3, "completions/max_terminated_length": 450.1333333333333, "completions/mean_length": 187.92501068115234, "completions/mean_terminated_length": 181.3502161661784, "completions/min_length": 55.9, "completions/min_terminated_length": 55.9, "entropy": 0.1250470625857512, "epoch": 0.4762085596020203, "eval/gt_acc_batch": 0.8052778164545695, "frac_reward_zero_std": 1.0, "grad_norm": 0.3301781415939331, "kd/policy_loss": 0.008609231926190357, "kd/rkl_advantage_mean": 0.7017073646110172, "kd/rkl_advantage_p95": 4.862059146165848, "kd/rkl_advantage_std": 2.1899453987677893, "kd/ssd_loss": 0.0, "learning_rate": 5.238294155622223e-07, "loss": 0.03443692525227864, "num_tokens": 410387755.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8052777727444966, "rewards/gt_logging_reward/std": 0.385347855091095, "sampling/importance_sampling_ratio/max": 1.9171698172887166, "sampling/importance_sampling_ratio/mean": 0.9953366061051686, "sampling/importance_sampling_ratio/min": 0.45803446918725965, "sampling/sampling_logp_difference/max": 0.3533571779727936, "sampling/sampling_logp_difference/mean": 0.0038906583096832035, "step": 12540, "step_time": 7.84950780516762, "student/entropy": 0.1250470625857512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01944444573794802, "completions/max_length": 504.56666666666666, "completions/max_terminated_length": 452.46666666666664, "completions/mean_length": 183.7888977050781, "completions/mean_terminated_length": 177.2657679239909, "completions/min_length": 54.166666666666664, "completions/min_terminated_length": 54.166666666666664, "entropy": 0.1373899048815171, "epoch": 0.47734781452929786, "eval/gt_acc_batch": 0.7880555689334869, "frac_reward_zero_std": 1.0, "grad_norm": 0.33327504992485046, "kd/policy_loss": 0.00919904092637201, "kd/rkl_advantage_mean": 0.7638780643620218, "kd/rkl_advantage_p95": 5.089945125579834, "kd/rkl_advantage_std": 2.272619946797689, "kd/ssd_loss": 0.0, "learning_rate": 5.226901606349448e-07, "loss": 0.036796164512634275, "num_tokens": 411348083.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7880555589993795, "rewards/gt_logging_reward/std": 0.3989095409711202, "sampling/importance_sampling_ratio/max": 1.9794055660565695, "sampling/importance_sampling_ratio/mean": 1.005633242925008, "sampling/importance_sampling_ratio/min": 0.4240633398294449, "sampling/sampling_logp_difference/max": 0.32150013049443565, "sampling/sampling_logp_difference/mean": 0.00413146597178032, "step": 12570, "step_time": 7.923313862433618, "student/entropy": 0.1373899048815171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778704961143, "completions/max_length": 457.6666666666667, "completions/max_terminated_length": 414.8333333333333, "completions/mean_length": 184.74723256429036, "completions/mean_terminated_length": 178.9701944986979, "completions/min_length": 53.766666666666666, "completions/min_terminated_length": 53.766666666666666, "entropy": 0.12524750089893738, "epoch": 0.4784870694565754, "eval/gt_acc_batch": 0.8036111334959666, "frac_reward_zero_std": 1.0, "grad_norm": 0.30214250087738037, "kd/policy_loss": 0.00841039073614714, "kd/rkl_advantage_mean": 0.7116778746208486, "kd/rkl_advantage_p95": 4.844099666674932, "kd/rkl_advantage_std": 2.1577556560436886, "kd/ssd_loss": 0.0, "learning_rate": 5.215509057076671e-07, "loss": 0.03364156484603882, "num_tokens": 412311421.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111096541086, "rewards/gt_logging_reward/std": 0.38855257133642834, "sampling/importance_sampling_ratio/max": 1.8663800239562989, "sampling/importance_sampling_ratio/mean": 1.0016922255357106, "sampling/importance_sampling_ratio/min": 0.43024966021378835, "sampling/sampling_logp_difference/max": 0.3467615048090617, "sampling/sampling_logp_difference/mean": 0.0039003147588421903, "step": 12600, "step_time": 7.655710810002347, "student/entropy": 0.12524750089893738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001297642788, "completions/max_length": 487.06666666666666, "completions/max_terminated_length": 456.1333333333333, "completions/mean_length": 188.10167744954427, "completions/mean_terminated_length": 180.73096466064453, "completions/min_length": 47.4, "completions/min_terminated_length": 47.4, "entropy": 0.13153769026199977, "epoch": 0.47962632438385294, "eval/gt_acc_batch": 0.7980555812517802, "frac_reward_zero_std": 1.0, "grad_norm": 0.2609873414039612, "kd/policy_loss": 0.008707795220349606, "kd/rkl_advantage_mean": 0.7694641329968969, "kd/rkl_advantage_p95": 5.108043116331101, "kd/rkl_advantage_std": 2.2635294953982035, "kd/ssd_loss": 0.0, "learning_rate": 5.204116507803896e-07, "loss": 0.03483118216196696, "num_tokens": 413298467.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7980555593967438, "rewards/gt_logging_reward/std": 0.3963110496600469, "sampling/importance_sampling_ratio/max": 1.8643792589505515, "sampling/importance_sampling_ratio/mean": 0.9989051381746928, "sampling/importance_sampling_ratio/min": 0.49423596262931824, "sampling/sampling_logp_difference/max": 0.3765718857447306, "sampling/sampling_logp_difference/mean": 0.004107233633597692, "step": 12630, "step_time": 7.970042690967967, "student/entropy": 0.13153769026199977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112429449956, "completions/max_length": 487.1333333333333, "completions/max_terminated_length": 453.3666666666667, "completions/mean_length": 186.78834228515626, "completions/mean_terminated_length": 179.8039764404297, "completions/min_length": 56.666666666666664, "completions/min_terminated_length": 56.666666666666664, "entropy": 0.12940102635572354, "epoch": 0.4807655793111305, "eval/gt_acc_batch": 0.8041666825612386, "frac_reward_zero_std": 1.0, "grad_norm": 0.28625544905662537, "kd/policy_loss": 0.00821543302348194, "kd/rkl_advantage_mean": 0.7745011765509844, "kd/rkl_advantage_p95": 5.061305189132691, "kd/rkl_advantage_std": 2.2277542104323707, "kd/ssd_loss": 0.0, "learning_rate": 5.19272395853112e-07, "loss": 0.03286173144976298, "num_tokens": 414264281.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666666666667, "rewards/gt_logging_reward/std": 0.38842560946941374, "sampling/importance_sampling_ratio/max": 1.7729122201601664, "sampling/importance_sampling_ratio/mean": 0.9926212906837464, "sampling/importance_sampling_ratio/min": 0.43396565119425456, "sampling/sampling_logp_difference/max": 0.32010950446128844, "sampling/sampling_logp_difference/mean": 0.004025701456703246, "step": 12660, "step_time": 7.752204642337165, "student/entropy": 0.12940102635572354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01194444519157211, "completions/max_length": 457.93333333333334, "completions/max_terminated_length": 430.2, "completions/mean_length": 179.58778737386066, "completions/mean_terminated_length": 175.55408274332683, "completions/min_length": 56.7, "completions/min_terminated_length": 56.7, "entropy": 0.1294696072116494, "epoch": 0.4819048342384081, "eval/gt_acc_batch": 0.8213889022668203, "frac_reward_zero_std": 1.0, "grad_norm": 0.25255241990089417, "kd/policy_loss": 0.008138923702063039, "kd/rkl_advantage_mean": 0.7223417156065504, "kd/rkl_advantage_p95": 4.890562552213669, "kd/rkl_advantage_std": 2.1737030218044917, "kd/ssd_loss": 0.0, "learning_rate": 5.181331409258344e-07, "loss": 0.032555697361628215, "num_tokens": 415213637.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8213888883590699, "rewards/gt_logging_reward/std": 0.377575746178627, "sampling/importance_sampling_ratio/max": 1.8993517756462097, "sampling/importance_sampling_ratio/mean": 1.0006264011065165, "sampling/importance_sampling_ratio/min": 0.38869684835275015, "sampling/sampling_logp_difference/max": 0.3556972702344259, "sampling/sampling_logp_difference/mean": 0.004051536849389473, "step": 12690, "step_time": 7.724716821066977, "student/entropy": 0.1294696072116494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223797192177, "completions/max_length": 481.43333333333334, "completions/max_terminated_length": 447.8666666666667, "completions/mean_length": 189.83000895182292, "completions/mean_terminated_length": 180.99695892333983, "completions/min_length": 60.233333333333334, "completions/min_terminated_length": 60.233333333333334, "entropy": 0.1336875431239605, "epoch": 0.48304408916568564, "eval/gt_acc_batch": 0.7966666917006174, "frac_reward_zero_std": 1.0, "grad_norm": 0.3290959298610687, "kd/policy_loss": 0.009111005294835195, "kd/rkl_advantage_mean": 0.7284327437790732, "kd/rkl_advantage_p95": 4.907591076691945, "kd/rkl_advantage_std": 2.1913512726624806, "kd/ssd_loss": 0.0, "learning_rate": 5.169938859985569e-07, "loss": 0.03644402821858724, "num_tokens": 416202113.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7966666638851165, "rewards/gt_logging_reward/std": 0.3919711252053579, "sampling/importance_sampling_ratio/max": 1.9949310580889383, "sampling/importance_sampling_ratio/mean": 0.9968347609043121, "sampling/importance_sampling_ratio/min": 0.44508445014556247, "sampling/sampling_logp_difference/max": 0.3466535607973735, "sampling/sampling_logp_difference/mean": 0.004015412616233031, "step": 12720, "step_time": 7.9249766730650055, "student/entropy": 0.1336875431239605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500001130004723, "completions/max_length": 475.3, "completions/max_terminated_length": 447.8, "completions/mean_length": 187.96195373535156, "completions/mean_terminated_length": 182.2034927368164, "completions/min_length": 54.5, "completions/min_terminated_length": 54.5, "entropy": 0.12608816946546236, "epoch": 0.4841833440929632, "eval/gt_acc_batch": 0.8080555876096089, "frac_reward_zero_std": 1.0, "grad_norm": 0.27184510231018066, "kd/policy_loss": 0.008268644030128296, "kd/rkl_advantage_mean": 0.8023717172443867, "kd/rkl_advantage_p95": 5.125519134600958, "kd/rkl_advantage_std": 2.2660691291093826, "kd/ssd_loss": 0.0, "learning_rate": 5.158546310712794e-07, "loss": 0.03307457566261292, "num_tokens": 417185480.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8080555578072866, "rewards/gt_logging_reward/std": 0.38630496760209404, "sampling/importance_sampling_ratio/max": 1.9522570927937826, "sampling/importance_sampling_ratio/mean": 1.0008424699306488, "sampling/importance_sampling_ratio/min": 0.4588800440231959, "sampling/sampling_logp_difference/max": 0.3224307656288147, "sampling/sampling_logp_difference/mean": 0.003932757986088594, "step": 12750, "step_time": 7.9768145864674205, "student/entropy": 0.12608816946546236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500001130004723, "completions/max_length": 480.8666666666667, "completions/max_terminated_length": 457.03333333333336, "completions/mean_length": 189.50750885009765, "completions/mean_terminated_length": 183.7904825846354, "completions/min_length": 48.56666666666667, "completions/min_terminated_length": 48.56666666666667, "entropy": 0.12585870946447056, "epoch": 0.4853225990202408, "eval/gt_acc_batch": 0.7869444568951924, "frac_reward_zero_std": 1.0, "grad_norm": 0.2548651695251465, "kd/policy_loss": 0.008568632546424244, "kd/rkl_advantage_mean": 0.7703784157832464, "kd/rkl_advantage_p95": 4.972911282380422, "kd/rkl_advantage_std": 2.2032086004813514, "kd/ssd_loss": 0.0, "learning_rate": 5.147153761440018e-07, "loss": 0.03427453438440959, "num_tokens": 418174051.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444410006206, "rewards/gt_logging_reward/std": 0.40290601750214894, "sampling/importance_sampling_ratio/max": 1.9557335098584494, "sampling/importance_sampling_ratio/mean": 0.9976895689964295, "sampling/importance_sampling_ratio/min": 0.3921566759546598, "sampling/sampling_logp_difference/max": 0.3479256014029185, "sampling/sampling_logp_difference/mean": 0.003929089677209656, "step": 12780, "step_time": 7.7869664665020535, "student/entropy": 0.12585870946447056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02416666786496838, "completions/max_length": 494.6666666666667, "completions/max_terminated_length": 454.96666666666664, "completions/mean_length": 192.06251017252603, "completions/mean_terminated_length": 184.00249735514322, "completions/min_length": 56.46666666666667, "completions/min_terminated_length": 56.46666666666667, "entropy": 0.12727266885340213, "epoch": 0.48646185394751834, "eval/gt_acc_batch": 0.7877777934074401, "frac_reward_zero_std": 1.0, "grad_norm": 0.3017065227031708, "kd/policy_loss": 0.008563455217517913, "kd/rkl_advantage_mean": 0.7222618437682589, "kd/rkl_advantage_p95": 5.002208779255549, "kd/rkl_advantage_std": 2.23451785047849, "kd/ssd_loss": 0.0, "learning_rate": 5.135761212167242e-07, "loss": 0.03425381978352864, "num_tokens": 419174812.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7877777775128683, "rewards/gt_logging_reward/std": 0.4024914781252543, "sampling/importance_sampling_ratio/max": 1.9597543239593507, "sampling/importance_sampling_ratio/mean": 1.0006189147631328, "sampling/importance_sampling_ratio/min": 0.4346261004606883, "sampling/sampling_logp_difference/max": 0.33620619773864746, "sampling/sampling_logp_difference/mean": 0.003976158075965941, "step": 12810, "step_time": 7.921080640230502, "student/entropy": 0.12727266885340213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111253499985, "completions/max_length": 483.6, "completions/max_terminated_length": 449.96666666666664, "completions/mean_length": 193.5452880859375, "completions/mean_terminated_length": 185.16425323486328, "completions/min_length": 53.2, "completions/min_terminated_length": 53.2, "entropy": 0.12653754372149706, "epoch": 0.4876011088747959, "eval/gt_acc_batch": 0.7844444572925567, "frac_reward_zero_std": 1.0, "grad_norm": 0.27356666326522827, "kd/policy_loss": 0.008159154884439582, "kd/rkl_advantage_mean": 0.7619230711211761, "kd/rkl_advantage_p95": 4.992198308308919, "kd/rkl_advantage_std": 2.2159988860289257, "kd/ssd_loss": 0.0, "learning_rate": 5.124368662894467e-07, "loss": 0.032636616627375284, "num_tokens": 420166175.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7844444453716278, "rewards/gt_logging_reward/std": 0.3987035363912582, "sampling/importance_sampling_ratio/max": 1.7942757527033488, "sampling/importance_sampling_ratio/mean": 0.9919310172398885, "sampling/importance_sampling_ratio/min": 0.4443650404612223, "sampling/sampling_logp_difference/max": 0.3363239884376526, "sampling/sampling_logp_difference/mean": 0.003919214537988106, "step": 12840, "step_time": 7.862965442899925, "student/entropy": 0.12653754372149706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334730317194, "completions/max_length": 487.8666666666667, "completions/max_terminated_length": 452.2, "completions/mean_length": 187.97667744954427, "completions/mean_terminated_length": 181.18448638916016, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "entropy": 0.1314614546795686, "epoch": 0.4887403638020734, "eval/gt_acc_batch": 0.7986111283302307, "frac_reward_zero_std": 1.0, "grad_norm": 0.283154159784317, "kd/policy_loss": 0.008840304324015355, "kd/rkl_advantage_mean": 0.7335600520173708, "kd/rkl_advantage_p95": 4.930224364995956, "kd/rkl_advantage_std": 2.2265256037314733, "kd/ssd_loss": 0.0, "learning_rate": 5.11297611362169e-07, "loss": 0.035361218452453616, "num_tokens": 421138467.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111124356587, "rewards/gt_logging_reward/std": 0.390252781411012, "sampling/importance_sampling_ratio/max": 1.9257989843686423, "sampling/importance_sampling_ratio/mean": 0.9939515431722005, "sampling/importance_sampling_ratio/min": 0.45312494138876597, "sampling/sampling_logp_difference/max": 0.3593298157056173, "sampling/sampling_logp_difference/mean": 0.004034339046726624, "step": 12870, "step_time": 7.775239194768559, "student/entropy": 0.1314614546795686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016944445421298346, "completions/max_length": 468.96666666666664, "completions/max_terminated_length": 444.53333333333336, "completions/mean_length": 182.70223134358724, "completions/mean_terminated_length": 177.14229532877604, "completions/min_length": 60.86666666666667, "completions/min_terminated_length": 60.86666666666667, "entropy": 0.126068641555806, "epoch": 0.489879618729351, "eval/gt_acc_batch": 0.8091666837533315, "frac_reward_zero_std": 1.0, "grad_norm": 0.24527336657047272, "kd/policy_loss": 0.008150560296295831, "kd/rkl_advantage_mean": 0.7703741757199168, "kd/rkl_advantage_p95": 4.987259052197138, "kd/rkl_advantage_std": 2.2059757729371388, "kd/ssd_loss": 0.0, "learning_rate": 5.101583564348916e-07, "loss": 0.03260224262873332, "num_tokens": 422110011.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.809166669845581, "rewards/gt_logging_reward/std": 0.3830762048562368, "sampling/importance_sampling_ratio/max": 1.8716726859410604, "sampling/importance_sampling_ratio/mean": 1.004055960973104, "sampling/importance_sampling_ratio/min": 0.4763409117857615, "sampling/sampling_logp_difference/max": 0.3607962926228841, "sampling/sampling_logp_difference/mean": 0.003923911298625171, "step": 12900, "step_time": 7.806473378799759, "student/entropy": 0.126068641555806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112404614686, "completions/max_length": 485.06666666666666, "completions/max_terminated_length": 451.76666666666665, "completions/mean_length": 190.2894541422526, "completions/mean_terminated_length": 182.7700190226237, "completions/min_length": 54.5, "completions/min_terminated_length": 54.5, "entropy": 0.12317278763900201, "epoch": 0.49101887365662855, "eval/gt_acc_batch": 0.7858333547910055, "frac_reward_zero_std": 1.0, "grad_norm": 0.35849878191947937, "kd/policy_loss": 0.008315546232430885, "kd/rkl_advantage_mean": 0.7614697139089306, "kd/rkl_advantage_p95": 4.975421154499054, "kd/rkl_advantage_std": 2.2033395061890286, "kd/ssd_loss": 0.0, "learning_rate": 5.09019101507614e-07, "loss": 0.03326218128204346, "num_tokens": 423100533.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.785833336909612, "rewards/gt_logging_reward/std": 0.39778612355391185, "sampling/importance_sampling_ratio/max": 1.9797397613525392, "sampling/importance_sampling_ratio/mean": 1.0031026562054952, "sampling/importance_sampling_ratio/min": 0.41837929636240007, "sampling/sampling_logp_difference/max": 0.3370972255865733, "sampling/sampling_logp_difference/mean": 0.003815776598639786, "step": 12930, "step_time": 7.9327052281655295, "student/entropy": 0.12317278763900201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055557192613683, "completions/max_length": 491.26666666666665, "completions/max_terminated_length": 464.3666666666667, "completions/mean_length": 197.05389963785808, "completions/mean_terminated_length": 189.74186147054036, "completions/min_length": 51.166666666666664, "completions/min_terminated_length": 51.166666666666664, "entropy": 0.1314670872564117, "epoch": 0.4921581285839061, "eval/gt_acc_batch": 0.7683333575725555, "frac_reward_zero_std": 1.0, "grad_norm": 0.2538837790489197, "kd/policy_loss": 0.009529152551355462, "kd/rkl_advantage_mean": 0.7642346562196811, "kd/rkl_advantage_p95": 5.036253313223521, "kd/rkl_advantage_std": 2.2237118860085805, "kd/ssd_loss": 0.0, "learning_rate": 5.078798465803365e-07, "loss": 0.0381166140238444, "num_tokens": 424124703.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7683333337306977, "rewards/gt_logging_reward/std": 0.41510116557280224, "sampling/importance_sampling_ratio/max": 2.0355565667152407, "sampling/importance_sampling_ratio/mean": 1.0035285254319508, "sampling/importance_sampling_ratio/min": 0.43620601296424866, "sampling/sampling_logp_difference/max": 0.33650799592336017, "sampling/sampling_logp_difference/mean": 0.004063276729236047, "step": 12960, "step_time": 8.049393393867165, "student/entropy": 0.1314670872564117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112559835116, "completions/max_length": 492.5, "completions/max_terminated_length": 468.4, "completions/mean_length": 193.4077885945638, "completions/mean_terminated_length": 185.9240966796875, "completions/min_length": 62.833333333333336, "completions/min_terminated_length": 62.833333333333336, "entropy": 0.1339859005063772, "epoch": 0.4932973835111837, "eval/gt_acc_batch": 0.7869444688161215, "frac_reward_zero_std": 1.0, "grad_norm": 0.2393186241388321, "kd/policy_loss": 0.008823801311276231, "kd/rkl_advantage_mean": 0.839279082044959, "kd/rkl_advantage_p95": 5.164686053991318, "kd/rkl_advantage_std": 2.234951266646385, "kd/ssd_loss": 0.0, "learning_rate": 5.067405916530588e-07, "loss": 0.03529520829518636, "num_tokens": 425119947.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.786944446961085, "rewards/gt_logging_reward/std": 0.39855003356933594, "sampling/importance_sampling_ratio/max": 1.9424442132314046, "sampling/importance_sampling_ratio/mean": 1.0008370399475097, "sampling/importance_sampling_ratio/min": 0.4217667758464813, "sampling/sampling_logp_difference/max": 0.33361701369285585, "sampling/sampling_logp_difference/mean": 0.004139342865285774, "step": 12990, "step_time": 7.963213203195482, "student/entropy": 0.1339859005063772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889035830895, "completions/max_length": 485.53333333333336, "completions/max_terminated_length": 439.9, "completions/mean_length": 183.5375089009603, "completions/mean_terminated_length": 174.85404434204102, "completions/min_length": 47.766666666666666, "completions/min_terminated_length": 47.766666666666666, "entropy": 0.12621417008340358, "epoch": 0.49443663843846125, "eval/gt_acc_batch": 0.8102777977784474, "frac_reward_zero_std": 1.0, "grad_norm": 0.26729056239128113, "kd/policy_loss": 0.00839853349995489, "kd/rkl_advantage_mean": 0.8338907292112708, "kd/rkl_advantage_p95": 5.200264549255371, "kd/rkl_advantage_std": 2.297726969917615, "kd/ssd_loss": 0.0, "learning_rate": 5.056013367257813e-07, "loss": 0.03359413146972656, "num_tokens": 426084866.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8102777739365895, "rewards/gt_logging_reward/std": 0.3791096925735474, "sampling/importance_sampling_ratio/max": 1.8559505621592203, "sampling/importance_sampling_ratio/mean": 0.9967754940191905, "sampling/importance_sampling_ratio/min": 0.40957562923431395, "sampling/sampling_logp_difference/max": 0.3283091644446055, "sampling/sampling_logp_difference/mean": 0.003932061100689073, "step": 13020, "step_time": 7.874777145999057, "student/entropy": 0.12621417008340358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03583333535740773, "completions/max_length": 492.06666666666666, "completions/max_terminated_length": 452.96666666666664, "completions/mean_length": 192.06612243652344, "completions/mean_terminated_length": 180.32630055745443, "completions/min_length": 57.9, "completions/min_terminated_length": 57.9, "entropy": 0.13460152813543877, "epoch": 0.4955758933657388, "eval/gt_acc_batch": 0.7827777981758117, "frac_reward_zero_std": 1.0, "grad_norm": 0.24928902089595795, "kd/policy_loss": 0.009149543100890393, "kd/rkl_advantage_mean": 0.7775705125648529, "kd/rkl_advantage_p95": 5.014631113409996, "kd/rkl_advantage_std": 2.2298164844512938, "kd/ssd_loss": 0.0, "learning_rate": 5.044620817985038e-07, "loss": 0.03659817377726237, "num_tokens": 427073112.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7827777783075969, "rewards/gt_logging_reward/std": 0.40819736023743947, "sampling/importance_sampling_ratio/max": 1.947151792049408, "sampling/importance_sampling_ratio/mean": 1.0023244043191275, "sampling/importance_sampling_ratio/min": 0.43849104394515354, "sampling/sampling_logp_difference/max": 0.3641825954119364, "sampling/sampling_logp_difference/mean": 0.004113110387697816, "step": 13050, "step_time": 7.878095649300667, "student/entropy": 0.13460152813543877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223834445078, "completions/max_length": 488.23333333333335, "completions/max_terminated_length": 448.5, "completions/mean_length": 190.5452885945638, "completions/mean_terminated_length": 180.74005076090495, "completions/min_length": 54.63333333333333, "completions/min_terminated_length": 54.63333333333333, "entropy": 0.13053163395573694, "epoch": 0.4967151482930164, "eval/gt_acc_batch": 0.7897222379843394, "frac_reward_zero_std": 1.0, "grad_norm": 0.3087827265262604, "kd/policy_loss": 0.008969450590666384, "kd/rkl_advantage_mean": 0.7714304440033932, "kd/rkl_advantage_p95": 5.002454785505931, "kd/rkl_advantage_std": 2.229687425494194, "kd/ssd_loss": 0.0, "learning_rate": 5.033228268712261e-07, "loss": 0.03587780396143595, "num_tokens": 428065531.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7897222161293029, "rewards/gt_logging_reward/std": 0.3968115518490473, "sampling/importance_sampling_ratio/max": 1.9843746145566306, "sampling/importance_sampling_ratio/mean": 1.0075724124908447, "sampling/importance_sampling_ratio/min": 0.4422375758488973, "sampling/sampling_logp_difference/max": 0.33616142670313515, "sampling/sampling_logp_difference/mean": 0.004063787730410695, "step": 13080, "step_time": 7.980096963732406, "student/entropy": 0.13053163395573694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029166668094694613, "completions/max_length": 478.8333333333333, "completions/max_terminated_length": 443.53333333333336, "completions/mean_length": 192.71334330240884, "completions/mean_terminated_length": 183.4984558105469, "completions/min_length": 56.7, "completions/min_terminated_length": 56.7, "entropy": 0.1282183055455486, "epoch": 0.49785440322029395, "eval/gt_acc_batch": 0.7905555705229441, "frac_reward_zero_std": 1.0, "grad_norm": 0.26518622040748596, "kd/policy_loss": 0.008455333282472565, "kd/rkl_advantage_mean": 0.754959103371948, "kd/rkl_advantage_p95": 5.012832333644231, "kd/rkl_advantage_std": 2.207040989398956, "kd/ssd_loss": 0.0, "learning_rate": 5.021835719439487e-07, "loss": 0.03382134040196737, "num_tokens": 429064755.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7905555566151937, "rewards/gt_logging_reward/std": 0.3973430961370468, "sampling/importance_sampling_ratio/max": 2.008651773134867, "sampling/importance_sampling_ratio/mean": 1.0014704843362172, "sampling/importance_sampling_ratio/min": 0.44071121712525685, "sampling/sampling_logp_difference/max": 0.3932512352863948, "sampling/sampling_logp_difference/mean": 0.004019444955823322, "step": 13110, "step_time": 7.904125590297918, "student/entropy": 0.1282183055455486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555557385087014, "completions/max_length": 487.96666666666664, "completions/max_terminated_length": 458.93333333333334, "completions/mean_length": 194.17306467692057, "completions/mean_terminated_length": 186.03732198079427, "completions/min_length": 53.7, "completions/min_terminated_length": 53.7, "entropy": 0.12837067128469545, "epoch": 0.49899365814757146, "eval/gt_acc_batch": 0.7908333480358124, "frac_reward_zero_std": 1.0, "grad_norm": 0.2535662353038788, "kd/policy_loss": 0.00853493712687244, "kd/rkl_advantage_mean": 0.8733937223131458, "kd/rkl_advantage_p95": 5.2161651968955995, "kd/rkl_advantage_std": 2.279207627971967, "kd/ssd_loss": 0.0, "learning_rate": 5.010443170166711e-07, "loss": 0.03413975238800049, "num_tokens": 430071914.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7908333341280619, "rewards/gt_logging_reward/std": 0.3961788425842921, "sampling/importance_sampling_ratio/max": 1.8649105827013652, "sampling/importance_sampling_ratio/mean": 0.995687089363734, "sampling/importance_sampling_ratio/min": 0.46317173341910045, "sampling/sampling_logp_difference/max": 0.33434557914733887, "sampling/sampling_logp_difference/mean": 0.003974920519006749, "step": 13140, "step_time": 8.044115194931996, "student/entropy": 0.12837067128469545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668051232895, "completions/max_length": 500.26666666666665, "completions/max_terminated_length": 458.8333333333333, "completions/mean_length": 191.49417622884116, "completions/mean_terminated_length": 183.69904022216798, "completions/min_length": 53.86666666666667, "completions/min_terminated_length": 53.86666666666667, "entropy": 0.1276354743167758, "epoch": 0.5001329130748491, "eval/gt_acc_batch": 0.7991666793823242, "frac_reward_zero_std": 1.0, "grad_norm": 0.24526314437389374, "kd/policy_loss": 0.008484186364027362, "kd/rkl_advantage_mean": 0.8408616100127498, "kd/rkl_advantage_p95": 5.179795014858246, "kd/rkl_advantage_std": 2.2809967319170634, "kd/ssd_loss": 0.0, "learning_rate": 4.999050620893935e-07, "loss": 0.03393675088882446, "num_tokens": 431072637.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7991666634877522, "rewards/gt_logging_reward/std": 0.3913499022523562, "sampling/importance_sampling_ratio/max": 1.9556130687395732, "sampling/importance_sampling_ratio/mean": 0.9997881213823955, "sampling/importance_sampling_ratio/min": 0.46809071600437163, "sampling/sampling_logp_difference/max": 0.3230780720710754, "sampling/sampling_logp_difference/mean": 0.003960460470989346, "step": 13170, "step_time": 8.203752201866397, "student/entropy": 0.1276354743167758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166667740792035, "completions/max_length": 487.03333333333336, "completions/max_terminated_length": 446.93333333333334, "completions/mean_length": 184.72917683919272, "completions/mean_terminated_length": 176.65811208089193, "completions/min_length": 54.4, "completions/min_terminated_length": 54.4, "entropy": 0.1375619273011883, "epoch": 0.5012721680021266, "eval/gt_acc_batch": 0.796111132701238, "frac_reward_zero_std": 1.0, "grad_norm": 0.2962196171283722, "kd/policy_loss": 0.009256092111657684, "kd/rkl_advantage_mean": 0.7987864051247016, "kd/rkl_advantage_p95": 5.174599754810333, "kd/rkl_advantage_std": 2.296146933237712, "kd/ssd_loss": 0.0, "learning_rate": 4.98765807162116e-07, "loss": 0.03702437480290731, "num_tokens": 432030750.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111108462016, "rewards/gt_logging_reward/std": 0.3956479440132777, "sampling/importance_sampling_ratio/max": 1.8867369532585143, "sampling/importance_sampling_ratio/mean": 1.009014755487442, "sampling/importance_sampling_ratio/min": 0.44499262471993767, "sampling/sampling_logp_difference/max": 0.3494084636370341, "sampling/sampling_logp_difference/mean": 0.0040081636669735115, "step": 13200, "step_time": 7.8094346632647405, "student/entropy": 0.1375619273011883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334668229023, "completions/max_length": 475.26666666666665, "completions/max_terminated_length": 419.8666666666667, "completions/mean_length": 192.1961196899414, "completions/mean_terminated_length": 185.57786509195964, "completions/min_length": 61.7, "completions/min_terminated_length": 61.7, "entropy": 0.1264710328852137, "epoch": 0.5024114229294042, "eval/gt_acc_batch": 0.7958333412806193, "frac_reward_zero_std": 1.0, "grad_norm": 0.24649715423583984, "kd/policy_loss": 0.008655014257722844, "kd/rkl_advantage_mean": 0.7202112230171527, "kd/rkl_advantage_p95": 4.943854818741481, "kd/rkl_advantage_std": 2.1738750924666723, "kd/ssd_loss": 0.0, "learning_rate": 4.976265522348384e-07, "loss": 0.03462006251017253, "num_tokens": 433034216.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7958333373069764, "rewards/gt_logging_reward/std": 0.39780582785606383, "sampling/importance_sampling_ratio/max": 1.9371222297350565, "sampling/importance_sampling_ratio/mean": 1.0032226105531057, "sampling/importance_sampling_ratio/min": 0.461353359123071, "sampling/sampling_logp_difference/max": 0.3145077735185623, "sampling/sampling_logp_difference/mean": 0.003950933421341081, "step": 13230, "step_time": 7.955366395799501, "student/entropy": 0.1264710328852137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01666666762903333, "completions/max_length": 466.6666666666667, "completions/max_terminated_length": 428.1, "completions/mean_length": 181.7775085449219, "completions/mean_terminated_length": 176.28002014160157, "completions/min_length": 56.833333333333336, "completions/min_terminated_length": 56.833333333333336, "entropy": 0.12771618279318014, "epoch": 0.5035506778566817, "eval/gt_acc_batch": 0.821111136674881, "frac_reward_zero_std": 1.0, "grad_norm": 0.2551063597202301, "kd/policy_loss": 0.008151335930839802, "kd/rkl_advantage_mean": 0.7570568180332581, "kd/rkl_advantage_p95": 4.934720222155253, "kd/rkl_advantage_std": 2.173758061726888, "kd/ssd_loss": 0.0, "learning_rate": 4.964872973075608e-07, "loss": 0.03260534604390462, "num_tokens": 433986207.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8211111108462016, "rewards/gt_logging_reward/std": 0.3782418469587962, "sampling/importance_sampling_ratio/max": 1.8684049169222514, "sampling/importance_sampling_ratio/mean": 1.0016039629777274, "sampling/importance_sampling_ratio/min": 0.46488766074180604, "sampling/sampling_logp_difference/max": 0.32926758925120037, "sampling/sampling_logp_difference/mean": 0.003971799688103298, "step": 13260, "step_time": 7.9125686012024135, "student/entropy": 0.12771618279318014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027500001589457194, "completions/max_length": 496.3666666666667, "completions/max_terminated_length": 461.0, "completions/mean_length": 192.24556630452474, "completions/mean_terminated_length": 183.40699971516926, "completions/min_length": 49.93333333333333, "completions/min_terminated_length": 49.93333333333333, "entropy": 0.13322918731719255, "epoch": 0.5046899327839592, "eval/gt_acc_batch": 0.7805555740992228, "frac_reward_zero_std": 1.0, "grad_norm": 0.31269320845603943, "kd/policy_loss": 0.00915673931594938, "kd/rkl_advantage_mean": 0.8193847555667162, "kd/rkl_advantage_p95": 5.115215700864792, "kd/rkl_advantage_std": 2.2608208298683166, "kd/ssd_loss": 0.0, "learning_rate": 4.953480423802833e-07, "loss": 0.036626954873402916, "num_tokens": 434982739.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7805555522441864, "rewards/gt_logging_reward/std": 0.407390163342158, "sampling/importance_sampling_ratio/max": 1.962827738126119, "sampling/importance_sampling_ratio/mean": 1.0062683562437693, "sampling/importance_sampling_ratio/min": 0.44755819290876386, "sampling/sampling_logp_difference/max": 0.368687907854716, "sampling/sampling_logp_difference/mean": 0.004105556119854251, "step": 13290, "step_time": 8.185467554962088, "student/entropy": 0.13322918731719255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889042039712, "completions/max_length": 494.8333333333333, "completions/max_terminated_length": 445.46666666666664, "completions/mean_length": 187.0783437093099, "completions/mean_terminated_length": 178.2075393676758, "completions/min_length": 49.53333333333333, "completions/min_terminated_length": 49.53333333333333, "entropy": 0.12515191187461217, "epoch": 0.5058291877112369, "eval/gt_acc_batch": 0.8002777894337972, "frac_reward_zero_std": 1.0, "grad_norm": 0.3359780013561249, "kd/policy_loss": 0.008154199146277581, "kd/rkl_advantage_mean": 0.7690290198971828, "kd/rkl_advantage_p95": 5.0485188364982605, "kd/rkl_advantage_std": 2.2545389433701835, "kd/ssd_loss": 0.0, "learning_rate": 4.942087874530057e-07, "loss": 0.03261679410934448, "num_tokens": 435949853.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8002777794996897, "rewards/gt_logging_reward/std": 0.3923842877149582, "sampling/importance_sampling_ratio/max": 1.8534108440081278, "sampling/importance_sampling_ratio/mean": 0.9955401321252187, "sampling/importance_sampling_ratio/min": 0.4455983271201452, "sampling/sampling_logp_difference/max": 0.34340945482254026, "sampling/sampling_logp_difference/mean": 0.003896466890970866, "step": 13320, "step_time": 7.997136927200093, "student/entropy": 0.12515191187461217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666668076068162, "completions/max_length": 488.1333333333333, "completions/max_terminated_length": 448.5, "completions/mean_length": 188.18973185221355, "completions/mean_terminated_length": 181.19383595784504, "completions/min_length": 50.86666666666667, "completions/min_terminated_length": 50.86666666666667, "entropy": 0.12467981179555257, "epoch": 0.5069684426385144, "eval/gt_acc_batch": 0.796111128727595, "frac_reward_zero_std": 1.0, "grad_norm": 0.2820405662059784, "kd/policy_loss": 0.008294938578425596, "kd/rkl_advantage_mean": 0.7662440652338167, "kd/rkl_advantage_p95": 5.072757263978322, "kd/rkl_advantage_std": 2.2323935051759083, "kd/ssd_loss": 0.0, "learning_rate": 4.930695325257282e-07, "loss": 0.03317975600560506, "num_tokens": 436928880.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111148198445, "rewards/gt_logging_reward/std": 0.39195047368605934, "sampling/importance_sampling_ratio/max": 1.992165982723236, "sampling/importance_sampling_ratio/mean": 1.0025160094102223, "sampling/importance_sampling_ratio/min": 0.4517238607009252, "sampling/sampling_logp_difference/max": 0.3380700091520945, "sampling/sampling_logp_difference/mean": 0.003911356379588445, "step": 13350, "step_time": 7.983294080166282, "student/entropy": 0.12467981179555257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02111111208796501, "completions/max_length": 484.1333333333333, "completions/max_terminated_length": 455.46666666666664, "completions/mean_length": 184.7255625406901, "completions/mean_terminated_length": 177.79320373535157, "completions/min_length": 59.3, "completions/min_terminated_length": 59.3, "entropy": 0.13155973901351292, "epoch": 0.508107697565792, "eval/gt_acc_batch": 0.8136111319065094, "frac_reward_zero_std": 1.0, "grad_norm": 0.3096168339252472, "kd/policy_loss": 0.008671039207062374, "kd/rkl_advantage_mean": 0.8796043239533902, "kd/rkl_advantage_p95": 5.234441208839416, "kd/rkl_advantage_std": 2.2961909343798954, "kd/ssd_loss": 0.0, "learning_rate": 4.919302775984505e-07, "loss": 0.034684157371520995, "num_tokens": 437896852.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8136111120382945, "rewards/gt_logging_reward/std": 0.38488171100616453, "sampling/importance_sampling_ratio/max": 1.8916655619939169, "sampling/importance_sampling_ratio/mean": 0.9952857553958893, "sampling/importance_sampling_ratio/min": 0.4136922722061475, "sampling/sampling_logp_difference/max": 0.3248528003692627, "sampling/sampling_logp_difference/mean": 0.004091426737916967, "step": 13380, "step_time": 7.948688119998648, "student/entropy": 0.13155973901351292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027500001123795907, "completions/max_length": 485.46666666666664, "completions/max_terminated_length": 457.03333333333336, "completions/mean_length": 191.8291763305664, "completions/mean_terminated_length": 182.94952646891275, "completions/min_length": 55.833333333333336, "completions/min_terminated_length": 55.833333333333336, "entropy": 0.1311343631396691, "epoch": 0.5092469524930695, "eval/gt_acc_batch": 0.7791666905085246, "frac_reward_zero_std": 1.0, "grad_norm": 0.28257858753204346, "kd/policy_loss": 0.008993870050956806, "kd/rkl_advantage_mean": 0.7776369160662094, "kd/rkl_advantage_p95": 5.067190390825272, "kd/rkl_advantage_std": 2.2339003572861356, "kd/ssd_loss": 0.0, "learning_rate": 4.90791022671173e-07, "loss": 0.035975476106007896, "num_tokens": 438903685.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7791666646798452, "rewards/gt_logging_reward/std": 0.4039033363262812, "sampling/importance_sampling_ratio/max": 1.9410647392272948, "sampling/importance_sampling_ratio/mean": 1.0030520061651866, "sampling/importance_sampling_ratio/min": 0.4271466612815857, "sampling/sampling_logp_difference/max": 0.36923479835192363, "sampling/sampling_logp_difference/mean": 0.004076689442930122, "step": 13410, "step_time": 8.302446158399107, "student/entropy": 0.1311343631396691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021839081952027207, "completions/max_length": 494.0689655172414, "completions/max_terminated_length": 460.7586206896552, "completions/mean_length": 190.31667564655172, "completions/mean_terminated_length": 183.24531081627154, "completions/min_length": 61.96551724137931, "completions/min_terminated_length": 61.96551724137931, "entropy": 0.12539255574088673, "epoch": 0.5103862074203471, "eval/gt_acc_batch": 0.789655181868323, "frac_reward_zero_std": 1.0, "grad_norm": 0.268333375453949, "kd/policy_loss": 0.008740880636996108, "kd/rkl_advantage_mean": 0.8316077430700434, "kd/rkl_advantage_p95": 5.138561659845813, "kd/rkl_advantage_std": 2.2401756623695634, "kd/ssd_loss": 0.0, "learning_rate": 4.896517677438955e-07, "loss": 0.03379807472229004, "num_tokens": 439854835.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7896551757023252, "rewards/gt_logging_reward/std": 0.39896534126380395, "sampling/importance_sampling_ratio/max": 1.9991547814730941, "sampling/importance_sampling_ratio/mean": 1.001974159273608, "sampling/importance_sampling_ratio/min": 0.4396031627367283, "sampling/sampling_logp_difference/max": 0.3317016229547303, "sampling/sampling_logp_difference/mean": 0.003919753093465135, "step": 13440, "step_time": 7.768348437467163, "student/entropy": 0.12539255574088673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333473652601, "completions/max_length": 476.73333333333335, "completions/max_terminated_length": 451.6, "completions/mean_length": 191.9269561767578, "completions/mean_terminated_length": 184.54794565836588, "completions/min_length": 59.4, "completions/min_terminated_length": 59.4, "entropy": 0.12801065780222415, "epoch": 0.5115254623476246, "eval/gt_acc_batch": 0.7947222510973613, "frac_reward_zero_std": 1.0, "grad_norm": 0.25952646136283875, "kd/policy_loss": 0.008443901053396985, "kd/rkl_advantage_mean": 0.8220498546957969, "kd/rkl_advantage_p95": 5.13026261528333, "kd/rkl_advantage_std": 2.2310816903909045, "kd/ssd_loss": 0.0, "learning_rate": 4.885125128166179e-07, "loss": 0.03377559979756673, "num_tokens": 440850932.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222232818604, "rewards/gt_logging_reward/std": 0.39274535824855167, "sampling/importance_sampling_ratio/max": 1.9392152627309163, "sampling/importance_sampling_ratio/mean": 0.9976689755916596, "sampling/importance_sampling_ratio/min": 0.41977618684371315, "sampling/sampling_logp_difference/max": 0.3255950371424357, "sampling/sampling_logp_difference/mean": 0.003986841967950265, "step": 13470, "step_time": 7.9586431915299425, "student/entropy": 0.12801065780222415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666785875956, "completions/max_length": 485.23333333333335, "completions/max_terminated_length": 448.43333333333334, "completions/mean_length": 189.1005650838216, "completions/mean_terminated_length": 181.95362141927083, "completions/min_length": 64.73333333333333, "completions/min_terminated_length": 64.73333333333333, "entropy": 0.11984459217637777, "epoch": 0.5126647172749023, "eval/gt_acc_batch": 0.8177777985731761, "frac_reward_zero_std": 1.0, "grad_norm": 0.24430814385414124, "kd/policy_loss": 0.007897363579832017, "kd/rkl_advantage_mean": 0.6872668864748751, "kd/rkl_advantage_p95": 4.864506145318349, "kd/rkl_advantage_std": 2.2011391202608745, "kd/ssd_loss": 0.0, "learning_rate": 4.873732578893404e-07, "loss": 0.031589452425638834, "num_tokens": 441846790.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8177777806917826, "rewards/gt_logging_reward/std": 0.37878380964199704, "sampling/importance_sampling_ratio/max": 1.893928297360738, "sampling/importance_sampling_ratio/mean": 0.9958478411038717, "sampling/importance_sampling_ratio/min": 0.41062778035799663, "sampling/sampling_logp_difference/max": 0.36681549151738485, "sampling/sampling_logp_difference/mean": 0.003774503362365067, "step": 13500, "step_time": 8.04497896546721, "student/entropy": 0.11984459217637777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555668969949, "completions/max_length": 478.1666666666667, "completions/max_terminated_length": 444.3666666666667, "completions/mean_length": 187.58250986735027, "completions/mean_terminated_length": 180.97928059895833, "completions/min_length": 56.733333333333334, "completions/min_terminated_length": 56.733333333333334, "entropy": 0.12203903887420893, "epoch": 0.5138039722021798, "eval/gt_acc_batch": 0.8155555705229441, "frac_reward_zero_std": 1.0, "grad_norm": 0.24134255945682526, "kd/policy_loss": 0.008116279230065023, "kd/rkl_advantage_mean": 0.842855021605889, "kd/rkl_advantage_p95": 5.158920633792877, "kd/rkl_advantage_std": 2.2467195520798366, "kd/ssd_loss": 0.0, "learning_rate": 4.862340029620628e-07, "loss": 0.03246511419614156, "num_tokens": 442819583.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8155555605888367, "rewards/gt_logging_reward/std": 0.37923822154601416, "sampling/importance_sampling_ratio/max": 1.8642985979715982, "sampling/importance_sampling_ratio/mean": 1.0044053276379903, "sampling/importance_sampling_ratio/min": 0.44336141645908356, "sampling/sampling_logp_difference/max": 0.3295977433522542, "sampling/sampling_logp_difference/mean": 0.003868533670902252, "step": 13530, "step_time": 7.901388114631603, "student/entropy": 0.12203903887420893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01916666782150666, "completions/max_length": 481.03333333333336, "completions/max_terminated_length": 439.8666666666667, "completions/mean_length": 185.97445424397787, "completions/mean_terminated_length": 179.71214243570964, "completions/min_length": 52.36666666666667, "completions/min_terminated_length": 52.36666666666667, "entropy": 0.13936131938050192, "epoch": 0.5149432271294573, "eval/gt_acc_batch": 0.8097222487131754, "frac_reward_zero_std": 1.0, "grad_norm": 0.26740798354148865, "kd/policy_loss": 0.009075922573295732, "kd/rkl_advantage_mean": 0.8259214186420043, "kd/rkl_advantage_p95": 5.127994169791539, "kd/rkl_advantage_std": 2.248959493637085, "kd/ssd_loss": 0.0, "learning_rate": 4.850947480347852e-07, "loss": 0.03630369504292806, "num_tokens": 443784379.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8097222248713175, "rewards/gt_logging_reward/std": 0.384277014931043, "sampling/importance_sampling_ratio/max": 1.8808781981468201, "sampling/importance_sampling_ratio/mean": 0.9989433268706004, "sampling/importance_sampling_ratio/min": 0.44252419571081797, "sampling/sampling_logp_difference/max": 0.3672982692718506, "sampling/sampling_logp_difference/mean": 0.0041104407981038095, "step": 13560, "step_time": 8.117828416832587, "student/entropy": 0.13936131938050192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030000001937150956, "completions/max_length": 494.3666666666667, "completions/max_terminated_length": 458.6, "completions/mean_length": 192.3291768391927, "completions/mean_terminated_length": 182.60846048990885, "completions/min_length": 46.8, "completions/min_terminated_length": 46.8, "entropy": 0.13288527689874172, "epoch": 0.5160824820567349, "eval/gt_acc_batch": 0.7997222344080607, "frac_reward_zero_std": 1.0, "grad_norm": 0.40810221433639526, "kd/policy_loss": 0.00876244815105262, "kd/rkl_advantage_mean": 0.849934043114384, "kd/rkl_advantage_p95": 5.208024096488953, "kd/rkl_advantage_std": 2.2789222965637843, "kd/ssd_loss": 0.0, "learning_rate": 4.839554931075077e-07, "loss": 0.03504979610443115, "num_tokens": 444785116.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7997222205003103, "rewards/gt_logging_reward/std": 0.38855270942052206, "sampling/importance_sampling_ratio/max": 1.9489187637964884, "sampling/importance_sampling_ratio/mean": 1.0011720597743987, "sampling/importance_sampling_ratio/min": 0.44456789642572403, "sampling/sampling_logp_difference/max": 0.3246934632460276, "sampling/sampling_logp_difference/mean": 0.00403062755552431, "step": 13590, "step_time": 8.327013289631576, "student/entropy": 0.13288527689874172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112243185442, "completions/max_length": 485.6, "completions/max_terminated_length": 440.56666666666666, "completions/mean_length": 187.0461212158203, "completions/mean_terminated_length": 180.1074244181315, "completions/min_length": 53.06666666666667, "completions/min_terminated_length": 53.06666666666667, "entropy": 0.12755054589360953, "epoch": 0.5172217369840124, "eval/gt_acc_batch": 0.7963889082272847, "frac_reward_zero_std": 1.0, "grad_norm": 0.2355193793773651, "kd/policy_loss": 0.008632331564634417, "kd/rkl_advantage_mean": 0.7591956538458665, "kd/rkl_advantage_p95": 5.05312252442042, "kd/rkl_advantage_std": 2.2291520784298577, "kd/ssd_loss": 0.0, "learning_rate": 4.828162381802301e-07, "loss": 0.03452932834625244, "num_tokens": 445768754.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888843854269, "rewards/gt_logging_reward/std": 0.39736758867899574, "sampling/importance_sampling_ratio/max": 1.8615408062934875, "sampling/importance_sampling_ratio/mean": 0.9970512727896372, "sampling/importance_sampling_ratio/min": 0.4248957335948944, "sampling/sampling_logp_difference/max": 0.3646218279997508, "sampling/sampling_logp_difference/mean": 0.003981468989513815, "step": 13620, "step_time": 8.052310594936232, "student/entropy": 0.12755054589360953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778673917055, "completions/max_length": 494.43333333333334, "completions/max_terminated_length": 461.2, "completions/mean_length": 186.54195454915364, "completions/mean_terminated_length": 180.6953145345052, "completions/min_length": 51.6, "completions/min_terminated_length": 51.6, "entropy": 0.1309233820065856, "epoch": 0.51836099191129, "eval/gt_acc_batch": 0.795000026623408, "frac_reward_zero_std": 1.0, "grad_norm": 0.26961806416511536, "kd/policy_loss": 0.008533356070984155, "kd/rkl_advantage_mean": 0.837415462732315, "kd/rkl_advantage_p95": 5.189280221859614, "kd/rkl_advantage_std": 2.290069223443667, "kd/ssd_loss": 0.0, "learning_rate": 4.816769832529525e-07, "loss": 0.03413343032201131, "num_tokens": 446737489.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7950000047683716, "rewards/gt_logging_reward/std": 0.39816976090272266, "sampling/importance_sampling_ratio/max": 1.9403555075327554, "sampling/importance_sampling_ratio/mean": 0.9946727136770884, "sampling/importance_sampling_ratio/min": 0.4119761834541957, "sampling/sampling_logp_difference/max": 0.35496972799301146, "sampling/sampling_logp_difference/mean": 0.004071328478554885, "step": 13650, "step_time": 7.865297824503311, "student/entropy": 0.1309233820065856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028888890178253253, "completions/max_length": 493.1333333333333, "completions/max_terminated_length": 454.8, "completions/mean_length": 192.88139851888022, "completions/mean_terminated_length": 183.4839848836263, "completions/min_length": 52.13333333333333, "completions/min_terminated_length": 52.13333333333333, "entropy": 0.13060363959521054, "epoch": 0.5195002468385675, "eval/gt_acc_batch": 0.8033333400885264, "frac_reward_zero_std": 1.0, "grad_norm": 0.35230961441993713, "kd/policy_loss": 0.008649666890657196, "kd/rkl_advantage_mean": 0.7920275257279475, "kd/rkl_advantage_p95": 5.1072557429472605, "kd/rkl_advantage_std": 2.236140798528989, "kd/ssd_loss": 0.0, "learning_rate": 4.80537728325675e-07, "loss": 0.03459866841634115, "num_tokens": 447734630.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.803333334128062, "rewards/gt_logging_reward/std": 0.3909426788489024, "sampling/importance_sampling_ratio/max": 2.00842391649882, "sampling/importance_sampling_ratio/mean": 0.9978987554709117, "sampling/importance_sampling_ratio/min": 0.4267040540774663, "sampling/sampling_logp_difference/max": 0.33734674056371056, "sampling/sampling_logp_difference/mean": 0.00404376497802635, "step": 13680, "step_time": 7.976594755735035, "student/entropy": 0.13060363959521054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02722222376614809, "completions/max_length": 475.23333333333335, "completions/max_terminated_length": 437.9, "completions/mean_length": 182.46251118977864, "completions/mean_terminated_length": 173.3874038696289, "completions/min_length": 54.2, "completions/min_terminated_length": 54.2, "entropy": 0.125187465796868, "epoch": 0.5206395017658452, "eval/gt_acc_batch": 0.8127777973810831, "frac_reward_zero_std": 1.0, "grad_norm": 0.30207359790802, "kd/policy_loss": 0.00835984144359827, "kd/rkl_advantage_mean": 0.7719736117869616, "kd/rkl_advantage_p95": 4.9903674066066745, "kd/rkl_advantage_std": 2.213218104839325, "kd/ssd_loss": 0.0, "learning_rate": 4.793984733983975e-07, "loss": 0.03343936602274577, "num_tokens": 448685207.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8127777735392253, "rewards/gt_logging_reward/std": 0.38102691968282065, "sampling/importance_sampling_ratio/max": 1.8996910373369853, "sampling/importance_sampling_ratio/mean": 1.0034263590971628, "sampling/importance_sampling_ratio/min": 0.45933054983615873, "sampling/sampling_logp_difference/max": 0.34427998860677084, "sampling/sampling_logp_difference/mean": 0.003889993011641006, "step": 13710, "step_time": 7.7680629976346, "student/entropy": 0.125187465796868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779251337052, "completions/max_length": 488.7, "completions/max_terminated_length": 441.1, "completions/mean_length": 190.2041763305664, "completions/mean_terminated_length": 181.9239466349284, "completions/min_length": 51.233333333333334, "completions/min_terminated_length": 51.233333333333334, "entropy": 0.12680780297766128, "epoch": 0.5217787566931227, "eval/gt_acc_batch": 0.8019444664319356, "frac_reward_zero_std": 1.0, "grad_norm": 0.32120800018310547, "kd/policy_loss": 0.00859815781780829, "kd/rkl_advantage_mean": 0.7344619212361674, "kd/rkl_advantage_p95": 4.955116772651673, "kd/rkl_advantage_std": 2.1920243273178737, "kd/ssd_loss": 0.0, "learning_rate": 4.782592184711198e-07, "loss": 0.03439263502756754, "num_tokens": 449686046.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444406032562, "rewards/gt_logging_reward/std": 0.39405519564946495, "sampling/importance_sampling_ratio/max": 1.9913106640179952, "sampling/importance_sampling_ratio/mean": 0.9999716798464457, "sampling/importance_sampling_ratio/min": 0.48339502811431884, "sampling/sampling_logp_difference/max": 0.340974493821462, "sampling/sampling_logp_difference/mean": 0.003958528100823363, "step": 13740, "step_time": 8.095022997328973, "student/entropy": 0.12680780297766128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02694444600492716, "completions/max_length": 489.9, "completions/max_terminated_length": 457.23333333333335, "completions/mean_length": 185.00639851888022, "completions/mean_terminated_length": 176.04859619140626, "completions/min_length": 54.733333333333334, "completions/min_terminated_length": 54.733333333333334, "entropy": 0.12894826233386994, "epoch": 0.5229180116204003, "eval/gt_acc_batch": 0.8038889030615489, "frac_reward_zero_std": 1.0, "grad_norm": 0.23327742516994476, "kd/policy_loss": 0.008628433052217587, "kd/rkl_advantage_mean": 0.8345113061368465, "kd/rkl_advantage_p95": 5.236396948496501, "kd/rkl_advantage_std": 2.315540524323781, "kd/ssd_loss": 0.0, "learning_rate": 4.771199635438423e-07, "loss": 0.03451374371846517, "num_tokens": 450652437.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8038888931274414, "rewards/gt_logging_reward/std": 0.3905547688404719, "sampling/importance_sampling_ratio/max": 2.0025027354558307, "sampling/importance_sampling_ratio/mean": 0.9950513343016306, "sampling/importance_sampling_ratio/min": 0.4557744642098745, "sampling/sampling_logp_difference/max": 0.30751564502716067, "sampling/sampling_logp_difference/mean": 0.0040189300430938605, "step": 13770, "step_time": 7.855921553161655, "student/entropy": 0.12894826233386994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02888889042660594, "completions/max_length": 480.3333333333333, "completions/max_terminated_length": 453.6333333333333, "completions/mean_length": 191.45362141927083, "completions/mean_terminated_length": 182.2514912923177, "completions/min_length": 60.166666666666664, "completions/min_terminated_length": 60.166666666666664, "entropy": 0.12216889367749294, "epoch": 0.5240572665476778, "eval/gt_acc_batch": 0.7891666809717814, "frac_reward_zero_std": 1.0, "grad_norm": 0.25446024537086487, "kd/policy_loss": 0.008162470910853396, "kd/rkl_advantage_mean": 0.7785483535379172, "kd/rkl_advantage_p95": 5.051048876841863, "kd/rkl_advantage_std": 2.222222508986791, "kd/ssd_loss": 0.0, "learning_rate": 4.7598070861656474e-07, "loss": 0.03264988859494527, "num_tokens": 451653094.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.789166663090388, "rewards/gt_logging_reward/std": 0.39659317086140317, "sampling/importance_sampling_ratio/max": 1.896891725063324, "sampling/importance_sampling_ratio/mean": 1.0005358974138896, "sampling/importance_sampling_ratio/min": 0.4740508397420247, "sampling/sampling_logp_difference/max": 0.3425781925519307, "sampling/sampling_logp_difference/mean": 0.0038158934796229003, "step": 13800, "step_time": 7.96781962163465, "student/entropy": 0.12216889367749294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02750000146528085, "completions/max_length": 498.3666666666667, "completions/max_terminated_length": 458.03333333333336, "completions/mean_length": 195.31862080891926, "completions/mean_terminated_length": 186.46502176920572, "completions/min_length": 61.166666666666664, "completions/min_terminated_length": 61.166666666666664, "entropy": 0.1278667830551664, "epoch": 0.5251965214749553, "eval/gt_acc_batch": 0.7825000186761221, "frac_reward_zero_std": 1.0, "grad_norm": 0.2778443396091461, "kd/policy_loss": 0.008574891627843802, "kd/rkl_advantage_mean": 0.8628701891750097, "kd/rkl_advantage_p95": 5.163269293308258, "kd/rkl_advantage_std": 2.2511018107334775, "kd/ssd_loss": 0.0, "learning_rate": 4.748414536892872e-07, "loss": 0.034299568335215254, "num_tokens": 452661297.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7824999988079071, "rewards/gt_logging_reward/std": 0.4052983740965525, "sampling/importance_sampling_ratio/max": 1.8894463141759237, "sampling/importance_sampling_ratio/mean": 0.9956264754136404, "sampling/importance_sampling_ratio/min": 0.44286731580893196, "sampling/sampling_logp_difference/max": 0.34516538778940836, "sampling/sampling_logp_difference/mean": 0.003961822646670043, "step": 13830, "step_time": 7.97011508703678, "student/entropy": 0.1278667830551664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02861111288269361, "completions/max_length": 494.73333333333335, "completions/max_terminated_length": 450.23333333333335, "completions/mean_length": 189.62362060546874, "completions/mean_terminated_length": 180.26937713623047, "completions/min_length": 48.7, "completions/min_terminated_length": 48.7, "entropy": 0.12593134908626477, "epoch": 0.526335776402233, "eval/gt_acc_batch": 0.8044444680213928, "frac_reward_zero_std": 1.0, "grad_norm": 0.26076647639274597, "kd/policy_loss": 0.008303049253299832, "kd/rkl_advantage_mean": 0.8527796275913715, "kd/rkl_advantage_p95": 5.199810290336609, "kd/rkl_advantage_std": 2.2878666977087656, "kd/ssd_loss": 0.0, "learning_rate": 4.737021987620096e-07, "loss": 0.033212198813756304, "num_tokens": 453644862.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8044444441795349, "rewards/gt_logging_reward/std": 0.39146906435489653, "sampling/importance_sampling_ratio/max": 1.8344443321228028, "sampling/importance_sampling_ratio/mean": 1.0011464973290762, "sampling/importance_sampling_ratio/min": 0.456776296099027, "sampling/sampling_logp_difference/max": 0.36933360596497855, "sampling/sampling_logp_difference/mean": 0.003948131886621316, "step": 13860, "step_time": 7.9857836399368045, "student/entropy": 0.12593134908626477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667573153974, "completions/max_length": 489.73333333333335, "completions/max_terminated_length": 465.6333333333333, "completions/mean_length": 193.71112111409505, "completions/mean_terminated_length": 187.5854288736979, "completions/min_length": 48.333333333333336, "completions/min_terminated_length": 48.333333333333336, "entropy": 0.13274628445506095, "epoch": 0.5274750313295105, "eval/gt_acc_batch": 0.785833348830541, "frac_reward_zero_std": 1.0, "grad_norm": 0.28369665145874023, "kd/policy_loss": 0.008904634231779103, "kd/rkl_advantage_mean": 0.7567404488179211, "kd/rkl_advantage_p95": 5.0734045525391895, "kd/rkl_advantage_std": 2.256464385986328, "kd/ssd_loss": 0.0, "learning_rate": 4.7256294383473204e-07, "loss": 0.035618539651234946, "num_tokens": 454650062.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.785833332935969, "rewards/gt_logging_reward/std": 0.3997292493780454, "sampling/importance_sampling_ratio/max": 2.008753768603007, "sampling/importance_sampling_ratio/mean": 0.9997224688529969, "sampling/importance_sampling_ratio/min": 0.4006120502948761, "sampling/sampling_logp_difference/max": 0.32309786081314085, "sampling/sampling_logp_difference/mean": 0.004097367667903503, "step": 13890, "step_time": 8.019089313566898, "student/entropy": 0.13274628445506095 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03166666819403569, "completions/max_length": 495.1666666666667, "completions/max_terminated_length": 451.3333333333333, "completions/mean_length": 196.13917744954426, "completions/mean_terminated_length": 186.0402374267578, "completions/min_length": 52.2, "completions/min_terminated_length": 52.2, "entropy": 0.1353298483416438, "epoch": 0.5286142862567881, "eval/gt_acc_batch": 0.7677778005599976, "frac_reward_zero_std": 1.0, "grad_norm": 0.27357378602027893, "kd/policy_loss": 0.009191409504273907, "kd/rkl_advantage_mean": 0.8681299385925134, "kd/rkl_advantage_p95": 5.179585138956706, "kd/rkl_advantage_std": 2.250325361887614, "kd/ssd_loss": 0.0, "learning_rate": 4.7142368890745446e-07, "loss": 0.03676563501358032, "num_tokens": 455673451.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7677777747313181, "rewards/gt_logging_reward/std": 0.41145606885353725, "sampling/importance_sampling_ratio/max": 1.9369396249453226, "sampling/importance_sampling_ratio/mean": 0.999887736638387, "sampling/importance_sampling_ratio/min": 0.42798667003711066, "sampling/sampling_logp_difference/max": 0.3296239177385966, "sampling/sampling_logp_difference/mean": 0.0041307649187122784, "step": 13920, "step_time": 8.22406125130025, "student/entropy": 0.1353298483416438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.012500000558793545, "completions/max_length": 461.23333333333335, "completions/max_terminated_length": 437.93333333333334, "completions/mean_length": 180.70612030029298, "completions/mean_terminated_length": 176.6969446818034, "completions/min_length": 64.03333333333333, "completions/min_terminated_length": 64.03333333333333, "entropy": 0.11860144517074028, "epoch": 0.5297535411840656, "eval/gt_acc_batch": 0.832500014702479, "frac_reward_zero_std": 1.0, "grad_norm": 0.2789967358112335, "kd/policy_loss": 0.0076336984037576865, "kd/rkl_advantage_mean": 0.7729518757512172, "kd/rkl_advantage_p95": 4.973008503516515, "kd/rkl_advantage_std": 2.1891183803478875, "kd/ssd_loss": 0.0, "learning_rate": 4.7028443398017693e-07, "loss": 0.030534797906875612, "num_tokens": 456616801.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8324999948342641, "rewards/gt_logging_reward/std": 0.3626602053642273, "sampling/importance_sampling_ratio/max": 1.908104125658671, "sampling/importance_sampling_ratio/mean": 1.0020939966042837, "sampling/importance_sampling_ratio/min": 0.48717716832955676, "sampling/sampling_logp_difference/max": 0.3304917881886164, "sampling/sampling_logp_difference/mean": 0.003745162549118201, "step": 13950, "step_time": 7.520095770598952, "student/entropy": 0.11860144517074028 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02500000149011612, "completions/max_length": 489.9, "completions/max_terminated_length": 448.6666666666667, "completions/mean_length": 186.6958435058594, "completions/mean_terminated_length": 178.47743428548176, "completions/min_length": 50.333333333333336, "completions/min_terminated_length": 50.333333333333336, "entropy": 0.13131221774965524, "epoch": 0.5308927961113432, "eval/gt_acc_batch": 0.7936111370722453, "frac_reward_zero_std": 1.0, "grad_norm": 0.34088826179504395, "kd/policy_loss": 0.009021756675792858, "kd/rkl_advantage_mean": 0.7565242799910872, "kd/rkl_advantage_p95": 5.030451864004135, "kd/rkl_advantage_std": 2.212914439042409, "kd/ssd_loss": 0.0, "learning_rate": 4.691451790528994e-07, "loss": 0.03608702023824056, "num_tokens": 457597810.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111132303874, "rewards/gt_logging_reward/std": 0.3957334746917089, "sampling/importance_sampling_ratio/max": 1.8170052886009216, "sampling/importance_sampling_ratio/mean": 1.0024266183376311, "sampling/importance_sampling_ratio/min": 0.47827940781911216, "sampling/sampling_logp_difference/max": 0.34440837303797406, "sampling/sampling_logp_difference/mean": 0.004038713787061473, "step": 13980, "step_time": 8.058347323072667, "student/entropy": 0.13131221774965524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555662761132, "completions/max_length": 479.03333333333336, "completions/max_terminated_length": 457.03333333333336, "completions/mean_length": 186.70167694091796, "completions/mean_terminated_length": 179.91555786132812, "completions/min_length": 57.4, "completions/min_terminated_length": 57.4, "entropy": 0.1322163434078296, "epoch": 0.5320320510386207, "eval/gt_acc_batch": 0.7925000170866648, "frac_reward_zero_std": 1.0, "grad_norm": 0.22778098285198212, "kd/policy_loss": 0.008738538060182084, "kd/rkl_advantage_mean": 0.8136056170798838, "kd/rkl_advantage_p95": 5.186686251560847, "kd/rkl_advantage_std": 2.271095664302508, "kd/ssd_loss": 0.0, "learning_rate": 4.680059241256218e-07, "loss": 0.03495415449142456, "num_tokens": 458570368.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7925000031789143, "rewards/gt_logging_reward/std": 0.3981808731953303, "sampling/importance_sampling_ratio/max": 1.7830043355623881, "sampling/importance_sampling_ratio/mean": 0.9996215383211772, "sampling/importance_sampling_ratio/min": 0.4313208987315496, "sampling/sampling_logp_difference/max": 0.32099093198776246, "sampling/sampling_logp_difference/mean": 0.004096685280092061, "step": 14010, "step_time": 7.837290285233272, "student/entropy": 0.1322163434078296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02750000162050128, "completions/max_length": 501.06666666666666, "completions/max_terminated_length": 467.56666666666666, "completions/mean_length": 190.10723215738932, "completions/mean_terminated_length": 181.12381388346355, "completions/min_length": 55.733333333333334, "completions/min_terminated_length": 55.733333333333334, "entropy": 0.13167274494965872, "epoch": 0.5331713059658983, "eval/gt_acc_batch": 0.7886111418406169, "frac_reward_zero_std": 1.0, "grad_norm": 0.2916111350059509, "kd/policy_loss": 0.009141125190459813, "kd/rkl_advantage_mean": 0.8818442208362588, "kd/rkl_advantage_p95": 5.244532811641693, "kd/rkl_advantage_std": 2.2945354243119556, "kd/ssd_loss": 0.0, "learning_rate": 4.668666691983443e-07, "loss": 0.03656450112660726, "num_tokens": 459559530.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.788611114025116, "rewards/gt_logging_reward/std": 0.40001068214575447, "sampling/importance_sampling_ratio/max": 2.031305734316508, "sampling/importance_sampling_ratio/mean": 1.0055910964806876, "sampling/importance_sampling_ratio/min": 0.45751590728759767, "sampling/sampling_logp_difference/max": 0.3302497982978821, "sampling/sampling_logp_difference/mean": 0.0040569905579711, "step": 14040, "step_time": 8.065026737634616, "student/entropy": 0.13167274494965872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001365939776, "completions/max_length": 500.56666666666666, "completions/max_terminated_length": 449.96666666666664, "completions/mean_length": 186.97139892578124, "completions/mean_terminated_length": 178.72216186523437, "completions/min_length": 48.733333333333334, "completions/min_terminated_length": 48.733333333333334, "entropy": 0.1330014217024048, "epoch": 0.5343105608931759, "eval/gt_acc_batch": 0.8100000182787578, "frac_reward_zero_std": 1.0, "grad_norm": 0.263800710439682, "kd/policy_loss": 0.008649396368612845, "kd/rkl_advantage_mean": 0.7830578133463859, "kd/rkl_advantage_p95": 5.160121321678162, "kd/rkl_advantage_std": 2.275480250517527, "kd/ssd_loss": 0.0, "learning_rate": 4.657274142710667e-07, "loss": 0.034597587585449216, "num_tokens": 460532555.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8100000023841858, "rewards/gt_logging_reward/std": 0.3837002421418826, "sampling/importance_sampling_ratio/max": 1.9510464270909627, "sampling/importance_sampling_ratio/mean": 0.9920869052410126, "sampling/importance_sampling_ratio/min": 0.4665325323740641, "sampling/sampling_logp_difference/max": 0.35459613998730977, "sampling/sampling_logp_difference/mean": 0.004093968775123358, "step": 14070, "step_time": 7.958294593698034, "student/entropy": 0.1330014217024048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334798614182, "completions/max_length": 482.4, "completions/max_terminated_length": 452.3666666666667, "completions/mean_length": 189.1511210123698, "completions/mean_terminated_length": 181.48666178385417, "completions/min_length": 59.8, "completions/min_terminated_length": 59.8, "entropy": 0.13095820005983114, "epoch": 0.5354498158204535, "eval/gt_acc_batch": 0.7755555748939514, "frac_reward_zero_std": 1.0, "grad_norm": 0.2728184163570404, "kd/policy_loss": 0.008942694230548416, "kd/rkl_advantage_mean": 0.7644301899398367, "kd/rkl_advantage_p95": 5.072128107150395, "kd/rkl_advantage_std": 2.260962744553884, "kd/ssd_loss": 0.0, "learning_rate": 4.645881593437891e-07, "loss": 0.03577077388763428, "num_tokens": 461515259.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.775555553038915, "rewards/gt_logging_reward/std": 0.4097857803106308, "sampling/importance_sampling_ratio/max": 1.908998962243398, "sampling/importance_sampling_ratio/mean": 1.0035688936710359, "sampling/importance_sampling_ratio/min": 0.4268093725045522, "sampling/sampling_logp_difference/max": 0.30399640202522277, "sampling/sampling_logp_difference/mean": 0.00404395826626569, "step": 14100, "step_time": 7.9358113301360085, "student/entropy": 0.13095820005983114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0188888899050653, "completions/max_length": 479.43333333333334, "completions/max_terminated_length": 434.6666666666667, "completions/mean_length": 185.80362091064453, "completions/mean_terminated_length": 179.755717976888, "completions/min_length": 52.4, "completions/min_terminated_length": 52.4, "entropy": 0.1281090594207247, "epoch": 0.536589070747731, "eval/gt_acc_batch": 0.7955555816491445, "frac_reward_zero_std": 1.0, "grad_norm": 0.3171338438987732, "kd/policy_loss": 0.00862113045101675, "kd/rkl_advantage_mean": 0.7178408570277194, "kd/rkl_advantage_p95": 4.922809422016144, "kd/rkl_advantage_std": 2.1948791911204655, "kd/ssd_loss": 0.0, "learning_rate": 4.634489044165116e-07, "loss": 0.034484525521596275, "num_tokens": 462496240.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555538336436, "rewards/gt_logging_reward/std": 0.3949325650930405, "sampling/importance_sampling_ratio/max": 1.82848379611969, "sampling/importance_sampling_ratio/mean": 0.9972746471563975, "sampling/importance_sampling_ratio/min": 0.4678614497184753, "sampling/sampling_logp_difference/max": 0.32258217136065165, "sampling/sampling_logp_difference/mean": 0.003982665009486178, "step": 14130, "step_time": 7.837802286339381, "student/entropy": 0.1281090594207247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030000001937150956, "completions/max_length": 498.1333333333333, "completions/max_terminated_length": 463.06666666666666, "completions/mean_length": 188.0269551595052, "completions/mean_terminated_length": 178.10547434488933, "completions/min_length": 49.4, "completions/min_terminated_length": 49.4, "entropy": 0.13191390068580708, "epoch": 0.5377283256750085, "eval/gt_acc_batch": 0.7911111195882161, "frac_reward_zero_std": 1.0, "grad_norm": 0.27249985933303833, "kd/policy_loss": 0.008935203154881796, "kd/rkl_advantage_mean": 0.7406061695267757, "kd/rkl_advantage_p95": 5.090488608678182, "kd/rkl_advantage_std": 2.2853286306063336, "kd/ssd_loss": 0.0, "learning_rate": 4.62309649489234e-07, "loss": 0.03574081261952718, "num_tokens": 463465217.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7911111136277517, "rewards/gt_logging_reward/std": 0.40392165780067446, "sampling/importance_sampling_ratio/max": 2.023034652074178, "sampling/importance_sampling_ratio/mean": 1.0014307816823325, "sampling/importance_sampling_ratio/min": 0.4638982613881429, "sampling/sampling_logp_difference/max": 0.32270373702049254, "sampling/sampling_logp_difference/mean": 0.004053582243310909, "step": 14160, "step_time": 7.916466237401377, "student/entropy": 0.13191390068580708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556720743575, "completions/max_length": 487.73333333333335, "completions/max_terminated_length": 451.8333333333333, "completions/mean_length": 189.17306518554688, "completions/mean_terminated_length": 182.4264673868815, "completions/min_length": 52.3, "completions/min_terminated_length": 52.3, "entropy": 0.1277829830845197, "epoch": 0.5388675806022861, "eval/gt_acc_batch": 0.7933333476384481, "frac_reward_zero_std": 1.0, "grad_norm": 0.2972525358200073, "kd/policy_loss": 0.008542922117824977, "kd/rkl_advantage_mean": 0.7868398564557234, "kd/rkl_advantage_p95": 5.109304853280386, "kd/rkl_advantage_std": 2.2591213792562486, "kd/ssd_loss": 0.0, "learning_rate": 4.6117039456195643e-07, "loss": 0.03417169253031413, "num_tokens": 464456416.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7933333337306976, "rewards/gt_logging_reward/std": 0.39858792622884115, "sampling/importance_sampling_ratio/max": 1.9485211769739788, "sampling/importance_sampling_ratio/mean": 1.0034557362397512, "sampling/importance_sampling_ratio/min": 0.47325532138347626, "sampling/sampling_logp_difference/max": 0.33223408460617065, "sampling/sampling_logp_difference/mean": 0.004001117854689558, "step": 14190, "step_time": 7.890491550864923, "student/entropy": 0.1277829830845197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556907008093, "completions/max_length": 487.3333333333333, "completions/max_terminated_length": 438.93333333333334, "completions/mean_length": 185.93195393880208, "completions/mean_terminated_length": 179.18723856608074, "completions/min_length": 53.0, "completions/min_terminated_length": 53.0, "entropy": 0.12861260082572698, "epoch": 0.5400068355295636, "eval/gt_acc_batch": 0.8047222455342611, "frac_reward_zero_std": 1.0, "grad_norm": 0.437302827835083, "kd/policy_loss": 0.008472359465667979, "kd/rkl_advantage_mean": 0.7876690263239046, "kd/rkl_advantage_p95": 5.097296951214473, "kd/rkl_advantage_std": 2.276056478420893, "kd/ssd_loss": 0.0, "learning_rate": 4.600311396346789e-07, "loss": 0.03388944466908773, "num_tokens": 465435395.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222236792246, "rewards/gt_logging_reward/std": 0.3868939866622289, "sampling/importance_sampling_ratio/max": 1.9082477251688639, "sampling/importance_sampling_ratio/mean": 0.9980798959732056, "sampling/importance_sampling_ratio/min": 0.4419197502235572, "sampling/sampling_logp_difference/max": 0.34326926271120706, "sampling/sampling_logp_difference/mean": 0.003999324445612728, "step": 14220, "step_time": 8.090203091269844, "student/entropy": 0.12861260082572698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778953313828, "completions/max_length": 484.93333333333334, "completions/max_terminated_length": 460.1666666666667, "completions/mean_length": 188.6252868652344, "completions/mean_terminated_length": 182.89507446289062, "completions/min_length": 50.766666666666666, "completions/min_terminated_length": 50.766666666666666, "entropy": 0.12615254260599612, "epoch": 0.5411460904568413, "eval/gt_acc_batch": 0.8033333460489909, "frac_reward_zero_std": 1.0, "grad_norm": 0.26796314120292664, "kd/policy_loss": 0.008588151833585774, "kd/rkl_advantage_mean": 0.824182416126132, "kd/rkl_advantage_p95": 5.134262442588806, "kd/rkl_advantage_std": 2.2486952185630797, "kd/ssd_loss": 0.0, "learning_rate": 4.588918847074013e-07, "loss": 0.034352608521779376, "num_tokens": 466412734.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8033333420753479, "rewards/gt_logging_reward/std": 0.3912434091170629, "sampling/importance_sampling_ratio/max": 1.8951197425524393, "sampling/importance_sampling_ratio/mean": 1.004019421339035, "sampling/importance_sampling_ratio/min": 0.42433348794778186, "sampling/sampling_logp_difference/max": 0.3242478271325429, "sampling/sampling_logp_difference/mean": 0.003908710445587834, "step": 14250, "step_time": 7.881029926898676, "student/entropy": 0.12615254260599612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111265917619, "completions/max_length": 473.6333333333333, "completions/max_terminated_length": 432.26666666666665, "completions/mean_length": 188.21778767903646, "completions/mean_terminated_length": 179.76077575683593, "completions/min_length": 44.733333333333334, "completions/min_terminated_length": 44.733333333333334, "entropy": 0.1246906536941727, "epoch": 0.5422853453841188, "eval/gt_acc_batch": 0.8105555931727092, "frac_reward_zero_std": 1.0, "grad_norm": 0.3046474754810333, "kd/policy_loss": 0.008073354056493069, "kd/rkl_advantage_mean": 0.7492210269284745, "kd/rkl_advantage_p95": 4.988565138975779, "kd/rkl_advantage_std": 2.2147785166899365, "kd/ssd_loss": 0.0, "learning_rate": 4.577526297801238e-07, "loss": 0.03229341109593709, "num_tokens": 467392974.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8105555534362793, "rewards/gt_logging_reward/std": 0.38291081885496775, "sampling/importance_sampling_ratio/max": 1.8150936563809712, "sampling/importance_sampling_ratio/mean": 0.9977585395177205, "sampling/importance_sampling_ratio/min": 0.4736273547013601, "sampling/sampling_logp_difference/max": 0.30562562942504884, "sampling/sampling_logp_difference/mean": 0.003924941543179254, "step": 14280, "step_time": 7.8026003944673, "student/entropy": 0.1246906536941727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03000000168879827, "completions/max_length": 501.5, "completions/max_terminated_length": 458.73333333333335, "completions/mean_length": 195.0900105794271, "completions/mean_terminated_length": 185.44005737304687, "completions/min_length": 57.06666666666667, "completions/min_terminated_length": 57.06666666666667, "entropy": 0.13724552188068628, "epoch": 0.5434246003113964, "eval/gt_acc_batch": 0.7741666952768962, "frac_reward_zero_std": 1.0, "grad_norm": 0.3184824287891388, "kd/policy_loss": 0.009162128332536667, "kd/rkl_advantage_mean": 0.8390109991033872, "kd/rkl_advantage_p95": 5.1096123615900675, "kd/rkl_advantage_std": 2.2231254210074742, "kd/ssd_loss": 0.0, "learning_rate": 4.566133748528462e-07, "loss": 0.03664851983388265, "num_tokens": 468398898.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7741666674613953, "rewards/gt_logging_reward/std": 0.4134669174750646, "sampling/importance_sampling_ratio/max": 2.0095699350039165, "sampling/importance_sampling_ratio/mean": 0.9991507867972056, "sampling/importance_sampling_ratio/min": 0.4148773342370987, "sampling/sampling_logp_difference/max": 0.3441101531187693, "sampling/sampling_logp_difference/mean": 0.004218136728741229, "step": 14310, "step_time": 8.046545662163407, "student/entropy": 0.13724552188068628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556658655404, "completions/max_length": 476.8, "completions/max_terminated_length": 433.8, "completions/mean_length": 182.37056477864584, "completions/mean_terminated_length": 175.6757598876953, "completions/min_length": 60.766666666666666, "completions/min_terminated_length": 60.766666666666666, "entropy": 0.1278589957083265, "epoch": 0.5445638552386739, "eval/gt_acc_batch": 0.8122222363948822, "frac_reward_zero_std": 1.0, "grad_norm": 0.2768915891647339, "kd/policy_loss": 0.008434132154798135, "kd/rkl_advantage_mean": 0.7375584746400515, "kd/rkl_advantage_p95": 4.934700310230255, "kd/rkl_advantage_std": 2.19104775985082, "kd/ssd_loss": 0.0, "learning_rate": 4.554741199255687e-07, "loss": 0.03373652696609497, "num_tokens": 469353400.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8122222205003102, "rewards/gt_logging_reward/std": 0.3801257664958636, "sampling/importance_sampling_ratio/max": 1.919525682926178, "sampling/importance_sampling_ratio/mean": 1.0021635254224142, "sampling/importance_sampling_ratio/min": 0.49672264953454337, "sampling/sampling_logp_difference/max": 0.3533597449461619, "sampling/sampling_logp_difference/mean": 0.003967809017437199, "step": 14340, "step_time": 7.827388152068791, "student/entropy": 0.1278589957083265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001198301712, "completions/max_length": 494.3666666666667, "completions/max_terminated_length": 458.6666666666667, "completions/mean_length": 185.60473124186197, "completions/mean_terminated_length": 178.9428741455078, "completions/min_length": 53.43333333333333, "completions/min_terminated_length": 53.43333333333333, "entropy": 0.13205894784380992, "epoch": 0.5457031101659515, "eval/gt_acc_batch": 0.7963889102141063, "frac_reward_zero_std": 1.0, "grad_norm": 0.2869984209537506, "kd/policy_loss": 0.008894563704961911, "kd/rkl_advantage_mean": 0.779936836163203, "kd/rkl_advantage_p95": 5.062445110082626, "kd/rkl_advantage_std": 2.2575415521860123, "kd/ssd_loss": 0.0, "learning_rate": 4.543348649982911e-07, "loss": 0.035578258832295734, "num_tokens": 470319465.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888883590698, "rewards/gt_logging_reward/std": 0.39498642285664876, "sampling/importance_sampling_ratio/max": 1.8591102759043376, "sampling/importance_sampling_ratio/mean": 1.0012077689170837, "sampling/importance_sampling_ratio/min": 0.4524426430463791, "sampling/sampling_logp_difference/max": 0.3474019070466359, "sampling/sampling_logp_difference/mean": 0.004075704608112574, "step": 14370, "step_time": 7.960193502201097, "student/entropy": 0.13205894784380992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779375513395, "completions/max_length": 481.56666666666666, "completions/max_terminated_length": 438.3, "completions/mean_length": 191.40167795817058, "completions/mean_terminated_length": 183.3941914876302, "completions/min_length": 61.233333333333334, "completions/min_terminated_length": 61.233333333333334, "entropy": 0.12285610934098562, "epoch": 0.546842365093229, "eval/gt_acc_batch": 0.7858333508173625, "frac_reward_zero_std": 1.0, "grad_norm": 0.3043651878833771, "kd/policy_loss": 0.00804751859783816, "kd/rkl_advantage_mean": 0.7266869453092416, "kd/rkl_advantage_p95": 4.945399794975916, "kd/rkl_advantage_std": 2.226421195268631, "kd/ssd_loss": 0.0, "learning_rate": 4.531956100710135e-07, "loss": 0.03219006856282552, "num_tokens": 471312687.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7858333349227905, "rewards/gt_logging_reward/std": 0.3974597712357839, "sampling/importance_sampling_ratio/max": 1.9460566480954489, "sampling/importance_sampling_ratio/mean": 0.9925326625506083, "sampling/importance_sampling_ratio/min": 0.4048585613568624, "sampling/sampling_logp_difference/max": 0.3424772421518962, "sampling/sampling_logp_difference/mean": 0.003805816119226317, "step": 14400, "step_time": 7.831805578096828, "student/entropy": 0.12285610934098562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026111112721264363, "completions/max_length": 492.1666666666667, "completions/max_terminated_length": 446.1333333333333, "completions/mean_length": 195.12417653401693, "completions/mean_terminated_length": 186.7240234375, "completions/min_length": 55.8, "completions/min_terminated_length": 55.8, "entropy": 0.12950461332996685, "epoch": 0.5479816200205065, "eval/gt_acc_batch": 0.7947222371896108, "frac_reward_zero_std": 1.0, "grad_norm": 0.26688554883003235, "kd/policy_loss": 0.008391196542652324, "kd/rkl_advantage_mean": 0.7083617351017892, "kd/rkl_advantage_p95": 4.934193418423335, "kd/rkl_advantage_std": 2.20128775537014, "kd/ssd_loss": 0.0, "learning_rate": 4.52056355143736e-07, "loss": 0.03356478611628214, "num_tokens": 472323566.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222212950389, "rewards/gt_logging_reward/std": 0.40067109763622283, "sampling/importance_sampling_ratio/max": 1.9461580753326415, "sampling/importance_sampling_ratio/mean": 1.000873827934265, "sampling/importance_sampling_ratio/min": 0.46605021754900616, "sampling/sampling_logp_difference/max": 0.3560246706008911, "sampling/sampling_logp_difference/mean": 0.003946274744036297, "step": 14430, "step_time": 8.065704048264402, "student/entropy": 0.12950461332996685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778934687375, "completions/max_length": 475.8, "completions/max_terminated_length": 437.56666666666666, "completions/mean_length": 183.84278869628906, "completions/mean_terminated_length": 176.41904907226564, "completions/min_length": 50.1, "completions/min_terminated_length": 50.1, "entropy": 0.12721151920656362, "epoch": 0.5491208749477842, "eval/gt_acc_batch": 0.7847222487131754, "frac_reward_zero_std": 1.0, "grad_norm": 0.31195974349975586, "kd/policy_loss": 0.008410358768499767, "kd/rkl_advantage_mean": 0.7828478263070185, "kd/rkl_advantage_p95": 5.053556867440542, "kd/rkl_advantage_std": 2.2377468019723894, "kd/ssd_loss": 0.0, "learning_rate": 4.509171002164584e-07, "loss": 0.03364144166310628, "num_tokens": 473290664.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7847222268581391, "rewards/gt_logging_reward/std": 0.40486281116803485, "sampling/importance_sampling_ratio/max": 1.8699603756268819, "sampling/importance_sampling_ratio/mean": 0.9962499896685283, "sampling/importance_sampling_ratio/min": 0.44797235280275344, "sampling/sampling_logp_difference/max": 0.35732428630193075, "sampling/sampling_logp_difference/mean": 0.003956443350762129, "step": 14460, "step_time": 7.799616241499219, "student/entropy": 0.12721151920656362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223865489165, "completions/max_length": 493.93333333333334, "completions/max_terminated_length": 447.06666666666666, "completions/mean_length": 193.12278747558594, "completions/mean_terminated_length": 183.2884979248047, "completions/min_length": 48.333333333333336, "completions/min_terminated_length": 48.333333333333336, "entropy": 0.13098448396970827, "epoch": 0.5502601298750617, "eval/gt_acc_batch": 0.796111128727595, "frac_reward_zero_std": 1.0, "grad_norm": 0.2744516432285309, "kd/policy_loss": 0.008811791877572734, "kd/rkl_advantage_mean": 0.7832028613736232, "kd/rkl_advantage_p95": 5.127536849180857, "kd/rkl_advantage_std": 2.2801956623792647, "kd/ssd_loss": 0.0, "learning_rate": 4.497778452891808e-07, "loss": 0.0352471669514974, "num_tokens": 474298570.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111108462016, "rewards/gt_logging_reward/std": 0.39008025676012037, "sampling/importance_sampling_ratio/max": 1.995458964506785, "sampling/importance_sampling_ratio/mean": 1.004594471057256, "sampling/importance_sampling_ratio/min": 0.4439615180095037, "sampling/sampling_logp_difference/max": 0.3124322195847829, "sampling/sampling_logp_difference/mean": 0.004005067663577696, "step": 14490, "step_time": 7.999133306264412, "student/entropy": 0.13098448396970827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029444446259488662, "completions/max_length": 495.23333333333335, "completions/max_terminated_length": 465.8, "completions/mean_length": 192.22445526123047, "completions/mean_terminated_length": 182.7338633219401, "completions/min_length": 50.3, "completions/min_terminated_length": 50.3, "entropy": 0.13106168750673533, "epoch": 0.5513993848023393, "eval/gt_acc_batch": 0.7808333575725556, "frac_reward_zero_std": 1.0, "grad_norm": 0.3008697032928467, "kd/policy_loss": 0.008835554384859279, "kd/rkl_advantage_mean": 0.852673387931039, "kd/rkl_advantage_p95": 5.156245501836141, "kd/rkl_advantage_std": 2.2747553139925003, "kd/ssd_loss": 0.0, "learning_rate": 4.486385903619033e-07, "loss": 0.03534222443898519, "num_tokens": 475294482.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7808333297570547, "rewards/gt_logging_reward/std": 0.4049136092265447, "sampling/importance_sampling_ratio/max": 1.979413139820099, "sampling/importance_sampling_ratio/mean": 1.0095176657040914, "sampling/importance_sampling_ratio/min": 0.45791137566169104, "sampling/sampling_logp_difference/max": 0.3763920903205872, "sampling/sampling_logp_difference/mean": 0.004075577156618238, "step": 14520, "step_time": 7.999782047532305, "student/entropy": 0.13106168750673533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111253499985, "completions/max_length": 474.9, "completions/max_terminated_length": 444.56666666666666, "completions/mean_length": 186.72278747558593, "completions/mean_terminated_length": 178.2408915201823, "completions/min_length": 54.96666666666667, "completions/min_terminated_length": 54.96666666666667, "entropy": 0.12860817952702444, "epoch": 0.5525386397296168, "eval/gt_acc_batch": 0.7933333655198415, "frac_reward_zero_std": 1.0, "grad_norm": 0.25711169838905334, "kd/policy_loss": 0.008543997213322049, "kd/rkl_advantage_mean": 0.7369885335365931, "kd/rkl_advantage_p95": 4.976098597049713, "kd/rkl_advantage_std": 2.230137065052986, "kd/ssd_loss": 0.0, "learning_rate": 4.4749933543462576e-07, "loss": 0.034175992012023926, "num_tokens": 476267316.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7933333277702331, "rewards/gt_logging_reward/std": 0.3977882832288742, "sampling/importance_sampling_ratio/max": 1.8473928332328797, "sampling/importance_sampling_ratio/mean": 0.997831251223882, "sampling/importance_sampling_ratio/min": 0.45278252263863883, "sampling/sampling_logp_difference/max": 0.32120487292607625, "sampling/sampling_logp_difference/mean": 0.003953506665614744, "step": 14550, "step_time": 7.86307699316676, "student/entropy": 0.12860817952702444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018333334662020206, "completions/max_length": 489.73333333333335, "completions/max_terminated_length": 452.8, "completions/mean_length": 186.9569544474284, "completions/mean_terminated_length": 181.07603352864584, "completions/min_length": 58.1, "completions/min_terminated_length": 58.1, "entropy": 0.12478051135937372, "epoch": 0.5536778946568944, "eval/gt_acc_batch": 0.8247222403685252, "frac_reward_zero_std": 1.0, "grad_norm": 0.1835320144891739, "kd/policy_loss": 0.007987252545232575, "kd/rkl_advantage_mean": 0.7970935344696045, "kd/rkl_advantage_p95": 5.052758999665579, "kd/rkl_advantage_std": 2.2401954541603724, "kd/ssd_loss": 0.0, "learning_rate": 4.463600805073482e-07, "loss": 0.031949015458424886, "num_tokens": 477251561.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8247222224871318, "rewards/gt_logging_reward/std": 0.37059103747208916, "sampling/importance_sampling_ratio/max": 1.9625274380048117, "sampling/importance_sampling_ratio/mean": 0.9889938294887543, "sampling/importance_sampling_ratio/min": 0.4346028923988342, "sampling/sampling_logp_difference/max": 0.31701832413673403, "sampling/sampling_logp_difference/mean": 0.0038976720068603756, "step": 14580, "step_time": 7.909220107533232, "student/entropy": 0.12478051135937372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.036666668858379124, "completions/max_length": 497.6, "completions/max_terminated_length": 465.23333333333335, "completions/mean_length": 196.66278737386068, "completions/mean_terminated_length": 184.81593119303386, "completions/min_length": 50.96666666666667, "completions/min_terminated_length": 50.96666666666667, "entropy": 0.13124432787299156, "epoch": 0.5548171495841719, "eval/gt_acc_batch": 0.7700000206629435, "frac_reward_zero_std": 1.0, "grad_norm": 0.27865561842918396, "kd/policy_loss": 0.00925178121542558, "kd/rkl_advantage_mean": 0.8589087894807259, "kd/rkl_advantage_p95": 5.242187841733297, "kd/rkl_advantage_std": 2.2911203374465305, "kd/ssd_loss": 0.0, "learning_rate": 4.4522082558007065e-07, "loss": 0.03700713316599528, "num_tokens": 478288491.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7700000047683716, "rewards/gt_logging_reward/std": 0.41718982458114623, "sampling/importance_sampling_ratio/max": 1.9514320850372315, "sampling/importance_sampling_ratio/mean": 0.9915979226430257, "sampling/importance_sampling_ratio/min": 0.41511506338914234, "sampling/sampling_logp_difference/max": 0.33625168800354005, "sampling/sampling_logp_difference/mean": 0.0040969335163633025, "step": 14610, "step_time": 8.259859662701395, "student/entropy": 0.13124432787299156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02388889032105605, "completions/max_length": 495.8, "completions/max_terminated_length": 442.2, "completions/mean_length": 186.1805648803711, "completions/mean_terminated_length": 178.3234405517578, "completions/min_length": 55.333333333333336, "completions/min_terminated_length": 55.333333333333336, "entropy": 0.12903570501754683, "epoch": 0.5559564045114496, "eval/gt_acc_batch": 0.8072222491105397, "frac_reward_zero_std": 1.0, "grad_norm": 0.2579575479030609, "kd/policy_loss": 0.008518146038598691, "kd/rkl_advantage_mean": 0.8103035018468897, "kd/rkl_advantage_p95": 5.015791630744934, "kd/rkl_advantage_std": 2.224917325377464, "kd/ssd_loss": 0.0, "learning_rate": 4.4408157065279307e-07, "loss": 0.034072589874267575, "num_tokens": 479263469.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222193082174, "rewards/gt_logging_reward/std": 0.38288371711969377, "sampling/importance_sampling_ratio/max": 1.8768938938776651, "sampling/importance_sampling_ratio/mean": 0.9962189773718516, "sampling/importance_sampling_ratio/min": 0.4684581900636355, "sampling/sampling_logp_difference/max": 0.328729252020518, "sampling/sampling_logp_difference/mean": 0.003971867356449365, "step": 14640, "step_time": 7.954169170030218, "student/entropy": 0.12903570501754683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016666667784253755, "completions/max_length": 477.53333333333336, "completions/max_terminated_length": 443.23333333333335, "completions/mean_length": 175.52084197998047, "completions/mean_terminated_length": 169.95115559895834, "completions/min_length": 49.56666666666667, "completions/min_terminated_length": 49.56666666666667, "entropy": 0.12994326800107955, "epoch": 0.5570956594387271, "eval/gt_acc_batch": 0.8216666917006175, "frac_reward_zero_std": 1.0, "grad_norm": 0.285945326089859, "kd/policy_loss": 0.008404760444924856, "kd/rkl_advantage_mean": 0.84225302208215, "kd/rkl_advantage_p95": 5.143693939844767, "kd/rkl_advantage_std": 2.2596986373265584, "kd/ssd_loss": 0.0, "learning_rate": 4.429423157255155e-07, "loss": 0.033619046211242676, "num_tokens": 480182752.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.821666665871938, "rewards/gt_logging_reward/std": 0.37485228578249613, "sampling/importance_sampling_ratio/max": 1.8609692295392355, "sampling/importance_sampling_ratio/mean": 0.9977297206719716, "sampling/importance_sampling_ratio/min": 0.4561636020739873, "sampling/sampling_logp_difference/max": 0.3249599575996399, "sampling/sampling_logp_difference/mean": 0.004067621318002542, "step": 14670, "step_time": 7.684766521669129, "student/entropy": 0.12994326800107955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030833335034549235, "completions/max_length": 482.4, "completions/max_terminated_length": 448.53333333333336, "completions/mean_length": 188.05167592366536, "completions/mean_terminated_length": 177.9848492940267, "completions/min_length": 54.86666666666667, "completions/min_terminated_length": 54.86666666666667, "entropy": 0.12949613040934008, "epoch": 0.5582349143660046, "eval/gt_acc_batch": 0.7963889141877493, "frac_reward_zero_std": 1.0, "grad_norm": 0.2759934663772583, "kd/policy_loss": 0.008721781603526324, "kd/rkl_advantage_mean": 0.8565758790820837, "kd/rkl_advantage_p95": 5.206394044558207, "kd/rkl_advantage_std": 2.277173594633738, "kd/ssd_loss": 0.0, "learning_rate": 4.418030607982379e-07, "loss": 0.03488712310791016, "num_tokens": 481170562.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888923327128, "rewards/gt_logging_reward/std": 0.3919052427013715, "sampling/importance_sampling_ratio/max": 1.9613359093666076, "sampling/importance_sampling_ratio/mean": 0.9974311470985413, "sampling/importance_sampling_ratio/min": 0.4282126009464264, "sampling/sampling_logp_difference/max": 0.36439983248710633, "sampling/sampling_logp_difference/mean": 0.00400510155595839, "step": 14700, "step_time": 7.857621572098287, "student/entropy": 0.12949613040934008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556969096264, "completions/max_length": 501.73333333333335, "completions/max_terminated_length": 462.6666666666667, "completions/mean_length": 183.54473215738932, "completions/mean_terminated_length": 176.74147338867186, "completions/min_length": 47.7, "completions/min_terminated_length": 47.7, "entropy": 0.13114777375012637, "epoch": 0.5593741692932822, "eval/gt_acc_batch": 0.8041666825612386, "frac_reward_zero_std": 1.0, "grad_norm": 0.3531908690929413, "kd/policy_loss": 0.008747491700341925, "kd/rkl_advantage_mean": 0.8482032224303111, "kd/rkl_advantage_p95": 5.114320764938991, "kd/rkl_advantage_std": 2.2371450712283454, "kd/ssd_loss": 0.0, "learning_rate": 4.4066380587096037e-07, "loss": 0.034989968935648603, "num_tokens": 482130275.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666646798452, "rewards/gt_logging_reward/std": 0.3930734773476919, "sampling/importance_sampling_ratio/max": 1.9159248232841493, "sampling/importance_sampling_ratio/mean": 1.0023906370004019, "sampling/importance_sampling_ratio/min": 0.4692353387673696, "sampling/sampling_logp_difference/max": 0.34857372045516966, "sampling/sampling_logp_difference/mean": 0.004072307166643441, "step": 14730, "step_time": 7.975029643499875, "student/entropy": 0.13114777375012637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018055556683490674, "completions/max_length": 479.3333333333333, "completions/max_terminated_length": 449.56666666666666, "completions/mean_length": 186.2350102742513, "completions/mean_terminated_length": 180.2590555826823, "completions/min_length": 57.6, "completions/min_terminated_length": 57.6, "entropy": 0.11867553809036811, "epoch": 0.5605134242205597, "eval/gt_acc_batch": 0.8286111334959666, "frac_reward_zero_std": 1.0, "grad_norm": 0.34131333231925964, "kd/policy_loss": 0.007528761878105191, "kd/rkl_advantage_mean": 0.7645862091332674, "kd/rkl_advantage_p95": 4.997155328591664, "kd/rkl_advantage_std": 2.2215456237395603, "kd/ssd_loss": 0.0, "learning_rate": 4.395245509436828e-07, "loss": 0.030115048090616863, "num_tokens": 483104113.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8286111176013946, "rewards/gt_logging_reward/std": 0.37212048371632894, "sampling/importance_sampling_ratio/max": 1.7891842603683472, "sampling/importance_sampling_ratio/mean": 0.993642270565033, "sampling/importance_sampling_ratio/min": 0.4611045718193054, "sampling/sampling_logp_difference/max": 0.36978780627250674, "sampling/sampling_logp_difference/mean": 0.003695269433471064, "step": 14760, "step_time": 7.818187504065766, "student/entropy": 0.11867553809036811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02555555726091067, "completions/max_length": 479.8, "completions/max_terminated_length": 447.8666666666667, "completions/mean_length": 188.87806599934896, "completions/mean_terminated_length": 180.53165588378906, "completions/min_length": 59.333333333333336, "completions/min_terminated_length": 59.333333333333336, "entropy": 0.1261564191430807, "epoch": 0.5616526791478373, "eval/gt_acc_batch": 0.8158333559830984, "frac_reward_zero_std": 1.0, "grad_norm": 0.25424909591674805, "kd/policy_loss": 0.008392402001967033, "kd/rkl_advantage_mean": 0.7963059127486001, "kd/rkl_advantage_p95": 5.059913158416748, "kd/rkl_advantage_std": 2.2320326904455823, "kd/ssd_loss": 0.0, "learning_rate": 4.383852960164052e-07, "loss": 0.03356961409250895, "num_tokens": 484087458.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8158333321412404, "rewards/gt_logging_reward/std": 0.37888095527887344, "sampling/importance_sampling_ratio/max": 1.8959337989489238, "sampling/importance_sampling_ratio/mean": 0.9944297870000204, "sampling/importance_sampling_ratio/min": 0.46082624047994614, "sampling/sampling_logp_difference/max": 0.315626589457194, "sampling/sampling_logp_difference/mean": 0.003889120346866548, "step": 14790, "step_time": 7.841765105200951, "student/entropy": 0.1261564191430807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445998718343, "completions/max_length": 489.3, "completions/max_terminated_length": 458.7, "completions/mean_length": 194.67389831542968, "completions/mean_terminated_length": 186.84115397135417, "completions/min_length": 58.233333333333334, "completions/min_terminated_length": 58.233333333333334, "entropy": 0.1269659071539839, "epoch": 0.5627919340751149, "eval/gt_acc_batch": 0.7727778057257334, "frac_reward_zero_std": 1.0, "grad_norm": 0.27688223123550415, "kd/policy_loss": 0.008319795640030254, "kd/rkl_advantage_mean": 0.8748870355387529, "kd/rkl_advantage_p95": 5.257646427551905, "kd/rkl_advantage_std": 2.2771026412645976, "kd/ssd_loss": 0.0, "learning_rate": 4.3724604108912773e-07, "loss": 0.03327918450037638, "num_tokens": 485093100.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.772777779897054, "rewards/gt_logging_reward/std": 0.4126951058705648, "sampling/importance_sampling_ratio/max": 1.892851503690084, "sampling/importance_sampling_ratio/mean": 0.9920689602692921, "sampling/importance_sampling_ratio/min": 0.42179983158906303, "sampling/sampling_logp_difference/max": 0.37629685203234353, "sampling/sampling_logp_difference/mean": 0.003959286616494259, "step": 14820, "step_time": 8.051334394432585, "student/entropy": 0.1269659071539839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333469927311, "completions/max_length": 486.3333333333333, "completions/max_terminated_length": 452.06666666666666, "completions/mean_length": 184.4652872721354, "completions/mean_terminated_length": 177.5471649169922, "completions/min_length": 55.06666666666667, "completions/min_terminated_length": 55.06666666666667, "entropy": 0.1252980480591456, "epoch": 0.5639311890023925, "eval/gt_acc_batch": 0.800555576880773, "frac_reward_zero_std": 1.0, "grad_norm": 0.2786824405193329, "kd/policy_loss": 0.008398722735000775, "kd/rkl_advantage_mean": 0.7946568898235759, "kd/rkl_advantage_p95": 5.08853451013565, "kd/rkl_advantage_std": 2.2454396029313406, "kd/ssd_loss": 0.0, "learning_rate": 4.3610678616185015e-07, "loss": 0.03359489440917969, "num_tokens": 486068359.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8005555550257365, "rewards/gt_logging_reward/std": 0.39454602003097533, "sampling/importance_sampling_ratio/max": 1.9525071382522583, "sampling/importance_sampling_ratio/mean": 1.0039041618506113, "sampling/importance_sampling_ratio/min": 0.45630282660325366, "sampling/sampling_logp_difference/max": 0.33644049962361655, "sampling/sampling_logp_difference/mean": 0.003926339807609717, "step": 14850, "step_time": 8.106788741063792, "student/entropy": 0.1252980480591456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016111112075547378, "completions/max_length": 467.9, "completions/max_terminated_length": 433.23333333333335, "completions/mean_length": 181.05889841715495, "completions/mean_terminated_length": 175.7219263712565, "completions/min_length": 56.766666666666666, "completions/min_terminated_length": 56.766666666666666, "entropy": 0.1295476840188106, "epoch": 0.56507044392967, "eval/gt_acc_batch": 0.7936111271381379, "frac_reward_zero_std": 1.0, "grad_norm": 0.34005385637283325, "kd/policy_loss": 0.008815631210260714, "kd/rkl_advantage_mean": 0.7409318139272121, "kd/rkl_advantage_p95": 4.880761867761612, "kd/rkl_advantage_std": 2.172141740719477, "kd/ssd_loss": 0.0, "learning_rate": 4.3496753123457257e-07, "loss": 0.035262529055277506, "num_tokens": 487019483.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111152172088, "rewards/gt_logging_reward/std": 0.39561836073795953, "sampling/importance_sampling_ratio/max": 1.8938074707984924, "sampling/importance_sampling_ratio/mean": 0.9934985737005869, "sampling/importance_sampling_ratio/min": 0.4813077827294668, "sampling/sampling_logp_difference/max": 0.3207790434360504, "sampling/sampling_logp_difference/mean": 0.003998855012468994, "step": 14880, "step_time": 7.7092128826329525, "student/entropy": 0.1295476840188106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016111111858238776, "completions/max_length": 474.06666666666666, "completions/max_terminated_length": 428.9, "completions/mean_length": 185.1150095621745, "completions/mean_terminated_length": 179.83917439778645, "completions/min_length": 53.266666666666666, "completions/min_terminated_length": 53.266666666666666, "entropy": 0.12999116114030282, "epoch": 0.5662096988569476, "eval/gt_acc_batch": 0.8072222471237183, "frac_reward_zero_std": 1.0, "grad_norm": 0.348001092672348, "kd/policy_loss": 0.008627593538646276, "kd/rkl_advantage_mean": 0.7754723818351825, "kd/rkl_advantage_p95": 4.995095139741897, "kd/rkl_advantage_std": 2.2205897559722265, "kd/ssd_loss": 0.0, "learning_rate": 4.3382827630729504e-07, "loss": 0.03451037804285685, "num_tokens": 487982817.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222173213959, "rewards/gt_logging_reward/std": 0.3874447226524353, "sampling/importance_sampling_ratio/max": 1.9067360440889993, "sampling/importance_sampling_ratio/mean": 1.0009732445081074, "sampling/importance_sampling_ratio/min": 0.45793266172210373, "sampling/sampling_logp_difference/max": 0.3747342546780904, "sampling/sampling_logp_difference/mean": 0.003987411200068891, "step": 14910, "step_time": 7.796049405565524, "student/entropy": 0.12999116114030282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112367361785, "completions/max_length": 492.2, "completions/max_terminated_length": 447.53333333333336, "completions/mean_length": 188.02945404052736, "completions/mean_terminated_length": 181.1200937906901, "completions/min_length": 50.7, "completions/min_terminated_length": 50.7, "entropy": 0.12577752713114024, "epoch": 0.5673489537842251, "eval/gt_acc_batch": 0.8113889118035634, "frac_reward_zero_std": 1.0, "grad_norm": 0.2444296032190323, "kd/policy_loss": 0.008076839686448995, "kd/rkl_advantage_mean": 0.8560708520313104, "kd/rkl_advantage_p95": 5.201135377089183, "kd/rkl_advantage_std": 2.245233385761579, "kd/ssd_loss": 0.0, "learning_rate": 4.3268902138001745e-07, "loss": 0.032307360569636026, "num_tokens": 488969923.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8113888959089915, "rewards/gt_logging_reward/std": 0.38616430858771006, "sampling/importance_sampling_ratio/max": 1.8590363343556722, "sampling/importance_sampling_ratio/mean": 0.9978529055913289, "sampling/importance_sampling_ratio/min": 0.45586527734994886, "sampling/sampling_logp_difference/max": 0.3754005750020345, "sampling/sampling_logp_difference/mean": 0.0038810459276040397, "step": 14940, "step_time": 8.214343106532276, "student/entropy": 0.12577752713114024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015833334314326444, "completions/max_length": 486.1, "completions/max_terminated_length": 443.9, "completions/mean_length": 183.20112050374348, "completions/mean_terminated_length": 177.9965021769206, "completions/min_length": 56.0, "completions/min_terminated_length": 56.0, "entropy": 0.1367311540370186, "epoch": 0.5684882087115026, "eval/gt_acc_batch": 0.7950000246365865, "frac_reward_zero_std": 1.0, "grad_norm": 0.34911081194877625, "kd/policy_loss": 0.009001181197042267, "kd/rkl_advantage_mean": 0.8047993111113707, "kd/rkl_advantage_p95": 5.087614411115647, "kd/rkl_advantage_std": 2.252368946870168, "kd/ssd_loss": 0.0, "learning_rate": 4.3154976645273987e-07, "loss": 0.03600472609202067, "num_tokens": 489929951.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7950000027815501, "rewards/gt_logging_reward/std": 0.395185653368632, "sampling/importance_sampling_ratio/max": 1.8461347222328186, "sampling/importance_sampling_ratio/mean": 0.9919510165850322, "sampling/importance_sampling_ratio/min": 0.44747891326745354, "sampling/sampling_logp_difference/max": 0.30000892877578733, "sampling/sampling_logp_difference/mean": 0.0041516999481245875, "step": 14970, "step_time": 7.974283711735916, "student/entropy": 0.1367311540370186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334326744078, "completions/max_length": 494.96666666666664, "completions/max_terminated_length": 459.2, "completions/mean_length": 190.02973124186198, "completions/mean_terminated_length": 183.3494140625, "completions/min_length": 53.2, "completions/min_terminated_length": 53.2, "entropy": 0.13136992050955693, "epoch": 0.5696274636387803, "eval/gt_acc_batch": 0.7938889006773631, "frac_reward_zero_std": 1.0, "grad_norm": 0.23492155969142914, "kd/policy_loss": 0.008779669877064104, "kd/rkl_advantage_mean": 0.8364458842823903, "kd/rkl_advantage_p95": 5.101889499028524, "kd/rkl_advantage_std": 2.2145146906375883, "kd/ssd_loss": 0.0, "learning_rate": 4.304105115254623e-07, "loss": 0.03511868715286255, "num_tokens": 490923466.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7938888887564342, "rewards/gt_logging_reward/std": 0.39249543224771816, "sampling/importance_sampling_ratio/max": 1.9053008516629537, "sampling/importance_sampling_ratio/mean": 1.0008631984392802, "sampling/importance_sampling_ratio/min": 0.4551946818828583, "sampling/sampling_logp_difference/max": 0.32081048091252645, "sampling/sampling_logp_difference/mean": 0.003950778356132408, "step": 15000, "step_time": 8.089994428228238, "student/entropy": 0.13136992050955693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01666666759798924, "completions/max_length": 481.96666666666664, "completions/max_terminated_length": 440.8333333333333, "completions/mean_length": 180.27362263997395, "completions/mean_terminated_length": 174.74376424153647, "completions/min_length": 56.4, "completions/min_terminated_length": 56.4, "entropy": 0.12867441729952891, "epoch": 0.5707667185660578, "eval/gt_acc_batch": 0.8041666845480601, "frac_reward_zero_std": 1.0, "grad_norm": 0.2652240991592407, "kd/policy_loss": 0.008298163465224207, "kd/rkl_advantage_mean": 0.7733783020327488, "kd/rkl_advantage_p95": 4.938387550910314, "kd/rkl_advantage_std": 2.206475105881691, "kd/ssd_loss": 0.0, "learning_rate": 4.2927125659818476e-07, "loss": 0.03319265643755595, "num_tokens": 491877467.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666646798452, "rewards/gt_logging_reward/std": 0.3914968465765317, "sampling/importance_sampling_ratio/max": 1.875240687529246, "sampling/importance_sampling_ratio/mean": 0.9921226640542348, "sampling/importance_sampling_ratio/min": 0.47616629501183827, "sampling/sampling_logp_difference/max": 0.32034258047739667, "sampling/sampling_logp_difference/mean": 0.003998813304739694, "step": 15030, "step_time": 7.998703920095674, "student/entropy": 0.12867441729952891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779381722212, "completions/max_length": 477.43333333333334, "completions/max_terminated_length": 446.6, "completions/mean_length": 191.46778717041016, "completions/mean_terminated_length": 182.48332875569662, "completions/min_length": 54.56666666666667, "completions/min_terminated_length": 54.56666666666667, "entropy": 0.13434038845201332, "epoch": 0.5719059734933354, "eval/gt_acc_batch": 0.7869444688161215, "frac_reward_zero_std": 1.0, "grad_norm": 0.2313460409641266, "kd/policy_loss": 0.009210808367546027, "kd/rkl_advantage_mean": 0.8026962720789015, "kd/rkl_advantage_p95": 5.114113074541092, "kd/rkl_advantage_std": 2.2509561757246654, "kd/ssd_loss": 0.0, "learning_rate": 4.281320016709072e-07, "loss": 0.03684323231379191, "num_tokens": 492871039.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444410006206, "rewards/gt_logging_reward/std": 0.3994880899786949, "sampling/importance_sampling_ratio/max": 1.912812642256419, "sampling/importance_sampling_ratio/mean": 1.0034723699092865, "sampling/importance_sampling_ratio/min": 0.4437927484512329, "sampling/sampling_logp_difference/max": 0.32970458070437114, "sampling/sampling_logp_difference/mean": 0.0040983060064415135, "step": 15060, "step_time": 7.941795165268316, "student/entropy": 0.13434038845201332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112336317697, "completions/max_length": 478.46666666666664, "completions/max_terminated_length": 443.7, "completions/mean_length": 179.22528737386068, "completions/mean_terminated_length": 172.2721201578776, "completions/min_length": 59.03333333333333, "completions/min_terminated_length": 59.03333333333333, "entropy": 0.13144735265523194, "epoch": 0.5730452284206129, "eval/gt_acc_batch": 0.8050000170866648, "frac_reward_zero_std": 1.0, "grad_norm": 0.24645313620567322, "kd/policy_loss": 0.008662938146153465, "kd/rkl_advantage_mean": 0.8035478123774131, "kd/rkl_advantage_p95": 5.112564738591512, "kd/rkl_advantage_std": 2.277611975868543, "kd/ssd_loss": 0.0, "learning_rate": 4.2699274674362965e-07, "loss": 0.034651756286621094, "num_tokens": 493822002.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8049999952316285, "rewards/gt_logging_reward/std": 0.38890370925267537, "sampling/importance_sampling_ratio/max": 1.9249722957611084, "sampling/importance_sampling_ratio/mean": 0.9933929920196534, "sampling/importance_sampling_ratio/min": 0.4324158291021983, "sampling/sampling_logp_difference/max": 0.36164778073628745, "sampling/sampling_logp_difference/mean": 0.004098411952145398, "step": 15090, "step_time": 8.00916341660001, "student/entropy": 0.13144735265523194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01972222321977218, "completions/max_length": 485.0, "completions/max_terminated_length": 447.46666666666664, "completions/mean_length": 185.01112111409506, "completions/mean_terminated_length": 178.6758270263672, "completions/min_length": 51.233333333333334, "completions/min_terminated_length": 51.233333333333334, "entropy": 0.12549574362734953, "epoch": 0.5741844833478905, "eval/gt_acc_batch": 0.8094444652398427, "frac_reward_zero_std": 1.0, "grad_norm": 0.28451910614967346, "kd/policy_loss": 0.008159764171189939, "kd/rkl_advantage_mean": 0.7780690651386977, "kd/rkl_advantage_p95": 5.0248590012391405, "kd/rkl_advantage_std": 2.2340725253025693, "kd/ssd_loss": 0.0, "learning_rate": 4.258534918163521e-07, "loss": 0.03263906240463257, "num_tokens": 494786922.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8094444374243418, "rewards/gt_logging_reward/std": 0.3869723528623581, "sampling/importance_sampling_ratio/max": 1.90249977906545, "sampling/importance_sampling_ratio/mean": 0.9972780883312226, "sampling/importance_sampling_ratio/min": 0.449739079674085, "sampling/sampling_logp_difference/max": 0.3390773852666219, "sampling/sampling_logp_difference/mean": 0.003907989881311854, "step": 15120, "step_time": 7.885476231901945, "student/entropy": 0.12549574362734953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445812453825, "completions/max_length": 488.7, "completions/max_terminated_length": 444.23333333333335, "completions/mean_length": 185.9105651855469, "completions/mean_terminated_length": 177.93930002848307, "completions/min_length": 49.666666666666664, "completions/min_terminated_length": 49.666666666666664, "entropy": 0.12933529913425446, "epoch": 0.575323738275168, "eval/gt_acc_batch": 0.8033333460489909, "frac_reward_zero_std": 1.0, "grad_norm": 0.30074068903923035, "kd/policy_loss": 0.008446535023783024, "kd/rkl_advantage_mean": 0.7386902132381995, "kd/rkl_advantage_p95": 4.996989820400874, "kd/rkl_advantage_std": 2.2234066396951677, "kd/ssd_loss": 0.0, "learning_rate": 4.2471423688907454e-07, "loss": 0.03378613789876302, "num_tokens": 495760600.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.803333334128062, "rewards/gt_logging_reward/std": 0.3894980102777481, "sampling/importance_sampling_ratio/max": 1.9617504874865215, "sampling/importance_sampling_ratio/mean": 0.9992002646128336, "sampling/importance_sampling_ratio/min": 0.480793189505736, "sampling/sampling_logp_difference/max": 0.2998013784488042, "sampling/sampling_logp_difference/mean": 0.003962013210790853, "step": 15150, "step_time": 8.034040543599986, "student/entropy": 0.12933529913425446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030833334910372892, "completions/max_length": 488.96666666666664, "completions/max_terminated_length": 448.9, "completions/mean_length": 193.179731241862, "completions/mean_terminated_length": 183.13263346354168, "completions/min_length": 56.233333333333334, "completions/min_terminated_length": 56.233333333333334, "entropy": 0.12690923418849706, "epoch": 0.5764629932024457, "eval/gt_acc_batch": 0.7891666829586029, "frac_reward_zero_std": 1.0, "grad_norm": 0.30078554153442383, "kd/policy_loss": 0.008580896196266015, "kd/rkl_advantage_mean": 0.8313922794846197, "kd/rkl_advantage_p95": 5.104433965682984, "kd/rkl_advantage_std": 2.2482741316159567, "kd/ssd_loss": 0.0, "learning_rate": 4.2357498196179695e-07, "loss": 0.03432358105977376, "num_tokens": 496761615.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7891666670640309, "rewards/gt_logging_reward/std": 0.4009433850646019, "sampling/importance_sampling_ratio/max": 1.950852676232656, "sampling/importance_sampling_ratio/mean": 1.0018661220868428, "sampling/importance_sampling_ratio/min": 0.4379060178995132, "sampling/sampling_logp_difference/max": 0.3031622886657715, "sampling/sampling_logp_difference/mean": 0.003907645051367581, "step": 15180, "step_time": 8.030876906170063, "student/entropy": 0.12690923418849706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223648180564, "completions/max_length": 484.23333333333335, "completions/max_terminated_length": 462.06666666666666, "completions/mean_length": 193.99834340413412, "completions/mean_terminated_length": 184.57589569091797, "completions/min_length": 58.46666666666667, "completions/min_terminated_length": 58.46666666666667, "entropy": 0.12933206483721732, "epoch": 0.5776022481297232, "eval/gt_acc_batch": 0.7861111183961232, "frac_reward_zero_std": 1.0, "grad_norm": 0.30495402216911316, "kd/policy_loss": 0.008787593555947144, "kd/rkl_advantage_mean": 0.7826602914681037, "kd/rkl_advantage_p95": 5.024501981337865, "kd/rkl_advantage_std": 2.2233079512914022, "kd/ssd_loss": 0.0, "learning_rate": 4.224357270345194e-07, "loss": 0.035150369008382164, "num_tokens": 497764161.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111104488373, "rewards/gt_logging_reward/std": 0.3969475731253624, "sampling/importance_sampling_ratio/max": 1.985982648531596, "sampling/importance_sampling_ratio/mean": 1.001385404666265, "sampling/importance_sampling_ratio/min": 0.4640386804938316, "sampling/sampling_logp_difference/max": 0.35490436255931856, "sampling/sampling_logp_difference/mean": 0.00395402725165089, "step": 15210, "step_time": 8.057391296929078, "student/entropy": 0.12933206483721732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334575096767, "completions/max_length": 465.3666666666667, "completions/max_terminated_length": 442.5, "completions/mean_length": 182.86639760335285, "completions/mean_terminated_length": 175.98800710042318, "completions/min_length": 61.1, "completions/min_terminated_length": 61.1, "entropy": 0.12440199851989746, "epoch": 0.5787415030570007, "eval/gt_acc_batch": 0.8213889102141062, "frac_reward_zero_std": 1.0, "grad_norm": 0.2680269479751587, "kd/policy_loss": 0.008077687015368913, "kd/rkl_advantage_mean": 0.8091907025625308, "kd/rkl_advantage_p95": 5.126834505796433, "kd/rkl_advantage_std": 2.2444133937358854, "kd/ssd_loss": 0.0, "learning_rate": 4.2129647210724184e-07, "loss": 0.03231074611345927, "num_tokens": 498731456.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8213888903458914, "rewards/gt_logging_reward/std": 0.3671550065279007, "sampling/importance_sampling_ratio/max": 1.8369230906168619, "sampling/importance_sampling_ratio/mean": 0.9957556307315827, "sampling/importance_sampling_ratio/min": 0.438019202152888, "sampling/sampling_logp_difference/max": 0.3281177262465159, "sampling/sampling_logp_difference/mean": 0.003868820828696092, "step": 15240, "step_time": 8.044955246132547, "student/entropy": 0.12440199851989746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02000000129143397, "completions/max_length": 478.93333333333334, "completions/max_terminated_length": 434.4, "completions/mean_length": 179.7725092569987, "completions/mean_terminated_length": 172.89996236165365, "completions/min_length": 56.43333333333333, "completions/min_terminated_length": 56.43333333333333, "entropy": 0.12350963912904263, "epoch": 0.5798807579842783, "eval/gt_acc_batch": 0.8244444668293, "frac_reward_zero_std": 1.0, "grad_norm": 0.2871145009994507, "kd/policy_loss": 0.007912115469419707, "kd/rkl_advantage_mean": 0.7846104147533576, "kd/rkl_advantage_p95": 5.068783607085546, "kd/rkl_advantage_std": 2.2352878113587695, "kd/ssd_loss": 0.0, "learning_rate": 4.2015721717996426e-07, "loss": 0.03164846102396647, "num_tokens": 499672925.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.824444442987442, "rewards/gt_logging_reward/std": 0.3752590358257294, "sampling/importance_sampling_ratio/max": 1.8502008438110351, "sampling/importance_sampling_ratio/mean": 1.0034859577814739, "sampling/importance_sampling_ratio/min": 0.49271436780691147, "sampling/sampling_logp_difference/max": 0.3523974299430847, "sampling/sampling_logp_difference/mean": 0.003831335878930986, "step": 15270, "step_time": 7.825866373730241, "student/entropy": 0.12350963912904263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02472222354263067, "completions/max_length": 489.3666666666667, "completions/max_terminated_length": 461.0, "completions/mean_length": 190.0438995361328, "completions/mean_terminated_length": 181.97462717692056, "completions/min_length": 47.766666666666666, "completions/min_terminated_length": 47.766666666666666, "entropy": 0.12655193104098242, "epoch": 0.5810200129115558, "eval/gt_acc_batch": 0.808611132701238, "frac_reward_zero_std": 1.0, "grad_norm": 0.2244393527507782, "kd/policy_loss": 0.00828826660872437, "kd/rkl_advantage_mean": 0.7538203207155069, "kd/rkl_advantage_p95": 5.002336754401525, "kd/rkl_advantage_std": 2.243658263484637, "kd/ssd_loss": 0.0, "learning_rate": 4.1901796225268673e-07, "loss": 0.03315306504567464, "num_tokens": 500664635.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8086111068725585, "rewards/gt_logging_reward/std": 0.3880385140577952, "sampling/importance_sampling_ratio/max": 1.8010735829671225, "sampling/importance_sampling_ratio/mean": 0.9997518122196197, "sampling/importance_sampling_ratio/min": 0.46314718425273893, "sampling/sampling_logp_difference/max": 0.3276507019996643, "sampling/sampling_logp_difference/mean": 0.0038854270707815884, "step": 15300, "step_time": 8.062427929797074, "student/entropy": 0.12655193104098242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334643393756, "completions/max_length": 476.4, "completions/max_terminated_length": 446.53333333333336, "completions/mean_length": 192.21389821370443, "completions/mean_terminated_length": 184.6425989786784, "completions/min_length": 57.56666666666667, "completions/min_terminated_length": 57.56666666666667, "entropy": 0.13125848242392144, "epoch": 0.5821592678388334, "eval/gt_acc_batch": 0.7833333591620127, "frac_reward_zero_std": 1.0, "grad_norm": 0.3745999038219452, "kd/policy_loss": 0.008662977618708585, "kd/rkl_advantage_mean": 0.7418439670155446, "kd/rkl_advantage_p95": 4.9431480666001635, "kd/rkl_advantage_std": 2.208835549155871, "kd/ssd_loss": 0.0, "learning_rate": 4.1787870732540915e-07, "loss": 0.03465191125869751, "num_tokens": 501654293.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7833333353201548, "rewards/gt_logging_reward/std": 0.4051589399576187, "sampling/importance_sampling_ratio/max": 1.931609364350637, "sampling/importance_sampling_ratio/mean": 0.996396827697754, "sampling/importance_sampling_ratio/min": 0.3903307015697161, "sampling/sampling_logp_difference/max": 0.36801263292630515, "sampling/sampling_logp_difference/mean": 0.004023080590801934, "step": 15330, "step_time": 8.047686284702893, "student/entropy": 0.13125848242392144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666785875956, "completions/max_length": 466.93333333333334, "completions/max_terminated_length": 429.6333333333333, "completions/mean_length": 188.75973358154297, "completions/mean_terminated_length": 181.6042037963867, "completions/min_length": 52.86666666666667, "completions/min_terminated_length": 52.86666666666667, "entropy": 0.12459179020176331, "epoch": 0.5832985227661109, "eval/gt_acc_batch": 0.8211111227671305, "frac_reward_zero_std": 1.0, "grad_norm": 0.253097802400589, "kd/policy_loss": 0.007671497717577343, "kd/rkl_advantage_mean": 0.6565991579865416, "kd/rkl_advantage_p95": 4.736712998151779, "kd/rkl_advantage_std": 2.145535032947858, "kd/ssd_loss": 0.0, "learning_rate": 4.167394523981316e-07, "loss": 0.03068599502245585, "num_tokens": 502636588.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8211111128330231, "rewards/gt_logging_reward/std": 0.367040874560674, "sampling/importance_sampling_ratio/max": 1.8182965159416198, "sampling/importance_sampling_ratio/mean": 0.9915614604949952, "sampling/importance_sampling_ratio/min": 0.44235172470410666, "sampling/sampling_logp_difference/max": 0.3187384198109309, "sampling/sampling_logp_difference/mean": 0.003881465387530625, "step": 15360, "step_time": 7.955239400363643, "student/entropy": 0.12459179020176331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333463718494, "completions/max_length": 491.03333333333336, "completions/max_terminated_length": 460.03333333333336, "completions/mean_length": 191.59417724609375, "completions/mean_terminated_length": 184.81543579101563, "completions/min_length": 52.833333333333336, "completions/min_terminated_length": 52.833333333333336, "entropy": 0.12691999059170483, "epoch": 0.5844377776933886, "eval/gt_acc_batch": 0.8077778001626332, "frac_reward_zero_std": 1.0, "grad_norm": 0.2624358534812927, "kd/policy_loss": 0.008400994296728944, "kd/rkl_advantage_mean": 0.7483067738202711, "kd/rkl_advantage_p95": 4.938881788651148, "kd/rkl_advantage_std": 2.18638753592968, "kd/ssd_loss": 0.0, "learning_rate": 4.1560019747085404e-07, "loss": 0.033603978157043454, "num_tokens": 503640695.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8077777783075969, "rewards/gt_logging_reward/std": 0.38772207001845044, "sampling/importance_sampling_ratio/max": 1.944317336877187, "sampling/importance_sampling_ratio/mean": 1.0001565019289653, "sampling/importance_sampling_ratio/min": 0.46641191144784294, "sampling/sampling_logp_difference/max": 0.3111032327016195, "sampling/sampling_logp_difference/mean": 0.0039324759893740215, "step": 15390, "step_time": 8.398528395097431, "student/entropy": 0.12691999059170483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001167257628, "completions/max_length": 480.6666666666667, "completions/max_terminated_length": 446.96666666666664, "completions/mean_length": 191.0069549560547, "completions/mean_terminated_length": 184.68834228515624, "completions/min_length": 58.166666666666664, "completions/min_terminated_length": 58.166666666666664, "entropy": 0.12796044796705247, "epoch": 0.5855770326206661, "eval/gt_acc_batch": 0.7891666928927104, "frac_reward_zero_std": 1.0, "grad_norm": 0.3009231984615326, "kd/policy_loss": 0.008347317232983186, "kd/rkl_advantage_mean": 0.7612043930683285, "kd/rkl_advantage_p95": 4.9628836691379545, "kd/rkl_advantage_std": 2.187847074866295, "kd/ssd_loss": 0.0, "learning_rate": 4.144609425435765e-07, "loss": 0.03338926235834758, "num_tokens": 504635168.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7891666670640309, "rewards/gt_logging_reward/std": 0.3944397394855817, "sampling/importance_sampling_ratio/max": 1.9309091250101724, "sampling/importance_sampling_ratio/mean": 0.9986812750498454, "sampling/importance_sampling_ratio/min": 0.4757702867190043, "sampling/sampling_logp_difference/max": 0.35686868826548257, "sampling/sampling_logp_difference/mean": 0.003986748921064039, "step": 15420, "step_time": 8.03136003853336, "student/entropy": 0.12796044796705247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027500001372148593, "completions/max_length": 489.3333333333333, "completions/max_terminated_length": 456.3333333333333, "completions/mean_length": 191.18417714436848, "completions/mean_terminated_length": 182.25890401204427, "completions/min_length": 55.9, "completions/min_terminated_length": 55.9, "entropy": 0.12386733833700418, "epoch": 0.5867162875479437, "eval/gt_acc_batch": 0.8175000170866649, "frac_reward_zero_std": 1.0, "grad_norm": 0.26517990231513977, "kd/policy_loss": 0.008066700918910404, "kd/rkl_advantage_mean": 0.796012317823867, "kd/rkl_advantage_p95": 5.085466355085373, "kd/rkl_advantage_std": 2.2516742517550785, "kd/ssd_loss": 0.0, "learning_rate": 4.133216876162989e-07, "loss": 0.03226680358250936, "num_tokens": 505621887.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8174999992052714, "rewards/gt_logging_reward/std": 0.3766180311640104, "sampling/importance_sampling_ratio/max": 1.930800437927246, "sampling/importance_sampling_ratio/mean": 1.0057152430216472, "sampling/importance_sampling_ratio/min": 0.4831873516241709, "sampling/sampling_logp_difference/max": 0.3149143815040588, "sampling/sampling_logp_difference/mean": 0.003839653815763692, "step": 15450, "step_time": 7.91720277016672, "student/entropy": 0.12386733833700418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112559835116, "completions/max_length": 472.3, "completions/max_terminated_length": 440.23333333333335, "completions/mean_length": 190.92889862060548, "completions/mean_terminated_length": 183.36648457845052, "completions/min_length": 54.766666666666666, "completions/min_terminated_length": 54.766666666666666, "entropy": 0.12998690145711103, "epoch": 0.5878555424752212, "eval/gt_acc_batch": 0.8069444576899211, "frac_reward_zero_std": 1.0, "grad_norm": 0.24584195017814636, "kd/policy_loss": 0.008579699695110321, "kd/rkl_advantage_mean": 0.729842017699654, "kd/rkl_advantage_p95": 5.007106119394303, "kd/rkl_advantage_std": 2.2357634802659354, "kd/ssd_loss": 0.0, "learning_rate": 4.1218243268902134e-07, "loss": 0.034318796793619794, "num_tokens": 506610655.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8069444477558136, "rewards/gt_logging_reward/std": 0.38022476732730864, "sampling/importance_sampling_ratio/max": 1.953475566705068, "sampling/importance_sampling_ratio/mean": 0.9904627660910289, "sampling/importance_sampling_ratio/min": 0.42225536455710727, "sampling/sampling_logp_difference/max": 0.3137604753176371, "sampling/sampling_logp_difference/mean": 0.004027486002693574, "step": 15480, "step_time": 7.956252208000903, "student/entropy": 0.12998690145711103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334513008596, "completions/max_length": 469.96666666666664, "completions/max_terminated_length": 433.73333333333335, "completions/mean_length": 185.16695454915364, "completions/mean_terminated_length": 178.34729614257813, "completions/min_length": 53.03333333333333, "completions/min_terminated_length": 53.03333333333333, "entropy": 0.12055580237259468, "epoch": 0.5889947974024987, "eval/gt_acc_batch": 0.8341666897137959, "frac_reward_zero_std": 1.0, "grad_norm": 0.24239039421081543, "kd/policy_loss": 0.007397564236695567, "kd/rkl_advantage_mean": 0.7565778630475203, "kd/rkl_advantage_p95": 5.046579605340957, "kd/rkl_advantage_std": 2.2380097011725106, "kd/ssd_loss": 0.0, "learning_rate": 4.110431777617438e-07, "loss": 0.029590253035227457, "num_tokens": 507588888.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8341666658719381, "rewards/gt_logging_reward/std": 0.3620256811380386, "sampling/importance_sampling_ratio/max": 1.795862909158071, "sampling/importance_sampling_ratio/mean": 0.9910203615824381, "sampling/importance_sampling_ratio/min": 0.45204316278298695, "sampling/sampling_logp_difference/max": 0.3272618462642034, "sampling/sampling_logp_difference/mean": 0.0037779625893260044, "step": 15510, "step_time": 7.996220321964938, "student/entropy": 0.12055580237259468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01944444567585985, "completions/max_length": 499.3, "completions/max_terminated_length": 460.43333333333334, "completions/mean_length": 189.09778798421223, "completions/mean_terminated_length": 182.6711908976237, "completions/min_length": 60.1, "completions/min_terminated_length": 60.1, "entropy": 0.12537546828389168, "epoch": 0.5901340523297763, "eval/gt_acc_batch": 0.8022222340106964, "frac_reward_zero_std": 1.0, "grad_norm": 0.36262214183807373, "kd/policy_loss": 0.008325079942975815, "kd/rkl_advantage_mean": 0.74627833639582, "kd/rkl_advantage_p95": 5.019305223226548, "kd/rkl_advantage_std": 2.2426316420237224, "kd/ssd_loss": 0.0, "learning_rate": 4.0990392283446623e-07, "loss": 0.03330031633377075, "num_tokens": 508576016.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8022222240765889, "rewards/gt_logging_reward/std": 0.3892242779334386, "sampling/importance_sampling_ratio/max": 1.85850590467453, "sampling/importance_sampling_ratio/mean": 1.003237009048462, "sampling/importance_sampling_ratio/min": 0.44135389626026156, "sampling/sampling_logp_difference/max": 0.3497146129608154, "sampling/sampling_logp_difference/mean": 0.0038993463230629763, "step": 15540, "step_time": 8.130781616765793, "student/entropy": 0.12537546828389168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03138889071221153, "completions/max_length": 495.1, "completions/max_terminated_length": 452.6666666666667, "completions/mean_length": 190.94417724609374, "completions/mean_terminated_length": 180.7211471557617, "completions/min_length": 54.666666666666664, "completions/min_terminated_length": 54.666666666666664, "entropy": 0.1305789766833186, "epoch": 0.5912733072570538, "eval/gt_acc_batch": 0.8094444652398427, "frac_reward_zero_std": 1.0, "grad_norm": 0.29261520504951477, "kd/policy_loss": 0.008603603423883518, "kd/rkl_advantage_mean": 0.8360791131854057, "kd/rkl_advantage_p95": 5.110711552699407, "kd/rkl_advantage_std": 2.2246329774459204, "kd/ssd_loss": 0.0, "learning_rate": 4.0876466790718865e-07, "loss": 0.03441441456476847, "num_tokens": 509566591.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8094444493452708, "rewards/gt_logging_reward/std": 0.38296848237514497, "sampling/importance_sampling_ratio/max": 1.888165549437205, "sampling/importance_sampling_ratio/mean": 0.9983424703280132, "sampling/importance_sampling_ratio/min": 0.44676081587870914, "sampling/sampling_logp_difference/max": 0.33730498552322385, "sampling/sampling_logp_difference/mean": 0.004016338824294507, "step": 15570, "step_time": 8.13543878403434, "student/entropy": 0.1305789766833186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029166668187826872, "completions/max_length": 494.6666666666667, "completions/max_terminated_length": 455.8666666666667, "completions/mean_length": 194.7055658976237, "completions/mean_terminated_length": 185.2229777018229, "completions/min_length": 48.9, "completions/min_terminated_length": 48.9, "entropy": 0.12321169320493937, "epoch": 0.5924125621843315, "eval/gt_acc_batch": 0.805277802546819, "frac_reward_zero_std": 1.0, "grad_norm": 0.20882534980773926, "kd/policy_loss": 0.007715173898031935, "kd/rkl_advantage_mean": 0.7663390563024829, "kd/rkl_advantage_p95": 5.0452071845531465, "kd/rkl_advantage_std": 2.225790709257126, "kd/ssd_loss": 0.0, "learning_rate": 4.076254129799111e-07, "loss": 0.030860700209935508, "num_tokens": 510572163.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8052777767181396, "rewards/gt_logging_reward/std": 0.3879568725824356, "sampling/importance_sampling_ratio/max": 1.8268786350886026, "sampling/importance_sampling_ratio/mean": 0.9874536097049713, "sampling/importance_sampling_ratio/min": 0.43498461643854774, "sampling/sampling_logp_difference/max": 0.3354682286580404, "sampling/sampling_logp_difference/mean": 0.0038333692975963157, "step": 15600, "step_time": 8.105524016135314, "student/entropy": 0.12321169320493937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01805555655931433, "completions/max_length": 467.43333333333334, "completions/max_terminated_length": 439.53333333333336, "completions/mean_length": 191.411398824056, "completions/mean_terminated_length": 185.55286153157553, "completions/min_length": 51.03333333333333, "completions/min_terminated_length": 51.03333333333333, "entropy": 0.127490694137911, "epoch": 0.593551817111609, "eval/gt_acc_batch": 0.7911111354827881, "frac_reward_zero_std": 1.0, "grad_norm": 0.2831970453262329, "kd/policy_loss": 0.008454525129248698, "kd/rkl_advantage_mean": 0.6735275062693593, "kd/rkl_advantage_p95": 4.846984126170477, "kd/rkl_advantage_std": 2.181845055023829, "kd/ssd_loss": 0.0, "learning_rate": 4.0648615805263354e-07, "loss": 0.033818097909291585, "num_tokens": 511572860.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7911111116409302, "rewards/gt_logging_reward/std": 0.39693643947442375, "sampling/importance_sampling_ratio/max": 1.9487972696622213, "sampling/importance_sampling_ratio/mean": 0.998158582051595, "sampling/importance_sampling_ratio/min": 0.4370227644840876, "sampling/sampling_logp_difference/max": 0.3262038350105286, "sampling/sampling_logp_difference/mean": 0.003971382509917021, "step": 15630, "step_time": 8.241968348264345, "student/entropy": 0.127490694137911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02277777884155512, "completions/max_length": 492.4, "completions/max_terminated_length": 467.1333333333333, "completions/mean_length": 186.03417561848957, "completions/mean_terminated_length": 178.43619842529296, "completions/min_length": 54.7, "completions/min_terminated_length": 54.7, "entropy": 0.12770710003872712, "epoch": 0.5946910720388866, "eval/gt_acc_batch": 0.8100000242392222, "frac_reward_zero_std": 1.0, "grad_norm": 0.26338621973991394, "kd/policy_loss": 0.008393448534964895, "kd/rkl_advantage_mean": 0.7845314490298431, "kd/rkl_advantage_p95": 5.015384034315745, "kd/rkl_advantage_std": 2.2345471580823264, "kd/ssd_loss": 0.0, "learning_rate": 4.05346903125356e-07, "loss": 0.03357380231221517, "num_tokens": 512543567.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8099999964237213, "rewards/gt_logging_reward/std": 0.38509528438250223, "sampling/importance_sampling_ratio/max": 1.941155727704366, "sampling/importance_sampling_ratio/mean": 0.9965835154056549, "sampling/importance_sampling_ratio/min": 0.47325592637062075, "sampling/sampling_logp_difference/max": 0.37603965600331624, "sampling/sampling_logp_difference/mean": 0.004009203240275383, "step": 15660, "step_time": 8.081632180299009, "student/entropy": 0.12770710003872712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0269444459117949, "completions/max_length": 485.1333333333333, "completions/max_terminated_length": 450.6333333333333, "completions/mean_length": 185.64778747558594, "completions/mean_terminated_length": 176.81735331217448, "completions/min_length": 55.333333333333336, "completions/min_terminated_length": 55.333333333333336, "entropy": 0.1290205366909504, "epoch": 0.5958303269661641, "eval/gt_acc_batch": 0.7994444568951925, "frac_reward_zero_std": 1.0, "grad_norm": 0.3199511766433716, "kd/policy_loss": 0.008379186360010256, "kd/rkl_advantage_mean": 0.7713562530775865, "kd/rkl_advantage_p95": 4.987062883377075, "kd/rkl_advantage_std": 2.2103486488262813, "kd/ssd_loss": 0.0, "learning_rate": 4.042076481980785e-07, "loss": 0.03351674874623616, "num_tokens": 513517003.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.799444442987442, "rewards/gt_logging_reward/std": 0.39264953633149463, "sampling/importance_sampling_ratio/max": 1.951776643594106, "sampling/importance_sampling_ratio/mean": 0.9988098621368409, "sampling/importance_sampling_ratio/min": 0.44740179081757864, "sampling/sampling_logp_difference/max": 0.31686970094839734, "sampling/sampling_logp_difference/mean": 0.0039893674508978925, "step": 15690, "step_time": 8.166299549134177, "student/entropy": 0.1290205366909504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334513008596, "completions/max_length": 479.73333333333335, "completions/max_terminated_length": 446.53333333333336, "completions/mean_length": 187.210009765625, "completions/mean_terminated_length": 180.52513682047527, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.12592077398051818, "epoch": 0.5969695818934417, "eval/gt_acc_batch": 0.8113888998826345, "frac_reward_zero_std": 1.0, "grad_norm": 0.26746585965156555, "kd/policy_loss": 0.008152361256846537, "kd/rkl_advantage_mean": 0.6917195445547502, "kd/rkl_advantage_p95": 4.850823251406352, "kd/rkl_advantage_std": 2.171912948290507, "kd/ssd_loss": 0.0, "learning_rate": 4.030683932708009e-07, "loss": 0.032609446843465166, "num_tokens": 514492239.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.811388889948527, "rewards/gt_logging_reward/std": 0.3829143851995468, "sampling/importance_sampling_ratio/max": 1.796720310052236, "sampling/importance_sampling_ratio/mean": 0.996383798122406, "sampling/importance_sampling_ratio/min": 0.46840039392312366, "sampling/sampling_logp_difference/max": 0.31974584857622784, "sampling/sampling_logp_difference/mean": 0.003891581227071583, "step": 15720, "step_time": 8.05172739102806, "student/entropy": 0.12592077398051818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333454405268, "completions/max_length": 481.5, "completions/max_terminated_length": 460.76666666666665, "completions/mean_length": 192.13250834147135, "completions/mean_terminated_length": 185.6835723876953, "completions/min_length": 48.766666666666666, "completions/min_terminated_length": 48.766666666666666, "entropy": 0.12436006100227436, "epoch": 0.5981088368207192, "eval/gt_acc_batch": 0.8022222379843394, "frac_reward_zero_std": 1.0, "grad_norm": 0.3053729236125946, "kd/policy_loss": 0.008275221484169985, "kd/rkl_advantage_mean": 0.6893599538365379, "kd/rkl_advantage_p95": 4.863330550988516, "kd/rkl_advantage_std": 2.172999294598897, "kd/ssd_loss": 0.0, "learning_rate": 4.019291383435233e-07, "loss": 0.03310088713963826, "num_tokens": 515492092.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8022222240765889, "rewards/gt_logging_reward/std": 0.3899690881371498, "sampling/importance_sampling_ratio/max": 2.0232393662134807, "sampling/importance_sampling_ratio/mean": 0.9972156465053559, "sampling/importance_sampling_ratio/min": 0.4345337028304736, "sampling/sampling_logp_difference/max": 0.374733978509903, "sampling/sampling_logp_difference/mean": 0.003858750166061024, "step": 15750, "step_time": 8.085787655336025, "student/entropy": 0.12436006100227436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02472222385307153, "completions/max_length": 482.43333333333334, "completions/max_terminated_length": 457.5, "completions/mean_length": 189.0444534301758, "completions/mean_terminated_length": 181.01937967936198, "completions/min_length": 58.2, "completions/min_terminated_length": 58.2, "entropy": 0.12686703130602836, "epoch": 0.5992480917479968, "eval/gt_acc_batch": 0.8077778041362762, "frac_reward_zero_std": 1.0, "grad_norm": 0.35016950964927673, "kd/policy_loss": 0.008368013927247375, "kd/rkl_advantage_mean": 0.7663965344429016, "kd/rkl_advantage_p95": 5.037122746308644, "kd/rkl_advantage_std": 2.2239587903022766, "kd/ssd_loss": 0.0, "learning_rate": 4.0078988341624573e-07, "loss": 0.03347205718358358, "num_tokens": 516471764.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8077777783075969, "rewards/gt_logging_reward/std": 0.3825808276732763, "sampling/importance_sampling_ratio/max": 1.8313717563947043, "sampling/importance_sampling_ratio/mean": 1.0008034884929657, "sampling/importance_sampling_ratio/min": 0.4530256375670433, "sampling/sampling_logp_difference/max": 0.3226268231868744, "sampling/sampling_logp_difference/mean": 0.003932269406504929, "step": 15780, "step_time": 7.904400261035092, "student/entropy": 0.12686703130602836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017222223151475192, "completions/max_length": 468.1666666666667, "completions/max_terminated_length": 422.96666666666664, "completions/mean_length": 182.78250935872396, "completions/mean_terminated_length": 177.13031209309895, "completions/min_length": 57.93333333333333, "completions/min_terminated_length": 57.93333333333333, "entropy": 0.12802215646952392, "epoch": 0.6003873466752744, "eval/gt_acc_batch": 0.8119444529215495, "frac_reward_zero_std": 1.0, "grad_norm": 0.31542521715164185, "kd/policy_loss": 0.00829076028506582, "kd/rkl_advantage_mean": 0.7512514657651385, "kd/rkl_advantage_p95": 4.8951058705647785, "kd/rkl_advantage_std": 2.1905432174603146, "kd/ssd_loss": 0.0, "learning_rate": 3.996506284889682e-07, "loss": 0.033163044850031534, "num_tokens": 517437629.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8119444429874421, "rewards/gt_logging_reward/std": 0.3818974624077479, "sampling/importance_sampling_ratio/max": 2.0094470143318177, "sampling/importance_sampling_ratio/mean": 1.0012769639492034, "sampling/importance_sampling_ratio/min": 0.49053241809209186, "sampling/sampling_logp_difference/max": 0.2979117691516876, "sampling/sampling_logp_difference/mean": 0.003966797220831116, "step": 15810, "step_time": 8.063624005065746, "student/entropy": 0.12802215646952392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112652967375, "completions/max_length": 474.73333333333335, "completions/max_terminated_length": 436.1, "completions/mean_length": 184.00000915527343, "completions/mean_terminated_length": 176.1875030517578, "completions/min_length": 58.36666666666667, "completions/min_terminated_length": 58.36666666666667, "entropy": 0.12783814184367656, "epoch": 0.6015266016025519, "eval/gt_acc_batch": 0.7919444640477499, "frac_reward_zero_std": 1.0, "grad_norm": 0.30936697125434875, "kd/policy_loss": 0.008336231739182647, "kd/rkl_advantage_mean": 0.7829390617087484, "kd/rkl_advantage_p95": 4.987817718585332, "kd/rkl_advantage_std": 2.1884882022937138, "kd/ssd_loss": 0.0, "learning_rate": 3.985113735616906e-07, "loss": 0.033344924449920654, "num_tokens": 518395061.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7919444481531779, "rewards/gt_logging_reward/std": 0.3997142573197683, "sampling/importance_sampling_ratio/max": 1.8273363908131917, "sampling/importance_sampling_ratio/mean": 0.9983710944652557, "sampling/importance_sampling_ratio/min": 0.481461164355278, "sampling/sampling_logp_difference/max": 0.30986491044362385, "sampling/sampling_logp_difference/mean": 0.0039452652679756285, "step": 15840, "step_time": 7.885465655365261, "student/entropy": 0.12783814184367656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112429449956, "completions/max_length": 481.56666666666666, "completions/max_terminated_length": 451.06666666666666, "completions/mean_length": 185.53750966389973, "completions/mean_terminated_length": 178.69170989990235, "completions/min_length": 50.56666666666667, "completions/min_terminated_length": 50.56666666666667, "entropy": 0.13005870059132577, "epoch": 0.6026658565298295, "eval/gt_acc_batch": 0.7866666873296102, "frac_reward_zero_std": 1.0, "grad_norm": 0.30338582396507263, "kd/policy_loss": 0.008863912901142612, "kd/rkl_advantage_mean": 0.7497934848070145, "kd/rkl_advantage_p95": 4.919593036174774, "kd/rkl_advantage_std": 2.184302517771721, "kd/ssd_loss": 0.0, "learning_rate": 3.9737211863441304e-07, "loss": 0.035455652077992755, "num_tokens": 519364340.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7866666674613952, "rewards/gt_logging_reward/std": 0.4025598704814911, "sampling/importance_sampling_ratio/max": 1.9306650360425313, "sampling/importance_sampling_ratio/mean": 0.9984888354937236, "sampling/importance_sampling_ratio/min": 0.4354955484469732, "sampling/sampling_logp_difference/max": 0.389822518825531, "sampling/sampling_logp_difference/mean": 0.0040501301952948175, "step": 15870, "step_time": 7.955905679598315, "student/entropy": 0.13005870059132577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333463718494, "completions/max_length": 485.46666666666664, "completions/max_terminated_length": 457.3333333333333, "completions/mean_length": 185.29973297119142, "completions/mean_terminated_length": 178.4042236328125, "completions/min_length": 59.43333333333333, "completions/min_terminated_length": 59.43333333333333, "entropy": 0.12818814075241486, "epoch": 0.603805111457107, "eval/gt_acc_batch": 0.8102777977784474, "frac_reward_zero_std": 1.0, "grad_norm": 0.3346809446811676, "kd/policy_loss": 0.008494082741284123, "kd/rkl_advantage_mean": 0.7604979100326698, "kd/rkl_advantage_p95": 4.951605884234111, "kd/rkl_advantage_std": 2.2038119047880174, "kd/ssd_loss": 0.0, "learning_rate": 3.962328637071355e-07, "loss": 0.03397633234659831, "num_tokens": 520343227.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8102777779102326, "rewards/gt_logging_reward/std": 0.386552095413208, "sampling/importance_sampling_ratio/max": 1.963563652833303, "sampling/importance_sampling_ratio/mean": 1.0053002417087555, "sampling/importance_sampling_ratio/min": 0.48579760591189064, "sampling/sampling_logp_difference/max": 0.30724371671676637, "sampling/sampling_logp_difference/mean": 0.0039039899439861376, "step": 15900, "step_time": 8.173177616965647, "student/entropy": 0.12818814075241486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01833333447575569, "completions/max_length": 483.93333333333334, "completions/max_terminated_length": 447.96666666666664, "completions/mean_length": 187.15750885009766, "completions/mean_terminated_length": 181.23476867675782, "completions/min_length": 59.833333333333336, "completions/min_terminated_length": 59.833333333333336, "entropy": 0.12395424172282218, "epoch": 0.6049443663843846, "eval/gt_acc_batch": 0.8111111362775166, "frac_reward_zero_std": 1.0, "grad_norm": 0.2872943878173828, "kd/policy_loss": 0.00802390878670849, "kd/rkl_advantage_mean": 0.7609133440225074, "kd/rkl_advantage_p95": 4.943732400735219, "kd/rkl_advantage_std": 2.190583861867587, "kd/ssd_loss": 0.0, "learning_rate": 3.95093608779858e-07, "loss": 0.03209563891092936, "num_tokens": 521325554.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8111111144224803, "rewards/gt_logging_reward/std": 0.3843908444046974, "sampling/importance_sampling_ratio/max": 1.9063910365104675, "sampling/importance_sampling_ratio/mean": 1.0015631159146627, "sampling/importance_sampling_ratio/min": 0.43512921035289764, "sampling/sampling_logp_difference/max": 0.320579868555069, "sampling/sampling_logp_difference/mean": 0.0038413260597735645, "step": 15930, "step_time": 8.073906587867532, "student/entropy": 0.12395424172282218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001365939776, "completions/max_length": 491.4, "completions/max_terminated_length": 456.8333333333333, "completions/mean_length": 196.49417521158855, "completions/mean_terminated_length": 188.44939066569012, "completions/min_length": 59.5, "completions/min_terminated_length": 59.5, "entropy": 0.12766238159189622, "epoch": 0.6060836213116622, "eval/gt_acc_batch": 0.7902777949968974, "frac_reward_zero_std": 1.0, "grad_norm": 0.256554514169693, "kd/policy_loss": 0.008613540757990753, "kd/rkl_advantage_mean": 0.7952773775905371, "kd/rkl_advantage_p95": 4.9991108993689215, "kd/rkl_advantage_std": 2.1992622951666516, "kd/ssd_loss": 0.0, "learning_rate": 3.939543538525804e-07, "loss": 0.03445416291554769, "num_tokens": 522338325.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.790277777115504, "rewards/gt_logging_reward/std": 0.40132998923460644, "sampling/importance_sampling_ratio/max": 1.815331264336904, "sampling/importance_sampling_ratio/mean": 0.996914803981781, "sampling/importance_sampling_ratio/min": 0.4390860259532928, "sampling/sampling_logp_difference/max": 0.3148394902547201, "sampling/sampling_logp_difference/mean": 0.003923622728325427, "step": 15960, "step_time": 8.230860503365694, "student/entropy": 0.12766238159189622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668175409238, "completions/max_length": 502.1, "completions/max_terminated_length": 465.06666666666666, "completions/mean_length": 187.34223327636718, "completions/mean_terminated_length": 179.3977081298828, "completions/min_length": 42.833333333333336, "completions/min_terminated_length": 42.833333333333336, "entropy": 0.1288131458684802, "epoch": 0.6072228762389398, "eval/gt_acc_batch": 0.7850000103314717, "frac_reward_zero_std": 1.0, "grad_norm": 0.23197075724601746, "kd/policy_loss": 0.008798420120729133, "kd/rkl_advantage_mean": 0.8196119910726944, "kd/rkl_advantage_p95": 5.166665718952815, "kd/rkl_advantage_std": 2.2843427121639253, "kd/ssd_loss": 0.0, "learning_rate": 3.9281509892530287e-07, "loss": 0.03519367774327596, "num_tokens": 523324885.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7849999984105428, "rewards/gt_logging_reward/std": 0.400286465883255, "sampling/importance_sampling_ratio/max": 1.8532754103342692, "sampling/importance_sampling_ratio/mean": 1.000154552857081, "sampling/importance_sampling_ratio/min": 0.4585703447461128, "sampling/sampling_logp_difference/max": 0.30954309900601706, "sampling/sampling_logp_difference/mean": 0.003955919916431109, "step": 15990, "step_time": 8.165640828472776, "student/entropy": 0.1288131458684802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223865489165, "completions/max_length": 483.06666666666666, "completions/max_terminated_length": 453.6, "completions/mean_length": 193.79167633056642, "completions/mean_terminated_length": 184.14105224609375, "completions/min_length": 60.266666666666666, "completions/min_terminated_length": 60.266666666666666, "entropy": 0.13027630367626747, "epoch": 0.6083621311662173, "eval/gt_acc_batch": 0.7891666889190674, "frac_reward_zero_std": 1.0, "grad_norm": 0.3508525788784027, "kd/policy_loss": 0.008583303424529731, "kd/rkl_advantage_mean": 0.7743471215168635, "kd/rkl_advantage_p95": 5.116559010744095, "kd/rkl_advantage_std": 2.2726360807816186, "kd/ssd_loss": 0.0, "learning_rate": 3.916758439980253e-07, "loss": 0.03433321317036946, "num_tokens": 524323783.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7891666690508524, "rewards/gt_logging_reward/std": 0.39616707861423495, "sampling/importance_sampling_ratio/max": 1.9412779728571574, "sampling/importance_sampling_ratio/mean": 0.9972905298074086, "sampling/importance_sampling_ratio/min": 0.4679200957218806, "sampling/sampling_logp_difference/max": 0.38267941276232403, "sampling/sampling_logp_difference/mean": 0.0039945051229248445, "step": 16020, "step_time": 8.05287980180389, "student/entropy": 0.13027630367626747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556944260994, "completions/max_length": 503.43333333333334, "completions/max_terminated_length": 456.43333333333334, "completions/mean_length": 188.30806528727214, "completions/mean_terminated_length": 180.6663553873698, "completions/min_length": 57.1, "completions/min_terminated_length": 57.1, "entropy": 0.1338476326316595, "epoch": 0.6095013860934948, "eval/gt_acc_batch": 0.7930555840333303, "frac_reward_zero_std": 1.0, "grad_norm": 0.4220879375934601, "kd/policy_loss": 0.008635141882890215, "kd/rkl_advantage_mean": 0.8012379263838132, "kd/rkl_advantage_p95": 5.078845445315043, "kd/rkl_advantage_std": 2.254502280553182, "kd/ssd_loss": 0.0, "learning_rate": 3.905365890707477e-07, "loss": 0.03454056978225708, "num_tokens": 525302716.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555601914724, "rewards/gt_logging_reward/std": 0.40146684149901074, "sampling/importance_sampling_ratio/max": 1.9790518363316854, "sampling/importance_sampling_ratio/mean": 0.9993346989154815, "sampling/importance_sampling_ratio/min": 0.4315601160128911, "sampling/sampling_logp_difference/max": 0.3418779889742533, "sampling/sampling_logp_difference/mean": 0.004098230266633133, "step": 16050, "step_time": 8.140646205830853, "student/entropy": 0.1338476326316595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015000000751266876, "completions/max_length": 478.1333333333333, "completions/max_terminated_length": 446.1666666666667, "completions/mean_length": 184.49778696695964, "completions/mean_terminated_length": 179.67063547770184, "completions/min_length": 58.56666666666667, "completions/min_terminated_length": 58.56666666666667, "entropy": 0.12882435272137324, "epoch": 0.6106406410207724, "eval/gt_acc_batch": 0.8152777969837188, "frac_reward_zero_std": 1.0, "grad_norm": 0.2989327609539032, "kd/policy_loss": 0.008497587560365597, "kd/rkl_advantage_mean": 0.791849190245072, "kd/rkl_advantage_p95": 4.986265883843104, "kd/rkl_advantage_std": 2.1854081749916077, "kd/ssd_loss": 0.0, "learning_rate": 3.8939733414347017e-07, "loss": 0.033990347385406496, "num_tokens": 526264268.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.815277777115504, "rewards/gt_logging_reward/std": 0.3724860509236654, "sampling/importance_sampling_ratio/max": 1.8757411281267802, "sampling/importance_sampling_ratio/mean": 1.000356262922287, "sampling/importance_sampling_ratio/min": 0.4390860805908839, "sampling/sampling_logp_difference/max": 0.3035271147886912, "sampling/sampling_logp_difference/mean": 0.0039834463813652595, "step": 16080, "step_time": 7.943859009534935, "student/entropy": 0.12882435272137324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445874541997, "completions/max_length": 491.6, "completions/max_terminated_length": 449.2, "completions/mean_length": 190.73528798421225, "completions/mean_terminated_length": 182.7814488728841, "completions/min_length": 56.13333333333333, "completions/min_terminated_length": 56.13333333333333, "entropy": 0.12868641093373298, "epoch": 0.6117798959480499, "eval/gt_acc_batch": 0.8080555737018585, "frac_reward_zero_std": 1.0, "grad_norm": 0.27515748143196106, "kd/policy_loss": 0.00831142831981803, "kd/rkl_advantage_mean": 0.7069383263587952, "kd/rkl_advantage_p95": 4.8676646769046785, "kd/rkl_advantage_std": 2.165761078397433, "kd/ssd_loss": 0.0, "learning_rate": 3.882580792161926e-07, "loss": 0.03324571847915649, "num_tokens": 527250147.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8080555538336436, "rewards/gt_logging_reward/std": 0.38587050437927245, "sampling/importance_sampling_ratio/max": 1.8219180742899577, "sampling/importance_sampling_ratio/mean": 1.0002155482769013, "sampling/importance_sampling_ratio/min": 0.4363971268137296, "sampling/sampling_logp_difference/max": 0.36372308135032655, "sampling/sampling_logp_difference/mean": 0.003929648982981841, "step": 16110, "step_time": 8.068288964136931, "student/entropy": 0.12868641093373298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026111112938572965, "completions/max_length": 478.06666666666666, "completions/max_terminated_length": 441.43333333333334, "completions/mean_length": 184.91612091064454, "completions/mean_terminated_length": 176.3590342203776, "completions/min_length": 47.96666666666667, "completions/min_terminated_length": 47.96666666666667, "entropy": 0.12874820275853077, "epoch": 0.6129191508753276, "eval/gt_acc_batch": 0.8080555816491445, "frac_reward_zero_std": 1.0, "grad_norm": 0.3159691095352173, "kd/policy_loss": 0.008389032234360154, "kd/rkl_advantage_mean": 0.7229133251744012, "kd/rkl_advantage_p95": 4.935847183068593, "kd/rkl_advantage_std": 2.2351902465025586, "kd/ssd_loss": 0.0, "learning_rate": 3.87118824288915e-07, "loss": 0.033556127548217775, "num_tokens": 528220125.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.808055559794108, "rewards/gt_logging_reward/std": 0.38251296877861024, "sampling/importance_sampling_ratio/max": 1.9146992802619933, "sampling/importance_sampling_ratio/mean": 1.001764863729477, "sampling/importance_sampling_ratio/min": 0.4380338748296102, "sampling/sampling_logp_difference/max": 0.329975155989329, "sampling/sampling_logp_difference/mean": 0.003968514820250372, "step": 16140, "step_time": 7.990069626832459, "student/entropy": 0.12874820275853077 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019444445644815764, "completions/max_length": 489.46666666666664, "completions/max_terminated_length": 449.3, "completions/mean_length": 187.7047317504883, "completions/mean_terminated_length": 181.44164326985677, "completions/min_length": 53.7, "completions/min_terminated_length": 53.7, "entropy": 0.13009910012284914, "epoch": 0.6140584058026051, "eval/gt_acc_batch": 0.8019444704055786, "frac_reward_zero_std": 1.0, "grad_norm": 0.259034663438797, "kd/policy_loss": 0.008422326315970471, "kd/rkl_advantage_mean": 0.7582660774389903, "kd/rkl_advantage_p95": 5.090832056601842, "kd/rkl_advantage_std": 2.267538712422053, "kd/ssd_loss": 0.0, "learning_rate": 3.859795693616374e-07, "loss": 0.03368930419286092, "num_tokens": 529189038.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8019444425900777, "rewards/gt_logging_reward/std": 0.3891359200080236, "sampling/importance_sampling_ratio/max": 1.8321316560109457, "sampling/importance_sampling_ratio/mean": 0.9985219260056813, "sampling/importance_sampling_ratio/min": 0.47985398570696514, "sampling/sampling_logp_difference/max": 0.3291045784950256, "sampling/sampling_logp_difference/mean": 0.003990324990202983, "step": 16170, "step_time": 7.981387337631896, "student/entropy": 0.13009910012284914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223927577337, "completions/max_length": 501.1, "completions/max_terminated_length": 457.93333333333334, "completions/mean_length": 190.8505650838216, "completions/mean_terminated_length": 181.13305104573567, "completions/min_length": 45.56666666666667, "completions/min_terminated_length": 45.56666666666667, "entropy": 0.12872015809019408, "epoch": 0.6151976607298827, "eval/gt_acc_batch": 0.7908333440621694, "frac_reward_zero_std": 1.0, "grad_norm": 0.317183256149292, "kd/policy_loss": 0.008498742411999653, "kd/rkl_advantage_mean": 0.8482374099393686, "kd/rkl_advantage_p95": 5.175614484151205, "kd/rkl_advantage_std": 2.278575149178505, "kd/ssd_loss": 0.0, "learning_rate": 3.8484031443435995e-07, "loss": 0.033994972705841064, "num_tokens": 530177612.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7908333361148834, "rewards/gt_logging_reward/std": 0.3989820212125778, "sampling/importance_sampling_ratio/max": 1.953744900226593, "sampling/importance_sampling_ratio/mean": 1.0024462521076203, "sampling/importance_sampling_ratio/min": 0.43393003443876904, "sampling/sampling_logp_difference/max": 0.3179595470428467, "sampling/sampling_logp_difference/mean": 0.003945093105236689, "step": 16200, "step_time": 8.175325140032024, "student/entropy": 0.12872015809019408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016944445483386517, "completions/max_length": 474.46666666666664, "completions/max_terminated_length": 442.5, "completions/mean_length": 178.73556416829427, "completions/mean_terminated_length": 173.11146901448566, "completions/min_length": 53.1, "completions/min_terminated_length": 53.1, "entropy": 0.12931654298057157, "epoch": 0.6163369156571602, "eval/gt_acc_batch": 0.8147222439448039, "frac_reward_zero_std": 1.0, "grad_norm": 0.3079073131084442, "kd/policy_loss": 0.008387955877697095, "kd/rkl_advantage_mean": 0.7893757448221247, "kd/rkl_advantage_p95": 5.00601985057195, "kd/rkl_advantage_std": 2.203719703356425, "kd/ssd_loss": 0.0, "learning_rate": 3.8370105950708237e-07, "loss": 0.03355183204015096, "num_tokens": 531124068.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8147222201029459, "rewards/gt_logging_reward/std": 0.3830281287431717, "sampling/importance_sampling_ratio/max": 1.9008373061815897, "sampling/importance_sampling_ratio/mean": 0.9966725329558055, "sampling/importance_sampling_ratio/min": 0.4917365392049154, "sampling/sampling_logp_difference/max": 0.3297815084457397, "sampling/sampling_logp_difference/mean": 0.0039880018138016265, "step": 16230, "step_time": 8.041853319767203, "student/entropy": 0.12931654298057157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556733161212, "completions/max_length": 492.4, "completions/max_terminated_length": 463.96666666666664, "completions/mean_length": 192.27362263997395, "completions/mean_terminated_length": 184.07227020263673, "completions/min_length": 56.7, "completions/min_terminated_length": 56.7, "entropy": 0.13239434839536746, "epoch": 0.6174761705844378, "eval/gt_acc_batch": 0.7844444632530212, "frac_reward_zero_std": 1.0, "grad_norm": 0.4053783118724823, "kd/policy_loss": 0.00906479285331443, "kd/rkl_advantage_mean": 0.7650435421693449, "kd/rkl_advantage_p95": 4.982504747311274, "kd/rkl_advantage_std": 2.234773101409276, "kd/ssd_loss": 0.0, "learning_rate": 3.825618045798048e-07, "loss": 0.036259170373280844, "num_tokens": 532106565.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7844444473584493, "rewards/gt_logging_reward/std": 0.4066020756959915, "sampling/importance_sampling_ratio/max": 1.9832794864972432, "sampling/importance_sampling_ratio/mean": 1.0032292147477468, "sampling/importance_sampling_ratio/min": 0.4702116395036379, "sampling/sampling_logp_difference/max": 0.3445621053377787, "sampling/sampling_logp_difference/mean": 0.004073410946875811, "step": 16260, "step_time": 8.044667978666258, "student/entropy": 0.13239434839536746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.032500001788139346, "completions/max_length": 499.6333333333333, "completions/max_terminated_length": 463.7, "completions/mean_length": 195.9894551595052, "completions/mean_terminated_length": 185.38331247965496, "completions/min_length": 50.13333333333333, "completions/min_terminated_length": 50.13333333333333, "entropy": 0.1281928534929951, "epoch": 0.6186154255117153, "eval/gt_acc_batch": 0.7594444692134857, "frac_reward_zero_std": 1.0, "grad_norm": 0.29615482687950134, "kd/policy_loss": 0.008919656649231911, "kd/rkl_advantage_mean": 0.7089311379939318, "kd/rkl_advantage_p95": 4.92143691778183, "kd/rkl_advantage_std": 2.1982248316208524, "kd/ssd_loss": 0.0, "learning_rate": 3.8142254965252725e-07, "loss": 0.03567862510681152, "num_tokens": 533121103.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7594444394111634, "rewards/gt_logging_reward/std": 0.42100370724995934, "sampling/importance_sampling_ratio/max": 1.9626025517781576, "sampling/importance_sampling_ratio/mean": 0.9974181075890859, "sampling/importance_sampling_ratio/min": 0.40109802583853404, "sampling/sampling_logp_difference/max": 0.3809069236119588, "sampling/sampling_logp_difference/mean": 0.003946674017546077, "step": 16290, "step_time": 8.21119038723021, "student/entropy": 0.1281928534929951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777903402845, "completions/max_length": 496.03333333333336, "completions/max_terminated_length": 453.5, "completions/mean_length": 189.37278747558594, "completions/mean_terminated_length": 181.0870559692383, "completions/min_length": 54.93333333333333, "completions/min_terminated_length": 54.93333333333333, "entropy": 0.13218690666059654, "epoch": 0.6197546804389928, "eval/gt_acc_batch": 0.8041666785875956, "frac_reward_zero_std": 1.0, "grad_norm": 0.23816561698913574, "kd/policy_loss": 0.008901640528347344, "kd/rkl_advantage_mean": 0.7862682494024436, "kd/rkl_advantage_p95": 5.0194881121317545, "kd/rkl_advantage_std": 2.2352590103944143, "kd/ssd_loss": 0.0, "learning_rate": 3.8028329472524967e-07, "loss": 0.03560655911763509, "num_tokens": 534108205.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666626930237, "rewards/gt_logging_reward/std": 0.3898299276828766, "sampling/importance_sampling_ratio/max": 2.0544403274854024, "sampling/importance_sampling_ratio/mean": 0.9976440032323202, "sampling/importance_sampling_ratio/min": 0.45273423691590625, "sampling/sampling_logp_difference/max": 0.3422394076983134, "sampling/sampling_logp_difference/mean": 0.004077131821153064, "step": 16320, "step_time": 8.01834190809944, "student/entropy": 0.13218690666059654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01750000094374021, "completions/max_length": 490.8333333333333, "completions/max_terminated_length": 454.1, "completions/mean_length": 188.10389912923176, "completions/mean_terminated_length": 182.32041676839194, "completions/min_length": 57.766666666666666, "completions/min_terminated_length": 57.766666666666666, "entropy": 0.13077647797763348, "epoch": 0.6208939353662705, "eval/gt_acc_batch": 0.796111132701238, "frac_reward_zero_std": 1.0, "grad_norm": 0.3429872691631317, "kd/policy_loss": 0.00862401796427245, "kd/rkl_advantage_mean": 0.7836837870379288, "kd/rkl_advantage_p95": 4.987880390882492, "kd/rkl_advantage_std": 2.2046846876541775, "kd/ssd_loss": 0.0, "learning_rate": 3.791440397979721e-07, "loss": 0.03449607690175374, "num_tokens": 535088643.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.796111112833023, "rewards/gt_logging_reward/std": 0.3942525893449783, "sampling/importance_sampling_ratio/max": 1.9104328632354737, "sampling/importance_sampling_ratio/mean": 0.9968473315238953, "sampling/importance_sampling_ratio/min": 0.45274848540623985, "sampling/sampling_logp_difference/max": 0.33758111397425333, "sampling/sampling_logp_difference/mean": 0.003958714737867315, "step": 16350, "step_time": 8.29930556039326, "student/entropy": 0.13077647797763348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018611112361152968, "completions/max_length": 469.06666666666666, "completions/max_terminated_length": 434.46666666666664, "completions/mean_length": 185.74639892578125, "completions/mean_terminated_length": 179.76029154459636, "completions/min_length": 53.86666666666667, "completions/min_terminated_length": 53.86666666666667, "entropy": 0.1272642644122243, "epoch": 0.622033190293548, "eval/gt_acc_batch": 0.796111136674881, "frac_reward_zero_std": 1.0, "grad_norm": 0.40915447473526, "kd/policy_loss": 0.008324152879261722, "kd/rkl_advantage_mean": 0.7545023831228416, "kd/rkl_advantage_p95": 4.930125202735265, "kd/rkl_advantage_std": 2.189967851837476, "kd/ssd_loss": 0.0, "learning_rate": 3.7800478487069456e-07, "loss": 0.0332966148853302, "num_tokens": 536062570.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111187934875, "rewards/gt_logging_reward/std": 0.3958402246236801, "sampling/importance_sampling_ratio/max": 1.903787942727407, "sampling/importance_sampling_ratio/mean": 0.9968758702278138, "sampling/importance_sampling_ratio/min": 0.45758094390233356, "sampling/sampling_logp_difference/max": 0.327685292561849, "sampling/sampling_logp_difference/mean": 0.003954170496823887, "step": 16380, "step_time": 8.029695574897536, "student/entropy": 0.1272642644122243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666810711225, "completions/max_length": 487.9, "completions/max_terminated_length": 452.76666666666665, "completions/mean_length": 186.55528818766277, "completions/mean_terminated_length": 179.54840901692708, "completions/min_length": 51.3, "completions/min_terminated_length": 51.3, "entropy": 0.12495134522517522, "epoch": 0.6231724452208256, "eval/gt_acc_batch": 0.809166685740153, "frac_reward_zero_std": 1.0, "grad_norm": 0.29333174228668213, "kd/policy_loss": 0.008216502511641011, "kd/rkl_advantage_mean": 0.7364287225607161, "kd/rkl_advantage_p95": 4.960745553175609, "kd/rkl_advantage_std": 2.2038942406574886, "kd/ssd_loss": 0.0, "learning_rate": 3.76865529943417e-07, "loss": 0.03286601305007934, "num_tokens": 537037009.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8091666618982951, "rewards/gt_logging_reward/std": 0.380447319149971, "sampling/importance_sampling_ratio/max": 1.89793594678243, "sampling/importance_sampling_ratio/mean": 1.0024449427922566, "sampling/importance_sampling_ratio/min": 0.4571714719136556, "sampling/sampling_logp_difference/max": 0.3684006631374359, "sampling/sampling_logp_difference/mean": 0.0038417460784936947, "step": 16410, "step_time": 8.15918818139762, "student/entropy": 0.12495134522517522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556553105513, "completions/max_length": 480.23333333333335, "completions/max_terminated_length": 446.43333333333334, "completions/mean_length": 186.05139872233073, "completions/mean_terminated_length": 180.98852081298827, "completions/min_length": 57.266666666666666, "completions/min_terminated_length": 57.266666666666666, "entropy": 0.12538273800164462, "epoch": 0.6243117001481031, "eval/gt_acc_batch": 0.7972222367922465, "frac_reward_zero_std": 1.0, "grad_norm": 0.3025725781917572, "kd/policy_loss": 0.008372696888788294, "kd/rkl_advantage_mean": 0.7380722592895229, "kd/rkl_advantage_p95": 4.966238047679266, "kd/rkl_advantage_std": 2.222761232654254, "kd/ssd_loss": 0.0, "learning_rate": 3.757262750161394e-07, "loss": 0.03349079291025798, "num_tokens": 538007818.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7972222248713176, "rewards/gt_logging_reward/std": 0.3965550710757573, "sampling/importance_sampling_ratio/max": 2.075756084918976, "sampling/importance_sampling_ratio/mean": 1.008975859483083, "sampling/importance_sampling_ratio/min": 0.49477258920669553, "sampling/sampling_logp_difference/max": 0.31146361827850344, "sampling/sampling_logp_difference/mean": 0.0038684601817900936, "step": 16440, "step_time": 7.9924219875683775, "student/entropy": 0.12538273800164462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444562618931, "completions/max_length": 492.23333333333335, "completions/max_terminated_length": 450.56666666666666, "completions/mean_length": 185.8975092569987, "completions/mean_terminated_length": 177.97116394042968, "completions/min_length": 46.13333333333333, "completions/min_terminated_length": 46.13333333333333, "entropy": 0.1298248674099644, "epoch": 0.6254509550753807, "eval/gt_acc_batch": 0.8111111223697662, "frac_reward_zero_std": 1.0, "grad_norm": 0.30794087052345276, "kd/policy_loss": 0.008177121874177828, "kd/rkl_advantage_mean": 0.7349569372701807, "kd/rkl_advantage_p95": 4.883764714002609, "kd/rkl_advantage_std": 2.1962475409110387, "kd/ssd_loss": 0.0, "learning_rate": 3.745870200888619e-07, "loss": 0.032708489894866945, "num_tokens": 538983249.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8111111104488373, "rewards/gt_logging_reward/std": 0.3826622838775317, "sampling/importance_sampling_ratio/max": 1.9064668615659077, "sampling/importance_sampling_ratio/mean": 0.9988359530766805, "sampling/importance_sampling_ratio/min": 0.4255235016345978, "sampling/sampling_logp_difference/max": 0.31299150983492535, "sampling/sampling_logp_difference/mean": 0.003951947235812743, "step": 16470, "step_time": 8.098057282835361, "student/entropy": 0.1298248674099644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500001130004723, "completions/max_length": 489.26666666666665, "completions/max_terminated_length": 452.1333333333333, "completions/mean_length": 182.30472920735676, "completions/mean_terminated_length": 176.4876256306966, "completions/min_length": 57.266666666666666, "completions/min_terminated_length": 57.266666666666666, "entropy": 0.12459181528538465, "epoch": 0.6265902100026582, "eval/gt_acc_batch": 0.822222230831782, "frac_reward_zero_std": 1.0, "grad_norm": 0.37383216619491577, "kd/policy_loss": 0.007679929916048422, "kd/rkl_advantage_mean": 0.8093613032872479, "kd/rkl_advantage_p95": 5.0979839861392975, "kd/rkl_advantage_std": 2.264000352223714, "kd/ssd_loss": 0.0, "learning_rate": 3.7344776516158434e-07, "loss": 0.03071972131729126, "num_tokens": 539945738.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8222222248713176, "rewards/gt_logging_reward/std": 0.37302082280317944, "sampling/importance_sampling_ratio/max": 1.8568974177042643, "sampling/importance_sampling_ratio/mean": 0.9992794672648112, "sampling/importance_sampling_ratio/min": 0.4493908127148946, "sampling/sampling_logp_difference/max": 0.3342802921930949, "sampling/sampling_logp_difference/mean": 0.003856910113245249, "step": 16500, "step_time": 8.131358830500782, "student/entropy": 0.12459181528538465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014444445197780927, "completions/max_length": 476.43333333333334, "completions/max_terminated_length": 436.23333333333335, "completions/mean_length": 184.18334350585937, "completions/mean_terminated_length": 179.54222971598307, "completions/min_length": 55.3, "completions/min_terminated_length": 55.3, "entropy": 0.12352447416633368, "epoch": 0.6277294649299359, "eval/gt_acc_batch": 0.8158333619435628, "frac_reward_zero_std": 1.0, "grad_norm": 0.28680166602134705, "kd/policy_loss": 0.007762846554396674, "kd/rkl_advantage_mean": 0.7396268751161794, "kd/rkl_advantage_p95": 4.9695456226666765, "kd/rkl_advantage_std": 2.1886898557345074, "kd/ssd_loss": 0.0, "learning_rate": 3.7230851023430675e-07, "loss": 0.031051385402679443, "num_tokens": 540908510.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8158333281675975, "rewards/gt_logging_reward/std": 0.38190851906935375, "sampling/importance_sampling_ratio/max": 1.9092254757881164, "sampling/importance_sampling_ratio/mean": 0.9973410189151763, "sampling/importance_sampling_ratio/min": 0.4904051899909973, "sampling/sampling_logp_difference/max": 0.30880643328030905, "sampling/sampling_logp_difference/mean": 0.0038363789596284428, "step": 16530, "step_time": 7.830856922963479, "student/entropy": 0.12352447416633368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666779667139, "completions/max_length": 471.76666666666665, "completions/max_terminated_length": 444.8333333333333, "completions/mean_length": 187.8586201985677, "completions/mean_terminated_length": 180.84404754638672, "completions/min_length": 55.666666666666664, "completions/min_terminated_length": 55.666666666666664, "entropy": 0.12806627259900172, "epoch": 0.6288687198572134, "eval/gt_acc_batch": 0.8008333524068196, "frac_reward_zero_std": 1.0, "grad_norm": 0.32657936215400696, "kd/policy_loss": 0.008411666882845262, "kd/rkl_advantage_mean": 0.758517024666071, "kd/rkl_advantage_p95": 5.004856856664022, "kd/rkl_advantage_std": 2.2214075615008673, "kd/ssd_loss": 0.0, "learning_rate": 3.7116925530702917e-07, "loss": 0.03364667495091756, "num_tokens": 541887969.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333265781402, "rewards/gt_logging_reward/std": 0.3943203012148539, "sampling/importance_sampling_ratio/max": 1.9125505487124126, "sampling/importance_sampling_ratio/mean": 1.0027405003706613, "sampling/importance_sampling_ratio/min": 0.47687788903713224, "sampling/sampling_logp_difference/max": 0.3559423327445984, "sampling/sampling_logp_difference/mean": 0.0039725710405036805, "step": 16560, "step_time": 7.865853979802341, "student/entropy": 0.12806627259900172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018333334227403006, "completions/max_length": 464.2, "completions/max_terminated_length": 433.2, "completions/mean_length": 181.95028686523438, "completions/mean_terminated_length": 175.93243611653645, "completions/min_length": 57.8, "completions/min_terminated_length": 57.8, "entropy": 0.1254055700575312, "epoch": 0.630007974784491, "eval/gt_acc_batch": 0.8194444636503856, "frac_reward_zero_std": 1.0, "grad_norm": 0.2698708772659302, "kd/policy_loss": 0.007787124531265969, "kd/rkl_advantage_mean": 0.6661436108251413, "kd/rkl_advantage_p95": 4.808536885182063, "kd/rkl_advantage_std": 2.1458729803562164, "kd/ssd_loss": 0.0, "learning_rate": 3.7003000037975164e-07, "loss": 0.031148497263590494, "num_tokens": 542842086.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8194444417953491, "rewards/gt_logging_reward/std": 0.3757772545019786, "sampling/importance_sampling_ratio/max": 1.8904400865236919, "sampling/importance_sampling_ratio/mean": 0.9985626379648844, "sampling/importance_sampling_ratio/min": 0.462022398908933, "sampling/sampling_logp_difference/max": 0.3346476912498474, "sampling/sampling_logp_difference/mean": 0.0038626741152256727, "step": 16590, "step_time": 7.899620647964184, "student/entropy": 0.1254055700575312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028888890768090883, "completions/max_length": 500.8666666666667, "completions/max_terminated_length": 450.6333333333333, "completions/mean_length": 190.79889729817708, "completions/mean_terminated_length": 181.4381851196289, "completions/min_length": 57.86666666666667, "completions/min_terminated_length": 57.86666666666667, "entropy": 0.12932433895766735, "epoch": 0.6311472297117685, "eval/gt_acc_batch": 0.771666685740153, "frac_reward_zero_std": 1.0, "grad_norm": 0.3497261703014374, "kd/policy_loss": 0.008726177737116814, "kd/rkl_advantage_mean": 0.7588584430176222, "kd/rkl_advantage_p95": 4.994052294890086, "kd/rkl_advantage_std": 2.223782985409101, "kd/ssd_loss": 0.0, "learning_rate": 3.6889074545247406e-07, "loss": 0.03490471045176188, "num_tokens": 543845170.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7716666678587596, "rewards/gt_logging_reward/std": 0.4156635344028473, "sampling/importance_sampling_ratio/max": 1.8684284408887228, "sampling/importance_sampling_ratio/mean": 0.9956352293491364, "sampling/importance_sampling_ratio/min": 0.40860153834025065, "sampling/sampling_logp_difference/max": 0.3387289623419444, "sampling/sampling_logp_difference/mean": 0.003987524417849878, "step": 16620, "step_time": 8.402167396065973, "student/entropy": 0.12932433895766735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888889780888953, "completions/max_length": 471.93333333333334, "completions/max_terminated_length": 439.2, "completions/mean_length": 184.36778666178387, "completions/mean_terminated_length": 178.2680196126302, "completions/min_length": 55.4, "completions/min_terminated_length": 55.4, "entropy": 0.12329084128141403, "epoch": 0.632286484639046, "eval/gt_acc_batch": 0.8200000246365865, "frac_reward_zero_std": 1.0, "grad_norm": 0.28595054149627686, "kd/policy_loss": 0.00826877610719142, "kd/rkl_advantage_mean": 0.7214710847785075, "kd/rkl_advantage_p95": 4.951477764050166, "kd/rkl_advantage_std": 2.1896701008081436, "kd/ssd_loss": 0.0, "learning_rate": 3.677514905251965e-07, "loss": 0.033075108130772905, "num_tokens": 544809774.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.82000000278155, "rewards/gt_logging_reward/std": 0.3703742951154709, "sampling/importance_sampling_ratio/max": 1.902244253953298, "sampling/importance_sampling_ratio/mean": 1.004546622435252, "sampling/importance_sampling_ratio/min": 0.48086852033933003, "sampling/sampling_logp_difference/max": 0.3271129488945007, "sampling/sampling_logp_difference/mean": 0.003816502292950948, "step": 16650, "step_time": 7.985863657931137, "student/entropy": 0.12329084128141403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026666668336838485, "completions/max_length": 494.56666666666666, "completions/max_terminated_length": 458.96666666666664, "completions/mean_length": 194.1894541422526, "completions/mean_terminated_length": 185.63214365641275, "completions/min_length": 54.06666666666667, "completions/min_terminated_length": 54.06666666666667, "entropy": 0.12744026804963748, "epoch": 0.6334257395663236, "eval/gt_acc_batch": 0.7958333532015482, "frac_reward_zero_std": 1.0, "grad_norm": 0.24376645684242249, "kd/policy_loss": 0.008584142930340022, "kd/rkl_advantage_mean": 0.8068246103823185, "kd/rkl_advantage_p95": 5.054715953270594, "kd/rkl_advantage_std": 2.216339596112569, "kd/ssd_loss": 0.0, "learning_rate": 3.6661223559791895e-07, "loss": 0.03433656692504883, "num_tokens": 545812568.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7958333333333333, "rewards/gt_logging_reward/std": 0.3990170627832413, "sampling/importance_sampling_ratio/max": 1.88309938510259, "sampling/importance_sampling_ratio/mean": 0.9974883834520976, "sampling/importance_sampling_ratio/min": 0.4514936794837316, "sampling/sampling_logp_difference/max": 0.33097117642561596, "sampling/sampling_logp_difference/mean": 0.003914167871698737, "step": 16680, "step_time": 8.12900812367055, "student/entropy": 0.12744026804963748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026111112783352534, "completions/max_length": 471.5, "completions/max_terminated_length": 440.03333333333336, "completions/mean_length": 181.84556427001954, "completions/mean_terminated_length": 173.0122843424479, "completions/min_length": 53.333333333333336, "completions/min_terminated_length": 53.333333333333336, "entropy": 0.1310638548185428, "epoch": 0.6345649944936012, "eval/gt_acc_batch": 0.8158333579699198, "frac_reward_zero_std": 1.0, "grad_norm": 0.2601781189441681, "kd/policy_loss": 0.008611928365038086, "kd/rkl_advantage_mean": 0.7426661500086387, "kd/rkl_advantage_p95": 4.938623595237732, "kd/rkl_advantage_std": 2.2072240600983304, "kd/ssd_loss": 0.0, "learning_rate": 3.6547298067064137e-07, "loss": 0.03444771766662598, "num_tokens": 546775068.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8158333341280619, "rewards/gt_logging_reward/std": 0.3750457356373469, "sampling/importance_sampling_ratio/max": 1.9897431572278341, "sampling/importance_sampling_ratio/mean": 0.994774212439855, "sampling/importance_sampling_ratio/min": 0.4466517319281896, "sampling/sampling_logp_difference/max": 0.3173393766085307, "sampling/sampling_logp_difference/mean": 0.004001408590314289, "step": 16710, "step_time": 7.8749277216091285, "student/entropy": 0.1310638548185428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445806245008, "completions/max_length": 466.1, "completions/max_terminated_length": 438.4, "completions/mean_length": 186.42001037597657, "completions/mean_terminated_length": 179.26867014567057, "completions/min_length": 48.03333333333333, "completions/min_terminated_length": 48.03333333333333, "entropy": 0.13055832019696634, "epoch": 0.6357042494208788, "eval/gt_acc_batch": 0.8147222459316253, "frac_reward_zero_std": 1.0, "grad_norm": 0.4370981752872467, "kd/policy_loss": 0.008759688444357986, "kd/rkl_advantage_mean": 0.7667011284036562, "kd/rkl_advantage_p95": 4.991180789470673, "kd/rkl_advantage_std": 2.200049901008606, "kd/ssd_loss": 0.0, "learning_rate": 3.6433372574336384e-07, "loss": 0.03503875335057576, "num_tokens": 547743812.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8147222260634105, "rewards/gt_logging_reward/std": 0.3804353604714076, "sampling/importance_sampling_ratio/max": 1.9546062390009562, "sampling/importance_sampling_ratio/mean": 0.9984741330146789, "sampling/importance_sampling_ratio/min": 0.44320075313250223, "sampling/sampling_logp_difference/max": 0.30802469650904335, "sampling/sampling_logp_difference/mean": 0.004014152040084203, "step": 16740, "step_time": 7.8832811877975475, "student/entropy": 0.13055832019696634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.033888890966773035, "completions/max_length": 492.7, "completions/max_terminated_length": 457.3666666666667, "completions/mean_length": 195.19250996907553, "completions/mean_terminated_length": 184.2764119466146, "completions/min_length": 61.2, "completions/min_terminated_length": 61.2, "entropy": 0.12950787084798018, "epoch": 0.6368435043481563, "eval/gt_acc_batch": 0.7961111227671306, "frac_reward_zero_std": 1.0, "grad_norm": 0.2133023887872696, "kd/policy_loss": 0.008775795251131058, "kd/rkl_advantage_mean": 0.7557705676803986, "kd/rkl_advantage_p95": 4.955019611120224, "kd/rkl_advantage_std": 2.217440726359685, "kd/ssd_loss": 0.0, "learning_rate": 3.631944708160863e-07, "loss": 0.03510317802429199, "num_tokens": 548760817.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111108462016, "rewards/gt_logging_reward/std": 0.39317729075749713, "sampling/importance_sampling_ratio/max": 1.9034319082895914, "sampling/importance_sampling_ratio/mean": 1.0023000856240591, "sampling/importance_sampling_ratio/min": 0.45405703485012056, "sampling/sampling_logp_difference/max": 0.31260815262794495, "sampling/sampling_logp_difference/mean": 0.003969692423318823, "step": 16770, "step_time": 8.309821190134, "student/entropy": 0.12950787084798018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890761882066, "completions/max_length": 496.56666666666666, "completions/max_terminated_length": 462.6666666666667, "completions/mean_length": 196.08389892578126, "completions/mean_terminated_length": 187.7781962076823, "completions/min_length": 58.43333333333333, "completions/min_terminated_length": 58.43333333333333, "entropy": 0.13090988372763, "epoch": 0.6379827592754339, "eval/gt_acc_batch": 0.7644444704055786, "frac_reward_zero_std": 1.0, "grad_norm": 0.28201431035995483, "kd/policy_loss": 0.009022985579213128, "kd/rkl_advantage_mean": 0.8293279712398847, "kd/rkl_advantage_p95": 5.189117453495661, "kd/rkl_advantage_std": 2.2660762131214143, "kd/ssd_loss": 0.0, "learning_rate": 3.620552158888087e-07, "loss": 0.036091947555541994, "num_tokens": 549787239.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7644444465637207, "rewards/gt_logging_reward/std": 0.4131416355570157, "sampling/importance_sampling_ratio/max": 1.9011201500892638, "sampling/importance_sampling_ratio/mean": 1.0022559384504954, "sampling/importance_sampling_ratio/min": 0.46953366200129193, "sampling/sampling_logp_difference/max": 0.3316683411598206, "sampling/sampling_logp_difference/mean": 0.004019130649976432, "step": 16800, "step_time": 8.357869617029792, "student/entropy": 0.13090988372763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026111112410823503, "completions/max_length": 484.43333333333334, "completions/max_terminated_length": 432.2, "completions/mean_length": 189.45001068115235, "completions/mean_terminated_length": 180.9223642985026, "completions/min_length": 54.96666666666667, "completions/min_terminated_length": 54.96666666666667, "entropy": 0.1302058808505535, "epoch": 0.6391220142027114, "eval/gt_acc_batch": 0.7902778029441834, "frac_reward_zero_std": 1.0, "grad_norm": 0.2863351106643677, "kd/policy_loss": 0.008717595421088239, "kd/rkl_advantage_mean": 0.7980396802226702, "kd/rkl_advantage_p95": 5.033965384960174, "kd/rkl_advantage_std": 2.2028055797020594, "kd/ssd_loss": 0.0, "learning_rate": 3.6091596096153114e-07, "loss": 0.03487038215001424, "num_tokens": 550770395.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.790277777115504, "rewards/gt_logging_reward/std": 0.4044573217630386, "sampling/importance_sampling_ratio/max": 1.9070892373720805, "sampling/importance_sampling_ratio/mean": 0.9986464281876882, "sampling/importance_sampling_ratio/min": 0.4457494924465815, "sampling/sampling_logp_difference/max": 0.3311859846115112, "sampling/sampling_logp_difference/mean": 0.004034633461075524, "step": 16830, "step_time": 8.132317678266554, "student/entropy": 0.1302058808505535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333482965827, "completions/max_length": 479.03333333333336, "completions/max_terminated_length": 449.23333333333335, "completions/mean_length": 190.50528818766276, "completions/mean_terminated_length": 182.91724853515626, "completions/min_length": 57.36666666666667, "completions/min_terminated_length": 57.36666666666667, "entropy": 0.12861852670709292, "epoch": 0.640261269129989, "eval/gt_acc_batch": 0.8044444660345713, "frac_reward_zero_std": 1.0, "grad_norm": 0.26407188177108765, "kd/policy_loss": 0.008428666713492324, "kd/rkl_advantage_mean": 0.8468839629242818, "kd/rkl_advantage_p95": 5.182201846440633, "kd/rkl_advantage_std": 2.256565195322037, "kd/ssd_loss": 0.0, "learning_rate": 3.5977670603425356e-07, "loss": 0.033714667956034346, "num_tokens": 551755390.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8044444441795349, "rewards/gt_logging_reward/std": 0.3888797665635745, "sampling/importance_sampling_ratio/max": 1.9703713377316794, "sampling/importance_sampling_ratio/mean": 0.9970546662807465, "sampling/importance_sampling_ratio/min": 0.46315187017122905, "sampling/sampling_logp_difference/max": 0.321373571952184, "sampling/sampling_logp_difference/mean": 0.003977269252451758, "step": 16860, "step_time": 8.021342025401342, "student/entropy": 0.12861852670709292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888889936109383, "completions/max_length": 475.26666666666665, "completions/max_terminated_length": 452.46666666666664, "completions/mean_length": 186.71417592366535, "completions/mean_terminated_length": 180.74634602864583, "completions/min_length": 58.56666666666667, "completions/min_terminated_length": 58.56666666666667, "entropy": 0.132790748650829, "epoch": 0.6414005240572666, "eval/gt_acc_batch": 0.8013889114061992, "frac_reward_zero_std": 1.0, "grad_norm": 0.33089545369148254, "kd/policy_loss": 0.00864062579542709, "kd/rkl_advantage_mean": 0.7926197704548638, "kd/rkl_advantage_p95": 4.967120695114136, "kd/rkl_advantage_std": 2.199816984931628, "kd/ssd_loss": 0.0, "learning_rate": 3.5863745110697603e-07, "loss": 0.03456250031789144, "num_tokens": 552724953.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888915379842, "rewards/gt_logging_reward/std": 0.39046584367752074, "sampling/importance_sampling_ratio/max": 1.8505292097727457, "sampling/importance_sampling_ratio/mean": 0.9931596696376801, "sampling/importance_sampling_ratio/min": 0.4621580223242442, "sampling/sampling_logp_difference/max": 0.313336169719696, "sampling/sampling_logp_difference/mean": 0.004056859753715495, "step": 16890, "step_time": 7.953456612932496, "student/entropy": 0.132790748650829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014722223052134116, "completions/max_length": 473.5, "completions/max_terminated_length": 442.9, "completions/mean_length": 181.39667510986328, "completions/mean_terminated_length": 176.60907135009765, "completions/min_length": 51.666666666666664, "completions/min_terminated_length": 51.666666666666664, "entropy": 0.12692044793317717, "epoch": 0.6425397789845441, "eval/gt_acc_batch": 0.8225000202655792, "frac_reward_zero_std": 1.0, "grad_norm": 0.29109060764312744, "kd/policy_loss": 0.008347628182188297, "kd/rkl_advantage_mean": 0.7645384796584646, "kd/rkl_advantage_p95": 4.921144419908524, "kd/rkl_advantage_std": 2.1904951632022858, "kd/ssd_loss": 0.0, "learning_rate": 3.5749819617969845e-07, "loss": 0.03339051405588786, "num_tokens": 553684949.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8225000003973643, "rewards/gt_logging_reward/std": 0.37071799784898757, "sampling/importance_sampling_ratio/max": 1.8854875445365906, "sampling/importance_sampling_ratio/mean": 0.9971880853176117, "sampling/importance_sampling_ratio/min": 0.4574228525161743, "sampling/sampling_logp_difference/max": 0.35574683745702107, "sampling/sampling_logp_difference/mean": 0.00394845602568239, "step": 16920, "step_time": 8.03289802436775, "student/entropy": 0.12692044793317717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03194444617256522, "completions/max_length": 500.2, "completions/max_terminated_length": 456.56666666666666, "completions/mean_length": 200.96084238688152, "completions/mean_terminated_length": 190.8514139811198, "completions/min_length": 62.93333333333333, "completions/min_terminated_length": 62.93333333333333, "entropy": 0.12581306000550588, "epoch": 0.6436790339118217, "eval/gt_acc_batch": 0.7830555697282156, "frac_reward_zero_std": 1.0, "grad_norm": 0.18658971786499023, "kd/policy_loss": 0.008529637957690284, "kd/rkl_advantage_mean": 0.7302560514460007, "kd/rkl_advantage_p95": 4.95717288851738, "kd/rkl_advantage_std": 2.184102706114451, "kd/ssd_loss": 0.0, "learning_rate": 3.5635894125242087e-07, "loss": 0.03411855300267538, "num_tokens": 554712192.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7830555617809296, "rewards/gt_logging_reward/std": 0.40173674523830416, "sampling/importance_sampling_ratio/max": 1.9459283788998922, "sampling/importance_sampling_ratio/mean": 1.0045204480489096, "sampling/importance_sampling_ratio/min": 0.4331026092171669, "sampling/sampling_logp_difference/max": 0.3185398558775584, "sampling/sampling_logp_difference/mean": 0.0038957555157442887, "step": 16950, "step_time": 8.106574442635369, "student/entropy": 0.12581306000550588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890103747448, "completions/max_length": 486.93333333333334, "completions/max_terminated_length": 450.8333333333333, "completions/mean_length": 189.81973215738932, "completions/mean_terminated_length": 182.23128204345704, "completions/min_length": 55.93333333333333, "completions/min_terminated_length": 55.93333333333333, "entropy": 0.12517688553780318, "epoch": 0.6448182888390992, "eval/gt_acc_batch": 0.7980555713176727, "frac_reward_zero_std": 1.0, "grad_norm": 0.23848868906497955, "kd/policy_loss": 0.00832369676985157, "kd/rkl_advantage_mean": 0.6480079291388392, "kd/rkl_advantage_p95": 4.901359939575196, "kd/rkl_advantage_std": 2.232854505379995, "kd/ssd_loss": 0.0, "learning_rate": 3.5521968632514334e-07, "loss": 0.03329478700955709, "num_tokens": 555697663.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7980555534362793, "rewards/gt_logging_reward/std": 0.3926996052265167, "sampling/importance_sampling_ratio/max": 1.9332472840944925, "sampling/importance_sampling_ratio/mean": 1.0043156842390697, "sampling/importance_sampling_ratio/min": 0.4467013200124105, "sampling/sampling_logp_difference/max": 0.3129841009775797, "sampling/sampling_logp_difference/mean": 0.0038931862683966754, "step": 16980, "step_time": 7.9257947397592945, "student/entropy": 0.12517688553780318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223865489165, "completions/max_length": 493.3666666666667, "completions/max_terminated_length": 442.1, "completions/mean_length": 189.5900100708008, "completions/mean_terminated_length": 179.7471923828125, "completions/min_length": 48.96666666666667, "completions/min_terminated_length": 48.96666666666667, "entropy": 0.12464943962792556, "epoch": 0.6459575437663768, "eval/gt_acc_batch": 0.8025000135103861, "frac_reward_zero_std": 1.0, "grad_norm": 0.27047643065452576, "kd/policy_loss": 0.008202873372162382, "kd/rkl_advantage_mean": 0.7720872076422286, "kd/rkl_advantage_p95": 5.037077375253042, "kd/rkl_advantage_std": 2.2325597325960795, "kd/ssd_loss": 0.0, "learning_rate": 3.5408043139786575e-07, "loss": 0.03281149864196777, "num_tokens": 556686859.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8025000035762787, "rewards/gt_logging_reward/std": 0.3918683489163717, "sampling/importance_sampling_ratio/max": 1.8840449412663778, "sampling/importance_sampling_ratio/mean": 0.9984977940718333, "sampling/importance_sampling_ratio/min": 0.42580653751889863, "sampling/sampling_logp_difference/max": 0.3353710254033407, "sampling/sampling_logp_difference/mean": 0.003849335752117137, "step": 17010, "step_time": 7.98488733356741, "student/entropy": 0.12464943962792556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890265176695, "completions/max_length": 494.23333333333335, "completions/max_terminated_length": 454.7, "completions/mean_length": 189.18251088460286, "completions/mean_terminated_length": 180.70476888020832, "completions/min_length": 50.46666666666667, "completions/min_terminated_length": 50.46666666666667, "entropy": 0.13043951236953338, "epoch": 0.6470967986936543, "eval/gt_acc_batch": 0.7861111323038737, "frac_reward_zero_std": 1.0, "grad_norm": 0.2660060226917267, "kd/policy_loss": 0.008643310008725773, "kd/rkl_advantage_mean": 0.7926251156100382, "kd/rkl_advantage_p95": 5.034605377912522, "kd/rkl_advantage_std": 2.2338148544232053, "kd/ssd_loss": 0.0, "learning_rate": 3.529411764705882e-07, "loss": 0.03457323710123698, "num_tokens": 557664068.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111124356588, "rewards/gt_logging_reward/std": 0.40044122536977134, "sampling/importance_sampling_ratio/max": 1.8600624283154805, "sampling/importance_sampling_ratio/mean": 1.0003699362277985, "sampling/importance_sampling_ratio/min": 0.4648956427971522, "sampling/sampling_logp_difference/max": 0.31121462980906167, "sampling/sampling_logp_difference/mean": 0.003996648173779249, "step": 17040, "step_time": 8.048448121971644, "student/entropy": 0.13043951236953338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02750000189989805, "completions/max_length": 481.03333333333336, "completions/max_terminated_length": 449.0, "completions/mean_length": 190.1408437093099, "completions/mean_terminated_length": 181.21231231689453, "completions/min_length": 55.7, "completions/min_terminated_length": 55.7, "entropy": 0.12673679540554683, "epoch": 0.648236053620932, "eval/gt_acc_batch": 0.8138889094193776, "frac_reward_zero_std": 1.0, "grad_norm": 0.3182868957519531, "kd/policy_loss": 0.008340878057060764, "kd/rkl_advantage_mean": 0.752524463708202, "kd/rkl_advantage_p95": 5.011719028155009, "kd/rkl_advantage_std": 2.199068667491277, "kd/ssd_loss": 0.0, "learning_rate": 3.518019215433107e-07, "loss": 0.03336351712544759, "num_tokens": 558667535.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8138888895511627, "rewards/gt_logging_reward/std": 0.3823909064133962, "sampling/importance_sampling_ratio/max": 1.8473361055056254, "sampling/importance_sampling_ratio/mean": 0.9927049557367961, "sampling/importance_sampling_ratio/min": 0.45785174518823624, "sampling/sampling_logp_difference/max": 0.4298693756262461, "sampling/sampling_logp_difference/mean": 0.00387502727098763, "step": 17070, "step_time": 8.16017706253139, "student/entropy": 0.12673679540554683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02777777925754587, "completions/max_length": 484.2, "completions/max_terminated_length": 451.6666666666667, "completions/mean_length": 182.57278747558593, "completions/mean_terminated_length": 173.11978403727213, "completions/min_length": 51.53333333333333, "completions/min_terminated_length": 51.53333333333333, "entropy": 0.129990264835457, "epoch": 0.6493753085482095, "eval/gt_acc_batch": 0.7991666853427887, "frac_reward_zero_std": 1.0, "grad_norm": 0.25680431723594666, "kd/policy_loss": 0.00845349082762065, "kd/rkl_advantage_mean": 0.7881077564321458, "kd/rkl_advantage_p95": 4.997448068857193, "kd/rkl_advantage_std": 2.2257494290669757, "kd/ssd_loss": 0.0, "learning_rate": 3.506626666160331e-07, "loss": 0.033813969294230146, "num_tokens": 559626861.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7991666615009307, "rewards/gt_logging_reward/std": 0.39131142099698385, "sampling/importance_sampling_ratio/max": 1.8216785033543905, "sampling/importance_sampling_ratio/mean": 0.9983015437920888, "sampling/importance_sampling_ratio/min": 0.4365095059076945, "sampling/sampling_logp_difference/max": 0.3363833675781886, "sampling/sampling_logp_difference/mean": 0.004028104168052475, "step": 17100, "step_time": 8.071341763609476, "student/entropy": 0.129990264835457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112621923287, "completions/max_length": 490.5, "completions/max_terminated_length": 458.93333333333334, "completions/mean_length": 185.7497309366862, "completions/mean_terminated_length": 178.0465296427409, "completions/min_length": 56.56666666666667, "completions/min_terminated_length": 56.56666666666667, "entropy": 0.12880011362334093, "epoch": 0.6505145634754871, "eval/gt_acc_batch": 0.8050000190734863, "frac_reward_zero_std": 1.0, "grad_norm": 0.2536056339740753, "kd/policy_loss": 0.0084313633677084, "kd/rkl_advantage_mean": 0.8496054710820318, "kd/rkl_advantage_p95": 5.23048862417539, "kd/rkl_advantage_std": 2.3166631162166595, "kd/ssd_loss": 0.0, "learning_rate": 3.4952341168875553e-07, "loss": 0.033725460370381675, "num_tokens": 560597712.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8050000031789144, "rewards/gt_logging_reward/std": 0.38492599030335745, "sampling/importance_sampling_ratio/max": 1.8876159270604451, "sampling/importance_sampling_ratio/mean": 0.9998113413651785, "sampling/importance_sampling_ratio/min": 0.45339525640010836, "sampling/sampling_logp_difference/max": 0.31308398246765134, "sampling/sampling_logp_difference/mean": 0.004005709810492893, "step": 17130, "step_time": 8.032414309601881, "student/entropy": 0.12880011362334093 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500001067916552, "completions/max_length": 486.1, "completions/max_terminated_length": 454.6333333333333, "completions/mean_length": 188.8425094604492, "completions/mean_terminated_length": 183.32730458577473, "completions/min_length": 55.166666666666664, "completions/min_terminated_length": 55.166666666666664, "entropy": 0.12512787009278933, "epoch": 0.6516538184027646, "eval/gt_acc_batch": 0.8175000270207723, "frac_reward_zero_std": 1.0, "grad_norm": 0.24812979996204376, "kd/policy_loss": 0.008056775713339449, "kd/rkl_advantage_mean": 0.709441336352999, "kd/rkl_advantage_p95": 4.9326574881871545, "kd/rkl_advantage_std": 2.214345129330953, "kd/ssd_loss": 0.0, "learning_rate": 3.48384156761478e-07, "loss": 0.03222710291544596, "num_tokens": 561584121.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8175000051657358, "rewards/gt_logging_reward/std": 0.3823355803887049, "sampling/importance_sampling_ratio/max": 1.8817971110343934, "sampling/importance_sampling_ratio/mean": 1.0015168885389965, "sampling/importance_sampling_ratio/min": 0.44526663223902385, "sampling/sampling_logp_difference/max": 0.375621239344279, "sampling/sampling_logp_difference/mean": 0.0038643741281703115, "step": 17160, "step_time": 7.939830409031129, "student/entropy": 0.12512787009278933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444565102458, "completions/max_length": 499.6, "completions/max_terminated_length": 469.46666666666664, "completions/mean_length": 192.77334238688152, "completions/mean_terminated_length": 185.7891591389974, "completions/min_length": 53.7, "completions/min_terminated_length": 53.7, "entropy": 0.12967084149519603, "epoch": 0.6527930733300421, "eval/gt_acc_batch": 0.7975000301996867, "frac_reward_zero_std": 1.0, "grad_norm": 0.3100634813308716, "kd/policy_loss": 0.008209549899523457, "kd/rkl_advantage_mean": 0.8336216099560261, "kd/rkl_advantage_p95": 5.1515682776769, "kd/rkl_advantage_std": 2.2405625651280086, "kd/ssd_loss": 0.0, "learning_rate": 3.472449018342004e-07, "loss": 0.03283819357554118, "num_tokens": 562584993.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7974999964237213, "rewards/gt_logging_reward/std": 0.3893975590666135, "sampling/importance_sampling_ratio/max": 1.8936812241872152, "sampling/importance_sampling_ratio/mean": 0.9946183045705159, "sampling/importance_sampling_ratio/min": 0.4260357399781545, "sampling/sampling_logp_difference/max": 0.3180759867032369, "sampling/sampling_logp_difference/mean": 0.003975346319687863, "step": 17190, "step_time": 8.159894287334948, "student/entropy": 0.12967084149519603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02527777897194028, "completions/max_length": 490.5, "completions/max_terminated_length": 450.43333333333334, "completions/mean_length": 184.88695475260417, "completions/mean_terminated_length": 176.6697738647461, "completions/min_length": 47.9, "completions/min_terminated_length": 47.9, "entropy": 0.12816196090231338, "epoch": 0.6539323282573197, "eval/gt_acc_batch": 0.8030555725097657, "frac_reward_zero_std": 1.0, "grad_norm": 0.39181390404701233, "kd/policy_loss": 0.008119095965715436, "kd/rkl_advantage_mean": 0.8203365982820591, "kd/rkl_advantage_p95": 5.122635807593664, "kd/rkl_advantage_std": 2.2617140690485638, "kd/ssd_loss": 0.0, "learning_rate": 3.4610564690692284e-07, "loss": 0.0324763834476471, "num_tokens": 563557042.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555546283722, "rewards/gt_logging_reward/std": 0.38667806386947634, "sampling/importance_sampling_ratio/max": 1.7836159467697144, "sampling/importance_sampling_ratio/mean": 0.9991901059945424, "sampling/importance_sampling_ratio/min": 0.4507124811410904, "sampling/sampling_logp_difference/max": 0.34797885616620383, "sampling/sampling_logp_difference/mean": 0.003964576971096297, "step": 17220, "step_time": 8.003491304906978, "student/entropy": 0.12816196090231338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02138888988023003, "completions/max_length": 482.9, "completions/max_terminated_length": 453.46666666666664, "completions/mean_length": 187.54806620279948, "completions/mean_terminated_length": 180.58749084472657, "completions/min_length": 46.9, "completions/min_terminated_length": 46.9, "entropy": 0.12660679935167232, "epoch": 0.6550715831845972, "eval/gt_acc_batch": 0.8147222479184468, "frac_reward_zero_std": 1.0, "grad_norm": 0.2492290884256363, "kd/policy_loss": 0.008193143360161532, "kd/rkl_advantage_mean": 0.6728134886361659, "kd/rkl_advantage_p95": 4.818406353394191, "kd/rkl_advantage_std": 2.173099362850189, "kd/ssd_loss": 0.0, "learning_rate": 3.4496639197964525e-07, "loss": 0.03277257283528646, "num_tokens": 564528575.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8147222181161244, "rewards/gt_logging_reward/std": 0.3841874122619629, "sampling/importance_sampling_ratio/max": 1.8976428707440693, "sampling/importance_sampling_ratio/mean": 1.0029410700003305, "sampling/importance_sampling_ratio/min": 0.4632793088754018, "sampling/sampling_logp_difference/max": 0.337332155307134, "sampling/sampling_logp_difference/mean": 0.003923756489530206, "step": 17250, "step_time": 7.93261963952779, "student/entropy": 0.12660679935167232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112156262, "completions/max_length": 485.6666666666667, "completions/max_terminated_length": 452.6, "completions/mean_length": 190.3930658976237, "completions/mean_terminated_length": 182.70277404785156, "completions/min_length": 54.733333333333334, "completions/min_terminated_length": 54.733333333333334, "entropy": 0.12307022170474131, "epoch": 0.6562108381118749, "eval/gt_acc_batch": 0.8147222439448039, "frac_reward_zero_std": 1.0, "grad_norm": 0.3426455855369568, "kd/policy_loss": 0.008079214851992826, "kd/rkl_advantage_mean": 0.7779317287107309, "kd/rkl_advantage_p95": 5.13288760582606, "kd/rkl_advantage_std": 2.256614649295807, "kd/ssd_loss": 0.0, "learning_rate": 3.438271370523677e-07, "loss": 0.03231686353683472, "num_tokens": 565513966.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8147222240765889, "rewards/gt_logging_reward/std": 0.38011477192242943, "sampling/importance_sampling_ratio/max": 1.8073400855064392, "sampling/importance_sampling_ratio/mean": 1.0021860977013906, "sampling/importance_sampling_ratio/min": 0.4736773798863093, "sampling/sampling_logp_difference/max": 0.3347885966300964, "sampling/sampling_logp_difference/mean": 0.0038397989235818388, "step": 17280, "step_time": 7.932388707695646, "student/entropy": 0.12307022170474131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055557515472174, "completions/max_length": 498.26666666666665, "completions/max_terminated_length": 457.3, "completions/mean_length": 186.28389790852864, "completions/mean_terminated_length": 176.88760172526042, "completions/min_length": 59.3, "completions/min_terminated_length": 59.3, "entropy": 0.12739157664279144, "epoch": 0.6573500930391524, "eval/gt_acc_batch": 0.8041666845480601, "frac_reward_zero_std": 1.0, "grad_norm": 0.26202431321144104, "kd/policy_loss": 0.008401907450752333, "kd/rkl_advantage_mean": 0.8452710379225512, "kd/rkl_advantage_p95": 5.186262981096903, "kd/rkl_advantage_std": 2.261862533291181, "kd/ssd_loss": 0.0, "learning_rate": 3.426878821250902e-07, "loss": 0.033607629934946696, "num_tokens": 566478956.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666646798452, "rewards/gt_logging_reward/std": 0.39313697020212807, "sampling/importance_sampling_ratio/max": 1.8522125442822774, "sampling/importance_sampling_ratio/mean": 0.9998597145080567, "sampling/importance_sampling_ratio/min": 0.41462819228569664, "sampling/sampling_logp_difference/max": 0.3409214456876119, "sampling/sampling_logp_difference/mean": 0.003935561732699474, "step": 17310, "step_time": 7.937692051599152, "student/entropy": 0.12739157664279144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01805555649722616, "completions/max_length": 463.06666666666666, "completions/max_terminated_length": 433.93333333333334, "completions/mean_length": 184.32056427001953, "completions/mean_terminated_length": 178.4252182006836, "completions/min_length": 56.7, "completions/min_terminated_length": 56.7, "entropy": 0.12021523397415876, "epoch": 0.65848934796643, "eval/gt_acc_batch": 0.8380555748939514, "frac_reward_zero_std": 1.0, "grad_norm": 0.2985924184322357, "kd/policy_loss": 0.007507918745977804, "kd/rkl_advantage_mean": 0.6578180702907654, "kd/rkl_advantage_p95": 4.787430010239283, "kd/rkl_advantage_std": 2.144655391573906, "kd/ssd_loss": 0.0, "learning_rate": 3.415486271978126e-07, "loss": 0.03003167708714803, "num_tokens": 567441918.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8380555609862009, "rewards/gt_logging_reward/std": 0.3616993298133214, "sampling/importance_sampling_ratio/max": 1.8850119471549989, "sampling/importance_sampling_ratio/mean": 0.9961894830067952, "sampling/importance_sampling_ratio/min": 0.4024794687827428, "sampling/sampling_logp_difference/max": 0.3355924646059672, "sampling/sampling_logp_difference/mean": 0.003739680373109877, "step": 17340, "step_time": 7.6842571621595805, "student/entropy": 0.12021523397415876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02694444628432393, "completions/max_length": 493.8666666666667, "completions/max_terminated_length": 458.46666666666664, "completions/mean_length": 190.57862091064453, "completions/mean_terminated_length": 181.90733388264974, "completions/min_length": 58.03333333333333, "completions/min_terminated_length": 58.03333333333333, "entropy": 0.12291164596875509, "epoch": 0.6596286028937075, "eval/gt_acc_batch": 0.8022222459316254, "frac_reward_zero_std": 1.0, "grad_norm": 0.1959981918334961, "kd/policy_loss": 0.008044641414502014, "kd/rkl_advantage_mean": 0.7724793929606676, "kd/rkl_advantage_p95": 5.11864776412646, "kd/rkl_advantage_std": 2.2524529536565145, "kd/ssd_loss": 0.0, "learning_rate": 3.404093722705351e-07, "loss": 0.03217856486638387, "num_tokens": 568429041.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8022222220897675, "rewards/gt_logging_reward/std": 0.39161916077136993, "sampling/importance_sampling_ratio/max": 2.0102009495099384, "sampling/importance_sampling_ratio/mean": 1.002364852031072, "sampling/importance_sampling_ratio/min": 0.4630582511425018, "sampling/sampling_logp_difference/max": 0.3216120441754659, "sampling/sampling_logp_difference/mean": 0.003817741402114431, "step": 17370, "step_time": 7.940266089568225, "student/entropy": 0.12291164596875509 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02277777884155512, "completions/max_length": 482.56666666666666, "completions/max_terminated_length": 444.03333333333336, "completions/mean_length": 184.4930669148763, "completions/mean_terminated_length": 176.88331502278646, "completions/min_length": 54.53333333333333, "completions/min_terminated_length": 54.53333333333333, "entropy": 0.1272055149078369, "epoch": 0.6607678578209851, "eval/gt_acc_batch": 0.8136111279328664, "frac_reward_zero_std": 1.0, "grad_norm": 0.2287134826183319, "kd/policy_loss": 0.008458635624265298, "kd/rkl_advantage_mean": 0.7212472068145871, "kd/rkl_advantage_p95": 4.909848507245382, "kd/rkl_advantage_std": 2.2212389171123506, "kd/ssd_loss": 0.0, "learning_rate": 3.392701173432575e-07, "loss": 0.033834540843963624, "num_tokens": 569398376.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813611110051473, "rewards/gt_logging_reward/std": 0.38213937332232795, "sampling/importance_sampling_ratio/max": 1.9362521370251973, "sampling/importance_sampling_ratio/mean": 1.0004419406255087, "sampling/importance_sampling_ratio/min": 0.46223601549863813, "sampling/sampling_logp_difference/max": 0.37812764644622804, "sampling/sampling_logp_difference/mean": 0.003940871551943322, "step": 17400, "step_time": 7.8952496529256075, "student/entropy": 0.1272055149078369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01750000109896064, "completions/max_length": 467.43333333333334, "completions/max_terminated_length": 439.3, "completions/mean_length": 184.19195353190105, "completions/mean_terminated_length": 178.53575083414714, "completions/min_length": 54.7, "completions/min_terminated_length": 54.7, "entropy": 0.12391647100448608, "epoch": 0.6619071127482626, "eval/gt_acc_batch": 0.8213889271020889, "frac_reward_zero_std": 1.0, "grad_norm": 0.29260334372520447, "kd/policy_loss": 0.007686402378991867, "kd/rkl_advantage_mean": 0.6604364017024636, "kd/rkl_advantage_p95": 4.822237334648768, "kd/rkl_advantage_std": 2.1723181307315826, "kd/ssd_loss": 0.0, "learning_rate": 3.381308624159799e-07, "loss": 0.03074561357498169, "num_tokens": 570366139.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8213888833920161, "rewards/gt_logging_reward/std": 0.3662651225924492, "sampling/importance_sampling_ratio/max": 1.8376686334609986, "sampling/importance_sampling_ratio/mean": 0.9948789397875468, "sampling/importance_sampling_ratio/min": 0.4434926142295202, "sampling/sampling_logp_difference/max": 0.3453893423080444, "sampling/sampling_logp_difference/mean": 0.003806181865123411, "step": 17430, "step_time": 7.843234439462928, "student/entropy": 0.12391647100448608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890283803145, "completions/max_length": 475.53333333333336, "completions/max_terminated_length": 449.1666666666667, "completions/mean_length": 183.92556610107422, "completions/mean_terminated_length": 177.04452412923177, "completions/min_length": 58.03333333333333, "completions/min_terminated_length": 58.03333333333333, "entropy": 0.12861754813541967, "epoch": 0.6630463676755401, "eval/gt_acc_batch": 0.804722249507904, "frac_reward_zero_std": 1.0, "grad_norm": 0.26947760581970215, "kd/policy_loss": 0.008294671089000378, "kd/rkl_advantage_mean": 0.7002710719282429, "kd/rkl_advantage_p95": 4.949443423748017, "kd/rkl_advantage_std": 2.189201246698697, "kd/ssd_loss": 0.0, "learning_rate": 3.369916074887024e-07, "loss": 0.0331786851088206, "num_tokens": 571330775.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222256660461, "rewards/gt_logging_reward/std": 0.38997937937577565, "sampling/importance_sampling_ratio/max": 1.8600077470143637, "sampling/importance_sampling_ratio/mean": 1.000118871529897, "sampling/importance_sampling_ratio/min": 0.48650990625222523, "sampling/sampling_logp_difference/max": 0.31823251247406004, "sampling/sampling_logp_difference/mean": 0.003964298738477131, "step": 17460, "step_time": 7.937490180737223, "student/entropy": 0.12861754813541967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001334895692, "completions/max_length": 494.8, "completions/max_terminated_length": 454.4, "completions/mean_length": 188.11528778076172, "completions/mean_terminated_length": 179.8384256998698, "completions/min_length": 52.9, "completions/min_terminated_length": 52.9, "entropy": 0.13024234250187874, "epoch": 0.6641856226028178, "eval/gt_acc_batch": 0.8066666821638743, "frac_reward_zero_std": 1.0, "grad_norm": 0.2412887066602707, "kd/policy_loss": 0.008754815525996188, "kd/rkl_advantage_mean": 0.7652453483392795, "kd/rkl_advantage_p95": 4.981199435393015, "kd/rkl_advantage_std": 2.1962982128063837, "kd/ssd_loss": 0.0, "learning_rate": 3.358523525614248e-07, "loss": 0.0350192666053772, "num_tokens": 572319078.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8066666702429454, "rewards/gt_logging_reward/std": 0.38653584371010463, "sampling/importance_sampling_ratio/max": 1.9506943027178447, "sampling/importance_sampling_ratio/mean": 1.0027129828929902, "sampling/importance_sampling_ratio/min": 0.4954705576101939, "sampling/sampling_logp_difference/max": 0.3265507996082306, "sampling/sampling_logp_difference/mean": 0.004002032754942775, "step": 17490, "step_time": 8.146531678866207, "student/entropy": 0.13024234250187874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777779027819633, "completions/max_length": 492.3333333333333, "completions/max_terminated_length": 441.9, "completions/mean_length": 181.60000864664713, "completions/mean_terminated_length": 174.14453786214193, "completions/min_length": 53.1, "completions/min_terminated_length": 53.1, "entropy": 0.12845517043024302, "epoch": 0.6653248775300953, "eval/gt_acc_batch": 0.8261111319065094, "frac_reward_zero_std": 1.0, "grad_norm": 0.4585038721561432, "kd/policy_loss": 0.008344640322805692, "kd/rkl_advantage_mean": 0.7679364129726309, "kd/rkl_advantage_p95": 5.108111164967219, "kd/rkl_advantage_std": 2.278198626637459, "kd/ssd_loss": 0.0, "learning_rate": 3.347130976341472e-07, "loss": 0.033378565311431886, "num_tokens": 573273638.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.826111110051473, "rewards/gt_logging_reward/std": 0.36905793895324074, "sampling/importance_sampling_ratio/max": 2.042654585838318, "sampling/importance_sampling_ratio/mean": 1.0005733370780945, "sampling/importance_sampling_ratio/min": 0.46291246712207795, "sampling/sampling_logp_difference/max": 0.3283746679623922, "sampling/sampling_logp_difference/mean": 0.003936907237706085, "step": 17520, "step_time": 7.962564455492733, "student/entropy": 0.12845517043024302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026666668181618055, "completions/max_length": 500.8, "completions/max_terminated_length": 447.8, "completions/mean_length": 190.62862243652344, "completions/mean_terminated_length": 181.86478678385416, "completions/min_length": 51.166666666666664, "completions/min_terminated_length": 51.166666666666664, "entropy": 0.13295833431184292, "epoch": 0.6664641324573729, "eval/gt_acc_batch": 0.7916666766007742, "frac_reward_zero_std": 1.0, "grad_norm": 0.3519384264945984, "kd/policy_loss": 0.008860076228544737, "kd/rkl_advantage_mean": 0.800262009104093, "kd/rkl_advantage_p95": 5.066384834051132, "kd/rkl_advantage_std": 2.2498834441105524, "kd/ssd_loss": 0.0, "learning_rate": 3.3357384270686964e-07, "loss": 0.03544031381607056, "num_tokens": 574262829.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7916666706403096, "rewards/gt_logging_reward/std": 0.40027037958304085, "sampling/importance_sampling_ratio/max": 1.8876842856407166, "sampling/importance_sampling_ratio/mean": 0.990189915895462, "sampling/importance_sampling_ratio/min": 0.39682947595914203, "sampling/sampling_logp_difference/max": 0.3215497016906738, "sampling/sampling_logp_difference/mean": 0.004083842597901821, "step": 17550, "step_time": 8.019523767268401, "student/entropy": 0.13295833431184292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889051352938, "completions/max_length": 490.0, "completions/max_terminated_length": 451.6333333333333, "completions/mean_length": 190.0511210123698, "completions/mean_terminated_length": 181.46961873372396, "completions/min_length": 53.96666666666667, "completions/min_terminated_length": 53.96666666666667, "entropy": 0.13090494039158027, "epoch": 0.6676033873846504, "eval/gt_acc_batch": 0.79277779062589, "frac_reward_zero_std": 1.0, "grad_norm": 0.24259047210216522, "kd/policy_loss": 0.008506533354132746, "kd/rkl_advantage_mean": 0.8372690501933296, "kd/rkl_advantage_p95": 5.053820435206095, "kd/rkl_advantage_std": 2.231183784206708, "kd/ssd_loss": 0.0, "learning_rate": 3.3243458777959217e-07, "loss": 0.034026137987772626, "num_tokens": 575247981.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7927777806917826, "rewards/gt_logging_reward/std": 0.398642701903979, "sampling/importance_sampling_ratio/max": 1.808309535185496, "sampling/importance_sampling_ratio/mean": 0.9883547385533651, "sampling/importance_sampling_ratio/min": 0.4439965049425761, "sampling/sampling_logp_difference/max": 0.31943644881248473, "sampling/sampling_logp_difference/mean": 0.0040115293969089786, "step": 17580, "step_time": 8.142879592701986, "student/entropy": 0.13090494039158027 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890221714973, "completions/max_length": 492.46666666666664, "completions/max_terminated_length": 451.9, "completions/mean_length": 187.41862080891926, "completions/mean_terminated_length": 180.32379760742188, "completions/min_length": 57.733333333333334, "completions/min_terminated_length": 57.733333333333334, "entropy": 0.127741383202374, "epoch": 0.668742642311928, "eval/gt_acc_batch": 0.8130555768807729, "frac_reward_zero_std": 1.0, "grad_norm": 0.23289693892002106, "kd/policy_loss": 0.008100119281637792, "kd/rkl_advantage_mean": 0.7520952455699443, "kd/rkl_advantage_p95": 4.958979135751724, "kd/rkl_advantage_std": 2.2088383555412294, "kd/ssd_loss": 0.0, "learning_rate": 3.312953328523146e-07, "loss": 0.03240047891934713, "num_tokens": 576230120.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8130555550257365, "rewards/gt_logging_reward/std": 0.38249687751134237, "sampling/importance_sampling_ratio/max": 1.8718886097272238, "sampling/importance_sampling_ratio/mean": 0.9942612111568451, "sampling/importance_sampling_ratio/min": 0.406381560365359, "sampling/sampling_logp_difference/max": 0.3465886195500692, "sampling/sampling_logp_difference/mean": 0.003957128711044788, "step": 17610, "step_time": 8.086330650762344, "student/entropy": 0.127741383202374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01750000100582838, "completions/max_length": 487.06666666666666, "completions/max_terminated_length": 452.3, "completions/mean_length": 179.71973215738933, "completions/mean_terminated_length": 173.82500508626302, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.1260816395903627, "epoch": 0.6698818972392055, "eval/gt_acc_batch": 0.8327778021494547, "frac_reward_zero_std": 1.0, "grad_norm": 0.3257884085178375, "kd/policy_loss": 0.00811724223700973, "kd/rkl_advantage_mean": 0.8224388501296441, "kd/rkl_advantage_p95": 5.134667722384135, "kd/rkl_advantage_std": 2.265880990028381, "kd/ssd_loss": 0.0, "learning_rate": 3.30156077925037e-07, "loss": 0.032468966643015545, "num_tokens": 577173455.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8327777763207753, "rewards/gt_logging_reward/std": 0.3665173803766569, "sampling/importance_sampling_ratio/max": 1.8417413552602133, "sampling/importance_sampling_ratio/mean": 0.9995017925898234, "sampling/importance_sampling_ratio/min": 0.4501650477449099, "sampling/sampling_logp_difference/max": 0.31654035250345863, "sampling/sampling_logp_difference/mean": 0.0038304586972420416, "step": 17640, "step_time": 7.8305025284372585, "student/entropy": 0.1260816395903627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028888890612870456, "completions/max_length": 495.06666666666666, "completions/max_terminated_length": 445.1666666666667, "completions/mean_length": 181.57084197998046, "completions/mean_terminated_length": 171.85578002929688, "completions/min_length": 49.333333333333336, "completions/min_terminated_length": 49.333333333333336, "entropy": 0.1256029965976874, "epoch": 0.6710211521664832, "eval/gt_acc_batch": 0.8119444608688354, "frac_reward_zero_std": 1.0, "grad_norm": 0.301008015871048, "kd/policy_loss": 0.008078069045829276, "kd/rkl_advantage_mean": 0.8618729834275048, "kd/rkl_advantage_p95": 5.155529946088791, "kd/rkl_advantage_std": 2.2501130123933155, "kd/ssd_loss": 0.0, "learning_rate": 3.2901682299775947e-07, "loss": 0.032312277952829996, "num_tokens": 578119862.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.811944446961085, "rewards/gt_logging_reward/std": 0.37728563447793323, "sampling/importance_sampling_ratio/max": 1.9106628537178039, "sampling/importance_sampling_ratio/mean": 0.9954687217871349, "sampling/importance_sampling_ratio/min": 0.41723570475975674, "sampling/sampling_logp_difference/max": 0.3137870629628499, "sampling/sampling_logp_difference/mean": 0.00384935907398661, "step": 17670, "step_time": 8.023740875794708, "student/entropy": 0.1256029965976874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02944444598009189, "completions/max_length": 495.6, "completions/max_terminated_length": 463.93333333333334, "completions/mean_length": 188.01751149495442, "completions/mean_terminated_length": 178.5489298502604, "completions/min_length": 59.46666666666667, "completions/min_terminated_length": 59.46666666666667, "entropy": 0.13582610028485456, "epoch": 0.6721604070937607, "eval/gt_acc_batch": 0.7941666881243388, "frac_reward_zero_std": 1.0, "grad_norm": 0.3141253888607025, "kd/policy_loss": 0.008911592460935935, "kd/rkl_advantage_mean": 0.7694632877906163, "kd/rkl_advantage_p95": 4.980625418821971, "kd/rkl_advantage_std": 2.2073015958070754, "kd/ssd_loss": 0.0, "learning_rate": 3.278775680704819e-07, "loss": 0.03564637104670207, "num_tokens": 579090677.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7941666722297669, "rewards/gt_logging_reward/std": 0.3942030747731527, "sampling/importance_sampling_ratio/max": 1.9174003680547078, "sampling/importance_sampling_ratio/mean": 1.0027616699536641, "sampling/importance_sampling_ratio/min": 0.4200414627790451, "sampling/sampling_logp_difference/max": 0.3492885649204254, "sampling/sampling_logp_difference/mean": 0.004133965959772468, "step": 17700, "step_time": 7.995640997001707, "student/entropy": 0.13582610028485456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667604198058, "completions/max_length": 481.03333333333336, "completions/max_terminated_length": 441.3333333333333, "completions/mean_length": 195.67639923095703, "completions/mean_terminated_length": 189.54854685465494, "completions/min_length": 60.43333333333333, "completions/min_terminated_length": 60.43333333333333, "entropy": 0.1291210301220417, "epoch": 0.6732996620210382, "eval/gt_acc_batch": 0.8036111334959666, "frac_reward_zero_std": 1.0, "grad_norm": 0.24315160512924194, "kd/policy_loss": 0.008430358967355762, "kd/rkl_advantage_mean": 0.7433569273600976, "kd/rkl_advantage_p95": 5.022708815336228, "kd/rkl_advantage_std": 2.2195707082748415, "kd/ssd_loss": 0.0, "learning_rate": 3.267383131432043e-07, "loss": 0.033721431096394854, "num_tokens": 580094472.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111076672872, "rewards/gt_logging_reward/std": 0.39150675336519875, "sampling/importance_sampling_ratio/max": 1.8727708498636881, "sampling/importance_sampling_ratio/mean": 0.9999435901641845, "sampling/importance_sampling_ratio/min": 0.474872417251269, "sampling/sampling_logp_difference/max": 0.3092583288749059, "sampling/sampling_logp_difference/mean": 0.0039841874114548165, "step": 17730, "step_time": 7.943550622064504, "student/entropy": 0.1291210301220417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888890029241642, "completions/max_length": 480.0, "completions/max_terminated_length": 453.1, "completions/mean_length": 187.85084279378256, "completions/mean_terminated_length": 181.81392720540364, "completions/min_length": 55.7, "completions/min_terminated_length": 55.7, "entropy": 0.12886380193134148, "epoch": 0.6744389169483158, "eval/gt_acc_batch": 0.8036111235618592, "frac_reward_zero_std": 1.0, "grad_norm": 0.27165961265563965, "kd/policy_loss": 0.00849630232163084, "kd/rkl_advantage_mean": 0.7642403537174687, "kd/rkl_advantage_p95": 5.006518101692199, "kd/rkl_advantage_std": 2.2279286036888757, "kd/ssd_loss": 0.0, "learning_rate": 3.255990582159268e-07, "loss": 0.03398520946502685, "num_tokens": 581075639.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111136277516, "rewards/gt_logging_reward/std": 0.38833606044451396, "sampling/importance_sampling_ratio/max": 1.8867033839225769, "sampling/importance_sampling_ratio/mean": 0.9968628287315369, "sampling/importance_sampling_ratio/min": 0.4064067636926969, "sampling/sampling_logp_difference/max": 0.327189177274704, "sampling/sampling_logp_difference/mean": 0.003958731726743281, "step": 17760, "step_time": 7.851146203500685, "student/entropy": 0.12886380193134148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02361111268401146, "completions/max_length": 490.46666666666664, "completions/max_terminated_length": 441.7, "completions/mean_length": 189.4336201985677, "completions/mean_terminated_length": 181.74365641276043, "completions/min_length": 54.56666666666667, "completions/min_terminated_length": 54.56666666666667, "entropy": 0.12563642027477423, "epoch": 0.6755781718755933, "eval/gt_acc_batch": 0.8052777846654257, "frac_reward_zero_std": 1.0, "grad_norm": 0.24795067310333252, "kd/policy_loss": 0.008076871250523254, "kd/rkl_advantage_mean": 0.847195222399508, "kd/rkl_advantage_p95": 5.172544811169306, "kd/rkl_advantage_std": 2.2653516153494517, "kd/ssd_loss": 0.0, "learning_rate": 3.244598032886492e-07, "loss": 0.03230748176574707, "num_tokens": 582063736.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8052777767181396, "rewards/gt_logging_reward/std": 0.3890812615553538, "sampling/importance_sampling_ratio/max": 1.8630433797836303, "sampling/importance_sampling_ratio/mean": 0.9973360737164815, "sampling/importance_sampling_ratio/min": 0.43950029611587527, "sampling/sampling_logp_difference/max": 0.31749341686566673, "sampling/sampling_logp_difference/mean": 0.0038571629983683428, "step": 17790, "step_time": 7.946709738534022, "student/entropy": 0.12563642027477423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028333335183560848, "completions/max_length": 484.4, "completions/max_terminated_length": 459.4, "completions/mean_length": 188.76167755126954, "completions/mean_terminated_length": 179.54772542317707, "completions/min_length": 45.06666666666667, "completions/min_terminated_length": 45.06666666666667, "entropy": 0.1328347146511078, "epoch": 0.676717426802871, "eval/gt_acc_batch": 0.7980555673440297, "frac_reward_zero_std": 1.0, "grad_norm": 0.26817095279693604, "kd/policy_loss": 0.008812753208136808, "kd/rkl_advantage_mean": 0.698044736062487, "kd/rkl_advantage_p95": 4.858969604969024, "kd/rkl_advantage_std": 2.2173392554124196, "kd/ssd_loss": 0.0, "learning_rate": 3.233205483613716e-07, "loss": 0.0352510134379069, "num_tokens": 583049494.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7980555554231008, "rewards/gt_logging_reward/std": 0.3944853434960047, "sampling/importance_sampling_ratio/max": 1.8414246519406636, "sampling/importance_sampling_ratio/mean": 0.997216937939326, "sampling/importance_sampling_ratio/min": 0.4437927315632502, "sampling/sampling_logp_difference/max": 0.33489974538485207, "sampling/sampling_logp_difference/mean": 0.004074512957595288, "step": 17820, "step_time": 8.03971927149687, "student/entropy": 0.1328347146511078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02555555688838164, "completions/max_length": 479.26666666666665, "completions/max_terminated_length": 441.1666666666667, "completions/mean_length": 187.09917755126952, "completions/mean_terminated_length": 178.61787923177084, "completions/min_length": 48.333333333333336, "completions/min_terminated_length": 48.333333333333336, "entropy": 0.1273341982314984, "epoch": 0.6778566817301485, "eval/gt_acc_batch": 0.7958333512147268, "frac_reward_zero_std": 1.0, "grad_norm": 0.3095136880874634, "kd/policy_loss": 0.008327373468394702, "kd/rkl_advantage_mean": 0.7624353275634348, "kd/rkl_advantage_p95": 5.011302584409714, "kd/rkl_advantage_std": 2.215672713518143, "kd/ssd_loss": 0.0, "learning_rate": 3.2218129343409414e-07, "loss": 0.03330949743588765, "num_tokens": 584026539.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7958333313465118, "rewards/gt_logging_reward/std": 0.3955939928690592, "sampling/importance_sampling_ratio/max": 1.9509803851445515, "sampling/importance_sampling_ratio/mean": 0.9963717103004456, "sampling/importance_sampling_ratio/min": 0.43224813615282376, "sampling/sampling_logp_difference/max": 0.31671832005182904, "sampling/sampling_logp_difference/mean": 0.003914232303698858, "step": 17850, "step_time": 7.873290835100731, "student/entropy": 0.1273341982314984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01694444523503383, "completions/max_length": 479.4, "completions/max_terminated_length": 442.9, "completions/mean_length": 184.20000966389975, "completions/mean_terminated_length": 178.5193364461263, "completions/min_length": 59.43333333333333, "completions/min_terminated_length": 59.43333333333333, "entropy": 0.1276683614899715, "epoch": 0.6789959366574261, "eval/gt_acc_batch": 0.8002777914206187, "frac_reward_zero_std": 1.0, "grad_norm": 0.31665438413619995, "kd/policy_loss": 0.00846880291743825, "kd/rkl_advantage_mean": 0.7790825539578994, "kd/rkl_advantage_p95": 5.007539200782776, "kd/rkl_advantage_std": 2.211745300889015, "kd/ssd_loss": 0.0, "learning_rate": 3.2104203850681655e-07, "loss": 0.03387520710627238, "num_tokens": 584985595.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8002777735392252, "rewards/gt_logging_reward/std": 0.38736618409554163, "sampling/importance_sampling_ratio/max": 1.995789150396983, "sampling/importance_sampling_ratio/mean": 1.007623302936554, "sampling/importance_sampling_ratio/min": 0.43800482352574666, "sampling/sampling_logp_difference/max": 0.3270734906196594, "sampling/sampling_logp_difference/mean": 0.0039473214186728, "step": 17880, "step_time": 7.911463196604745, "student/entropy": 0.1276683614899715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02472222357367476, "completions/max_length": 488.73333333333335, "completions/max_terminated_length": 446.96666666666664, "completions/mean_length": 187.4747319539388, "completions/mean_terminated_length": 179.40337320963542, "completions/min_length": 58.2, "completions/min_terminated_length": 58.2, "entropy": 0.12990355882793664, "epoch": 0.6801351915847036, "eval/gt_acc_batch": 0.8163889070351918, "frac_reward_zero_std": 1.0, "grad_norm": 0.45883074402809143, "kd/policy_loss": 0.008337611479995151, "kd/rkl_advantage_mean": 0.812320146523416, "kd/rkl_advantage_p95": 5.132954478263855, "kd/rkl_advantage_std": 2.2567496170600254, "kd/ssd_loss": 0.0, "learning_rate": 3.1990278357953897e-07, "loss": 0.03335044781366984, "num_tokens": 585963952.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8163888891537984, "rewards/gt_logging_reward/std": 0.38127917846043907, "sampling/importance_sampling_ratio/max": 1.9115824182828267, "sampling/importance_sampling_ratio/mean": 0.9972885489463806, "sampling/importance_sampling_ratio/min": 0.43443761169910433, "sampling/sampling_logp_difference/max": 0.3232491175333659, "sampling/sampling_logp_difference/mean": 0.00402447241358459, "step": 17910, "step_time": 7.961873410265738, "student/entropy": 0.12990355882793664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556584149598, "completions/max_length": 472.26666666666665, "completions/max_terminated_length": 437.4, "completions/mean_length": 186.0219528198242, "completions/mean_terminated_length": 180.986528523763, "completions/min_length": 50.733333333333334, "completions/min_terminated_length": 50.733333333333334, "entropy": 0.12586873341351748, "epoch": 0.6812744465119812, "eval/gt_acc_batch": 0.7975000301996867, "frac_reward_zero_std": 1.0, "grad_norm": 0.22059522569179535, "kd/policy_loss": 0.008376707745871197, "kd/rkl_advantage_mean": 0.6987958053592592, "kd/rkl_advantage_p95": 4.8451431731383, "kd/rkl_advantage_std": 2.1673806617657343, "kd/ssd_loss": 0.0, "learning_rate": 3.187635286522614e-07, "loss": 0.03350683450698853, "num_tokens": 586943623.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7975000003973644, "rewards/gt_logging_reward/std": 0.393998991449674, "sampling/importance_sampling_ratio/max": 1.8826738317807516, "sampling/importance_sampling_ratio/mean": 0.9981965343157451, "sampling/importance_sampling_ratio/min": 0.4577892005443573, "sampling/sampling_logp_difference/max": 0.2787577311197917, "sampling/sampling_logp_difference/mean": 0.0038746320564920704, "step": 17940, "step_time": 7.868905318831094, "student/entropy": 0.12586873341351748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02000000107412537, "completions/max_length": 483.96666666666664, "completions/max_terminated_length": 431.0, "completions/mean_length": 186.6716761271159, "completions/mean_terminated_length": 180.1531992594401, "completions/min_length": 51.43333333333333, "completions/min_terminated_length": 51.43333333333333, "entropy": 0.12898040575285752, "epoch": 0.6824137014392587, "eval/gt_acc_batch": 0.8141666968663533, "frac_reward_zero_std": 1.0, "grad_norm": 0.25929930806159973, "kd/policy_loss": 0.008538192180761446, "kd/rkl_advantage_mean": 0.7483343536655108, "kd/rkl_advantage_p95": 5.078054487705231, "kd/rkl_advantage_std": 2.2675636768341065, "kd/ssd_loss": 0.0, "learning_rate": 3.1762427372498386e-07, "loss": 0.034152770042419435, "num_tokens": 587915505.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.814166667064031, "rewards/gt_logging_reward/std": 0.3828497757514318, "sampling/importance_sampling_ratio/max": 1.8483790596326193, "sampling/importance_sampling_ratio/mean": 1.002558410167694, "sampling/importance_sampling_ratio/min": 0.4654486685991287, "sampling/sampling_logp_difference/max": 0.30356703599294027, "sampling/sampling_logp_difference/mean": 0.003995450166985393, "step": 17970, "step_time": 7.946609368337279, "student/entropy": 0.12898040575285752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02138889009753863, "completions/max_length": 495.93333333333334, "completions/max_terminated_length": 459.73333333333335, "completions/mean_length": 187.5788996378581, "completions/mean_terminated_length": 180.5467519124349, "completions/min_length": 48.266666666666666, "completions/min_terminated_length": 48.266666666666666, "entropy": 0.13041285661359628, "epoch": 0.6835529563665362, "eval/gt_acc_batch": 0.807500014702479, "frac_reward_zero_std": 1.0, "grad_norm": 0.25505396723747253, "kd/policy_loss": 0.008540194052814816, "kd/rkl_advantage_mean": 0.8163311689471205, "kd/rkl_advantage_p95": 5.127550554275513, "kd/rkl_advantage_std": 2.2722776492436725, "kd/ssd_loss": 0.0, "learning_rate": 3.164850187977063e-07, "loss": 0.03416076898574829, "num_tokens": 588902909.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8075000007947286, "rewards/gt_logging_reward/std": 0.3871312712629636, "sampling/importance_sampling_ratio/max": 1.916427528858185, "sampling/importance_sampling_ratio/mean": 0.9972895920276642, "sampling/importance_sampling_ratio/min": 0.45244673391183216, "sampling/sampling_logp_difference/max": 0.3035268763701121, "sampling/sampling_logp_difference/mean": 0.003981121652759612, "step": 18000, "step_time": 8.257267534701775, "student/entropy": 0.13041285661359628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779220292965, "completions/max_length": 492.2, "completions/max_terminated_length": 449.3666666666667, "completions/mean_length": 185.27278645833334, "completions/mean_terminated_length": 177.00454813639323, "completions/min_length": 54.4, "completions/min_terminated_length": 54.4, "entropy": 0.13612093056241673, "epoch": 0.6846922112938139, "eval/gt_acc_batch": 0.7966666877269745, "frac_reward_zero_std": 1.0, "grad_norm": 0.3508550226688385, "kd/policy_loss": 0.009112033044220879, "kd/rkl_advantage_mean": 0.8423595079531272, "kd/rkl_advantage_p95": 5.182273242870966, "kd/rkl_advantage_std": 2.26228499909242, "kd/ssd_loss": 0.0, "learning_rate": 3.153457638704287e-07, "loss": 0.036448129018147785, "num_tokens": 589863291.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7966666698455811, "rewards/gt_logging_reward/std": 0.3900836298863093, "sampling/importance_sampling_ratio/max": 1.9809180736541747, "sampling/importance_sampling_ratio/mean": 1.0028761307398477, "sampling/importance_sampling_ratio/min": 0.4263595978418986, "sampling/sampling_logp_difference/max": 0.31735443870226543, "sampling/sampling_logp_difference/mean": 0.004101817108069857, "step": 18030, "step_time": 7.969392490602331, "student/entropy": 0.13612093056241673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444571932157, "completions/max_length": 482.3, "completions/max_terminated_length": 457.5, "completions/mean_length": 191.6047332763672, "completions/mean_terminated_length": 183.73922729492188, "completions/min_length": 51.43333333333333, "completions/min_terminated_length": 51.43333333333333, "entropy": 0.1245240855962038, "epoch": 0.6858314662210914, "eval/gt_acc_batch": 0.7952778081099192, "frac_reward_zero_std": 1.0, "grad_norm": 0.2773171663284302, "kd/policy_loss": 0.00806598345710275, "kd/rkl_advantage_mean": 0.769089749113967, "kd/rkl_advantage_p95": 4.931274106105168, "kd/rkl_advantage_std": 2.1733868777751923, "kd/ssd_loss": 0.0, "learning_rate": 3.1420650894315117e-07, "loss": 0.032263930638631186, "num_tokens": 590860468.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777723471324, "rewards/gt_logging_reward/std": 0.39235001703103384, "sampling/importance_sampling_ratio/max": 1.9071621457735697, "sampling/importance_sampling_ratio/mean": 0.9987006743748983, "sampling/importance_sampling_ratio/min": 0.45650686224301656, "sampling/sampling_logp_difference/max": 0.3229703962802887, "sampling/sampling_logp_difference/mean": 0.00384234170584629, "step": 18060, "step_time": 8.15610914560578, "student/entropy": 0.1245240855962038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388889787097774, "completions/max_length": 489.4, "completions/max_terminated_length": 447.1333333333333, "completions/mean_length": 189.81778767903646, "completions/mean_terminated_length": 182.89275410970052, "completions/min_length": 50.833333333333336, "completions/min_terminated_length": 50.833333333333336, "entropy": 0.126194712271293, "epoch": 0.686970721148369, "eval/gt_acc_batch": 0.7936111350854238, "frac_reward_zero_std": 1.0, "grad_norm": 0.3101235628128052, "kd/policy_loss": 0.008460897182036813, "kd/rkl_advantage_mean": 0.7577066428959369, "kd/rkl_advantage_p95": 4.963706485430399, "kd/rkl_advantage_std": 2.208876476685206, "kd/ssd_loss": 0.0, "learning_rate": 3.130672540158736e-07, "loss": 0.033843588829040525, "num_tokens": 591858756.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111132303874, "rewards/gt_logging_reward/std": 0.39424096842606865, "sampling/importance_sampling_ratio/max": 1.953086825211843, "sampling/importance_sampling_ratio/mean": 0.9994511743386586, "sampling/importance_sampling_ratio/min": 0.4649672736724218, "sampling/sampling_logp_difference/max": 0.3292308966318766, "sampling/sampling_logp_difference/mean": 0.0038776816024134555, "step": 18090, "step_time": 8.18072950176235, "student/entropy": 0.126194712271293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0100000006146729, "completions/max_length": 468.56666666666666, "completions/max_terminated_length": 443.1, "completions/mean_length": 179.47917785644532, "completions/mean_terminated_length": 176.17530619303386, "completions/min_length": 62.36666666666667, "completions/min_terminated_length": 62.36666666666667, "entropy": 0.13196246325969696, "epoch": 0.6881099760756465, "eval/gt_acc_batch": 0.8144444723924001, "frac_reward_zero_std": 1.0, "grad_norm": 0.23561455309391022, "kd/policy_loss": 0.008443762305735921, "kd/rkl_advantage_mean": 0.7640486222381393, "kd/rkl_advantage_p95": 5.045073366165161, "kd/rkl_advantage_std": 2.2376035432020824, "kd/ssd_loss": 0.0, "learning_rate": 3.11927999088596e-07, "loss": 0.0337750514348348, "num_tokens": 592803241.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8144444465637207, "rewards/gt_logging_reward/std": 0.38380595048268634, "sampling/importance_sampling_ratio/max": 1.866903058687846, "sampling/importance_sampling_ratio/mean": 0.9974591275056203, "sampling/importance_sampling_ratio/min": 0.4490860124429067, "sampling/sampling_logp_difference/max": 0.3206980268160502, "sampling/sampling_logp_difference/mean": 0.004080821884175142, "step": 18120, "step_time": 7.840722847765816, "student/entropy": 0.13196246325969696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890283803145, "completions/max_length": 482.8, "completions/max_terminated_length": 446.23333333333335, "completions/mean_length": 189.53834431966146, "completions/mean_terminated_length": 182.65403696695964, "completions/min_length": 51.266666666666666, "completions/min_terminated_length": 51.266666666666666, "entropy": 0.12737260311841964, "epoch": 0.6892492310029241, "eval/gt_acc_batch": 0.7991666853427887, "frac_reward_zero_std": 1.0, "grad_norm": 0.3646838963031769, "kd/policy_loss": 0.008553746433851, "kd/rkl_advantage_mean": 0.7762618708851127, "kd/rkl_advantage_p95": 5.111948704719543, "kd/rkl_advantage_std": 2.248295947909355, "kd/ssd_loss": 0.0, "learning_rate": 3.107887441613185e-07, "loss": 0.034214981396993, "num_tokens": 593786811.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7991666694482168, "rewards/gt_logging_reward/std": 0.3915697505076726, "sampling/importance_sampling_ratio/max": 1.948603904247284, "sampling/importance_sampling_ratio/mean": 0.9995978891849517, "sampling/importance_sampling_ratio/min": 0.46480485796928406, "sampling/sampling_logp_difference/max": 0.35054067373275755, "sampling/sampling_logp_difference/mean": 0.003951631983121236, "step": 18150, "step_time": 7.887799388097482, "student/entropy": 0.12737260311841964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016944445483386517, "completions/max_length": 482.1, "completions/max_terminated_length": 462.93333333333334, "completions/mean_length": 191.54806772867838, "completions/mean_terminated_length": 186.09014841715495, "completions/min_length": 48.833333333333336, "completions/min_terminated_length": 48.833333333333336, "entropy": 0.13261126037687063, "epoch": 0.6903884859302016, "eval/gt_acc_batch": 0.7861111263434092, "frac_reward_zero_std": 1.0, "grad_norm": 0.3227193057537079, "kd/policy_loss": 0.009115590118259813, "kd/rkl_advantage_mean": 0.6986501305053632, "kd/rkl_advantage_p95": 4.9062960088253025, "kd/rkl_advantage_std": 2.1996230045954386, "kd/ssd_loss": 0.0, "learning_rate": 3.0964948923404094e-07, "loss": 0.036462366580963135, "num_tokens": 594785592.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111144224803, "rewards/gt_logging_reward/std": 0.40417528450489043, "sampling/importance_sampling_ratio/max": 1.9665724754333496, "sampling/importance_sampling_ratio/mean": 1.005012478431066, "sampling/importance_sampling_ratio/min": 0.4638899366060893, "sampling/sampling_logp_difference/max": 0.3497488498687744, "sampling/sampling_logp_difference/mean": 0.00411854285436372, "step": 18180, "step_time": 7.984861002365748, "student/entropy": 0.13261126037687063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944445539265872, "completions/max_length": 471.1, "completions/max_terminated_length": 446.5, "completions/mean_length": 192.70528818766277, "completions/mean_terminated_length": 183.98812052408854, "completions/min_length": 57.266666666666666, "completions/min_terminated_length": 57.266666666666666, "entropy": 0.1292762207488219, "epoch": 0.6915277408574793, "eval/gt_acc_batch": 0.7858333508173625, "frac_reward_zero_std": 1.0, "grad_norm": 0.22560624778270721, "kd/policy_loss": 0.008570722960090886, "kd/rkl_advantage_mean": 0.7268507499868672, "kd/rkl_advantage_p95": 4.936966741085053, "kd/rkl_advantage_std": 2.1818471868832905, "kd/ssd_loss": 0.0, "learning_rate": 3.0851023430676336e-07, "loss": 0.034282898902893065, "num_tokens": 595774907.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7858333309491475, "rewards/gt_logging_reward/std": 0.4002948120236397, "sampling/importance_sampling_ratio/max": 1.87258491118749, "sampling/importance_sampling_ratio/mean": 0.9989691098531087, "sampling/importance_sampling_ratio/min": 0.44489890038967134, "sampling/sampling_logp_difference/max": 0.3068312029043833, "sampling/sampling_logp_difference/mean": 0.003979794300782184, "step": 18210, "step_time": 7.911307927898209, "student/entropy": 0.1292762207488219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02305555638546745, "completions/max_length": 490.53333333333336, "completions/max_terminated_length": 446.4, "completions/mean_length": 187.82417704264324, "completions/mean_terminated_length": 180.14046478271484, "completions/min_length": 61.96666666666667, "completions/min_terminated_length": 61.96666666666667, "entropy": 0.1282666942725579, "epoch": 0.6926669957847568, "eval/gt_acc_batch": 0.8011111299196879, "frac_reward_zero_std": 1.0, "grad_norm": 0.26365894079208374, "kd/policy_loss": 0.008515885472297669, "kd/rkl_advantage_mean": 0.7651928345983227, "kd/rkl_advantage_p95": 5.047714086373647, "kd/rkl_advantage_std": 2.237474959095319, "kd/ssd_loss": 0.0, "learning_rate": 3.0737097937948583e-07, "loss": 0.034063541889190675, "num_tokens": 596756042.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8011111120382944, "rewards/gt_logging_reward/std": 0.3891572400927544, "sampling/importance_sampling_ratio/max": 1.9272836089134215, "sampling/importance_sampling_ratio/mean": 0.9976880351702372, "sampling/importance_sampling_ratio/min": 0.45717196762561796, "sampling/sampling_logp_difference/max": 0.3149083495140076, "sampling/sampling_logp_difference/mean": 0.003947656229138374, "step": 18240, "step_time": 8.085744847095338, "student/entropy": 0.1282666942725579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0269444459117949, "completions/max_length": 495.3, "completions/max_terminated_length": 448.8, "completions/mean_length": 189.26639862060546, "completions/mean_terminated_length": 180.30506591796876, "completions/min_length": 50.6, "completions/min_terminated_length": 50.6, "entropy": 0.12943279383083184, "epoch": 0.6938062507120343, "eval/gt_acc_batch": 0.8138888875643412, "frac_reward_zero_std": 1.0, "grad_norm": 0.409175306558609, "kd/policy_loss": 0.008587579252586389, "kd/rkl_advantage_mean": 0.8263406481749068, "kd/rkl_advantage_p95": 5.0353517214457195, "kd/rkl_advantage_std": 2.2363624374071756, "kd/ssd_loss": 0.0, "learning_rate": 3.0623172445220825e-07, "loss": 0.03435031970342, "num_tokens": 597737969.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8138888895511627, "rewards/gt_logging_reward/std": 0.3789021243651708, "sampling/importance_sampling_ratio/max": 1.9318847378094992, "sampling/importance_sampling_ratio/mean": 1.000699096918106, "sampling/importance_sampling_ratio/min": 0.4502990633249283, "sampling/sampling_logp_difference/max": 0.3312809189160665, "sampling/sampling_logp_difference/mean": 0.003978108917362988, "step": 18270, "step_time": 8.075375041169657, "student/entropy": 0.12943279383083184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223387410244, "completions/max_length": 474.7, "completions/max_terminated_length": 434.76666666666665, "completions/mean_length": 185.9627888997396, "completions/mean_terminated_length": 177.73510131835937, "completions/min_length": 57.36666666666667, "completions/min_terminated_length": 57.36666666666667, "entropy": 0.1315958693002661, "epoch": 0.6949455056393119, "eval/gt_acc_batch": 0.7936111231644948, "frac_reward_zero_std": 1.0, "grad_norm": 0.2774946987628937, "kd/policy_loss": 0.008677724758551145, "kd/rkl_advantage_mean": 0.6989152099005878, "kd/rkl_advantage_p95": 4.906930540005366, "kd/rkl_advantage_std": 2.2174929777781167, "kd/ssd_loss": 0.0, "learning_rate": 3.0509246952493067e-07, "loss": 0.034710899988810225, "num_tokens": 598717803.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111132303874, "rewards/gt_logging_reward/std": 0.39885537773370744, "sampling/importance_sampling_ratio/max": 1.8834928035736085, "sampling/importance_sampling_ratio/mean": 0.9972959180672963, "sampling/importance_sampling_ratio/min": 0.4221644302209218, "sampling/sampling_logp_difference/max": 0.34836281538009645, "sampling/sampling_logp_difference/mean": 0.004001338826492428, "step": 18300, "step_time": 8.091076636928483, "student/entropy": 0.1315958693002661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01916666782150666, "completions/max_length": 466.43333333333334, "completions/max_terminated_length": 427.3666666666667, "completions/mean_length": 184.68111928304037, "completions/mean_terminated_length": 178.45380859375, "completions/min_length": 53.333333333333336, "completions/min_terminated_length": 53.333333333333336, "entropy": 0.12554976145426433, "epoch": 0.6960847605665894, "eval/gt_acc_batch": 0.8102778017520904, "frac_reward_zero_std": 1.0, "grad_norm": 0.2703593969345093, "kd/policy_loss": 0.008044734443925942, "kd/rkl_advantage_mean": 0.7189149659126997, "kd/rkl_advantage_p95": 4.897382819652558, "kd/rkl_advantage_std": 2.1929639448722202, "kd/ssd_loss": 0.0, "learning_rate": 3.039532145976531e-07, "loss": 0.03217894236246745, "num_tokens": 599694919.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8102777798970541, "rewards/gt_logging_reward/std": 0.38523093461990354, "sampling/importance_sampling_ratio/max": 1.8721401969591775, "sampling/importance_sampling_ratio/mean": 0.994475652774175, "sampling/importance_sampling_ratio/min": 0.46803710410992305, "sampling/sampling_logp_difference/max": 0.3130121290683746, "sampling/sampling_logp_difference/mean": 0.003861436798858146, "step": 18330, "step_time": 8.048193770062062, "student/entropy": 0.12554976145426433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112150053183, "completions/max_length": 491.53333333333336, "completions/max_terminated_length": 455.6333333333333, "completions/mean_length": 193.2144551595052, "completions/mean_terminated_length": 186.54857635498047, "completions/min_length": 68.3, "completions/min_terminated_length": 68.3, "entropy": 0.12933761949340503, "epoch": 0.697224015493867, "eval/gt_acc_batch": 0.7947222411632537, "frac_reward_zero_std": 1.0, "grad_norm": 0.5516051650047302, "kd/policy_loss": 0.008925543913695341, "kd/rkl_advantage_mean": 0.7979816662768523, "kd/rkl_advantage_p95": 5.079685970147451, "kd/rkl_advantage_std": 2.2260403553644816, "kd/ssd_loss": 0.0, "learning_rate": 3.0281395967037556e-07, "loss": 0.03570217688878377, "num_tokens": 600690475.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222193082174, "rewards/gt_logging_reward/std": 0.3942178308963776, "sampling/importance_sampling_ratio/max": 1.936139424641927, "sampling/importance_sampling_ratio/mean": 0.9973698814709981, "sampling/importance_sampling_ratio/min": 0.4176129331191381, "sampling/sampling_logp_difference/max": 0.3905489524205526, "sampling/sampling_logp_difference/mean": 0.0040051479591056705, "step": 18360, "step_time": 8.016814453700015, "student/entropy": 0.12933761949340503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779065072538, "completions/max_length": 477.53333333333336, "completions/max_terminated_length": 448.56666666666666, "completions/mean_length": 186.31389770507812, "completions/mean_terminated_length": 178.13377431233724, "completions/min_length": 58.233333333333334, "completions/min_terminated_length": 58.233333333333334, "entropy": 0.12646811020871004, "epoch": 0.6983632704211445, "eval/gt_acc_batch": 0.8013889054457347, "frac_reward_zero_std": 1.0, "grad_norm": 0.35066571831703186, "kd/policy_loss": 0.008118312124861405, "kd/rkl_advantage_mean": 0.8019596740603447, "kd/rkl_advantage_p95": 5.090736709038416, "kd/rkl_advantage_std": 2.251822625597318, "kd/ssd_loss": 0.0, "learning_rate": 3.0167470474309797e-07, "loss": 0.032473246256510414, "num_tokens": 601664925.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888895511627, "rewards/gt_logging_reward/std": 0.3846681584914525, "sampling/importance_sampling_ratio/max": 1.8872914075851441, "sampling/importance_sampling_ratio/mean": 1.000750050942103, "sampling/importance_sampling_ratio/min": 0.4506725122531255, "sampling/sampling_logp_difference/max": 0.3208147803942362, "sampling/sampling_logp_difference/mean": 0.003907567247127494, "step": 18390, "step_time": 7.933137268692372, "student/entropy": 0.12646811020871004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02472222372889519, "completions/max_length": 498.8333333333333, "completions/max_terminated_length": 447.03333333333336, "completions/mean_length": 192.03473154703775, "completions/mean_terminated_length": 183.96815643310546, "completions/min_length": 54.86666666666667, "completions/min_terminated_length": 54.86666666666667, "entropy": 0.13556651708980402, "epoch": 0.6995025253484222, "eval/gt_acc_batch": 0.7788889169692993, "frac_reward_zero_std": 1.0, "grad_norm": 0.24275439977645874, "kd/policy_loss": 0.009344894105258088, "kd/rkl_advantage_mean": 0.9469573999444644, "kd/rkl_advantage_p95": 5.400123810768127, "kd/rkl_advantage_std": 2.326257519920667, "kd/ssd_loss": 0.0, "learning_rate": 3.0053544981582044e-07, "loss": 0.0373795747756958, "num_tokens": 602674386.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7788888871669769, "rewards/gt_logging_reward/std": 0.4094826857248942, "sampling/importance_sampling_ratio/max": 1.9177453756332397, "sampling/importance_sampling_ratio/mean": 1.003491848707199, "sampling/importance_sampling_ratio/min": 0.47103525400161744, "sampling/sampling_logp_difference/max": 0.35701558788617455, "sampling/sampling_logp_difference/mean": 0.004133848768348495, "step": 18420, "step_time": 8.422095310839358, "student/entropy": 0.13556651708980402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01833333441366752, "completions/max_length": 492.0, "completions/max_terminated_length": 449.1666666666667, "completions/mean_length": 184.79445241292316, "completions/mean_terminated_length": 178.81987965901692, "completions/min_length": 55.333333333333336, "completions/min_terminated_length": 55.333333333333336, "entropy": 0.12976668663322927, "epoch": 0.7006417802756997, "eval/gt_acc_batch": 0.8116666833559673, "frac_reward_zero_std": 1.0, "grad_norm": 0.19064480066299438, "kd/policy_loss": 0.00818750433002909, "kd/rkl_advantage_mean": 0.8144569670160612, "kd/rkl_advantage_p95": 5.0740978936354315, "kd/rkl_advantage_std": 2.2395573715368906, "kd/ssd_loss": 0.0, "learning_rate": 2.993961948885429e-07, "loss": 0.032750014464060465, "num_tokens": 603643062.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8116666674613953, "rewards/gt_logging_reward/std": 0.3838582346836726, "sampling/importance_sampling_ratio/max": 1.8573846260706584, "sampling/importance_sampling_ratio/mean": 1.0007722099622092, "sampling/importance_sampling_ratio/min": 0.4489586422840754, "sampling/sampling_logp_difference/max": 0.32277405659357705, "sampling/sampling_logp_difference/mean": 0.0039819115307182075, "step": 18450, "step_time": 8.155393820236592, "student/entropy": 0.12976668663322927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03166666865969698, "completions/max_length": 500.46666666666664, "completions/max_terminated_length": 455.8, "completions/mean_length": 189.86362050374348, "completions/mean_terminated_length": 179.49260711669922, "completions/min_length": 52.333333333333336, "completions/min_terminated_length": 52.333333333333336, "entropy": 0.12812492387990157, "epoch": 0.7017810352029773, "eval/gt_acc_batch": 0.7822222292423249, "frac_reward_zero_std": 1.0, "grad_norm": 0.2484414130449295, "kd/policy_loss": 0.00853080946059587, "kd/rkl_advantage_mean": 0.8934882594272494, "kd/rkl_advantage_p95": 5.1840221385161085, "kd/rkl_advantage_std": 2.2489487419525784, "kd/ssd_loss": 0.0, "learning_rate": 2.9825693996126533e-07, "loss": 0.03412323792775472, "num_tokens": 604632027.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7822222193082173, "rewards/gt_logging_reward/std": 0.40344332754611967, "sampling/importance_sampling_ratio/max": 1.9275755167007447, "sampling/importance_sampling_ratio/mean": 0.9969559113184611, "sampling/importance_sampling_ratio/min": 0.43207543790340425, "sampling/sampling_logp_difference/max": 0.32221940557161965, "sampling/sampling_logp_difference/mean": 0.0038992984918877482, "step": 18480, "step_time": 8.170352719763953, "student/entropy": 0.12812492387990157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223468124866, "completions/max_length": 493.2, "completions/max_terminated_length": 450.2, "completions/mean_length": 180.8238972981771, "completions/mean_terminated_length": 174.27778065999348, "completions/min_length": 53.333333333333336, "completions/min_terminated_length": 53.333333333333336, "entropy": 0.12932220920920373, "epoch": 0.7029202901302548, "eval/gt_acc_batch": 0.8027777949968974, "frac_reward_zero_std": 1.0, "grad_norm": 0.3711436986923218, "kd/policy_loss": 0.00817838727380149, "kd/rkl_advantage_mean": 0.7520150609935323, "kd/rkl_advantage_p95": 4.985353815555572, "kd/rkl_advantage_std": 2.212083871165911, "kd/ssd_loss": 0.0, "learning_rate": 2.9711768503398775e-07, "loss": 0.03271355231602987, "num_tokens": 605592721.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8027777791023254, "rewards/gt_logging_reward/std": 0.38588288525740305, "sampling/importance_sampling_ratio/max": 1.7852956732114156, "sampling/importance_sampling_ratio/mean": 0.9960488061110179, "sampling/importance_sampling_ratio/min": 0.4664416174093882, "sampling/sampling_logp_difference/max": 0.31902802785237633, "sampling/sampling_logp_difference/mean": 0.003971459033588569, "step": 18510, "step_time": 8.14868825719847, "student/entropy": 0.12932220920920373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055557298163572, "completions/max_length": 492.23333333333335, "completions/max_terminated_length": 451.46666666666664, "completions/mean_length": 191.56251068115233, "completions/mean_terminated_length": 182.4316899617513, "completions/min_length": 54.86666666666667, "completions/min_terminated_length": 54.86666666666667, "entropy": 0.13414881179730098, "epoch": 0.7040595450575323, "eval/gt_acc_batch": 0.7952777922153473, "frac_reward_zero_std": 1.0, "grad_norm": 0.34319522976875305, "kd/policy_loss": 0.008842326467856765, "kd/rkl_advantage_mean": 0.8037486221641302, "kd/rkl_advantage_p95": 5.101621939738592, "kd/rkl_advantage_std": 2.2420857548713684, "kd/ssd_loss": 0.0, "learning_rate": 2.959784301067102e-07, "loss": 0.03536931276321411, "num_tokens": 606588770.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777743339539, "rewards/gt_logging_reward/std": 0.3961900889873505, "sampling/importance_sampling_ratio/max": 1.8794790426890056, "sampling/importance_sampling_ratio/mean": 0.9971237679322561, "sampling/importance_sampling_ratio/min": 0.45948495815197626, "sampling/sampling_logp_difference/max": 0.3388438562552134, "sampling/sampling_logp_difference/mean": 0.004106915928423405, "step": 18540, "step_time": 8.14916591863827, "student/entropy": 0.13414881179730098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556875964005, "completions/max_length": 483.7, "completions/max_terminated_length": 450.5, "completions/mean_length": 182.25167643229167, "completions/mean_terminated_length": 175.31168263753256, "completions/min_length": 43.266666666666666, "completions/min_terminated_length": 43.266666666666666, "entropy": 0.12910201642662286, "epoch": 0.7051987999848099, "eval/gt_acc_batch": 0.7994444727897644, "frac_reward_zero_std": 1.0, "grad_norm": 0.3604404628276825, "kd/policy_loss": 0.008293058729032054, "kd/rkl_advantage_mean": 0.8595464533815781, "kd/rkl_advantage_p95": 5.2220585187276205, "kd/rkl_advantage_std": 2.274797679980596, "kd/ssd_loss": 0.0, "learning_rate": 2.9483917517943264e-07, "loss": 0.03317224184672038, "num_tokens": 607555940.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7994444449742635, "rewards/gt_logging_reward/std": 0.3926816691954931, "sampling/importance_sampling_ratio/max": 1.836533002058665, "sampling/importance_sampling_ratio/mean": 1.0025392770767212, "sampling/importance_sampling_ratio/min": 0.408458536863327, "sampling/sampling_logp_difference/max": 0.32733068664868675, "sampling/sampling_logp_difference/mean": 0.003943121107295155, "step": 18570, "step_time": 8.205209675896914, "student/entropy": 0.12910201642662286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02305555666486422, "completions/max_length": 490.1333333333333, "completions/max_terminated_length": 452.1333333333333, "completions/mean_length": 184.77862091064452, "completions/mean_terminated_length": 176.99105682373047, "completions/min_length": 57.4, "completions/min_terminated_length": 57.4, "entropy": 0.1270523158212503, "epoch": 0.7063380549120875, "eval/gt_acc_batch": 0.8108333508173625, "frac_reward_zero_std": 1.0, "grad_norm": 0.38210341334342957, "kd/policy_loss": 0.008407066892444467, "kd/rkl_advantage_mean": 0.792829422528545, "kd/rkl_advantage_p95": 5.054631618658702, "kd/rkl_advantage_std": 2.22312508225441, "kd/ssd_loss": 0.0, "learning_rate": 2.9369992025215506e-07, "loss": 0.03362826903661092, "num_tokens": 608522495.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8108333289623261, "rewards/gt_logging_reward/std": 0.3892966320117315, "sampling/importance_sampling_ratio/max": 1.9868064363797506, "sampling/importance_sampling_ratio/mean": 1.001477736234665, "sampling/importance_sampling_ratio/min": 0.4532947967449824, "sampling/sampling_logp_difference/max": 0.31704949140548705, "sampling/sampling_logp_difference/mean": 0.003910982678644359, "step": 18600, "step_time": 8.06083397672628, "student/entropy": 0.1270523158212503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03111111276472608, "completions/max_length": 492.1, "completions/max_terminated_length": 433.73333333333335, "completions/mean_length": 185.83334401448568, "completions/mean_terminated_length": 175.3925013224284, "completions/min_length": 56.333333333333336, "completions/min_terminated_length": 56.333333333333336, "entropy": 0.12957003855456908, "epoch": 0.7074773098393651, "eval/gt_acc_batch": 0.8138889054457347, "frac_reward_zero_std": 1.0, "grad_norm": 0.30980634689331055, "kd/policy_loss": 0.008473196000947307, "kd/rkl_advantage_mean": 0.8185330686469873, "kd/rkl_advantage_p95": 5.123369429508845, "kd/rkl_advantage_std": 2.2605281730492908, "kd/ssd_loss": 0.0, "learning_rate": 2.9256066532487747e-07, "loss": 0.03389278650283813, "num_tokens": 609487983.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8138888935248058, "rewards/gt_logging_reward/std": 0.3765319590767225, "sampling/importance_sampling_ratio/max": 1.911739722887675, "sampling/importance_sampling_ratio/mean": 0.9965421557426453, "sampling/importance_sampling_ratio/min": 0.4455436944961548, "sampling/sampling_logp_difference/max": 0.3370644966761271, "sampling/sampling_logp_difference/mean": 0.003933078943130871, "step": 18630, "step_time": 8.13633809129921, "student/entropy": 0.12957003855456908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014722223207354546, "completions/max_length": 480.93333333333334, "completions/max_terminated_length": 439.1666666666667, "completions/mean_length": 187.73000946044922, "completions/mean_terminated_length": 182.9732630411784, "completions/min_length": 54.8, "completions/min_terminated_length": 54.8, "entropy": 0.13046239241957663, "epoch": 0.7086165647666426, "eval/gt_acc_batch": 0.7822222371896108, "frac_reward_zero_std": 1.0, "grad_norm": 0.33178022503852844, "kd/policy_loss": 0.008701390627538785, "kd/rkl_advantage_mean": 0.8307648420333862, "kd/rkl_advantage_p95": 5.046725553274155, "kd/rkl_advantage_std": 2.2208530684312184, "kd/ssd_loss": 0.0, "learning_rate": 2.9142141039759994e-07, "loss": 0.03480556011199951, "num_tokens": 610465251.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7822222193082173, "rewards/gt_logging_reward/std": 0.4049077779054642, "sampling/importance_sampling_ratio/max": 2.023377617200216, "sampling/importance_sampling_ratio/mean": 1.003094643354416, "sampling/importance_sampling_ratio/min": 0.47097368637720743, "sampling/sampling_logp_difference/max": 0.34797490934530895, "sampling/sampling_logp_difference/mean": 0.003945659101009369, "step": 18660, "step_time": 8.05546142287009, "student/entropy": 0.13046239241957663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112305273613, "completions/max_length": 493.53333333333336, "completions/max_terminated_length": 433.5, "completions/mean_length": 177.9100102742513, "completions/mean_terminated_length": 170.84434509277344, "completions/min_length": 59.43333333333333, "completions/min_terminated_length": 59.43333333333333, "entropy": 0.12462754286825657, "epoch": 0.7097558196939202, "eval/gt_acc_batch": 0.8141666849454244, "frac_reward_zero_std": 1.0, "grad_norm": 0.2696957588195801, "kd/policy_loss": 0.008103168156230823, "kd/rkl_advantage_mean": 0.790746673506995, "kd/rkl_advantage_p95": 5.004683154821396, "kd/rkl_advantage_std": 2.228380317489306, "kd/ssd_loss": 0.0, "learning_rate": 2.902821554703224e-07, "loss": 0.03241267204284668, "num_tokens": 611399623.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8141666650772095, "rewards/gt_logging_reward/std": 0.3812244971593221, "sampling/importance_sampling_ratio/max": 1.9128605365753173, "sampling/importance_sampling_ratio/mean": 1.0031359374523163, "sampling/importance_sampling_ratio/min": 0.4329400186737378, "sampling/sampling_logp_difference/max": 0.31673192779223125, "sampling/sampling_logp_difference/mean": 0.0038473621166000764, "step": 18690, "step_time": 7.918726723168705, "student/entropy": 0.12462754286825657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01333333421498537, "completions/max_length": 469.8, "completions/max_terminated_length": 445.73333333333335, "completions/mean_length": 183.21084238688152, "completions/mean_terminated_length": 178.8848424275716, "completions/min_length": 62.6, "completions/min_terminated_length": 62.6, "entropy": 0.12514458366980155, "epoch": 0.7108950746211977, "eval/gt_acc_batch": 0.7986111283302307, "frac_reward_zero_std": 1.0, "grad_norm": 0.1981181800365448, "kd/policy_loss": 0.008164219639729709, "kd/rkl_advantage_mean": 0.6318269823988278, "kd/rkl_advantage_p95": 4.650505977869034, "kd/rkl_advantage_std": 2.136773830652237, "kd/ssd_loss": 0.0, "learning_rate": 2.8914290054304483e-07, "loss": 0.032656880219777425, "num_tokens": 612357110.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111164093017, "rewards/gt_logging_reward/std": 0.39082243541876477, "sampling/importance_sampling_ratio/max": 1.8958666284879049, "sampling/importance_sampling_ratio/mean": 0.997365798552831, "sampling/importance_sampling_ratio/min": 0.4414369732141495, "sampling/sampling_logp_difference/max": 0.3165182371934255, "sampling/sampling_logp_difference/mean": 0.0038914022501558066, "step": 18720, "step_time": 7.783124997171884, "student/entropy": 0.12514458366980155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02305555666486422, "completions/max_length": 492.26666666666665, "completions/max_terminated_length": 454.76666666666665, "completions/mean_length": 180.53695271809895, "completions/mean_terminated_length": 172.75641276041668, "completions/min_length": 45.1, "completions/min_terminated_length": 45.1, "entropy": 0.13477735910564662, "epoch": 0.7120343295484753, "eval/gt_acc_batch": 0.8066666901111603, "frac_reward_zero_std": 1.0, "grad_norm": 0.3227863609790802, "kd/policy_loss": 0.008505434593341002, "kd/rkl_advantage_mean": 0.8152099175999562, "kd/rkl_advantage_p95": 5.114200355609258, "kd/rkl_advantage_std": 2.284795719385147, "kd/ssd_loss": 0.0, "learning_rate": 2.880036456157673e-07, "loss": 0.03402174313863118, "num_tokens": 613318547.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8066666702429454, "rewards/gt_logging_reward/std": 0.39032358129819233, "sampling/importance_sampling_ratio/max": 1.8816378394762674, "sampling/importance_sampling_ratio/mean": 0.9994307418664297, "sampling/importance_sampling_ratio/min": 0.45114853531122207, "sampling/sampling_logp_difference/max": 0.292666886250178, "sampling/sampling_logp_difference/mean": 0.0038967344366634886, "step": 18750, "step_time": 8.062442385536269, "student/entropy": 0.13477735910564662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333454405268, "completions/max_length": 472.9, "completions/max_terminated_length": 448.1333333333333, "completions/mean_length": 188.7452865600586, "completions/mean_terminated_length": 182.1093729654948, "completions/min_length": 56.2, "completions/min_terminated_length": 56.2, "entropy": 0.1276249962548415, "epoch": 0.7131735844757529, "eval/gt_acc_batch": 0.7980555812517802, "frac_reward_zero_std": 1.0, "grad_norm": 0.27223095297813416, "kd/policy_loss": 0.00844787698976385, "kd/rkl_advantage_mean": 0.7390564364691575, "kd/rkl_advantage_p95": 4.908010822534561, "kd/rkl_advantage_std": 2.1594148268302282, "kd/ssd_loss": 0.0, "learning_rate": 2.868643906884897e-07, "loss": 0.033791510264078777, "num_tokens": 614296246.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7980555514494578, "rewards/gt_logging_reward/std": 0.3897443945209185, "sampling/importance_sampling_ratio/max": 1.9309635996818542, "sampling/importance_sampling_ratio/mean": 1.003533265988032, "sampling/importance_sampling_ratio/min": 0.4657454987366994, "sampling/sampling_logp_difference/max": 0.3028684695561727, "sampling/sampling_logp_difference/mean": 0.003919799307671686, "step": 18780, "step_time": 7.997021963363902, "student/entropy": 0.1276249962548415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02694444606701533, "completions/max_length": 485.7, "completions/max_terminated_length": 456.1333333333333, "completions/mean_length": 190.79973297119142, "completions/mean_terminated_length": 182.16696370442708, "completions/min_length": 47.833333333333336, "completions/min_terminated_length": 47.833333333333336, "entropy": 0.13260359621296328, "epoch": 0.7143128394030304, "eval/gt_acc_batch": 0.7833333532015483, "frac_reward_zero_std": 1.0, "grad_norm": 0.2590438425540924, "kd/policy_loss": 0.008870318803625802, "kd/rkl_advantage_mean": 0.7966099987427394, "kd/rkl_advantage_p95": 5.080254073937734, "kd/rkl_advantage_std": 2.2536922067403795, "kd/ssd_loss": 0.0, "learning_rate": 2.8572513576121214e-07, "loss": 0.03548127810160319, "num_tokens": 615299197.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7833333293596904, "rewards/gt_logging_reward/std": 0.4032714645067851, "sampling/importance_sampling_ratio/max": 1.98001473347346, "sampling/importance_sampling_ratio/mean": 1.003070205450058, "sampling/importance_sampling_ratio/min": 0.462103167672952, "sampling/sampling_logp_difference/max": 0.34302109479904175, "sampling/sampling_logp_difference/mean": 0.004037820198573172, "step": 18810, "step_time": 8.259865591799219, "student/entropy": 0.13260359621296328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03583333566784859, "completions/max_length": 505.96666666666664, "completions/max_terminated_length": 458.8666666666667, "completions/mean_length": 192.21556599934897, "completions/mean_terminated_length": 180.47741139729817, "completions/min_length": 55.3, "completions/min_terminated_length": 55.3, "entropy": 0.12838153758396706, "epoch": 0.715452094330308, "eval/gt_acc_batch": 0.799444454908371, "frac_reward_zero_std": 1.0, "grad_norm": 0.3584807813167572, "kd/policy_loss": 0.008528082284222667, "kd/rkl_advantage_mean": 0.7936316055556139, "kd/rkl_advantage_p95": 5.129116360346476, "kd/rkl_advantage_std": 2.2658176998297375, "kd/ssd_loss": 0.0, "learning_rate": 2.845858808339346e-07, "loss": 0.03411232630411784, "num_tokens": 616289661.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.799444442987442, "rewards/gt_logging_reward/std": 0.3923456440369288, "sampling/importance_sampling_ratio/max": 1.9391244411468507, "sampling/importance_sampling_ratio/mean": 0.9995732545852661, "sampling/importance_sampling_ratio/min": 0.43518434166908265, "sampling/sampling_logp_difference/max": 0.325477659702301, "sampling/sampling_logp_difference/mean": 0.003898715685742597, "step": 18840, "step_time": 8.2121027410952, "student/entropy": 0.12838153758396706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02388889022792379, "completions/max_length": 489.43333333333334, "completions/max_terminated_length": 438.0, "completions/mean_length": 189.53973185221355, "completions/mean_terminated_length": 181.6225336710612, "completions/min_length": 56.1, "completions/min_terminated_length": 56.1, "entropy": 0.12492978939165672, "epoch": 0.7165913492575855, "eval/gt_acc_batch": 0.8058333535989125, "frac_reward_zero_std": 1.0, "grad_norm": 0.3185071349143982, "kd/policy_loss": 0.007842431634586925, "kd/rkl_advantage_mean": 0.8254496946930885, "kd/rkl_advantage_p95": 5.119173242648443, "kd/rkl_advantage_std": 2.2329300304253894, "kd/ssd_loss": 0.0, "learning_rate": 2.83446625906657e-07, "loss": 0.03136972188949585, "num_tokens": 617275940.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8058333377043406, "rewards/gt_logging_reward/std": 0.3841863547762235, "sampling/importance_sampling_ratio/max": 1.9304044922192891, "sampling/importance_sampling_ratio/mean": 1.0006747742493947, "sampling/importance_sampling_ratio/min": 0.4503905822833379, "sampling/sampling_logp_difference/max": 0.3059377352396647, "sampling/sampling_logp_difference/mean": 0.003828979500879844, "step": 18870, "step_time": 8.13756590153207, "student/entropy": 0.12492978939165672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777779307216407, "completions/max_length": 487.73333333333335, "completions/max_terminated_length": 453.53333333333336, "completions/mean_length": 186.90306498209637, "completions/mean_terminated_length": 179.29770100911458, "completions/min_length": 55.733333333333334, "completions/min_terminated_length": 55.733333333333334, "entropy": 0.12325595089544852, "epoch": 0.7177306041848631, "eval/gt_acc_batch": 0.8061111450195313, "frac_reward_zero_std": 1.0, "grad_norm": 0.25789549946784973, "kd/policy_loss": 0.008228989958297461, "kd/rkl_advantage_mean": 0.7814322414807975, "kd/rkl_advantage_p95": 5.042327606678009, "kd/rkl_advantage_std": 2.2370506087938944, "kd/ssd_loss": 0.0, "learning_rate": 2.8230737097937944e-07, "loss": 0.03291596372922261, "num_tokens": 618255327.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8061111092567443, "rewards/gt_logging_reward/std": 0.3886197497447332, "sampling/importance_sampling_ratio/max": 1.8504010756810507, "sampling/importance_sampling_ratio/mean": 0.9934372584025065, "sampling/importance_sampling_ratio/min": 0.43379037578900653, "sampling/sampling_logp_difference/max": 0.33207580049832663, "sampling/sampling_logp_difference/mean": 0.00377567324321717, "step": 18900, "step_time": 8.166459698903298, "student/entropy": 0.12325595089544852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026666668492058912, "completions/max_length": 486.6, "completions/max_terminated_length": 453.2, "completions/mean_length": 189.51862080891928, "completions/mean_terminated_length": 180.69070536295573, "completions/min_length": 48.233333333333334, "completions/min_terminated_length": 48.233333333333334, "entropy": 0.13149923322101434, "epoch": 0.7188698591121406, "eval/gt_acc_batch": 0.7886111438274384, "frac_reward_zero_std": 1.0, "grad_norm": 0.31630635261535645, "kd/policy_loss": 0.00903317701498357, "kd/rkl_advantage_mean": 0.7808940712362528, "kd/rkl_advantage_p95": 5.026230070988337, "kd/rkl_advantage_std": 2.2258971919616064, "kd/ssd_loss": 0.0, "learning_rate": 2.811681160521019e-07, "loss": 0.03613270918528239, "num_tokens": 619246842.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7886111080646515, "rewards/gt_logging_reward/std": 0.39939035177230836, "sampling/importance_sampling_ratio/max": 1.9848928292592367, "sampling/importance_sampling_ratio/mean": 1.004356865088145, "sampling/importance_sampling_ratio/min": 0.4334350248177846, "sampling/sampling_logp_difference/max": 0.32934614419937136, "sampling/sampling_logp_difference/mean": 0.003987930668517947, "step": 18930, "step_time": 8.205049830965192, "student/entropy": 0.13149923322101434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334835867087, "completions/max_length": 492.4, "completions/max_terminated_length": 451.03333333333336, "completions/mean_length": 189.304177347819, "completions/mean_terminated_length": 180.7540730794271, "completions/min_length": 57.833333333333336, "completions/min_terminated_length": 57.833333333333336, "entropy": 0.12802238712708155, "epoch": 0.7200091140394183, "eval/gt_acc_batch": 0.8077777942021688, "frac_reward_zero_std": 1.0, "grad_norm": 0.2956252694129944, "kd/policy_loss": 0.00838618869893253, "kd/rkl_advantage_mean": 0.825469506594042, "kd/rkl_advantage_p95": 5.062785450617472, "kd/rkl_advantage_std": 2.2340629398822784, "kd/ssd_loss": 0.0, "learning_rate": 2.800288611248244e-07, "loss": 0.03354475498199463, "num_tokens": 620224449.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8077777743339538, "rewards/gt_logging_reward/std": 0.3875200092792511, "sampling/importance_sampling_ratio/max": 1.9556353489557903, "sampling/importance_sampling_ratio/mean": 1.0024316608905792, "sampling/importance_sampling_ratio/min": 0.45567113558451333, "sampling/sampling_logp_difference/max": 0.3246176858743032, "sampling/sampling_logp_difference/mean": 0.0039634844986721875, "step": 18960, "step_time": 7.991382958001728, "student/entropy": 0.12802238712708155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778680125872, "completions/max_length": 485.6333333333333, "completions/max_terminated_length": 438.96666666666664, "completions/mean_length": 184.6269536336263, "completions/mean_terminated_length": 177.86489969889323, "completions/min_length": 54.36666666666667, "completions/min_terminated_length": 54.36666666666667, "entropy": 0.12631419083724418, "epoch": 0.7211483689666958, "eval/gt_acc_batch": 0.8161111255486806, "frac_reward_zero_std": 1.0, "grad_norm": 0.26834091544151306, "kd/policy_loss": 0.0079883647384122, "kd/rkl_advantage_mean": 0.7527104582637548, "kd/rkl_advantage_p95": 4.93022270600001, "kd/rkl_advantage_std": 2.200076950589816, "kd/ssd_loss": 0.0, "learning_rate": 2.788896061975468e-07, "loss": 0.031953457991282144, "num_tokens": 621186274.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8161111156145732, "rewards/gt_logging_reward/std": 0.38258205552895863, "sampling/importance_sampling_ratio/max": 1.8114596803983052, "sampling/importance_sampling_ratio/mean": 0.9981013258298238, "sampling/importance_sampling_ratio/min": 0.47251108090082805, "sampling/sampling_logp_difference/max": 0.30775660077730815, "sampling/sampling_logp_difference/mean": 0.0038826532584304613, "step": 18990, "step_time": 8.040193271801884, "student/entropy": 0.12631419083724418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112497746944, "completions/max_length": 498.96666666666664, "completions/max_terminated_length": 461.8333333333333, "completions/mean_length": 190.69167734781902, "completions/mean_terminated_length": 182.9242935180664, "completions/min_length": 56.96666666666667, "completions/min_terminated_length": 56.96666666666667, "entropy": 0.12708753732343514, "epoch": 0.7222876238939734, "eval/gt_acc_batch": 0.797222246726354, "frac_reward_zero_std": 1.0, "grad_norm": 0.24790042638778687, "kd/policy_loss": 0.00854038818506524, "kd/rkl_advantage_mean": 0.7310247719443093, "kd/rkl_advantage_p95": 4.959832914670309, "kd/rkl_advantage_std": 2.200832423567772, "kd/ssd_loss": 0.0, "learning_rate": 2.777503512702692e-07, "loss": 0.03416155576705933, "num_tokens": 622180892.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7972222228844961, "rewards/gt_logging_reward/std": 0.39433205425739287, "sampling/importance_sampling_ratio/max": 1.904612406094869, "sampling/importance_sampling_ratio/mean": 1.004942035675049, "sampling/importance_sampling_ratio/min": 0.4705708851416906, "sampling/sampling_logp_difference/max": 0.3227923889954885, "sampling/sampling_logp_difference/mean": 0.0038692069705575705, "step": 19020, "step_time": 8.128762692765061, "student/entropy": 0.12708753732343514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667641450962, "completions/max_length": 487.96666666666664, "completions/max_terminated_length": 454.6, "completions/mean_length": 191.6055658976237, "completions/mean_terminated_length": 184.59213460286458, "completions/min_length": 46.7, "completions/min_terminated_length": 46.7, "entropy": 0.13156115698317686, "epoch": 0.7234268788212509, "eval/gt_acc_batch": 0.783611128727595, "frac_reward_zero_std": 1.0, "grad_norm": 0.27806273102760315, "kd/policy_loss": 0.008740294222176696, "kd/rkl_advantage_mean": 0.8223446052211026, "kd/rkl_advantage_p95": 5.094746828079224, "kd/rkl_advantage_std": 2.2412268936634065, "kd/ssd_loss": 0.0, "learning_rate": 2.766110963429917e-07, "loss": 0.0349611759185791, "num_tokens": 623187664.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7836111148198446, "rewards/gt_logging_reward/std": 0.40551029741764066, "sampling/importance_sampling_ratio/max": 1.8870457649230956, "sampling/importance_sampling_ratio/mean": 1.0015461405118307, "sampling/importance_sampling_ratio/min": 0.43552455902099607, "sampling/sampling_logp_difference/max": 0.3065626154343287, "sampling/sampling_logp_difference/mean": 0.00402178760462751, "step": 19050, "step_time": 8.26893453420295, "student/entropy": 0.13156115698317686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778959522645, "completions/max_length": 490.3, "completions/max_terminated_length": 455.3666666666667, "completions/mean_length": 186.85306498209636, "completions/mean_terminated_length": 180.2141881306966, "completions/min_length": 53.333333333333336, "completions/min_terminated_length": 53.333333333333336, "entropy": 0.13149737535665432, "epoch": 0.7245661337485284, "eval/gt_acc_batch": 0.8100000202655793, "frac_reward_zero_std": 1.0, "grad_norm": 0.36305731534957886, "kd/policy_loss": 0.00878937779343687, "kd/rkl_advantage_mean": 0.7868192375792812, "kd/rkl_advantage_p95": 4.974683066209157, "kd/rkl_advantage_std": 2.2175570358832677, "kd/ssd_loss": 0.0, "learning_rate": 2.754718414157141e-07, "loss": 0.035157513618469236, "num_tokens": 624165951.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8100000023841858, "rewards/gt_logging_reward/std": 0.385286819934845, "sampling/importance_sampling_ratio/max": 2.0065584341684977, "sampling/importance_sampling_ratio/mean": 1.0079496184984842, "sampling/importance_sampling_ratio/min": 0.477982164422671, "sampling/sampling_logp_difference/max": 0.3064212123552958, "sampling/sampling_logp_difference/mean": 0.004017945557522277, "step": 19080, "step_time": 8.072730235867978, "student/entropy": 0.13149737535665432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666816920042, "completions/max_length": 479.8666666666667, "completions/max_terminated_length": 428.3, "completions/mean_length": 184.008620707194, "completions/mean_terminated_length": 176.95087381998698, "completions/min_length": 58.233333333333334, "completions/min_terminated_length": 58.233333333333334, "entropy": 0.13473638215412695, "epoch": 0.725705388675806, "eval/gt_acc_batch": 0.7941666801770528, "frac_reward_zero_std": 1.0, "grad_norm": 0.2604370415210724, "kd/policy_loss": 0.008903717984988664, "kd/rkl_advantage_mean": 0.7919267082897326, "kd/rkl_advantage_p95": 5.022930471102397, "kd/rkl_advantage_std": 2.1952030271291734, "kd/ssd_loss": 0.0, "learning_rate": 2.743325864884365e-07, "loss": 0.03561487197875977, "num_tokens": 625126926.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7941666682561238, "rewards/gt_logging_reward/std": 0.3958113143841426, "sampling/importance_sampling_ratio/max": 1.9125473300615947, "sampling/importance_sampling_ratio/mean": 1.0062893827756245, "sampling/importance_sampling_ratio/min": 0.4518770967920621, "sampling/sampling_logp_difference/max": 0.31133819818496705, "sampling/sampling_logp_difference/mean": 0.004126173739011089, "step": 19110, "step_time": 7.9330710192007245, "student/entropy": 0.13473638215412695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223474333683, "completions/max_length": 486.3666666666667, "completions/max_terminated_length": 447.03333333333336, "completions/mean_length": 187.34500986735026, "completions/mean_terminated_length": 180.0403055826823, "completions/min_length": 52.63333333333333, "completions/min_terminated_length": 52.63333333333333, "entropy": 0.12583524299164614, "epoch": 0.7268446436030835, "eval/gt_acc_batch": 0.8100000242392222, "frac_reward_zero_std": 1.0, "grad_norm": 0.277184396982193, "kd/policy_loss": 0.0081218947423622, "kd/rkl_advantage_mean": 0.7302921103934447, "kd/rkl_advantage_p95": 4.979685960213343, "kd/rkl_advantage_std": 2.2320472339789075, "kd/ssd_loss": 0.0, "learning_rate": 2.73193331561159e-07, "loss": 0.03248758316040039, "num_tokens": 626109848.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8100000023841858, "rewards/gt_logging_reward/std": 0.38400597472985587, "sampling/importance_sampling_ratio/max": 1.8010416547457377, "sampling/importance_sampling_ratio/mean": 1.0017314811547597, "sampling/importance_sampling_ratio/min": 0.4824485868215561, "sampling/sampling_logp_difference/max": 0.33777461846669515, "sampling/sampling_logp_difference/mean": 0.003825637154902021, "step": 19140, "step_time": 8.055420617297447, "student/entropy": 0.12583524299164614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667573153974, "completions/max_length": 473.06666666666666, "completions/max_terminated_length": 434.6, "completions/mean_length": 180.18889872233072, "completions/mean_terminated_length": 173.82629801432293, "completions/min_length": 52.266666666666666, "completions/min_terminated_length": 52.266666666666666, "entropy": 0.12961737333486478, "epoch": 0.7279838985303612, "eval/gt_acc_batch": 0.8111111323038737, "frac_reward_zero_std": 1.0, "grad_norm": 0.3378889560699463, "kd/policy_loss": 0.008563372966212532, "kd/rkl_advantage_mean": 0.7370686170955499, "kd/rkl_advantage_p95": 4.972315973043441, "kd/rkl_advantage_std": 2.218448344866435, "kd/ssd_loss": 0.0, "learning_rate": 2.720540766338814e-07, "loss": 0.03425349394480388, "num_tokens": 627058544.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8111111124356588, "rewards/gt_logging_reward/std": 0.38053431262572607, "sampling/importance_sampling_ratio/max": 1.8748007933298747, "sampling/importance_sampling_ratio/mean": 1.0007248123486836, "sampling/importance_sampling_ratio/min": 0.43037160535653435, "sampling/sampling_logp_difference/max": 0.32828500668207805, "sampling/sampling_logp_difference/mean": 0.003980298472257952, "step": 19170, "step_time": 7.904598762301612, "student/entropy": 0.12961737333486478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017500000881652038, "completions/max_length": 487.1666666666667, "completions/max_terminated_length": 442.6333333333333, "completions/mean_length": 189.279177347819, "completions/mean_terminated_length": 183.5934844970703, "completions/min_length": 55.5, "completions/min_terminated_length": 55.5, "entropy": 0.12582531521717708, "epoch": 0.7291231534576387, "eval/gt_acc_batch": 0.8188888986905416, "frac_reward_zero_std": 1.0, "grad_norm": 0.30019691586494446, "kd/policy_loss": 0.008307304477784782, "kd/rkl_advantage_mean": 0.7365399579207103, "kd/rkl_advantage_p95": 4.931020029385885, "kd/rkl_advantage_std": 2.18638577858607, "kd/ssd_loss": 0.0, "learning_rate": 2.7091482170660383e-07, "loss": 0.033229213953018186, "num_tokens": 628033277.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8188888907432557, "rewards/gt_logging_reward/std": 0.3786193976799647, "sampling/importance_sampling_ratio/max": 1.8907293796539306, "sampling/importance_sampling_ratio/mean": 1.0033932030200958, "sampling/importance_sampling_ratio/min": 0.44517294863859813, "sampling/sampling_logp_difference/max": 0.33348788420359293, "sampling/sampling_logp_difference/mean": 0.0038342515705153345, "step": 19200, "step_time": 7.915141133273331, "student/entropy": 0.12582531521717708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02183908179145435, "completions/max_length": 467.0, "completions/max_terminated_length": 423.51724137931035, "completions/mean_length": 183.6396642224542, "completions/mean_terminated_length": 176.42342297784214, "completions/min_length": 56.55172413793103, "completions/min_terminated_length": 56.55172413793103, "entropy": 0.13032252219473495, "epoch": 0.7302624083849163, "eval/gt_acc_batch": 0.8000000221975918, "frac_reward_zero_std": 1.0, "grad_norm": 0.34196704626083374, "kd/policy_loss": 0.008231781555564496, "kd/rkl_advantage_mean": 0.806012436747551, "kd/rkl_advantage_p95": 4.985342181962112, "kd/rkl_advantage_std": 2.1841452573907785, "kd/ssd_loss": 0.0, "learning_rate": 2.6977556677932636e-07, "loss": 0.031829559803009035, "num_tokens": 628959751.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7999999995889335, "rewards/gt_logging_reward/std": 0.3913719690051572, "sampling/importance_sampling_ratio/max": 1.979108530899574, "sampling/importance_sampling_ratio/mean": 1.0015781316263923, "sampling/importance_sampling_ratio/min": 0.4514777095153414, "sampling/sampling_logp_difference/max": 0.3490932193295709, "sampling/sampling_logp_difference/mean": 0.004045210329109225, "step": 19230, "step_time": 7.560036902467255, "student/entropy": 0.13032252219473495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030555557242284218, "completions/max_length": 497.1, "completions/max_terminated_length": 450.76666666666665, "completions/mean_length": 188.30167592366536, "completions/mean_terminated_length": 178.19794413248698, "completions/min_length": 54.7, "completions/min_terminated_length": 54.7, "entropy": 0.13210447101543346, "epoch": 0.7314016633121938, "eval/gt_acc_batch": 0.8083333472410837, "frac_reward_zero_std": 1.0, "grad_norm": 0.317132830619812, "kd/policy_loss": 0.008624310583885138, "kd/rkl_advantage_mean": 0.8453960252304872, "kd/rkl_advantage_p95": 5.1813263416290285, "kd/rkl_advantage_std": 2.283019922176997, "kd/ssd_loss": 0.0, "learning_rate": 2.6863631185204877e-07, "loss": 0.03449724515279134, "num_tokens": 629943533.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8083333392937978, "rewards/gt_logging_reward/std": 0.38801578680674237, "sampling/importance_sampling_ratio/max": 1.8775590419769288, "sampling/importance_sampling_ratio/mean": 1.0010281205177307, "sampling/importance_sampling_ratio/min": 0.45583374400933585, "sampling/sampling_logp_difference/max": 0.304938202102979, "sampling/sampling_logp_difference/mean": 0.003977814138246079, "step": 19260, "step_time": 8.143341417360352, "student/entropy": 0.13210447101543346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02250000120451053, "completions/max_length": 494.56666666666666, "completions/max_terminated_length": 450.73333333333335, "completions/mean_length": 189.23750864664714, "completions/mean_terminated_length": 182.0042526245117, "completions/min_length": 58.1, "completions/min_terminated_length": 58.1, "entropy": 0.12999957849582036, "epoch": 0.7325409182394714, "eval/gt_acc_batch": 0.8072222451368968, "frac_reward_zero_std": 1.0, "grad_norm": 0.2692919373512268, "kd/policy_loss": 0.008633186602188895, "kd/rkl_advantage_mean": 0.7971085583791137, "kd/rkl_advantage_p95": 5.064895957708359, "kd/rkl_advantage_std": 2.224267645676931, "kd/ssd_loss": 0.0, "learning_rate": 2.674970569247712e-07, "loss": 0.03453274965286255, "num_tokens": 630926788.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222272555033, "rewards/gt_logging_reward/std": 0.3865293741226196, "sampling/importance_sampling_ratio/max": 1.9388410170873007, "sampling/importance_sampling_ratio/mean": 0.999427487452825, "sampling/importance_sampling_ratio/min": 0.43871418833732606, "sampling/sampling_logp_difference/max": 0.3171274503072103, "sampling/sampling_logp_difference/mean": 0.003936924366280436, "step": 19290, "step_time": 8.110810611000245, "student/entropy": 0.12999957849582036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030833334724108378, "completions/max_length": 480.3666666666667, "completions/max_terminated_length": 451.9, "completions/mean_length": 191.11278737386067, "completions/mean_terminated_length": 181.02879842122397, "completions/min_length": 56.46666666666667, "completions/min_terminated_length": 56.46666666666667, "entropy": 0.13327580944945414, "epoch": 0.7336801731667489, "eval/gt_acc_batch": 0.7813889046510061, "frac_reward_zero_std": 1.0, "grad_norm": 0.45897042751312256, "kd/policy_loss": 0.00898876841335247, "kd/rkl_advantage_mean": 0.7612913464506467, "kd/rkl_advantage_p95": 4.937490085760753, "kd/rkl_advantage_std": 2.1859763006369275, "kd/ssd_loss": 0.0, "learning_rate": 2.6635780199749366e-07, "loss": 0.03595507542292277, "num_tokens": 631923970.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7813888867696126, "rewards/gt_logging_reward/std": 0.40441614935795467, "sampling/importance_sampling_ratio/max": 1.8986808816591898, "sampling/importance_sampling_ratio/mean": 0.9979060252507528, "sampling/importance_sampling_ratio/min": 0.3856195519367854, "sampling/sampling_logp_difference/max": 0.3505401690800985, "sampling/sampling_logp_difference/mean": 0.0040578966417039435, "step": 19320, "step_time": 8.161738468461165, "student/entropy": 0.13327580944945414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334860702357, "completions/max_length": 486.8, "completions/max_terminated_length": 464.4, "completions/mean_length": 185.22834370930988, "completions/mean_terminated_length": 177.51339060465494, "completions/min_length": 50.733333333333334, "completions/min_terminated_length": 50.733333333333334, "entropy": 0.13296963156511385, "epoch": 0.7348194280940266, "eval/gt_acc_batch": 0.8008333504199981, "frac_reward_zero_std": 1.0, "grad_norm": 0.3559264838695526, "kd/policy_loss": 0.008843824835882213, "kd/rkl_advantage_mean": 0.8328322950129707, "kd/rkl_advantage_p95": 5.180996797482172, "kd/rkl_advantage_std": 2.2722189724445343, "kd/ssd_loss": 0.0, "learning_rate": 2.652185470702161e-07, "loss": 0.035375301043192545, "num_tokens": 632909160.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333345254263, "rewards/gt_logging_reward/std": 0.39144307176272075, "sampling/importance_sampling_ratio/max": 1.9314284245173137, "sampling/importance_sampling_ratio/mean": 0.9984660903612773, "sampling/importance_sampling_ratio/min": 0.4762565518418948, "sampling/sampling_logp_difference/max": 0.3511220246553421, "sampling/sampling_logp_difference/mean": 0.004058385244570672, "step": 19350, "step_time": 8.137571561970011, "student/entropy": 0.13296963156511385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01472222302109003, "completions/max_length": 474.96666666666664, "completions/max_terminated_length": 433.5, "completions/mean_length": 179.2697311401367, "completions/mean_terminated_length": 174.45407358805338, "completions/min_length": 60.53333333333333, "completions/min_terminated_length": 60.53333333333333, "entropy": 0.12542189123729866, "epoch": 0.7359586830213041, "eval/gt_acc_batch": 0.821666689713796, "frac_reward_zero_std": 1.0, "grad_norm": 0.24359279870986938, "kd/policy_loss": 0.00789370912243612, "kd/rkl_advantage_mean": 0.7711478068027645, "kd/rkl_advantage_p95": 5.076553642749786, "kd/rkl_advantage_std": 2.228021421035131, "kd/ssd_loss": 0.0, "learning_rate": 2.640792921429385e-07, "loss": 0.03157483537991842, "num_tokens": 633853499.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.821666665871938, "rewards/gt_logging_reward/std": 0.3759480704863866, "sampling/importance_sampling_ratio/max": 1.8693518439928691, "sampling/importance_sampling_ratio/mean": 1.0034370879332224, "sampling/importance_sampling_ratio/min": 0.48747678299744923, "sampling/sampling_logp_difference/max": 0.31028189659118655, "sampling/sampling_logp_difference/mean": 0.003851521434262395, "step": 19380, "step_time": 7.751471898734841, "student/entropy": 0.12542189123729866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555662761132, "completions/max_length": 488.23333333333335, "completions/max_terminated_length": 435.8333333333333, "completions/mean_length": 184.27556559244792, "completions/mean_terminated_length": 177.53091227213542, "completions/min_length": 52.9, "completions/min_terminated_length": 52.9, "entropy": 0.13226532358676196, "epoch": 0.7370979379485816, "eval/gt_acc_batch": 0.7975000282128651, "frac_reward_zero_std": 1.0, "grad_norm": 0.2965153753757477, "kd/policy_loss": 0.008964819001266732, "kd/rkl_advantage_mean": 0.7819308566550414, "kd/rkl_advantage_p95": 5.041588701804479, "kd/rkl_advantage_std": 2.2478778918584186, "kd/ssd_loss": 0.0, "learning_rate": 2.629400372156609e-07, "loss": 0.035859278837839764, "num_tokens": 634810579.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7975000003973644, "rewards/gt_logging_reward/std": 0.39203281899293263, "sampling/importance_sampling_ratio/max": 1.943582812945048, "sampling/importance_sampling_ratio/mean": 0.9999742945035298, "sampling/importance_sampling_ratio/min": 0.4360542764266332, "sampling/sampling_logp_difference/max": 0.2960124413172404, "sampling/sampling_logp_difference/mean": 0.004066380734244982, "step": 19410, "step_time": 7.89203226553218, "student/entropy": 0.13226532358676196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556789040567, "completions/max_length": 490.1, "completions/max_terminated_length": 438.8666666666667, "completions/mean_length": 188.46862131754557, "completions/mean_terminated_length": 180.8783218383789, "completions/min_length": 57.36666666666667, "completions/min_terminated_length": 57.36666666666667, "entropy": 0.12879397366195916, "epoch": 0.7382371928758592, "eval/gt_acc_batch": 0.7966666877269745, "frac_reward_zero_std": 1.0, "grad_norm": 0.27390459179878235, "kd/policy_loss": 0.008536899815468738, "kd/rkl_advantage_mean": 0.7296721260373791, "kd/rkl_advantage_p95": 4.964423048496246, "kd/rkl_advantage_std": 2.232636331518491, "kd/ssd_loss": 0.0, "learning_rate": 2.618007822883834e-07, "loss": 0.034147604306538897, "num_tokens": 635793250.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7966666698455811, "rewards/gt_logging_reward/std": 0.39449888120094934, "sampling/importance_sampling_ratio/max": 1.904247530301412, "sampling/importance_sampling_ratio/mean": 1.0002993623415628, "sampling/importance_sampling_ratio/min": 0.41962619324525197, "sampling/sampling_logp_difference/max": 0.37840380469957985, "sampling/sampling_logp_difference/mean": 0.003960203550135096, "step": 19440, "step_time": 8.012527602303695, "student/entropy": 0.12879397366195916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445899377267, "completions/max_length": 482.1333333333333, "completions/max_terminated_length": 437.8, "completions/mean_length": 186.4313990275065, "completions/mean_terminated_length": 179.18441212972004, "completions/min_length": 55.7, "completions/min_terminated_length": 55.7, "entropy": 0.12804486316939195, "epoch": 0.7393764478031367, "eval/gt_acc_batch": 0.801111121972402, "frac_reward_zero_std": 1.0, "grad_norm": 0.30575495958328247, "kd/policy_loss": 0.008544389662953714, "kd/rkl_advantage_mean": 0.7090903775456051, "kd/rkl_advantage_p95": 4.897886943817139, "kd/rkl_advantage_std": 2.1874799112478893, "kd/ssd_loss": 0.0, "learning_rate": 2.606615273611058e-07, "loss": 0.03417756160100301, "num_tokens": 636768787.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.801111110051473, "rewards/gt_logging_reward/std": 0.39319525261720023, "sampling/importance_sampling_ratio/max": 1.8935348550478617, "sampling/importance_sampling_ratio/mean": 0.9975704828898112, "sampling/importance_sampling_ratio/min": 0.4313364734252294, "sampling/sampling_logp_difference/max": 0.32644379933675133, "sampling/sampling_logp_difference/mean": 0.003922679756457607, "step": 19470, "step_time": 8.154283256965574, "student/entropy": 0.12804486316939195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02555555695046981, "completions/max_length": 495.0, "completions/max_terminated_length": 458.3333333333333, "completions/mean_length": 193.44140014648437, "completions/mean_terminated_length": 185.09754333496093, "completions/min_length": 46.2, "completions/min_terminated_length": 46.2, "entropy": 0.12722696823378404, "epoch": 0.7405157027304143, "eval/gt_acc_batch": 0.7925000150998434, "frac_reward_zero_std": 1.0, "grad_norm": 0.29781484603881836, "kd/policy_loss": 0.008491856288552905, "kd/rkl_advantage_mean": 0.7590056039082508, "kd/rkl_advantage_p95": 5.078929046789805, "kd/rkl_advantage_std": 2.2637422968943914, "kd/ssd_loss": 0.0, "learning_rate": 2.595222724338282e-07, "loss": 0.03396742343902588, "num_tokens": 637774840.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7924999952316284, "rewards/gt_logging_reward/std": 0.39845245629549025, "sampling/importance_sampling_ratio/max": 1.9071011543273926, "sampling/importance_sampling_ratio/mean": 1.0040442963441214, "sampling/importance_sampling_ratio/min": 0.4842746168375015, "sampling/sampling_logp_difference/max": 0.3291155139605204, "sampling/sampling_logp_difference/mean": 0.0038875183168177803, "step": 19500, "step_time": 8.320591368034366, "student/entropy": 0.12722696823378404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010833334022512039, "completions/max_length": 471.1666666666667, "completions/max_terminated_length": 444.03333333333336, "completions/mean_length": 181.7894546508789, "completions/mean_terminated_length": 178.21392720540365, "completions/min_length": 48.766666666666666, "completions/min_terminated_length": 48.766666666666666, "entropy": 0.12836513525495927, "epoch": 0.7416549576576918, "eval/gt_acc_batch": 0.8163889070351918, "frac_reward_zero_std": 1.0, "grad_norm": 0.7235389947891235, "kd/policy_loss": 0.00831828751640084, "kd/rkl_advantage_mean": 0.7598971753536413, "kd/rkl_advantage_p95": 4.980448603630066, "kd/rkl_advantage_std": 2.2115337401628494, "kd/ssd_loss": 0.0, "learning_rate": 2.5838301750655074e-07, "loss": 0.033273152510325116, "num_tokens": 638721074.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8163888891537984, "rewards/gt_logging_reward/std": 0.38240728676319125, "sampling/importance_sampling_ratio/max": 1.865463423728943, "sampling/importance_sampling_ratio/mean": 1.0002119521299997, "sampling/importance_sampling_ratio/min": 0.4542728583017985, "sampling/sampling_logp_difference/max": 0.29543550809224445, "sampling/sampling_logp_difference/mean": 0.003938382646689813, "step": 19530, "step_time": 8.033308284730689, "student/entropy": 0.12836513525495927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556813875834, "completions/max_length": 478.9, "completions/max_terminated_length": 455.3666666666667, "completions/mean_length": 185.1644551595052, "completions/mean_terminated_length": 178.39063618977863, "completions/min_length": 55.733333333333334, "completions/min_terminated_length": 55.733333333333334, "entropy": 0.12620271053165197, "epoch": 0.7427942125849695, "eval/gt_acc_batch": 0.8083333472410837, "frac_reward_zero_std": 1.0, "grad_norm": 0.28973111510276794, "kd/policy_loss": 0.008260869867323587, "kd/rkl_advantage_mean": 0.7570522269544502, "kd/rkl_advantage_p95": 5.011263100306193, "kd/rkl_advantage_std": 2.23166080613931, "kd/ssd_loss": 0.0, "learning_rate": 2.5724376257927316e-07, "loss": 0.03304347991943359, "num_tokens": 639686698.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8083333293596904, "rewards/gt_logging_reward/std": 0.3868274211883545, "sampling/importance_sampling_ratio/max": 1.9244820396105449, "sampling/importance_sampling_ratio/mean": 0.9966843008995057, "sampling/importance_sampling_ratio/min": 0.45407118648290634, "sampling/sampling_logp_difference/max": 0.3307483692963918, "sampling/sampling_logp_difference/mean": 0.0038931434197972217, "step": 19560, "step_time": 8.120924717635111, "student/entropy": 0.12620271053165197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778897434474, "completions/max_length": 469.43333333333334, "completions/max_terminated_length": 445.6333333333333, "completions/mean_length": 186.55889892578125, "completions/mean_terminated_length": 179.91060740152994, "completions/min_length": 53.266666666666666, "completions/min_terminated_length": 53.266666666666666, "entropy": 0.12711496371775866, "epoch": 0.743933467512247, "eval/gt_acc_batch": 0.8002777973810832, "frac_reward_zero_std": 1.0, "grad_norm": 0.2639774680137634, "kd/policy_loss": 0.007949693681439385, "kd/rkl_advantage_mean": 0.7916605478773514, "kd/rkl_advantage_p95": 5.156017396847407, "kd/rkl_advantage_std": 2.257316647966703, "kd/ssd_loss": 0.0, "learning_rate": 2.561045076519956e-07, "loss": 0.031798774003982545, "num_tokens": 640671270.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8002777735392252, "rewards/gt_logging_reward/std": 0.3884796549876531, "sampling/importance_sampling_ratio/max": 1.8561038851737977, "sampling/importance_sampling_ratio/mean": 0.9944007635116577, "sampling/importance_sampling_ratio/min": 0.4689375470081965, "sampling/sampling_logp_difference/max": 0.3065548559029897, "sampling/sampling_logp_difference/mean": 0.003914768473866085, "step": 19590, "step_time": 8.207603816769552, "student/entropy": 0.12711496371775866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889042039712, "completions/max_length": 487.56666666666666, "completions/max_terminated_length": 455.6333333333333, "completions/mean_length": 190.75223236083986, "completions/mean_terminated_length": 181.9724858601888, "completions/min_length": 61.266666666666666, "completions/min_terminated_length": 61.266666666666666, "entropy": 0.1298315931111574, "epoch": 0.7450727224395246, "eval/gt_acc_batch": 0.7841666837533315, "frac_reward_zero_std": 1.0, "grad_norm": 0.34257563948631287, "kd/policy_loss": 0.008503481411995988, "kd/rkl_advantage_mean": 0.7953653243059914, "kd/rkl_advantage_p95": 5.075889070828755, "kd/rkl_advantage_std": 2.244518346587817, "kd/ssd_loss": 0.0, "learning_rate": 2.5496525272471805e-07, "loss": 0.03401393095652262, "num_tokens": 641665874.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7841666678587595, "rewards/gt_logging_reward/std": 0.40789507031440736, "sampling/importance_sampling_ratio/max": 1.9015865643819174, "sampling/importance_sampling_ratio/mean": 0.997674963871638, "sampling/importance_sampling_ratio/min": 0.4555261512597402, "sampling/sampling_logp_difference/max": 0.32917863726615904, "sampling/sampling_logp_difference/mean": 0.003975241111281017, "step": 19620, "step_time": 8.2731588326608, "student/entropy": 0.1298315931111574 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112559835116, "completions/max_length": 496.06666666666666, "completions/max_terminated_length": 449.3333333333333, "completions/mean_length": 183.2702875773112, "completions/mean_terminated_length": 175.51051432291666, "completions/min_length": 57.3, "completions/min_terminated_length": 57.3, "entropy": 0.12850976524253685, "epoch": 0.7462119773668021, "eval/gt_acc_batch": 0.7930555740992228, "frac_reward_zero_std": 1.0, "grad_norm": 0.310396671295166, "kd/policy_loss": 0.00876221347716637, "kd/rkl_advantage_mean": 0.7948073625875016, "kd/rkl_advantage_p95": 5.062232597668966, "kd/rkl_advantage_std": 2.254484365383784, "kd/ssd_loss": 0.0, "learning_rate": 2.5382599779744047e-07, "loss": 0.035048862298329674, "num_tokens": 642623279.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7930555542310079, "rewards/gt_logging_reward/std": 0.3992587695519129, "sampling/importance_sampling_ratio/max": 1.9491121967633565, "sampling/importance_sampling_ratio/mean": 1.00113951365153, "sampling/importance_sampling_ratio/min": 0.4520309418439865, "sampling/sampling_logp_difference/max": 0.3498302161693573, "sampling/sampling_logp_difference/mean": 0.003959835188773771, "step": 19650, "step_time": 8.238327447032983, "student/entropy": 0.12850976524253685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333469927311, "completions/max_length": 480.96666666666664, "completions/max_terminated_length": 443.8333333333333, "completions/mean_length": 187.39334411621093, "completions/mean_terminated_length": 180.5657964070638, "completions/min_length": 52.666666666666664, "completions/min_terminated_length": 52.666666666666664, "entropy": 0.12679010362674792, "epoch": 0.7473512322940796, "eval/gt_acc_batch": 0.8216666936874389, "frac_reward_zero_std": 1.0, "grad_norm": 0.3078823685646057, "kd/policy_loss": 0.007990691098772611, "kd/rkl_advantage_mean": 0.6734390700856845, "kd/rkl_advantage_p95": 4.862349017461141, "kd/rkl_advantage_std": 2.1925534665584565, "kd/ssd_loss": 0.0, "learning_rate": 2.526867428701629e-07, "loss": 0.031962766249974565, "num_tokens": 643605887.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8216666718324025, "rewards/gt_logging_reward/std": 0.379154505332311, "sampling/importance_sampling_ratio/max": 1.7827107350031535, "sampling/importance_sampling_ratio/mean": 0.9989740530649821, "sampling/importance_sampling_ratio/min": 0.4922326495250066, "sampling/sampling_logp_difference/max": 0.32603750824928285, "sampling/sampling_logp_difference/mean": 0.0038839524999881784, "step": 19680, "step_time": 8.076102761201522, "student/entropy": 0.12679010362674792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778959522645, "completions/max_length": 495.2, "completions/max_terminated_length": 464.03333333333336, "completions/mean_length": 186.2997319539388, "completions/mean_terminated_length": 179.6750966389974, "completions/min_length": 59.53333333333333, "completions/min_terminated_length": 59.53333333333333, "entropy": 0.12673180686930816, "epoch": 0.7484904872213572, "eval/gt_acc_batch": 0.8013889094193777, "frac_reward_zero_std": 1.0, "grad_norm": 0.3209072947502136, "kd/policy_loss": 0.008358414876662815, "kd/rkl_advantage_mean": 0.7763866887117425, "kd/rkl_advantage_p95": 5.094975358247757, "kd/rkl_advantage_std": 2.26398828625679, "kd/ssd_loss": 0.0, "learning_rate": 2.515474879428853e-07, "loss": 0.03343366781870524, "num_tokens": 644582014.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888915379842, "rewards/gt_logging_reward/std": 0.3899892215927442, "sampling/importance_sampling_ratio/max": 1.8391324917475382, "sampling/importance_sampling_ratio/mean": 0.993171199162801, "sampling/importance_sampling_ratio/min": 0.4423011526465416, "sampling/sampling_logp_difference/max": 0.3218663235505422, "sampling/sampling_logp_difference/mean": 0.003897423429104189, "step": 19710, "step_time": 8.218176582028779, "student/entropy": 0.12673180686930816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01500000072022279, "completions/max_length": 481.6333333333333, "completions/max_terminated_length": 431.03333333333336, "completions/mean_length": 182.3047302246094, "completions/mean_terminated_length": 177.2401555379232, "completions/min_length": 55.733333333333334, "completions/min_terminated_length": 55.733333333333334, "entropy": 0.13139221432308357, "epoch": 0.7496297421486348, "eval/gt_acc_batch": 0.817222245534261, "frac_reward_zero_std": 1.0, "grad_norm": 0.26999449729919434, "kd/policy_loss": 0.008433338138274848, "kd/rkl_advantage_mean": 0.8774571143711607, "kd/rkl_advantage_p95": 5.226573189099629, "kd/rkl_advantage_std": 2.2509266316890715, "kd/ssd_loss": 0.0, "learning_rate": 2.5040823301560777e-07, "loss": 0.03373335202534993, "num_tokens": 645526575.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8172222236792247, "rewards/gt_logging_reward/std": 0.37583180169264474, "sampling/importance_sampling_ratio/max": 1.9272438486417134, "sampling/importance_sampling_ratio/mean": 1.001880031824112, "sampling/importance_sampling_ratio/min": 0.446943587064743, "sampling/sampling_logp_difference/max": 0.3117194930712382, "sampling/sampling_logp_difference/mean": 0.003981241281144321, "step": 19740, "step_time": 7.984754919894233, "student/entropy": 0.13139221432308357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445526848235, "completions/max_length": 484.8, "completions/max_terminated_length": 457.76666666666665, "completions/mean_length": 188.80750885009766, "completions/mean_terminated_length": 181.6355412801107, "completions/min_length": 54.766666666666666, "completions/min_terminated_length": 54.766666666666666, "entropy": 0.13366201426833868, "epoch": 0.7507689970759124, "eval/gt_acc_batch": 0.7861111303170522, "frac_reward_zero_std": 1.0, "grad_norm": 0.2261309027671814, "kd/policy_loss": 0.00899652723261776, "kd/rkl_advantage_mean": 0.7244734970231851, "kd/rkl_advantage_p95": 4.98481352130572, "kd/rkl_advantage_std": 2.2169239113728207, "kd/ssd_loss": 0.0, "learning_rate": 2.4926897808833024e-07, "loss": 0.03598611354827881, "num_tokens": 646511834.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7861111104488373, "rewards/gt_logging_reward/std": 0.4022670795520147, "sampling/importance_sampling_ratio/max": 1.9687163670857748, "sampling/importance_sampling_ratio/mean": 1.0022607147693634, "sampling/importance_sampling_ratio/min": 0.4264992376168569, "sampling/sampling_logp_difference/max": 0.3464935580889384, "sampling/sampling_logp_difference/mean": 0.004049272874059776, "step": 19770, "step_time": 8.177774197265778, "student/entropy": 0.13366201426833868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112243185442, "completions/max_length": 481.5, "completions/max_terminated_length": 447.43333333333334, "completions/mean_length": 187.53334248860676, "completions/mean_terminated_length": 180.5811014811198, "completions/min_length": 57.46666666666667, "completions/min_terminated_length": 57.46666666666667, "entropy": 0.1255962581684192, "epoch": 0.7519082520031899, "eval/gt_acc_batch": 0.8030555784702301, "frac_reward_zero_std": 1.0, "grad_norm": 0.28510990738868713, "kd/policy_loss": 0.007905050068317602, "kd/rkl_advantage_mean": 0.7714004440504747, "kd/rkl_advantage_p95": 4.989827267328898, "kd/rkl_advantage_std": 2.204768345753352, "kd/ssd_loss": 0.0, "learning_rate": 2.4812972316105266e-07, "loss": 0.03162020047505697, "num_tokens": 647495882.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555526415507, "rewards/gt_logging_reward/std": 0.3913274695475896, "sampling/importance_sampling_ratio/max": 1.8493103663126627, "sampling/importance_sampling_ratio/mean": 0.9994418303171794, "sampling/importance_sampling_ratio/min": 0.44341179231802624, "sampling/sampling_logp_difference/max": 0.35790724754333497, "sampling/sampling_logp_difference/mean": 0.003836966158511738, "step": 19800, "step_time": 8.272996518494134, "student/entropy": 0.1255962581684192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015000000813355048, "completions/max_length": 477.7, "completions/max_terminated_length": 450.3666666666667, "completions/mean_length": 186.86639760335285, "completions/mean_terminated_length": 182.005815633138, "completions/min_length": 63.5, "completions/min_terminated_length": 63.5, "entropy": 0.1222544901072979, "epoch": 0.7530475069304675, "eval/gt_acc_batch": 0.8122222483158111, "frac_reward_zero_std": 1.0, "grad_norm": 0.3483871519565582, "kd/policy_loss": 0.00798060125671327, "kd/rkl_advantage_mean": 0.8047697834049662, "kd/rkl_advantage_p95": 5.064788546164831, "kd/rkl_advantage_std": 2.2264504611492155, "kd/ssd_loss": 0.0, "learning_rate": 2.469904682337751e-07, "loss": 0.03192240993181864, "num_tokens": 648464081.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8122222224871317, "rewards/gt_logging_reward/std": 0.37681624641021094, "sampling/importance_sampling_ratio/max": 1.9371894598007202, "sampling/importance_sampling_ratio/mean": 1.0028741717338563, "sampling/importance_sampling_ratio/min": 0.45523566504319507, "sampling/sampling_logp_difference/max": 0.3247182905673981, "sampling/sampling_logp_difference/mean": 0.0037766827270388605, "step": 19830, "step_time": 8.067869361765528, "student/entropy": 0.1222544901072979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779251337052, "completions/max_length": 483.6666666666667, "completions/max_terminated_length": 448.06666666666666, "completions/mean_length": 188.09195353190105, "completions/mean_terminated_length": 179.71124064127605, "completions/min_length": 54.5, "completions/min_terminated_length": 54.5, "entropy": 0.12388698402792216, "epoch": 0.754186761857745, "eval/gt_acc_batch": 0.7913889149824779, "frac_reward_zero_std": 1.0, "grad_norm": 0.388114333152771, "kd/policy_loss": 0.008181636427373936, "kd/rkl_advantage_mean": 0.8452639158815145, "kd/rkl_advantage_p95": 5.1399304111798605, "kd/rkl_advantage_std": 2.2439719259738924, "kd/ssd_loss": 0.0, "learning_rate": 2.4585121330649755e-07, "loss": 0.032726548115412396, "num_tokens": 649452996.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7913888911406199, "rewards/gt_logging_reward/std": 0.3983986794948578, "sampling/importance_sampling_ratio/max": 1.802250858147939, "sampling/importance_sampling_ratio/mean": 0.9979208807150522, "sampling/importance_sampling_ratio/min": 0.45878295799096425, "sampling/sampling_logp_difference/max": 0.3567464590072632, "sampling/sampling_logp_difference/mean": 0.003797354370666047, "step": 19860, "step_time": 8.333770604800277, "student/entropy": 0.12388698402792216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02305555691321691, "completions/max_length": 489.1333333333333, "completions/max_terminated_length": 459.06666666666666, "completions/mean_length": 189.98862050374348, "completions/mean_terminated_length": 182.51639404296876, "completions/min_length": 55.4, "completions/min_terminated_length": 55.4, "entropy": 0.13450771756470203, "epoch": 0.7553260167850226, "eval/gt_acc_batch": 0.7919444640477499, "frac_reward_zero_std": 1.0, "grad_norm": 0.25787797570228577, "kd/policy_loss": 0.008987945194045702, "kd/rkl_advantage_mean": 0.7270278632019956, "kd/rkl_advantage_p95": 4.947769403457642, "kd/rkl_advantage_std": 2.2166728963454565, "kd/ssd_loss": 0.0, "learning_rate": 2.4471195837921997e-07, "loss": 0.03595178524653117, "num_tokens": 650448595.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7919444421927134, "rewards/gt_logging_reward/std": 0.39638554453849795, "sampling/importance_sampling_ratio/max": 1.9669509530067444, "sampling/importance_sampling_ratio/mean": 1.0036982774734498, "sampling/importance_sampling_ratio/min": 0.4349796627958616, "sampling/sampling_logp_difference/max": 0.2999541759490967, "sampling/sampling_logp_difference/mean": 0.004082537163048983, "step": 19890, "step_time": 8.344489080503505, "student/entropy": 0.13450771756470203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.031666668504476546, "completions/max_length": 485.1333333333333, "completions/max_terminated_length": 451.3666666666667, "completions/mean_length": 190.0919537862142, "completions/mean_terminated_length": 179.92456741333007, "completions/min_length": 53.6, "completions/min_terminated_length": 53.6, "entropy": 0.1305525541305542, "epoch": 0.7564652717123002, "eval/gt_acc_batch": 0.796111128727595, "frac_reward_zero_std": 1.0, "grad_norm": 0.37085169553756714, "kd/policy_loss": 0.008563092273349563, "kd/rkl_advantage_mean": 0.7898605638494094, "kd/rkl_advantage_p95": 4.977546898523967, "kd/rkl_advantage_std": 2.191403034329414, "kd/ssd_loss": 0.0, "learning_rate": 2.4357270345194244e-07, "loss": 0.03425236543019613, "num_tokens": 651434334.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7961111187934875, "rewards/gt_logging_reward/std": 0.3882455547650655, "sampling/importance_sampling_ratio/max": 1.9246365110079446, "sampling/importance_sampling_ratio/mean": 1.0015799363454183, "sampling/importance_sampling_ratio/min": 0.4641634831825892, "sampling/sampling_logp_difference/max": 0.331625239054362, "sampling/sampling_logp_difference/mean": 0.003939435662080844, "step": 19920, "step_time": 8.291937018596219, "student/entropy": 0.1305525541305542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277779176831247, "completions/max_length": 485.1, "completions/max_terminated_length": 449.3333333333333, "completions/mean_length": 186.81389973958332, "completions/mean_terminated_length": 180.28101450602213, "completions/min_length": 55.46666666666667, "completions/min_terminated_length": 55.46666666666667, "entropy": 0.12847357988357544, "epoch": 0.7576045266395777, "eval/gt_acc_batch": 0.7977778017520905, "frac_reward_zero_std": 1.0, "grad_norm": 0.28305739164352417, "kd/policy_loss": 0.008177311635032916, "kd/rkl_advantage_mean": 0.8303352258478601, "kd/rkl_advantage_p95": 5.0494480351607, "kd/rkl_advantage_std": 2.2440278003613154, "kd/ssd_loss": 0.0, "learning_rate": 2.4243344852466486e-07, "loss": 0.0327092448870341, "num_tokens": 652401904.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.797777779897054, "rewards/gt_logging_reward/std": 0.3916856919725736, "sampling/importance_sampling_ratio/max": 1.8594923535982768, "sampling/importance_sampling_ratio/mean": 0.998709503809611, "sampling/importance_sampling_ratio/min": 0.45400549868742623, "sampling/sampling_logp_difference/max": 0.34520491361618044, "sampling/sampling_logp_difference/mean": 0.0039034358380983275, "step": 19950, "step_time": 8.159155991429968, "student/entropy": 0.12847357988357544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111265917619, "completions/max_length": 475.8333333333333, "completions/max_terminated_length": 446.26666666666665, "completions/mean_length": 191.04556681315105, "completions/mean_terminated_length": 182.55660909016927, "completions/min_length": 56.166666666666664, "completions/min_terminated_length": 56.166666666666664, "entropy": 0.13232018916557234, "epoch": 0.7587437815668553, "eval/gt_acc_batch": 0.7775000174840291, "frac_reward_zero_std": 1.0, "grad_norm": 0.27149832248687744, "kd/policy_loss": 0.00895920314748461, "kd/rkl_advantage_mean": 0.7623641296289861, "kd/rkl_advantage_p95": 5.0228143016497295, "kd/rkl_advantage_std": 2.245679821570714, "kd/ssd_loss": 0.0, "learning_rate": 2.4129419359738727e-07, "loss": 0.035836819807688394, "num_tokens": 653384572.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7775000055631002, "rewards/gt_logging_reward/std": 0.40682370265324913, "sampling/importance_sampling_ratio/max": 1.9168914357821147, "sampling/importance_sampling_ratio/mean": 1.0050819873809815, "sampling/importance_sampling_ratio/min": 0.47154206931591036, "sampling/sampling_logp_difference/max": 0.3092679738998413, "sampling/sampling_logp_difference/mean": 0.0040049545389289655, "step": 19980, "step_time": 8.21666705206153, "student/entropy": 0.13232018916557234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.031111112982034683, "completions/max_length": 497.8, "completions/max_terminated_length": 469.6666666666667, "completions/mean_length": 199.01361999511718, "completions/mean_terminated_length": 189.04684804280598, "completions/min_length": 51.4, "completions/min_terminated_length": 51.4, "entropy": 0.13351108152419328, "epoch": 0.7598830364941328, "eval/gt_acc_batch": 0.7825000186761221, "frac_reward_zero_std": 1.0, "grad_norm": 0.22841492295265198, "kd/policy_loss": 0.008406857892987318, "kd/rkl_advantage_mean": 0.727429825005432, "kd/rkl_advantage_p95": 4.921079760789871, "kd/rkl_advantage_std": 2.187630224227905, "kd/ssd_loss": 0.0, "learning_rate": 2.4015493867010974e-07, "loss": 0.033627438545227054, "num_tokens": 654400485.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7824999948342641, "rewards/gt_logging_reward/std": 0.4033319503068924, "sampling/importance_sampling_ratio/max": 1.8490193168322244, "sampling/importance_sampling_ratio/mean": 0.9912253399689992, "sampling/importance_sampling_ratio/min": 0.4354811171690623, "sampling/sampling_logp_difference/max": 0.31484848658243814, "sampling/sampling_logp_difference/mean": 0.003932747027526299, "step": 20010, "step_time": 8.47252599527031, "student/entropy": 0.13351108152419328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556882172822, "completions/max_length": 497.3666666666667, "completions/max_terminated_length": 444.3, "completions/mean_length": 185.99861958821614, "completions/mean_terminated_length": 178.42871907552083, "completions/min_length": 54.93333333333333, "completions/min_terminated_length": 54.93333333333333, "entropy": 0.1296739850193262, "epoch": 0.7610222914214104, "eval/gt_acc_batch": 0.8061111330986023, "frac_reward_zero_std": 1.0, "grad_norm": 0.24005495011806488, "kd/policy_loss": 0.008468670580380906, "kd/rkl_advantage_mean": 0.8473821502178908, "kd/rkl_advantage_p95": 5.088946072260539, "kd/rkl_advantage_std": 2.235593613982201, "kd/ssd_loss": 0.0, "learning_rate": 2.3901568374283216e-07, "loss": 0.03387468655904134, "num_tokens": 655380480.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8061111092567443, "rewards/gt_logging_reward/std": 0.38893579741319023, "sampling/importance_sampling_ratio/max": 1.8896080414454142, "sampling/importance_sampling_ratio/mean": 0.9988877773284912, "sampling/importance_sampling_ratio/min": 0.46019707520802816, "sampling/sampling_logp_difference/max": 0.3136323591073354, "sampling/sampling_logp_difference/mean": 0.003967275563627482, "step": 20040, "step_time": 8.341845629169256, "student/entropy": 0.1296739850193262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277779344469307, "completions/max_length": 491.6, "completions/max_terminated_length": 454.8666666666667, "completions/mean_length": 193.0166768391927, "completions/mean_terminated_length": 184.76156158447264, "completions/min_length": 57.733333333333334, "completions/min_terminated_length": 57.733333333333334, "entropy": 0.1395179795101285, "epoch": 0.7621615463486879, "eval/gt_acc_batch": 0.7966666877269745, "frac_reward_zero_std": 1.0, "grad_norm": 0.2531838119029999, "kd/policy_loss": 0.00913243896405523, "kd/rkl_advantage_mean": 0.7581034948428472, "kd/rkl_advantage_p95": 5.002742932240168, "kd/rkl_advantage_std": 2.2148460666338603, "kd/ssd_loss": 0.0, "learning_rate": 2.378764288155546e-07, "loss": 0.036529759565989174, "num_tokens": 656380084.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7966666678587596, "rewards/gt_logging_reward/std": 0.3985517054796219, "sampling/importance_sampling_ratio/max": 2.022644905249278, "sampling/importance_sampling_ratio/mean": 1.0091464837392172, "sampling/importance_sampling_ratio/min": 0.44264207084973656, "sampling/sampling_logp_difference/max": 0.3095769762992859, "sampling/sampling_logp_difference/mean": 0.004059667830976347, "step": 20070, "step_time": 8.255384985994898, "student/entropy": 0.1395179795101285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028333335183560848, "completions/max_length": 489.3333333333333, "completions/max_terminated_length": 461.46666666666664, "completions/mean_length": 195.10306599934896, "completions/mean_terminated_length": 186.05101013183594, "completions/min_length": 54.96666666666667, "completions/min_terminated_length": 54.96666666666667, "entropy": 0.12498614713549613, "epoch": 0.7633008012759656, "eval/gt_acc_batch": 0.7902777949968974, "frac_reward_zero_std": 1.0, "grad_norm": 0.30757397413253784, "kd/policy_loss": 0.008221563725965098, "kd/rkl_advantage_mean": 0.7908283287038406, "kd/rkl_advantage_p95": 5.207660073041916, "kd/rkl_advantage_std": 2.297394529978434, "kd/ssd_loss": 0.0, "learning_rate": 2.3673717388827705e-07, "loss": 0.03288624882698059, "num_tokens": 657392151.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.790277773141861, "rewards/gt_logging_reward/std": 0.39762401282787324, "sampling/importance_sampling_ratio/max": 1.818430209159851, "sampling/importance_sampling_ratio/mean": 1.0007449785868328, "sampling/importance_sampling_ratio/min": 0.43533316552639006, "sampling/sampling_logp_difference/max": 0.31465014616648357, "sampling/sampling_logp_difference/mean": 0.0038063563018416366, "step": 20100, "step_time": 8.334112043300411, "student/entropy": 0.12498614713549613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013611111976206302, "completions/max_length": 463.76666666666665, "completions/max_terminated_length": 445.73333333333335, "completions/mean_length": 189.67667541503906, "completions/mean_terminated_length": 185.3237274169922, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.12397052260736624, "epoch": 0.7644400562032431, "eval/gt_acc_batch": 0.8011111279328664, "frac_reward_zero_std": 1.0, "grad_norm": 0.29247042536735535, "kd/policy_loss": 0.008016562626774732, "kd/rkl_advantage_mean": 0.7292116859927773, "kd/rkl_advantage_p95": 4.931886845827103, "kd/rkl_advantage_std": 2.1652101506789525, "kd/ssd_loss": 0.0, "learning_rate": 2.3559791896099952e-07, "loss": 0.03206625580787659, "num_tokens": 658383123.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8011111080646515, "rewards/gt_logging_reward/std": 0.38807089775800707, "sampling/importance_sampling_ratio/max": 1.7525275588035583, "sampling/importance_sampling_ratio/mean": 0.9953193624814352, "sampling/importance_sampling_ratio/min": 0.482170761624972, "sampling/sampling_logp_difference/max": 0.3035338441530863, "sampling/sampling_logp_difference/mean": 0.0037959063891321422, "step": 20130, "step_time": 8.211766632875273, "student/entropy": 0.12397052260736624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02183908182356892, "completions/max_length": 465.8965517241379, "completions/max_terminated_length": 443.9310344827586, "completions/mean_length": 187.06409007105333, "completions/mean_terminated_length": 179.79784209152749, "completions/min_length": 55.41379310344828, "completions/min_terminated_length": 55.41379310344828, "entropy": 0.12227650230814671, "epoch": 0.7655793111305207, "eval/gt_acc_batch": 0.8068965735106632, "frac_reward_zero_std": 1.0, "grad_norm": 0.23376566171646118, "kd/policy_loss": 0.008023715923251263, "kd/rkl_advantage_mean": 0.6686869071764422, "kd/rkl_advantage_p95": 4.7949009928210025, "kd/rkl_advantage_std": 2.1588204564719367, "kd/ssd_loss": 0.0, "learning_rate": 2.3445866403372194e-07, "loss": 0.031025036176045736, "num_tokens": 659322130.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8068965488466723, "rewards/gt_logging_reward/std": 0.38692529756447364, "sampling/importance_sampling_ratio/max": 1.9866512561666554, "sampling/importance_sampling_ratio/mean": 1.0014176964759827, "sampling/importance_sampling_ratio/min": 0.48028547702164487, "sampling/sampling_logp_difference/max": 0.32457565028091956, "sampling/sampling_logp_difference/mean": 0.0037395433314016155, "step": 20160, "step_time": 7.794941440600087, "student/entropy": 0.12227650230814671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026666668461014828, "completions/max_length": 481.1, "completions/max_terminated_length": 457.3666666666667, "completions/mean_length": 182.7791768391927, "completions/mean_terminated_length": 173.97730509440103, "completions/min_length": 53.5, "completions/min_terminated_length": 53.5, "entropy": 0.12608017306774855, "epoch": 0.7667185660577982, "eval/gt_acc_batch": 0.810277795791626, "frac_reward_zero_std": 1.0, "grad_norm": 0.29796749353408813, "kd/policy_loss": 0.008385820490851377, "kd/rkl_advantage_mean": 0.7675553504610434, "kd/rkl_advantage_p95": 5.103670253356298, "kd/rkl_advantage_std": 2.2750742544730502, "kd/ssd_loss": 0.0, "learning_rate": 2.3331940910644438e-07, "loss": 0.03354328473409017, "num_tokens": 660276631.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8102777818838756, "rewards/gt_logging_reward/std": 0.37860896239678066, "sampling/importance_sampling_ratio/max": 1.9109833280245463, "sampling/importance_sampling_ratio/mean": 0.9987269083658854, "sampling/importance_sampling_ratio/min": 0.46298396537701286, "sampling/sampling_logp_difference/max": 0.3181571920712789, "sampling/sampling_logp_difference/mean": 0.0038858320641641814, "step": 20190, "step_time": 8.140770838894726, "student/entropy": 0.12608017306774855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277779021610817, "completions/max_length": 471.53333333333336, "completions/max_terminated_length": 446.8333333333333, "completions/mean_length": 186.3833429972331, "completions/mean_terminated_length": 179.8139862060547, "completions/min_length": 50.43333333333333, "completions/min_terminated_length": 50.43333333333333, "entropy": 0.12775179861734312, "epoch": 0.7678578209850757, "eval/gt_acc_batch": 0.7977778037389119, "frac_reward_zero_std": 1.0, "grad_norm": 0.2782203257083893, "kd/policy_loss": 0.008399450307479128, "kd/rkl_advantage_mean": 0.8026129634430011, "kd/rkl_advantage_p95": 5.132111656665802, "kd/rkl_advantage_std": 2.269930386543274, "kd/ssd_loss": 0.0, "learning_rate": 2.321801541791668e-07, "loss": 0.03359781106313069, "num_tokens": 661247083.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.797777775923411, "rewards/gt_logging_reward/std": 0.392411212126414, "sampling/importance_sampling_ratio/max": 1.8501734932263691, "sampling/importance_sampling_ratio/mean": 1.0059687654177347, "sampling/importance_sampling_ratio/min": 0.4661195605993271, "sampling/sampling_logp_difference/max": 0.34161778887112937, "sampling/sampling_logp_difference/mean": 0.003893120974923174, "step": 20220, "step_time": 8.126092381765678, "student/entropy": 0.12775179861734312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222224045544863, "completions/max_length": 482.3666666666667, "completions/max_terminated_length": 446.23333333333335, "completions/mean_length": 190.9275095621745, "completions/mean_terminated_length": 182.14341939290364, "completions/min_length": 52.06666666666667, "completions/min_terminated_length": 52.06666666666667, "entropy": 0.12853542870531479, "epoch": 0.7689970759123533, "eval/gt_acc_batch": 0.7969444513320922, "frac_reward_zero_std": 1.0, "grad_norm": 0.2571999430656433, "kd/policy_loss": 0.008592863984328384, "kd/rkl_advantage_mean": 0.7894450464596351, "kd/rkl_advantage_p95": 5.052782917022705, "kd/rkl_advantage_std": 2.229092053572337, "kd/ssd_loss": 0.0, "learning_rate": 2.3104089925188924e-07, "loss": 0.03437145948410034, "num_tokens": 662240878.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444453716278, "rewards/gt_logging_reward/std": 0.39511067668596905, "sampling/importance_sampling_ratio/max": 1.9363954504330954, "sampling/importance_sampling_ratio/mean": 0.994359815120697, "sampling/importance_sampling_ratio/min": 0.4603819489479065, "sampling/sampling_logp_difference/max": 0.31499665379524233, "sampling/sampling_logp_difference/mean": 0.0038775968520591655, "step": 20250, "step_time": 8.336811973161335, "student/entropy": 0.12853542870531479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017222223213563363, "completions/max_length": 481.0, "completions/max_terminated_length": 436.53333333333336, "completions/mean_length": 184.62362009684244, "completions/mean_terminated_length": 178.977587890625, "completions/min_length": 53.833333333333336, "completions/min_terminated_length": 53.833333333333336, "entropy": 0.13090084809809924, "epoch": 0.7701363308396308, "eval/gt_acc_batch": 0.7847222447395324, "frac_reward_zero_std": 1.0, "grad_norm": 0.39712971448898315, "kd/policy_loss": 0.008615159635276844, "kd/rkl_advantage_mean": 0.7762270097931226, "kd/rkl_advantage_p95": 5.071673744916916, "kd/rkl_advantage_std": 2.2615598797798158, "kd/ssd_loss": 0.0, "learning_rate": 2.2990164432461171e-07, "loss": 0.03446063995361328, "num_tokens": 663207931.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7847222288449606, "rewards/gt_logging_reward/std": 0.40106662213802335, "sampling/importance_sampling_ratio/max": 1.8396191795667012, "sampling/importance_sampling_ratio/mean": 0.9960631469885508, "sampling/importance_sampling_ratio/min": 0.4672684828440348, "sampling/sampling_logp_difference/max": 0.3500140905380249, "sampling/sampling_logp_difference/mean": 0.00397274117761602, "step": 20280, "step_time": 8.209912867564707, "student/entropy": 0.13090084809809924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001012037198, "completions/max_length": 468.46666666666664, "completions/max_terminated_length": 447.76666666666665, "completions/mean_length": 189.07000986735025, "completions/mean_terminated_length": 182.74456837972005, "completions/min_length": 56.43333333333333, "completions/min_terminated_length": 56.43333333333333, "entropy": 0.1293072992314895, "epoch": 0.7712755857669085, "eval/gt_acc_batch": 0.7947222491105398, "frac_reward_zero_std": 1.0, "grad_norm": 0.4622788727283478, "kd/policy_loss": 0.008500051653633515, "kd/rkl_advantage_mean": 0.6777686673022496, "kd/rkl_advantage_p95": 4.867590502897898, "kd/rkl_advantage_std": 2.205088542898496, "kd/ssd_loss": 0.0, "learning_rate": 2.2876238939733413e-07, "loss": 0.034000213940938315, "num_tokens": 664196015.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222173213959, "rewards/gt_logging_reward/std": 0.3935557991266251, "sampling/importance_sampling_ratio/max": 1.8839170257250468, "sampling/importance_sampling_ratio/mean": 0.9971115310986837, "sampling/importance_sampling_ratio/min": 0.43642234603563945, "sampling/sampling_logp_difference/max": 0.3154336074988047, "sampling/sampling_logp_difference/mean": 0.003927576045195262, "step": 20310, "step_time": 7.975428503796381, "student/entropy": 0.1293072992314895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029166668467223645, "completions/max_length": 488.73333333333335, "completions/max_terminated_length": 448.26666666666665, "completions/mean_length": 194.82667592366536, "completions/mean_terminated_length": 185.24785766601562, "completions/min_length": 57.7, "completions/min_terminated_length": 57.7, "entropy": 0.12776682718346516, "epoch": 0.772414840694186, "eval/gt_acc_batch": 0.7863889078299204, "frac_reward_zero_std": 1.0, "grad_norm": 0.2926277816295624, "kd/policy_loss": 0.00847606971122635, "kd/rkl_advantage_mean": 0.7514126753279319, "kd/rkl_advantage_p95": 5.007332567373911, "kd/rkl_advantage_std": 2.2179048667351404, "kd/ssd_loss": 0.0, "learning_rate": 2.2762313447005658e-07, "loss": 0.033904282251993816, "num_tokens": 665201407.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7863888899485271, "rewards/gt_logging_reward/std": 0.4020795678098997, "sampling/importance_sampling_ratio/max": 1.9262135942777, "sampling/importance_sampling_ratio/mean": 1.0002436061700186, "sampling/importance_sampling_ratio/min": 0.43968753814697265, "sampling/sampling_logp_difference/max": 0.34240060249964394, "sampling/sampling_logp_difference/mean": 0.0038962303117538494, "step": 20340, "step_time": 8.116740273094425, "student/entropy": 0.12776682718346516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667573153974, "completions/max_length": 475.3, "completions/max_terminated_length": 434.46666666666664, "completions/mean_length": 182.8372314453125, "completions/mean_terminated_length": 176.41494903564453, "completions/min_length": 49.733333333333334, "completions/min_terminated_length": 49.733333333333334, "entropy": 0.12733068155745667, "epoch": 0.7735540956214636, "eval/gt_acc_batch": 0.807500010728836, "frac_reward_zero_std": 1.0, "grad_norm": 0.30256038904190063, "kd/policy_loss": 0.0083961200589935, "kd/rkl_advantage_mean": 0.7129727742324273, "kd/rkl_advantage_p95": 4.96230438152949, "kd/rkl_advantage_std": 2.2137958923975627, "kd/ssd_loss": 0.0, "learning_rate": 2.26483879542779e-07, "loss": 0.03358448346455892, "num_tokens": 666163197.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8075000007947286, "rewards/gt_logging_reward/std": 0.38715618004401525, "sampling/importance_sampling_ratio/max": 1.8037002364794412, "sampling/importance_sampling_ratio/mean": 0.9960067252318064, "sampling/importance_sampling_ratio/min": 0.4721009537577629, "sampling/sampling_logp_difference/max": 0.3322459717591604, "sampling/sampling_logp_difference/mean": 0.003890217173223694, "step": 20370, "step_time": 8.01645794492506, "student/entropy": 0.12733068155745667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112212141354, "completions/max_length": 486.2, "completions/max_terminated_length": 455.3, "completions/mean_length": 184.96889902750652, "completions/mean_terminated_length": 178.1186945597331, "completions/min_length": 57.13333333333333, "completions/min_terminated_length": 57.13333333333333, "entropy": 0.12820448856800795, "epoch": 0.7746933505487411, "eval/gt_acc_batch": 0.8083333551883698, "frac_reward_zero_std": 1.0, "grad_norm": 0.25930845737457275, "kd/policy_loss": 0.008388680642625938, "kd/rkl_advantage_mean": 0.8331831268966198, "kd/rkl_advantage_p95": 5.090759726365407, "kd/rkl_advantage_std": 2.2311219443877537, "kd/ssd_loss": 0.0, "learning_rate": 2.2534462461550144e-07, "loss": 0.03355472087860108, "num_tokens": 667127925.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8083333333333333, "rewards/gt_logging_reward/std": 0.38204803665479026, "sampling/importance_sampling_ratio/max": 1.9556318958600363, "sampling/importance_sampling_ratio/mean": 1.0001160661379496, "sampling/importance_sampling_ratio/min": 0.47620013455549876, "sampling/sampling_logp_difference/max": 0.35184895396232607, "sampling/sampling_logp_difference/mean": 0.003915569256059825, "step": 20400, "step_time": 7.957684768066974, "student/entropy": 0.12820448856800795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0302777794500192, "completions/max_length": 507.3666666666667, "completions/max_terminated_length": 474.1333333333333, "completions/mean_length": 195.3394546508789, "completions/mean_terminated_length": 185.4649622599284, "completions/min_length": 52.5, "completions/min_terminated_length": 52.5, "entropy": 0.13439681877692541, "epoch": 0.7758326054760187, "eval/gt_acc_batch": 0.7800000230471293, "frac_reward_zero_std": 1.0, "grad_norm": 0.31644415855407715, "kd/policy_loss": 0.008886761129057655, "kd/rkl_advantage_mean": 0.7976116010298332, "kd/rkl_advantage_p95": 5.1088983873526255, "kd/rkl_advantage_std": 2.2580823649962745, "kd/ssd_loss": 0.0, "learning_rate": 2.242053696882239e-07, "loss": 0.03554704189300537, "num_tokens": 668134595.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7800000031789144, "rewards/gt_logging_reward/std": 0.406439150373141, "sampling/importance_sampling_ratio/max": 1.919628377755483, "sampling/importance_sampling_ratio/mean": 0.9980144759019216, "sampling/importance_sampling_ratio/min": 0.43747264544169107, "sampling/sampling_logp_difference/max": 0.3267417629559835, "sampling/sampling_logp_difference/mean": 0.004038412061830362, "step": 20430, "step_time": 8.264020851098273, "student/entropy": 0.13439681877692541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556795249384, "completions/max_length": 497.06666666666666, "completions/max_terminated_length": 447.1, "completions/mean_length": 191.98750915527344, "completions/mean_terminated_length": 183.74403330485026, "completions/min_length": 55.63333333333333, "completions/min_terminated_length": 55.63333333333333, "entropy": 0.13024160744001467, "epoch": 0.7769718604032962, "eval/gt_acc_batch": 0.7975000202655792, "frac_reward_zero_std": 1.0, "grad_norm": 0.29700151085853577, "kd/policy_loss": 0.008485517602336283, "kd/rkl_advantage_mean": 0.8106122590601444, "kd/rkl_advantage_p95": 5.158261066675186, "kd/rkl_advantage_std": 2.2664575318495435, "kd/ssd_loss": 0.0, "learning_rate": 2.2306611476094633e-07, "loss": 0.03394207159678141, "num_tokens": 669134918.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7975000043710073, "rewards/gt_logging_reward/std": 0.39130662952860196, "sampling/importance_sampling_ratio/max": 1.8729846835136414, "sampling/importance_sampling_ratio/mean": 0.9984130283196767, "sampling/importance_sampling_ratio/min": 0.46468472182750703, "sampling/sampling_logp_difference/max": 0.3190427303314209, "sampling/sampling_logp_difference/mean": 0.003921035102879008, "step": 20460, "step_time": 8.35740302723813, "student/entropy": 0.13024160744001467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013055556453764439, "completions/max_length": 469.4, "completions/max_terminated_length": 447.4, "completions/mean_length": 182.32528737386068, "completions/mean_terminated_length": 178.07462615966796, "completions/min_length": 56.9, "completions/min_terminated_length": 56.9, "entropy": 0.12661665181318918, "epoch": 0.7781111153305738, "eval/gt_acc_batch": 0.8161111215750376, "frac_reward_zero_std": 1.0, "grad_norm": 0.3114558756351471, "kd/policy_loss": 0.0079575753537938, "kd/rkl_advantage_mean": 0.7409959439188242, "kd/rkl_advantage_p95": 4.903533246119817, "kd/rkl_advantage_std": 2.1717200110356014, "kd/ssd_loss": 0.0, "learning_rate": 2.2192685983366877e-07, "loss": 0.031830302874247235, "num_tokens": 670085777.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8161111096541087, "rewards/gt_logging_reward/std": 0.37958431939284004, "sampling/importance_sampling_ratio/max": 1.8112470666567484, "sampling/importance_sampling_ratio/mean": 0.99882164200147, "sampling/importance_sampling_ratio/min": 0.477987410624822, "sampling/sampling_logp_difference/max": 0.32637261152267455, "sampling/sampling_logp_difference/mean": 0.003871117765083909, "step": 20490, "step_time": 7.794535872465349, "student/entropy": 0.12661665181318918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888889936109383, "completions/max_length": 479.4, "completions/max_terminated_length": 443.4, "completions/mean_length": 188.68028767903647, "completions/mean_terminated_length": 182.5912094116211, "completions/min_length": 48.1, "completions/min_terminated_length": 48.1, "entropy": 0.12709727038939794, "epoch": 0.7792503702578514, "eval/gt_acc_batch": 0.811388905843099, "frac_reward_zero_std": 1.0, "grad_norm": 0.2774048447608948, "kd/policy_loss": 0.00833214686329787, "kd/rkl_advantage_mean": 0.7962485716678203, "kd/rkl_advantage_p95": 5.080854288736979, "kd/rkl_advantage_std": 2.2329776336749396, "kd/ssd_loss": 0.0, "learning_rate": 2.207876049063912e-07, "loss": 0.0333285927772522, "num_tokens": 671072186.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.811388889948527, "rewards/gt_logging_reward/std": 0.3784083108107249, "sampling/importance_sampling_ratio/max": 1.937760063012441, "sampling/importance_sampling_ratio/mean": 1.0034212589263916, "sampling/importance_sampling_ratio/min": 0.4434123381972313, "sampling/sampling_logp_difference/max": 0.36574379205703733, "sampling/sampling_logp_difference/mean": 0.0038859563802058497, "step": 20520, "step_time": 8.137478721667625, "student/entropy": 0.12709727038939794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015277778760840495, "completions/max_length": 470.4, "completions/max_terminated_length": 436.7, "completions/mean_length": 179.57667643229166, "completions/mean_terminated_length": 174.49473317464194, "completions/min_length": 58.6, "completions/min_terminated_length": 58.6, "entropy": 0.1250147915755709, "epoch": 0.7803896251851289, "eval/gt_acc_batch": 0.8272222359975179, "frac_reward_zero_std": 1.0, "grad_norm": 0.2733175456523895, "kd/policy_loss": 0.008075692469719797, "kd/rkl_advantage_mean": 0.7823130888553957, "kd/rkl_advantage_p95": 5.107070451974868, "kd/rkl_advantage_std": 2.2532985190550487, "kd/ssd_loss": 0.0, "learning_rate": 2.1964834997911366e-07, "loss": 0.03230277299880981, "num_tokens": 672022550.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.827222224076589, "rewards/gt_logging_reward/std": 0.3677007665236791, "sampling/importance_sampling_ratio/max": 1.8743006904919943, "sampling/importance_sampling_ratio/mean": 0.9976526101430258, "sampling/importance_sampling_ratio/min": 0.4566743363936742, "sampling/sampling_logp_difference/max": 0.32676708896954854, "sampling/sampling_logp_difference/mean": 0.003806903787578146, "step": 20550, "step_time": 7.949342761800896, "student/entropy": 0.1250147915755709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0280555572360754, "completions/max_length": 491.6666666666667, "completions/max_terminated_length": 445.7, "completions/mean_length": 190.12417551676432, "completions/mean_terminated_length": 180.85047912597656, "completions/min_length": 53.63333333333333, "completions/min_terminated_length": 53.63333333333333, "entropy": 0.12474353766689698, "epoch": 0.7815288801124065, "eval/gt_acc_batch": 0.8022222499052684, "frac_reward_zero_std": 1.0, "grad_norm": 0.25872039794921875, "kd/policy_loss": 0.007990100781898945, "kd/rkl_advantage_mean": 0.7386558180054029, "kd/rkl_advantage_p95": 4.91678938070933, "kd/rkl_advantage_std": 2.1653066982825595, "kd/ssd_loss": 0.0, "learning_rate": 2.185090950518361e-07, "loss": 0.03196040789286295, "num_tokens": 673025021.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8022222240765889, "rewards/gt_logging_reward/std": 0.3924072136481603, "sampling/importance_sampling_ratio/max": 1.7982855637868245, "sampling/importance_sampling_ratio/mean": 0.9973934431870778, "sampling/importance_sampling_ratio/min": 0.4604299326737722, "sampling/sampling_logp_difference/max": 0.32739702065785725, "sampling/sampling_logp_difference/mean": 0.0038345685228705407, "step": 20580, "step_time": 8.336836162526742, "student/entropy": 0.12474353766689698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0247222236978511, "completions/max_length": 486.93333333333334, "completions/max_terminated_length": 446.43333333333334, "completions/mean_length": 189.30612030029297, "completions/mean_terminated_length": 181.31784311930338, "completions/min_length": 52.03333333333333, "completions/min_terminated_length": 52.03333333333333, "entropy": 0.12915779805431762, "epoch": 0.782668135039684, "eval/gt_acc_batch": 0.8061111191908519, "frac_reward_zero_std": 1.0, "grad_norm": 0.34086868166923523, "kd/policy_loss": 0.008408397401217372, "kd/rkl_advantage_mean": 0.8313202444463968, "kd/rkl_advantage_p95": 5.174305558204651, "kd/rkl_advantage_std": 2.2662012706200283, "kd/ssd_loss": 0.0, "learning_rate": 2.1736984012455852e-07, "loss": 0.03363358577092489, "num_tokens": 674018059.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8061111072699229, "rewards/gt_logging_reward/std": 0.3844926506280899, "sampling/importance_sampling_ratio/max": 1.9471496105194093, "sampling/importance_sampling_ratio/mean": 1.001315047343572, "sampling/importance_sampling_ratio/min": 0.43048441807428994, "sampling/sampling_logp_difference/max": 0.31256946325302126, "sampling/sampling_logp_difference/mean": 0.003987694329892596, "step": 20610, "step_time": 8.228230388935966, "student/entropy": 0.12915779805431762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018055556528270244, "completions/max_length": 495.03333333333336, "completions/max_terminated_length": 455.53333333333336, "completions/mean_length": 183.38612009684246, "completions/mean_terminated_length": 177.21105550130207, "completions/min_length": 50.03333333333333, "completions/min_terminated_length": 50.03333333333333, "entropy": 0.1295291451116403, "epoch": 0.7838073899669616, "eval/gt_acc_batch": 0.8027778168519338, "frac_reward_zero_std": 1.0, "grad_norm": 0.2735559940338135, "kd/policy_loss": 0.008408554876223207, "kd/rkl_advantage_mean": 0.7839258414382736, "kd/rkl_advantage_p95": 5.0856968522071835, "kd/rkl_advantage_std": 2.2452591558297477, "kd/ssd_loss": 0.0, "learning_rate": 2.1623058519728096e-07, "loss": 0.03363422552744547, "num_tokens": 674975385.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8027777830759685, "rewards/gt_logging_reward/std": 0.3929351935784022, "sampling/importance_sampling_ratio/max": 1.967348035176595, "sampling/importance_sampling_ratio/mean": 1.0070539156595866, "sampling/importance_sampling_ratio/min": 0.4514552672704061, "sampling/sampling_logp_difference/max": 0.3462051272392273, "sampling/sampling_logp_difference/mean": 0.003984668385237455, "step": 20640, "step_time": 8.08317957483911, "student/entropy": 0.1295291451116403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111250395576, "completions/max_length": 494.3, "completions/max_terminated_length": 465.4, "completions/mean_length": 192.83167622884113, "completions/mean_terminated_length": 184.36162770589192, "completions/min_length": 52.46666666666667, "completions/min_terminated_length": 52.46666666666667, "entropy": 0.12714999597519636, "epoch": 0.7849466448942392, "eval/gt_acc_batch": 0.8063889145851135, "frac_reward_zero_std": 1.0, "grad_norm": 0.29043424129486084, "kd/policy_loss": 0.008480954273060585, "kd/rkl_advantage_mean": 0.8095453088482221, "kd/rkl_advantage_p95": 5.142375206947326, "kd/rkl_advantage_std": 2.258004918694496, "kd/ssd_loss": 0.0, "learning_rate": 2.150913302700034e-07, "loss": 0.0339238166809082, "num_tokens": 675976443.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8063888847827911, "rewards/gt_logging_reward/std": 0.3893670529127121, "sampling/importance_sampling_ratio/max": 1.9321671009063721, "sampling/importance_sampling_ratio/mean": 0.9970778902371724, "sampling/importance_sampling_ratio/min": 0.4497093717257182, "sampling/sampling_logp_difference/max": 0.3071626683076223, "sampling/sampling_logp_difference/mean": 0.0038644756268089017, "step": 20670, "step_time": 8.20494957366803, "student/entropy": 0.12714999597519636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001198301712, "completions/max_length": 482.76666666666665, "completions/max_terminated_length": 449.7, "completions/mean_length": 181.37723185221355, "completions/mean_terminated_length": 174.77051493326823, "completions/min_length": 50.5, "completions/min_terminated_length": 50.5, "entropy": 0.13238883428275586, "epoch": 0.7860858998215168, "eval/gt_acc_batch": 0.7886111338933309, "frac_reward_zero_std": 1.0, "grad_norm": 0.31483423709869385, "kd/policy_loss": 0.009092929549903298, "kd/rkl_advantage_mean": 0.6969040374892453, "kd/rkl_advantage_p95": 4.834889809290568, "kd/rkl_advantage_std": 2.177247029542923, "kd/ssd_loss": 0.0, "learning_rate": 2.1395207534272585e-07, "loss": 0.036371715863545734, "num_tokens": 676940585.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7886111179987589, "rewards/gt_logging_reward/std": 0.3999864846467972, "sampling/importance_sampling_ratio/max": 1.8913215080897012, "sampling/importance_sampling_ratio/mean": 0.991120582818985, "sampling/importance_sampling_ratio/min": 0.45748386283715564, "sampling/sampling_logp_difference/max": 0.32482681969801586, "sampling/sampling_logp_difference/mean": 0.0039768167305737736, "step": 20700, "step_time": 8.2495124487672, "student/entropy": 0.13238883428275586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277779114743075, "completions/max_length": 480.0, "completions/max_terminated_length": 431.6666666666667, "completions/mean_length": 183.4636225382487, "completions/mean_terminated_length": 176.64421895345052, "completions/min_length": 52.13333333333333, "completions/min_terminated_length": 52.13333333333333, "entropy": 0.12650155425071716, "epoch": 0.7872251547487943, "eval/gt_acc_batch": 0.8083333512147267, "frac_reward_zero_std": 1.0, "grad_norm": 0.30248844623565674, "kd/policy_loss": 0.007829507708083838, "kd/rkl_advantage_mean": 0.766255919697384, "kd/rkl_advantage_p95": 4.961375705401102, "kd/rkl_advantage_std": 2.2081872185071307, "kd/ssd_loss": 0.0, "learning_rate": 2.128128204154483e-07, "loss": 0.03131803274154663, "num_tokens": 677900590.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8083333273728689, "rewards/gt_logging_reward/std": 0.38521882792313894, "sampling/importance_sampling_ratio/max": 1.9455652077992758, "sampling/importance_sampling_ratio/mean": 1.0036179065704345, "sampling/importance_sampling_ratio/min": 0.4743225465218226, "sampling/sampling_logp_difference/max": 0.336443022886912, "sampling/sampling_logp_difference/mean": 0.003864831360988319, "step": 20730, "step_time": 7.845900852660027, "student/entropy": 0.12650155425071716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02138889003545046, "completions/max_length": 501.8333333333333, "completions/max_terminated_length": 464.26666666666665, "completions/mean_length": 191.30612131754557, "completions/mean_terminated_length": 184.2746790568034, "completions/min_length": 55.93333333333333, "completions/min_terminated_length": 55.93333333333333, "entropy": 0.13384631363054117, "epoch": 0.7883644096760718, "eval/gt_acc_batch": 0.7922222356001536, "frac_reward_zero_std": 1.0, "grad_norm": 0.32767125964164734, "kd/policy_loss": 0.009036393758530417, "kd/rkl_advantage_mean": 0.7906889160474141, "kd/rkl_advantage_p95": 5.1145099322001135, "kd/rkl_advantage_std": 2.268378095825513, "kd/ssd_loss": 0.0, "learning_rate": 2.1167356548817071e-07, "loss": 0.03614558378855388, "num_tokens": 678891324.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222236792247, "rewards/gt_logging_reward/std": 0.3980928326646487, "sampling/importance_sampling_ratio/max": 1.939883021513621, "sampling/importance_sampling_ratio/mean": 1.003896524508794, "sampling/importance_sampling_ratio/min": 0.46015554865201314, "sampling/sampling_logp_difference/max": 0.307533323764801, "sampling/sampling_logp_difference/mean": 0.004090004476408163, "step": 20760, "step_time": 8.102029238430744, "student/entropy": 0.13384631363054117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166667989144724, "completions/max_length": 505.8666666666667, "completions/max_terminated_length": 475.43333333333334, "completions/mean_length": 191.73639831542968, "completions/mean_terminated_length": 183.8676737467448, "completions/min_length": 54.03333333333333, "completions/min_terminated_length": 54.03333333333333, "entropy": 0.13296898659318684, "epoch": 0.7895036646033494, "eval/gt_acc_batch": 0.7927778005599976, "frac_reward_zero_std": 1.0, "grad_norm": 0.28788962960243225, "kd/policy_loss": 0.009044903571096559, "kd/rkl_advantage_mean": 0.7999830225327362, "kd/rkl_advantage_p95": 5.118143264452616, "kd/rkl_advantage_std": 2.284919469555219, "kd/ssd_loss": 0.0, "learning_rate": 2.1053431056089316e-07, "loss": 0.036179614067077634, "num_tokens": 679898695.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7927777747313182, "rewards/gt_logging_reward/std": 0.4014110763867696, "sampling/importance_sampling_ratio/max": 1.9107422947883606, "sampling/importance_sampling_ratio/mean": 0.999746960401535, "sampling/importance_sampling_ratio/min": 0.4534277538458506, "sampling/sampling_logp_difference/max": 0.3153413494427999, "sampling/sampling_logp_difference/mean": 0.004006055183708668, "step": 20790, "step_time": 8.461855065626635, "student/entropy": 0.13296898659318684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112305273613, "completions/max_length": 472.6333333333333, "completions/max_terminated_length": 446.56666666666666, "completions/mean_length": 192.06973317464193, "completions/mean_terminated_length": 185.31256256103515, "completions/min_length": 57.266666666666666, "completions/min_terminated_length": 57.266666666666666, "entropy": 0.12618211389829714, "epoch": 0.7906429195306269, "eval/gt_acc_batch": 0.7769444624582926, "frac_reward_zero_std": 1.0, "grad_norm": 0.28828006982803345, "kd/policy_loss": 0.008607525440553824, "kd/rkl_advantage_mean": 0.7057499185282116, "kd/rkl_advantage_p95": 4.88022495508194, "kd/rkl_advantage_std": 2.193454028169314, "kd/ssd_loss": 0.0, "learning_rate": 2.0939505563361563e-07, "loss": 0.03443010250727336, "num_tokens": 680896458.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7769444425900777, "rewards/gt_logging_reward/std": 0.4092552165190379, "sampling/importance_sampling_ratio/max": 1.9662815928459167, "sampling/importance_sampling_ratio/mean": 1.0038348774115244, "sampling/importance_sampling_ratio/min": 0.46886705557505287, "sampling/sampling_logp_difference/max": 0.33966626624266305, "sampling/sampling_logp_difference/mean": 0.0038516991461316746, "step": 20820, "step_time": 8.054982615999567, "student/entropy": 0.12618211389829714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01833333441366752, "completions/max_length": 460.2, "completions/max_terminated_length": 425.76666666666665, "completions/mean_length": 179.75167694091797, "completions/mean_terminated_length": 173.8018834431966, "completions/min_length": 54.93333333333333, "completions/min_terminated_length": 54.93333333333333, "entropy": 0.12987176875273387, "epoch": 0.7917821744579046, "eval/gt_acc_batch": 0.7980555733044942, "frac_reward_zero_std": 1.0, "grad_norm": 0.2645399868488312, "kd/policy_loss": 0.008225047215819358, "kd/rkl_advantage_mean": 0.6820656571472985, "kd/rkl_advantage_p95": 4.794006917874018, "kd/rkl_advantage_std": 2.186598411202431, "kd/ssd_loss": 0.0, "learning_rate": 2.0825580070633805e-07, "loss": 0.03290019035339355, "num_tokens": 681843796.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7980555554231008, "rewards/gt_logging_reward/std": 0.3875097299615542, "sampling/importance_sampling_ratio/max": 1.973408003648122, "sampling/importance_sampling_ratio/mean": 0.9969303965568542, "sampling/importance_sampling_ratio/min": 0.4358108381430308, "sampling/sampling_logp_difference/max": 0.3313009242216746, "sampling/sampling_logp_difference/mean": 0.0039783066449066, "step": 20850, "step_time": 8.147455522969056, "student/entropy": 0.12987176875273387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223927577337, "completions/max_length": 475.76666666666665, "completions/max_terminated_length": 450.6, "completions/mean_length": 190.93195343017578, "completions/mean_terminated_length": 181.2626159667969, "completions/min_length": 56.0, "completions/min_terminated_length": 56.0, "entropy": 0.13212150962402422, "epoch": 0.7929214293851821, "eval/gt_acc_batch": 0.77333336075147, "frac_reward_zero_std": 1.0, "grad_norm": 0.2849154770374298, "kd/policy_loss": 0.009008160023950041, "kd/rkl_advantage_mean": 0.7475448032375425, "kd/rkl_advantage_p95": 4.969313657283783, "kd/rkl_advantage_std": 2.1991303543249767, "kd/ssd_loss": 0.0, "learning_rate": 2.071165457790605e-07, "loss": 0.036032644907633464, "num_tokens": 682848863.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7733333309491476, "rewards/gt_logging_reward/std": 0.4101488202810287, "sampling/importance_sampling_ratio/max": 1.9111072738965353, "sampling/importance_sampling_ratio/mean": 0.9972355405489604, "sampling/importance_sampling_ratio/min": 0.45075533737738926, "sampling/sampling_logp_difference/max": 0.3197549968957901, "sampling/sampling_logp_difference/mean": 0.003992000004897515, "step": 20880, "step_time": 8.52326264863465, "student/entropy": 0.13212150962402422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013611111789941788, "completions/max_length": 486.23333333333335, "completions/max_terminated_length": 440.5, "completions/mean_length": 179.97862091064454, "completions/mean_terminated_length": 175.42652079264323, "completions/min_length": 59.4, "completions/min_terminated_length": 59.4, "entropy": 0.12789772152900697, "epoch": 0.7940606843124597, "eval/gt_acc_batch": 0.8172222415606181, "frac_reward_zero_std": 1.0, "grad_norm": 0.29409629106521606, "kd/policy_loss": 0.008386804023757577, "kd/rkl_advantage_mean": 0.7272642849013209, "kd/rkl_advantage_p95": 4.896261469523112, "kd/rkl_advantage_std": 2.197756173213323, "kd/ssd_loss": 0.0, "learning_rate": 2.059772908517829e-07, "loss": 0.03354722261428833, "num_tokens": 683800610.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8172222216924031, "rewards/gt_logging_reward/std": 0.3801286409298579, "sampling/importance_sampling_ratio/max": 1.8273634274800619, "sampling/importance_sampling_ratio/mean": 0.998584904273351, "sampling/importance_sampling_ratio/min": 0.4629117200771968, "sampling/sampling_logp_difference/max": 0.3172488133112589, "sampling/sampling_logp_difference/mean": 0.0038750738138332965, "step": 20910, "step_time": 8.344998847233365, "student/entropy": 0.12789772152900697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030000001844018697, "completions/max_length": 490.1333333333333, "completions/max_terminated_length": 459.4, "completions/mean_length": 194.3577880859375, "completions/mean_terminated_length": 184.73109486897786, "completions/min_length": 51.6, "completions/min_terminated_length": 51.6, "entropy": 0.1318827496220668, "epoch": 0.7951999392397372, "eval/gt_acc_batch": 0.7775000095367431, "frac_reward_zero_std": 1.0, "grad_norm": 0.2835569977760315, "kd/policy_loss": 0.008599493947500984, "kd/rkl_advantage_mean": 0.7423185429225365, "kd/rkl_advantage_p95": 4.951168139775594, "kd/rkl_advantage_std": 2.206764267881711, "kd/ssd_loss": 0.0, "learning_rate": 2.0483803592450535e-07, "loss": 0.03439797957738241, "num_tokens": 684809058.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7775000015894572, "rewards/gt_logging_reward/std": 0.40780978202819823, "sampling/importance_sampling_ratio/max": 1.8538680871327717, "sampling/importance_sampling_ratio/mean": 0.9940573831399282, "sampling/importance_sampling_ratio/min": 0.42840509017308553, "sampling/sampling_logp_difference/max": 0.29723299741744996, "sampling/sampling_logp_difference/mean": 0.0039596220711246135, "step": 20940, "step_time": 8.397919805869849, "student/entropy": 0.1318827496220668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055557068437336, "completions/max_length": 499.8, "completions/max_terminated_length": 472.26666666666665, "completions/mean_length": 191.39917755126953, "completions/mean_terminated_length": 183.81275024414063, "completions/min_length": 53.9, "completions/min_terminated_length": 53.9, "entropy": 0.130138896095256, "epoch": 0.7963391941670148, "eval/gt_acc_batch": 0.781944461663564, "frac_reward_zero_std": 1.0, "grad_norm": 0.3593582212924957, "kd/policy_loss": 0.008704493575108548, "kd/rkl_advantage_mean": 0.8481934975832701, "kd/rkl_advantage_p95": 5.217655406395594, "kd/rkl_advantage_std": 2.2757728179295857, "kd/ssd_loss": 0.0, "learning_rate": 2.0369878099722782e-07, "loss": 0.03481797377268473, "num_tokens": 685803983.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7819444437821707, "rewards/gt_logging_reward/std": 0.4063135474920273, "sampling/importance_sampling_ratio/max": 1.846528740723928, "sampling/importance_sampling_ratio/mean": 0.9944670220216115, "sampling/importance_sampling_ratio/min": 0.41029917597770693, "sampling/sampling_logp_difference/max": 0.3373536547025045, "sampling/sampling_logp_difference/mean": 0.003911465282241503, "step": 20970, "step_time": 8.462876100897363, "student/entropy": 0.130138896095256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018333334382623433, "completions/max_length": 490.8666666666667, "completions/max_terminated_length": 463.3, "completions/mean_length": 185.86528727213542, "completions/mean_terminated_length": 179.8620819091797, "completions/min_length": 58.86666666666667, "completions/min_terminated_length": 58.86666666666667, "entropy": 0.12768895191450913, "epoch": 0.7974784490942923, "eval/gt_acc_batch": 0.8013888955116272, "frac_reward_zero_std": 1.0, "grad_norm": 0.29044830799102783, "kd/policy_loss": 0.008308052979797746, "kd/rkl_advantage_mean": 0.8175796000039858, "kd/rkl_advantage_p95": 5.062593924999237, "kd/rkl_advantage_std": 2.2508513212203978, "kd/ssd_loss": 0.0, "learning_rate": 2.0255952606995024e-07, "loss": 0.03323222398757934, "num_tokens": 686774394.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888915379842, "rewards/gt_logging_reward/std": 0.3929796705643336, "sampling/importance_sampling_ratio/max": 1.8803833603858948, "sampling/importance_sampling_ratio/mean": 1.0027410606543223, "sampling/importance_sampling_ratio/min": 0.4828127185503642, "sampling/sampling_logp_difference/max": 0.3038428246974945, "sampling/sampling_logp_difference/mean": 0.0038897542515769603, "step": 21000, "step_time": 8.314082260394935, "student/entropy": 0.12768895191450913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111253499985, "completions/max_length": 489.43333333333334, "completions/max_terminated_length": 457.06666666666666, "completions/mean_length": 187.55806427001954, "completions/mean_terminated_length": 178.97872772216797, "completions/min_length": 53.266666666666666, "completions/min_terminated_length": 53.266666666666666, "entropy": 0.13096216283738613, "epoch": 0.7986177040215698, "eval/gt_acc_batch": 0.8055555860201518, "frac_reward_zero_std": 1.0, "grad_norm": 0.31705352663993835, "kd/policy_loss": 0.00852491333692645, "kd/rkl_advantage_mean": 0.7536177281290293, "kd/rkl_advantage_p95": 5.037333605686824, "kd/rkl_advantage_std": 2.2187119563420614, "kd/ssd_loss": 0.0, "learning_rate": 2.0142027114267268e-07, "loss": 0.034099654356638594, "num_tokens": 687753307.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8055555542310079, "rewards/gt_logging_reward/std": 0.3906245400508245, "sampling/importance_sampling_ratio/max": 1.9551949659983316, "sampling/importance_sampling_ratio/mean": 0.9988263050715128, "sampling/importance_sampling_ratio/min": 0.43537881374359133, "sampling/sampling_logp_difference/max": 0.29422687689463295, "sampling/sampling_logp_difference/mean": 0.003993981385913988, "step": 21030, "step_time": 8.255806891002187, "student/entropy": 0.13096216283738613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112528791032, "completions/max_length": 481.23333333333335, "completions/max_terminated_length": 440.2, "completions/mean_length": 191.4452891031901, "completions/mean_terminated_length": 183.75846354166666, "completions/min_length": 56.93333333333333, "completions/min_terminated_length": 56.93333333333333, "entropy": 0.13012899328023195, "epoch": 0.7997569589488475, "eval/gt_acc_batch": 0.800000011920929, "frac_reward_zero_std": 1.0, "grad_norm": 0.5345389246940613, "kd/policy_loss": 0.00909100923454389, "kd/rkl_advantage_mean": 0.7432518806308508, "kd/rkl_advantage_p95": 4.99458401799202, "kd/rkl_advantage_std": 2.2186006546020507, "kd/ssd_loss": 0.0, "learning_rate": 2.0028101621539513e-07, "loss": 0.03636404275894165, "num_tokens": 688751294.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8, "rewards/gt_logging_reward/std": 0.39726589719454447, "sampling/importance_sampling_ratio/max": 1.916915241877238, "sampling/importance_sampling_ratio/mean": 0.9994440277417501, "sampling/importance_sampling_ratio/min": 0.45043237805366515, "sampling/sampling_logp_difference/max": 0.3412542382876078, "sampling/sampling_logp_difference/mean": 0.003987054449195663, "step": 21060, "step_time": 8.216286845761351, "student/entropy": 0.13012899328023195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02750000162050128, "completions/max_length": 501.4, "completions/max_terminated_length": 476.5, "completions/mean_length": 197.3202885945638, "completions/mean_terminated_length": 188.42169952392578, "completions/min_length": 55.2, "completions/min_terminated_length": 55.2, "entropy": 0.12855937325706085, "epoch": 0.800896213876125, "eval/gt_acc_batch": 0.7777777930100759, "frac_reward_zero_std": 1.0, "grad_norm": 0.39833003282546997, "kd/policy_loss": 0.00861167978340139, "kd/rkl_advantage_mean": 0.7904579558720192, "kd/rkl_advantage_p95": 5.086360319455465, "kd/rkl_advantage_std": 2.248361203074455, "kd/ssd_loss": 0.0, "learning_rate": 1.9914176128811755e-07, "loss": 0.03444672425587972, "num_tokens": 689762527.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.777777777115504, "rewards/gt_logging_reward/std": 0.40930961569150287, "sampling/importance_sampling_ratio/max": 1.9196407755215963, "sampling/importance_sampling_ratio/mean": 0.9992976903915405, "sampling/importance_sampling_ratio/min": 0.4622038662433624, "sampling/sampling_logp_difference/max": 0.31345277627309165, "sampling/sampling_logp_difference/mean": 0.00391203563194722, "step": 21090, "step_time": 8.27229422122958, "student/entropy": 0.12855937325706085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013750000856816769, "completions/max_length": 463.2, "completions/max_terminated_length": 419.85, "completions/mean_length": 186.58417587280275, "completions/mean_terminated_length": 182.1340789794922, "completions/min_length": 58.65, "completions/min_terminated_length": 58.65, "entropy": 0.12939047580584884, "epoch": 0.8020354688034026, "eval/gt_acc_batch": 0.7875000208616256, "frac_reward_zero_std": 1.0, "grad_norm": 0.24375629425048828, "kd/policy_loss": 0.008131009031785652, "kd/rkl_advantage_mean": 0.6957314797269646, "kd/rkl_advantage_p95": 4.799916455149651, "kd/rkl_advantage_std": 2.151377236843109, "kd/ssd_loss": 0.0, "learning_rate": 1.9800250636084002e-07, "loss": 0.03252404034137726, "num_tokens": 690754535.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7874999940395355, "rewards/gt_logging_reward/std": 0.40385905653238297, "sampling/importance_sampling_ratio/max": 1.8248178124427796, "sampling/importance_sampling_ratio/mean": 0.9947557032108307, "sampling/importance_sampling_ratio/min": 0.4518837913870811, "sampling/sampling_logp_difference/max": 0.3027741551399231, "sampling/sampling_logp_difference/mean": 0.003884607832878828, "step": 21120, "step_time": 7.913813526606828, "student/entropy": 0.12939047580584884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112336317697, "completions/max_length": 489.43333333333334, "completions/max_terminated_length": 453.3666666666667, "completions/mean_length": 192.8969533284505, "completions/mean_terminated_length": 186.09339345296223, "completions/min_length": 58.666666666666664, "completions/min_terminated_length": 58.666666666666664, "entropy": 0.13123528690387806, "epoch": 0.8031747237306801, "eval/gt_acc_batch": 0.7786111295223236, "frac_reward_zero_std": 1.0, "grad_norm": 0.29372453689575195, "kd/policy_loss": 0.008425119538636257, "kd/rkl_advantage_mean": 0.7806499759977062, "kd/rkl_advantage_p95": 4.984885656833649, "kd/rkl_advantage_std": 2.219469228386879, "kd/ssd_loss": 0.0, "learning_rate": 1.9686325143356243e-07, "loss": 0.03370047807693481, "num_tokens": 691742708.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7786111116409302, "rewards/gt_logging_reward/std": 0.40786136786142985, "sampling/importance_sampling_ratio/max": 1.9203442653020224, "sampling/importance_sampling_ratio/mean": 0.9948749522368113, "sampling/importance_sampling_ratio/min": 0.44870459934075674, "sampling/sampling_logp_difference/max": 0.315538227558136, "sampling/sampling_logp_difference/mean": 0.003905812643157939, "step": 21150, "step_time": 7.937665155903475, "student/entropy": 0.13123528690387806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02888889048869411, "completions/max_length": 493.1333333333333, "completions/max_terminated_length": 456.1333333333333, "completions/mean_length": 194.62501017252603, "completions/mean_terminated_length": 185.605344136556, "completions/min_length": 53.8, "completions/min_terminated_length": 53.8, "entropy": 0.13520498418559632, "epoch": 0.8043139786579577, "eval/gt_acc_batch": 0.781388912598292, "frac_reward_zero_std": 1.0, "grad_norm": 0.24039152264595032, "kd/policy_loss": 0.009118565280611316, "kd/rkl_advantage_mean": 0.7751866079866886, "kd/rkl_advantage_p95": 5.142460225025813, "kd/rkl_advantage_std": 2.269443773229917, "kd/ssd_loss": 0.0, "learning_rate": 1.9572399650628488e-07, "loss": 0.03647425969441732, "num_tokens": 692755294.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7813888847827911, "rewards/gt_logging_reward/std": 0.40438465078671776, "sampling/importance_sampling_ratio/max": 1.9347715894381206, "sampling/importance_sampling_ratio/mean": 1.0023068030675253, "sampling/importance_sampling_ratio/min": 0.4184630736708641, "sampling/sampling_logp_difference/max": 0.3452309250831604, "sampling/sampling_logp_difference/mean": 0.004065493059655031, "step": 21180, "step_time": 8.157987681033168, "student/entropy": 0.13520498418559632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02833333509042859, "completions/max_length": 498.96666666666664, "completions/max_terminated_length": 451.2, "completions/mean_length": 191.89945424397786, "completions/mean_terminated_length": 182.65874532063802, "completions/min_length": 47.333333333333336, "completions/min_terminated_length": 47.333333333333336, "entropy": 0.13326559762159984, "epoch": 0.8054532335852352, "eval/gt_acc_batch": 0.7869444648424785, "frac_reward_zero_std": 1.0, "grad_norm": 0.2724500894546509, "kd/policy_loss": 0.008758121812328075, "kd/rkl_advantage_mean": 0.8229799826939901, "kd/rkl_advantage_p95": 5.141605557998021, "kd/rkl_advantage_std": 2.256368734439214, "kd/ssd_loss": 0.0, "learning_rate": 1.9458474157900732e-07, "loss": 0.03503248294194539, "num_tokens": 693749308.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444489479065, "rewards/gt_logging_reward/std": 0.3995523348450661, "sampling/importance_sampling_ratio/max": 1.9108855247497558, "sampling/importance_sampling_ratio/mean": 0.9965784549713135, "sampling/importance_sampling_ratio/min": 0.40940639277299246, "sampling/sampling_logp_difference/max": 0.3992016871770223, "sampling/sampling_logp_difference/mean": 0.003991637953246633, "step": 21210, "step_time": 8.045575109260001, "student/entropy": 0.13326559762159984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028888890519738197, "completions/max_length": 482.3, "completions/max_terminated_length": 448.8333333333333, "completions/mean_length": 191.83834381103514, "completions/mean_terminated_length": 182.60110880533855, "completions/min_length": 54.733333333333334, "completions/min_terminated_length": 54.733333333333334, "entropy": 0.13152330443263055, "epoch": 0.8065924885125129, "eval/gt_acc_batch": 0.7869444588820139, "frac_reward_zero_std": 1.0, "grad_norm": 0.24029015004634857, "kd/policy_loss": 0.008610709855565802, "kd/rkl_advantage_mean": 0.7209165226047237, "kd/rkl_advantage_p95": 4.917790903647741, "kd/rkl_advantage_std": 2.2127308348814645, "kd/ssd_loss": 0.0, "learning_rate": 1.9344548665172977e-07, "loss": 0.034442838033040366, "num_tokens": 694756566.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7869444449742635, "rewards/gt_logging_reward/std": 0.40343409578005474, "sampling/importance_sampling_ratio/max": 1.8957260092099508, "sampling/importance_sampling_ratio/mean": 0.9964794476826986, "sampling/importance_sampling_ratio/min": 0.44753661900758746, "sampling/sampling_logp_difference/max": 0.3381652812163035, "sampling/sampling_logp_difference/mean": 0.003952939948067069, "step": 21240, "step_time": 8.051577260938938, "student/entropy": 0.13152330443263055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02222222350537777, "completions/max_length": 499.26666666666665, "completions/max_terminated_length": 461.96666666666664, "completions/mean_length": 194.73473154703777, "completions/mean_terminated_length": 187.49263712565104, "completions/min_length": 58.2, "completions/min_terminated_length": 58.2, "entropy": 0.1273087098573645, "epoch": 0.8077317434397904, "eval/gt_acc_batch": 0.7975000202655792, "frac_reward_zero_std": 1.0, "grad_norm": 0.28558242321014404, "kd/policy_loss": 0.008055321513287103, "kd/rkl_advantage_mean": 0.7246089035955569, "kd/rkl_advantage_p95": 4.897933711608251, "kd/rkl_advantage_std": 2.1819740047057468, "kd/ssd_loss": 0.0, "learning_rate": 1.923062317244522e-07, "loss": 0.0322212815284729, "num_tokens": 695762723.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7974999944368998, "rewards/gt_logging_reward/std": 0.3955924371878306, "sampling/importance_sampling_ratio/max": 1.965640119711558, "sampling/importance_sampling_ratio/mean": 0.9942101458708446, "sampling/importance_sampling_ratio/min": 0.45960119664669036, "sampling/sampling_logp_difference/max": 0.2868567486604055, "sampling/sampling_logp_difference/mean": 0.0038575592217966912, "step": 21270, "step_time": 7.991447956441941, "student/entropy": 0.1273087098573645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018055556590358415, "completions/max_length": 484.73333333333335, "completions/max_terminated_length": 449.3, "completions/mean_length": 183.897509765625, "completions/mean_terminated_length": 177.93607533772786, "completions/min_length": 48.666666666666664, "completions/min_terminated_length": 48.666666666666664, "entropy": 0.1233679989973704, "epoch": 0.8088709983670679, "eval/gt_acc_batch": 0.8072222411632538, "frac_reward_zero_std": 1.0, "grad_norm": 0.33473846316337585, "kd/policy_loss": 0.007935358013492077, "kd/rkl_advantage_mean": 0.6881830565631389, "kd/rkl_advantage_p95": 4.779024289051692, "kd/rkl_advantage_std": 2.1548807621002197, "kd/ssd_loss": 0.0, "learning_rate": 1.9116697679717463e-07, "loss": 0.03174143234888713, "num_tokens": 696724722.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222173213959, "rewards/gt_logging_reward/std": 0.3881600668032964, "sampling/importance_sampling_ratio/max": 1.8383774201075236, "sampling/importance_sampling_ratio/mean": 0.9970783611138662, "sampling/importance_sampling_ratio/min": 0.45794594784577686, "sampling/sampling_logp_difference/max": 0.3310838023821513, "sampling/sampling_logp_difference/mean": 0.0037320230699454744, "step": 21300, "step_time": 7.909751482368059, "student/entropy": 0.1233679989973704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778673917055, "completions/max_length": 490.3, "completions/max_terminated_length": 456.06666666666666, "completions/mean_length": 187.6077870686849, "completions/mean_terminated_length": 181.94668833414713, "completions/min_length": 59.06666666666667, "completions/min_terminated_length": 59.06666666666667, "entropy": 0.13464793724318344, "epoch": 0.8100102532943455, "eval/gt_acc_batch": 0.7986111342906952, "frac_reward_zero_std": 1.0, "grad_norm": 0.4245620667934418, "kd/policy_loss": 0.008983809013928597, "kd/rkl_advantage_mean": 0.7822890217726429, "kd/rkl_advantage_p95": 5.114395962158839, "kd/rkl_advantage_std": 2.2638967007398607, "kd/ssd_loss": 0.0, "learning_rate": 1.9002772186989707e-07, "loss": 0.03593524297078451, "num_tokens": 697699550.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111084620158, "rewards/gt_logging_reward/std": 0.3967046856880188, "sampling/importance_sampling_ratio/max": 1.8580148855845133, "sampling/importance_sampling_ratio/mean": 0.9994392613569896, "sampling/importance_sampling_ratio/min": 0.4498531957467397, "sampling/sampling_logp_difference/max": 0.362181959549586, "sampling/sampling_logp_difference/mean": 0.004044527358685931, "step": 21330, "step_time": 7.825559357461558, "student/entropy": 0.13464793724318344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445775200924, "completions/max_length": 488.3, "completions/max_terminated_length": 445.3333333333333, "completions/mean_length": 190.79334411621093, "completions/mean_terminated_length": 183.86802571614584, "completions/min_length": 54.3, "completions/min_terminated_length": 54.3, "entropy": 0.1274471778422594, "epoch": 0.811149508221623, "eval/gt_acc_batch": 0.7816666762034098, "frac_reward_zero_std": 1.0, "grad_norm": 0.25712937116622925, "kd/policy_loss": 0.008506203854146103, "kd/rkl_advantage_mean": 0.7519914568712314, "kd/rkl_advantage_p95": 5.061613750457764, "kd/rkl_advantage_std": 2.237518349289894, "kd/ssd_loss": 0.0, "learning_rate": 1.8888846694261952e-07, "loss": 0.03402482271194458, "num_tokens": 698701814.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7816666662693024, "rewards/gt_logging_reward/std": 0.4059631168842316, "sampling/importance_sampling_ratio/max": 1.8578107714653016, "sampling/importance_sampling_ratio/mean": 0.9978854934374491, "sampling/importance_sampling_ratio/min": 0.4645260771115621, "sampling/sampling_logp_difference/max": 0.335895444949468, "sampling/sampling_logp_difference/mean": 0.003906302984493474, "step": 21360, "step_time": 8.11195277600588, "student/entropy": 0.1274471778422594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112280438343, "completions/max_length": 501.76666666666665, "completions/max_terminated_length": 453.0, "completions/mean_length": 182.49695332845053, "completions/mean_terminated_length": 174.50060526529947, "completions/min_length": 52.86666666666667, "completions/min_terminated_length": 52.86666666666667, "entropy": 0.1298230215907097, "epoch": 0.8122887631489006, "eval/gt_acc_batch": 0.8013889074325562, "frac_reward_zero_std": 1.0, "grad_norm": 0.315913587808609, "kd/policy_loss": 0.008438163456351807, "kd/rkl_advantage_mean": 0.7535806958253185, "kd/rkl_advantage_p95": 5.06130144794782, "kd/rkl_advantage_std": 2.258491728703181, "kd/ssd_loss": 0.0, "learning_rate": 1.8774921201534196e-07, "loss": 0.03375264803568522, "num_tokens": 699659107.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888875643412, "rewards/gt_logging_reward/std": 0.39203139742215476, "sampling/importance_sampling_ratio/max": 1.8814522663752238, "sampling/importance_sampling_ratio/mean": 1.0021841903527577, "sampling/importance_sampling_ratio/min": 0.4689969380696615, "sampling/sampling_logp_difference/max": 0.32055510183175406, "sampling/sampling_logp_difference/mean": 0.003913690041129788, "step": 21390, "step_time": 7.9487288718043905, "student/entropy": 0.1298230215907097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556789040567, "completions/max_length": 482.56666666666666, "completions/max_terminated_length": 456.9, "completions/mean_length": 190.2841771443685, "completions/mean_terminated_length": 182.85309397379558, "completions/min_length": 51.266666666666666, "completions/min_terminated_length": 51.266666666666666, "entropy": 0.12893607821315528, "epoch": 0.8134280180761781, "eval/gt_acc_batch": 0.8005555808544159, "frac_reward_zero_std": 1.0, "grad_norm": 0.24541622400283813, "kd/policy_loss": 0.007977969773734609, "kd/rkl_advantage_mean": 0.7155021130902848, "kd/rkl_advantage_p95": 4.905288430054982, "kd/rkl_advantage_std": 2.205449614922206, "kd/ssd_loss": 0.0, "learning_rate": 1.866099570880644e-07, "loss": 0.031911879777908325, "num_tokens": 700643418.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8005555510520935, "rewards/gt_logging_reward/std": 0.39207402169704436, "sampling/importance_sampling_ratio/max": 1.8734707554181418, "sampling/importance_sampling_ratio/mean": 0.9976408084233602, "sampling/importance_sampling_ratio/min": 0.46584392885367076, "sampling/sampling_logp_difference/max": 0.3085514783859253, "sampling/sampling_logp_difference/mean": 0.0038716528797522187, "step": 21420, "step_time": 7.978377699369836, "student/entropy": 0.12893607821315528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02138889003545046, "completions/max_length": 488.76666666666665, "completions/max_terminated_length": 463.96666666666664, "completions/mean_length": 195.21862030029297, "completions/mean_terminated_length": 188.56088358561198, "completions/min_length": 53.166666666666664, "completions/min_terminated_length": 53.166666666666664, "entropy": 0.1308124886204799, "epoch": 0.8145672730034558, "eval/gt_acc_batch": 0.784166689713796, "frac_reward_zero_std": 1.0, "grad_norm": 0.25789692997932434, "kd/policy_loss": 0.00879050512254859, "kd/rkl_advantage_mean": 0.7033161686422924, "kd/rkl_advantage_p95": 4.891208859284719, "kd/rkl_advantage_std": 2.1766762415568035, "kd/ssd_loss": 0.0, "learning_rate": 1.8547070216078682e-07, "loss": 0.03516202767690023, "num_tokens": 701649581.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7841666638851166, "rewards/gt_logging_reward/std": 0.401549357175827, "sampling/importance_sampling_ratio/max": 1.9654080549875894, "sampling/importance_sampling_ratio/mean": 0.9998113751411438, "sampling/importance_sampling_ratio/min": 0.45466448962688444, "sampling/sampling_logp_difference/max": 0.3273669362068176, "sampling/sampling_logp_difference/mean": 0.003996564641905328, "step": 21450, "step_time": 7.951938066908042, "student/entropy": 0.1308124886204799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223250816264, "completions/max_length": 485.9, "completions/max_terminated_length": 441.43333333333334, "completions/mean_length": 188.1672337849935, "completions/mean_terminated_length": 181.79902038574218, "completions/min_length": 61.4, "completions/min_terminated_length": 61.4, "entropy": 0.1277515217040976, "epoch": 0.8157065279307333, "eval/gt_acc_batch": 0.8108333647251129, "frac_reward_zero_std": 1.0, "grad_norm": 0.23814278841018677, "kd/policy_loss": 0.008341093972558156, "kd/rkl_advantage_mean": 0.7082819820245884, "kd/rkl_advantage_p95": 4.880656522512436, "kd/rkl_advantage_std": 2.175767601529757, "kd/ssd_loss": 0.0, "learning_rate": 1.8433144723350927e-07, "loss": 0.03336437543233236, "num_tokens": 702627831.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8108333349227905, "rewards/gt_logging_reward/std": 0.3865358720223109, "sampling/importance_sampling_ratio/max": 1.9074494043986003, "sampling/importance_sampling_ratio/mean": 1.0041028002897898, "sampling/importance_sampling_ratio/min": 0.41827363818883895, "sampling/sampling_logp_difference/max": 0.31163255174954735, "sampling/sampling_logp_difference/mean": 0.0039051345627134043, "step": 21480, "step_time": 7.952675979033423, "student/entropy": 0.1277515217040976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666795189182, "completions/max_length": 488.06666666666666, "completions/max_terminated_length": 441.1333333333333, "completions/mean_length": 185.55945485432943, "completions/mean_terminated_length": 178.31074015299478, "completions/min_length": 54.2, "completions/min_terminated_length": 54.2, "entropy": 0.12891173884272575, "epoch": 0.8168457828580109, "eval/gt_acc_batch": 0.8130555709203084, "frac_reward_zero_std": 1.0, "grad_norm": 0.3037206530570984, "kd/policy_loss": 0.008567850538141405, "kd/rkl_advantage_mean": 0.7182860840189581, "kd/rkl_advantage_p95": 4.831400942802429, "kd/rkl_advantage_std": 2.179238090912501, "kd/ssd_loss": 0.0, "learning_rate": 1.8319219230623174e-07, "loss": 0.0342713991800944, "num_tokens": 703598509.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813055549065272, "rewards/gt_logging_reward/std": 0.3845926821231842, "sampling/importance_sampling_ratio/max": 1.9487300634384155, "sampling/importance_sampling_ratio/mean": 1.012421230475108, "sampling/importance_sampling_ratio/min": 0.4840393697222074, "sampling/sampling_logp_difference/max": 0.315477458635966, "sampling/sampling_logp_difference/mean": 0.0039002410136163234, "step": 21510, "step_time": 7.865735871833749, "student/entropy": 0.12891173884272575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112715055546, "completions/max_length": 496.1, "completions/max_terminated_length": 459.8, "completions/mean_length": 189.77889811197917, "completions/mean_terminated_length": 182.0191212972005, "completions/min_length": 50.766666666666666, "completions/min_terminated_length": 50.766666666666666, "entropy": 0.12962638940662147, "epoch": 0.8179850377852884, "eval/gt_acc_batch": 0.785833356777827, "frac_reward_zero_std": 1.0, "grad_norm": 0.3081289827823639, "kd/policy_loss": 0.008272406097967177, "kd/rkl_advantage_mean": 0.802731096620361, "kd/rkl_advantage_p95": 5.093313046296438, "kd/rkl_advantage_std": 2.2428653965393703, "kd/ssd_loss": 0.0, "learning_rate": 1.8205293737895416e-07, "loss": 0.03308962186177571, "num_tokens": 704585529.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.785833332935969, "rewards/gt_logging_reward/std": 0.40284262001514437, "sampling/importance_sampling_ratio/max": 1.8613884488741557, "sampling/importance_sampling_ratio/mean": 0.9970019578933715, "sampling/importance_sampling_ratio/min": 0.4254110644261042, "sampling/sampling_logp_difference/max": 0.3769297202428182, "sampling/sampling_logp_difference/mean": 0.003939149117407699, "step": 21540, "step_time": 8.198885394664831, "student/entropy": 0.12962638940662147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0369444467437764, "completions/max_length": 496.73333333333335, "completions/max_terminated_length": 466.1666666666667, "completions/mean_length": 196.07695414225262, "completions/mean_terminated_length": 184.15743509928384, "completions/min_length": 56.56666666666667, "completions/min_terminated_length": 56.56666666666667, "entropy": 0.13428463619202374, "epoch": 0.8191242927125659, "eval/gt_acc_batch": 0.7850000162919363, "frac_reward_zero_std": 1.0, "grad_norm": 0.22197501361370087, "kd/policy_loss": 0.008920211750470723, "kd/rkl_advantage_mean": 0.7505288790911436, "kd/rkl_advantage_p95": 5.039597378174464, "kd/rkl_advantage_std": 2.2287869215011598, "kd/ssd_loss": 0.0, "learning_rate": 1.809136824516766e-07, "loss": 0.035680846373240156, "num_tokens": 705599998.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7849999984105428, "rewards/gt_logging_reward/std": 0.4009191979964574, "sampling/importance_sampling_ratio/max": 1.9622156262397765, "sampling/importance_sampling_ratio/mean": 1.0072839776674907, "sampling/importance_sampling_ratio/min": 0.4721508582433065, "sampling/sampling_logp_difference/max": 0.2947230875492096, "sampling/sampling_logp_difference/mean": 0.0040404104006787145, "step": 21570, "step_time": 8.311679606260926, "student/entropy": 0.13428463619202374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001390775046, "completions/max_length": 473.7, "completions/max_terminated_length": 452.26666666666665, "completions/mean_length": 192.9594533284505, "completions/mean_terminated_length": 185.8704396565755, "completions/min_length": 52.96666666666667, "completions/min_terminated_length": 52.96666666666667, "entropy": 0.12730694226920605, "epoch": 0.8202635476398435, "eval/gt_acc_batch": 0.7891666889190674, "frac_reward_zero_std": 1.0, "grad_norm": 0.2751025855541229, "kd/policy_loss": 0.008675953743901725, "kd/rkl_advantage_mean": 0.7575737199435632, "kd/rkl_advantage_p95": 4.971784994999568, "kd/rkl_advantage_std": 2.1853046278158823, "kd/ssd_loss": 0.0, "learning_rate": 1.7977442752439904e-07, "loss": 0.034703818957010905, "num_tokens": 706605836.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7891666611035665, "rewards/gt_logging_reward/std": 0.39875810643037163, "sampling/importance_sampling_ratio/max": 1.951008645693461, "sampling/importance_sampling_ratio/mean": 1.0007342358430227, "sampling/importance_sampling_ratio/min": 0.4613120873769124, "sampling/sampling_logp_difference/max": 0.297554948925972, "sampling/sampling_logp_difference/mean": 0.003863970749080181, "step": 21600, "step_time": 8.151641402333432, "student/entropy": 0.12730694226920605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02500000155220429, "completions/max_length": 474.96666666666664, "completions/max_terminated_length": 444.8, "completions/mean_length": 185.7738993326823, "completions/mean_terminated_length": 177.5375951131185, "completions/min_length": 47.93333333333333, "completions/min_terminated_length": 47.93333333333333, "entropy": 0.13689549683282773, "epoch": 0.8214028025671211, "eval/gt_acc_batch": 0.7941666821638743, "frac_reward_zero_std": 1.0, "grad_norm": 0.281383216381073, "kd/policy_loss": 0.0085822031852634, "kd/rkl_advantage_mean": 0.7488358478372296, "kd/rkl_advantage_p95": 4.940113818645477, "kd/rkl_advantage_std": 2.190806539853414, "kd/ssd_loss": 0.0, "learning_rate": 1.7863517259712146e-07, "loss": 0.03432881434758504, "num_tokens": 707579078.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7941666702429454, "rewards/gt_logging_reward/std": 0.39418881783882775, "sampling/importance_sampling_ratio/max": 1.8925624092419941, "sampling/importance_sampling_ratio/mean": 0.9996952394644419, "sampling/importance_sampling_ratio/min": 0.4581642975409826, "sampling/sampling_logp_difference/max": 0.3225720008214315, "sampling/sampling_logp_difference/mean": 0.004093757535641392, "step": 21630, "step_time": 8.003894641640363, "student/entropy": 0.13689549683282773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334488173325, "completions/max_length": 498.93333333333334, "completions/max_terminated_length": 446.93333333333334, "completions/mean_length": 188.41612091064454, "completions/mean_terminated_length": 180.75244954427083, "completions/min_length": 54.56666666666667, "completions/min_terminated_length": 54.56666666666667, "entropy": 0.13225641225775084, "epoch": 0.8225420574943987, "eval/gt_acc_batch": 0.7808333535989126, "frac_reward_zero_std": 1.0, "grad_norm": 0.3649830222129822, "kd/policy_loss": 0.008655792102217674, "kd/rkl_advantage_mean": 0.7970033611481389, "kd/rkl_advantage_p95": 5.0687026103337605, "kd/rkl_advantage_std": 2.246586122115453, "kd/ssd_loss": 0.0, "learning_rate": 1.7749591766984393e-07, "loss": 0.0346231738726298, "num_tokens": 708555840.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7808333337306976, "rewards/gt_logging_reward/std": 0.40490318139394127, "sampling/importance_sampling_ratio/max": 1.9233240127563476, "sampling/importance_sampling_ratio/mean": 1.0001326898733776, "sampling/importance_sampling_ratio/min": 0.46862563292185466, "sampling/sampling_logp_difference/max": 0.30760535796483357, "sampling/sampling_logp_difference/mean": 0.003980898717418313, "step": 21660, "step_time": 8.109931588194256, "student/entropy": 0.13225641225775084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016388889960944654, "completions/max_length": 483.06666666666666, "completions/max_terminated_length": 444.06666666666666, "completions/mean_length": 179.68028615315754, "completions/mean_terminated_length": 174.20833892822264, "completions/min_length": 56.266666666666666, "completions/min_terminated_length": 56.266666666666666, "entropy": 0.1339274414504568, "epoch": 0.8236813124216762, "eval/gt_acc_batch": 0.8047222435474396, "frac_reward_zero_std": 1.0, "grad_norm": 0.2913224697113037, "kd/policy_loss": 0.008849023721025636, "kd/rkl_advantage_mean": 0.821362184608976, "kd/rkl_advantage_p95": 5.158576792478561, "kd/rkl_advantage_std": 2.2609405746062596, "kd/ssd_loss": 0.0, "learning_rate": 1.7635666274256635e-07, "loss": 0.035396103064219156, "num_tokens": 709502313.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222216924032, "rewards/gt_logging_reward/std": 0.38899335960547127, "sampling/importance_sampling_ratio/max": 1.9313522100448608, "sampling/importance_sampling_ratio/mean": 0.9980152169863383, "sampling/importance_sampling_ratio/min": 0.45628765324751536, "sampling/sampling_logp_difference/max": 0.3049539844195048, "sampling/sampling_logp_difference/mean": 0.004048929231551786, "step": 21690, "step_time": 7.927884894470723, "student/entropy": 0.1339274414504568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555668969949, "completions/max_length": 492.76666666666665, "completions/max_terminated_length": 458.56666666666666, "completions/mean_length": 194.79250996907552, "completions/mean_terminated_length": 188.20710042317708, "completions/min_length": 54.46666666666667, "completions/min_terminated_length": 54.46666666666667, "entropy": 0.13108997823049626, "epoch": 0.8248205673489538, "eval/gt_acc_batch": 0.7819444755713145, "frac_reward_zero_std": 1.0, "grad_norm": 0.2820308804512024, "kd/policy_loss": 0.008627937355777248, "kd/rkl_advantage_mean": 0.7535503803907583, "kd/rkl_advantage_p95": 4.941602178414663, "kd/rkl_advantage_std": 2.194266207019488, "kd/ssd_loss": 0.0, "learning_rate": 1.752174078152888e-07, "loss": 0.034511748949686685, "num_tokens": 710505710.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7819444378217061, "rewards/gt_logging_reward/std": 0.4088441570599874, "sampling/importance_sampling_ratio/max": 1.9086131691932677, "sampling/importance_sampling_ratio/mean": 0.9902771373589834, "sampling/importance_sampling_ratio/min": 0.45349346896012627, "sampling/sampling_logp_difference/max": 0.32025028665860494, "sampling/sampling_logp_difference/mean": 0.003978815209120512, "step": 21720, "step_time": 8.124112357834626, "student/entropy": 0.13108997823049626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025277778909852108, "completions/max_length": 485.5, "completions/max_terminated_length": 459.03333333333336, "completions/mean_length": 192.97945454915364, "completions/mean_terminated_length": 184.81041514078777, "completions/min_length": 57.56666666666667, "completions/min_terminated_length": 57.56666666666667, "entropy": 0.13384941754241783, "epoch": 0.8259598222762313, "eval/gt_acc_batch": 0.7816666762034098, "frac_reward_zero_std": 1.0, "grad_norm": 0.2847552001476288, "kd/policy_loss": 0.008700640420041357, "kd/rkl_advantage_mean": 0.7606034453958272, "kd/rkl_advantage_p95": 4.917656515041987, "kd/rkl_advantage_std": 2.1838763962189356, "kd/ssd_loss": 0.0, "learning_rate": 1.7407815288801124e-07, "loss": 0.0348025639851888, "num_tokens": 711521572.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7816666662693024, "rewards/gt_logging_reward/std": 0.4054734428723653, "sampling/importance_sampling_ratio/max": 1.9535720467567443, "sampling/importance_sampling_ratio/mean": 1.0012876788775127, "sampling/importance_sampling_ratio/min": 0.45312662521998087, "sampling/sampling_logp_difference/max": 0.313635923465093, "sampling/sampling_logp_difference/mean": 0.004057838302105665, "step": 21750, "step_time": 8.487316269236423, "student/entropy": 0.13384941754241783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0188888899050653, "completions/max_length": 489.96666666666664, "completions/max_terminated_length": 451.1, "completions/mean_length": 187.47917683919272, "completions/mean_terminated_length": 181.3087422688802, "completions/min_length": 55.266666666666666, "completions/min_terminated_length": 55.266666666666666, "entropy": 0.12976673282682896, "epoch": 0.827099077203509, "eval/gt_acc_batch": 0.8027777930100759, "frac_reward_zero_std": 1.0, "grad_norm": 0.2743168771266937, "kd/policy_loss": 0.008542749425396323, "kd/rkl_advantage_mean": 0.7472719506671032, "kd/rkl_advantage_p95": 5.019225203990937, "kd/rkl_advantage_std": 2.2214262624581655, "kd/ssd_loss": 0.0, "learning_rate": 1.7293889796073366e-07, "loss": 0.034171001116434736, "num_tokens": 712493337.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8027777810891469, "rewards/gt_logging_reward/std": 0.38906519263982775, "sampling/importance_sampling_ratio/max": 1.891133761405945, "sampling/importance_sampling_ratio/mean": 1.005589618285497, "sampling/importance_sampling_ratio/min": 0.46531993001699445, "sampling/sampling_logp_difference/max": 0.3254246711730957, "sampling/sampling_logp_difference/mean": 0.003973469836637378, "step": 21780, "step_time": 8.066514998096197, "student/entropy": 0.12976673282682896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001396983863, "completions/max_length": 496.46666666666664, "completions/max_terminated_length": 463.3, "completions/mean_length": 188.13945465087892, "completions/mean_terminated_length": 179.76343790690103, "completions/min_length": 51.833333333333336, "completions/min_terminated_length": 51.833333333333336, "entropy": 0.12750049320360024, "epoch": 0.8282383321307865, "eval/gt_acc_batch": 0.796666689713796, "frac_reward_zero_std": 1.0, "grad_norm": 0.3356053829193115, "kd/policy_loss": 0.008072864638719087, "kd/rkl_advantage_mean": 0.8099790142228206, "kd/rkl_advantage_p95": 5.022656569878261, "kd/rkl_advantage_std": 2.2168882469336193, "kd/ssd_loss": 0.0, "learning_rate": 1.7179964303345613e-07, "loss": 0.03229145606358846, "num_tokens": 713468895.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7966666698455811, "rewards/gt_logging_reward/std": 0.39292255143324534, "sampling/importance_sampling_ratio/max": 1.7541900078455608, "sampling/importance_sampling_ratio/mean": 0.9984619041283925, "sampling/importance_sampling_ratio/min": 0.4451137820879618, "sampling/sampling_logp_difference/max": 0.32538766860961915, "sampling/sampling_logp_difference/mean": 0.0039301776637633646, "step": 21810, "step_time": 8.180330047561437, "student/entropy": 0.12750049320360024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026666668492058912, "completions/max_length": 487.8666666666667, "completions/max_terminated_length": 456.03333333333336, "completions/mean_length": 194.3425099690755, "completions/mean_terminated_length": 185.69802551269532, "completions/min_length": 51.0, "completions/min_terminated_length": 51.0, "entropy": 0.13061127346009016, "epoch": 0.8293775870580641, "eval/gt_acc_batch": 0.7963889102141063, "frac_reward_zero_std": 1.0, "grad_norm": 0.2515995502471924, "kd/policy_loss": 0.008087613041667888, "kd/rkl_advantage_mean": 0.7912805884766082, "kd/rkl_advantage_p95": 5.046743402878444, "kd/rkl_advantage_std": 2.2295059730609257, "kd/ssd_loss": 0.0, "learning_rate": 1.7066038810617854e-07, "loss": 0.03235045274098714, "num_tokens": 714466640.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888843854269, "rewards/gt_logging_reward/std": 0.39457153479258217, "sampling/importance_sampling_ratio/max": 1.941011361281077, "sampling/importance_sampling_ratio/mean": 0.9991806228955586, "sampling/importance_sampling_ratio/min": 0.40545007785161336, "sampling/sampling_logp_difference/max": 0.309822682539622, "sampling/sampling_logp_difference/mean": 0.003951842547394335, "step": 21840, "step_time": 8.082345298602013, "student/entropy": 0.13061127346009016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016388889712591965, "completions/max_length": 464.8, "completions/max_terminated_length": 434.03333333333336, "completions/mean_length": 186.94056396484376, "completions/mean_terminated_length": 181.6878860473633, "completions/min_length": 53.53333333333333, "completions/min_terminated_length": 53.53333333333333, "entropy": 0.1262030846749743, "epoch": 0.8305168419853416, "eval/gt_acc_batch": 0.8091666956742605, "frac_reward_zero_std": 1.0, "grad_norm": 0.257894903421402, "kd/policy_loss": 0.00782846058330809, "kd/rkl_advantage_mean": 0.6854274295891325, "kd/rkl_advantage_p95": 4.829876321554184, "kd/rkl_advantage_std": 2.1377787311871845, "kd/ssd_loss": 0.0, "learning_rate": 1.69521133178901e-07, "loss": 0.03131384452184041, "num_tokens": 715433242.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8091666618982951, "rewards/gt_logging_reward/std": 0.38261617223421734, "sampling/importance_sampling_ratio/max": 1.7856500267982482, "sampling/importance_sampling_ratio/mean": 0.9956918517748515, "sampling/importance_sampling_ratio/min": 0.4791534016529719, "sampling/sampling_logp_difference/max": 0.3144776701927185, "sampling/sampling_logp_difference/mean": 0.0038534905295819046, "step": 21870, "step_time": 7.846956185001181, "student/entropy": 0.1262030846749743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778673917055, "completions/max_length": 482.3, "completions/max_terminated_length": 451.0, "completions/mean_length": 183.95000864664715, "completions/mean_terminated_length": 178.10933634440104, "completions/min_length": 48.7, "completions/min_terminated_length": 48.7, "entropy": 0.12232558038085699, "epoch": 0.8316560969126191, "eval/gt_acc_batch": 0.8183333575725555, "frac_reward_zero_std": 1.0, "grad_norm": 0.26129063963890076, "kd/policy_loss": 0.007748832454672084, "kd/rkl_advantage_mean": 0.7645033324758211, "kd/rkl_advantage_p95": 5.0221162418524425, "kd/rkl_advantage_std": 2.235727126399676, "kd/ssd_loss": 0.0, "learning_rate": 1.6838187825162343e-07, "loss": 0.030995333194732667, "num_tokens": 716392270.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8183333277702332, "rewards/gt_logging_reward/std": 0.3790676787495613, "sampling/importance_sampling_ratio/max": 1.8259406447410584, "sampling/importance_sampling_ratio/mean": 0.9997441112995148, "sampling/importance_sampling_ratio/min": 0.49157346884409586, "sampling/sampling_logp_difference/max": 0.3152689814567566, "sampling/sampling_logp_difference/mean": 0.0037906839822729427, "step": 21900, "step_time": 7.896479809162944, "student/entropy": 0.12232558038085699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223387410244, "completions/max_length": 477.56666666666666, "completions/max_terminated_length": 450.53333333333336, "completions/mean_length": 194.79640045166016, "completions/mean_terminated_length": 186.9627919514974, "completions/min_length": 56.666666666666664, "completions/min_terminated_length": 56.666666666666664, "entropy": 0.1300292134905855, "epoch": 0.8327953518398967, "eval/gt_acc_batch": 0.7891666909058889, "frac_reward_zero_std": 1.0, "grad_norm": 0.2436823844909668, "kd/policy_loss": 0.00861390216741711, "kd/rkl_advantage_mean": 0.7133435395080596, "kd/rkl_advantage_p95": 4.949803876876831, "kd/rkl_advantage_std": 2.178648580114047, "kd/ssd_loss": 0.0, "learning_rate": 1.6724262332434588e-07, "loss": 0.034455609321594236, "num_tokens": 717407377.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7891666690508524, "rewards/gt_logging_reward/std": 0.39803616801897684, "sampling/importance_sampling_ratio/max": 1.941292687257131, "sampling/importance_sampling_ratio/mean": 0.9965042908986409, "sampling/importance_sampling_ratio/min": 0.42858380675315855, "sampling/sampling_logp_difference/max": 0.3182550092538198, "sampling/sampling_logp_difference/mean": 0.003997007485789557, "step": 21930, "step_time": 8.166973190870097, "student/entropy": 0.1300292134905855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277779021610817, "completions/max_length": 481.53333333333336, "completions/max_terminated_length": 450.23333333333335, "completions/mean_length": 188.7033442179362, "completions/mean_terminated_length": 182.20982767740887, "completions/min_length": 49.03333333333333, "completions/min_terminated_length": 49.03333333333333, "entropy": 0.13779965521146853, "epoch": 0.8339346067671742, "eval/gt_acc_batch": 0.7833333472410838, "frac_reward_zero_std": 1.0, "grad_norm": 0.2729966938495636, "kd/policy_loss": 0.008726057532476262, "kd/rkl_advantage_mean": 0.795087803589801, "kd/rkl_advantage_p95": 4.997982154289882, "kd/rkl_advantage_std": 2.199200164278348, "kd/ssd_loss": 0.0, "learning_rate": 1.6610336839706832e-07, "loss": 0.034904225667317705, "num_tokens": 718383077.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7833333373069763, "rewards/gt_logging_reward/std": 0.4009580830732981, "sampling/importance_sampling_ratio/max": 1.8972206672032674, "sampling/importance_sampling_ratio/mean": 0.9964913626511892, "sampling/importance_sampling_ratio/min": 0.48788846135139463, "sampling/sampling_logp_difference/max": 0.35476712783177694, "sampling/sampling_logp_difference/mean": 0.004088336912294229, "step": 21960, "step_time": 8.053667612030404, "student/entropy": 0.13779965521146853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026111112472911677, "completions/max_length": 477.23333333333335, "completions/max_terminated_length": 433.93333333333334, "completions/mean_length": 186.49667612711588, "completions/mean_terminated_length": 177.95895334879557, "completions/min_length": 55.53333333333333, "completions/min_terminated_length": 55.53333333333333, "entropy": 0.13304974629233282, "epoch": 0.8350738616944519, "eval/gt_acc_batch": 0.7994444787502288, "frac_reward_zero_std": 1.0, "grad_norm": 0.2885095179080963, "kd/policy_loss": 0.008483153051929548, "kd/rkl_advantage_mean": 0.7997201787307858, "kd/rkl_advantage_p95": 5.041120620568593, "kd/rkl_advantage_std": 2.2363890896240872, "kd/ssd_loss": 0.0, "learning_rate": 1.6496411346979074e-07, "loss": 0.033932614326477054, "num_tokens": 719365665.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7994444449742635, "rewards/gt_logging_reward/std": 0.3897714138031006, "sampling/importance_sampling_ratio/max": 1.7875962177912395, "sampling/importance_sampling_ratio/mean": 0.9909016092618307, "sampling/importance_sampling_ratio/min": 0.4791289319594701, "sampling/sampling_logp_difference/max": 0.33542196949323017, "sampling/sampling_logp_difference/mean": 0.004028124874457717, "step": 21990, "step_time": 8.312347773298582, "student/entropy": 0.13304974629233282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02305555697530508, "completions/max_length": 486.46666666666664, "completions/max_terminated_length": 434.46666666666664, "completions/mean_length": 185.91667785644532, "completions/mean_terminated_length": 178.40764414469402, "completions/min_length": 59.03333333333333, "completions/min_terminated_length": 59.03333333333333, "entropy": 0.1312519058585167, "epoch": 0.8362131166217294, "eval/gt_acc_batch": 0.7986111303170522, "frac_reward_zero_std": 1.0, "grad_norm": 0.3779628276824951, "kd/policy_loss": 0.008799421459358806, "kd/rkl_advantage_mean": 0.7534091283877691, "kd/rkl_advantage_p95": 4.972088917096456, "kd/rkl_advantage_std": 2.2000416457653045, "kd/ssd_loss": 0.0, "learning_rate": 1.6382485854251318e-07, "loss": 0.03519768714904785, "num_tokens": 720333317.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111104488373, "rewards/gt_logging_reward/std": 0.39396326839923856, "sampling/importance_sampling_ratio/max": 1.9584973971048991, "sampling/importance_sampling_ratio/mean": 1.0005714197953541, "sampling/importance_sampling_ratio/min": 0.4431596711277962, "sampling/sampling_logp_difference/max": 0.3609576880931854, "sampling/sampling_logp_difference/mean": 0.003970329145280023, "step": 22020, "step_time": 8.084651628466478, "student/entropy": 0.1312519058585167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112621923287, "completions/max_length": 486.73333333333335, "completions/max_terminated_length": 449.53333333333336, "completions/mean_length": 185.02500915527344, "completions/mean_terminated_length": 177.12046966552734, "completions/min_length": 45.63333333333333, "completions/min_terminated_length": 45.63333333333333, "entropy": 0.12716341987252236, "epoch": 0.837352371549007, "eval/gt_acc_batch": 0.7969444632530213, "frac_reward_zero_std": 1.0, "grad_norm": 0.39055272936820984, "kd/policy_loss": 0.008252689254004509, "kd/rkl_advantage_mean": 0.6966283041362961, "kd/rkl_advantage_p95": 4.919476302464803, "kd/rkl_advantage_std": 2.2218957523504894, "kd/ssd_loss": 0.0, "learning_rate": 1.6268560361523563e-07, "loss": 0.033010758956273395, "num_tokens": 721296471.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444493452708, "rewards/gt_logging_reward/std": 0.39337474505106607, "sampling/importance_sampling_ratio/max": 1.9045813123385111, "sampling/importance_sampling_ratio/mean": 1.0026594996452332, "sampling/importance_sampling_ratio/min": 0.4484321688612302, "sampling/sampling_logp_difference/max": 0.35066500306129456, "sampling/sampling_logp_difference/mean": 0.003907778665112952, "step": 22050, "step_time": 7.9944261062288815, "student/entropy": 0.12716341987252236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01805555655931433, "completions/max_length": 485.8666666666667, "completions/max_terminated_length": 437.3666666666667, "completions/mean_length": 187.33556518554687, "completions/mean_terminated_length": 181.45034891764323, "completions/min_length": 49.36666666666667, "completions/min_terminated_length": 49.36666666666667, "entropy": 0.1274328616137306, "epoch": 0.8384916264762845, "eval/gt_acc_batch": 0.8175000190734864, "frac_reward_zero_std": 1.0, "grad_norm": 0.28067290782928467, "kd/policy_loss": 0.007994890061672777, "kd/rkl_advantage_mean": 0.7603579053034385, "kd/rkl_advantage_p95": 4.991238663593928, "kd/rkl_advantage_std": 2.2179590155680975, "kd/ssd_loss": 0.0, "learning_rate": 1.6154634868795807e-07, "loss": 0.03197956085205078, "num_tokens": 722273159.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8175000031789144, "rewards/gt_logging_reward/std": 0.3745128850142161, "sampling/importance_sampling_ratio/max": 1.8468910058339436, "sampling/importance_sampling_ratio/mean": 0.9936999877293905, "sampling/importance_sampling_ratio/min": 0.47307549317677816, "sampling/sampling_logp_difference/max": 0.32768351038297017, "sampling/sampling_logp_difference/mean": 0.003869962653455635, "step": 22080, "step_time": 8.107962816328897, "student/entropy": 0.1274328616137306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778897434474, "completions/max_length": 484.56666666666666, "completions/max_terminated_length": 443.2, "completions/mean_length": 182.75973205566407, "completions/mean_terminated_length": 176.09385782877604, "completions/min_length": 54.4, "completions/min_terminated_length": 54.4, "entropy": 0.1325146062920491, "epoch": 0.8396308814035621, "eval/gt_acc_batch": 0.7986111323038737, "frac_reward_zero_std": 1.0, "grad_norm": 0.4273342788219452, "kd/policy_loss": 0.008730388956610113, "kd/rkl_advantage_mean": 0.8023306039472421, "kd/rkl_advantage_p95": 5.143548009792964, "kd/rkl_advantage_std": 2.2659287889798483, "kd/ssd_loss": 0.0, "learning_rate": 1.6040709376068051e-07, "loss": 0.03492155869801839, "num_tokens": 723232398.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111104488373, "rewards/gt_logging_reward/std": 0.3939600576957067, "sampling/importance_sampling_ratio/max": 1.909198788801829, "sampling/importance_sampling_ratio/mean": 1.0009625931580861, "sampling/importance_sampling_ratio/min": 0.418133877714475, "sampling/sampling_logp_difference/max": 0.33799819548924764, "sampling/sampling_logp_difference/mean": 0.004049147890570263, "step": 22110, "step_time": 8.173245803065948, "student/entropy": 0.1325146062920491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023333334426085154, "completions/max_length": 485.1, "completions/max_terminated_length": 461.5, "completions/mean_length": 195.76389923095704, "completions/mean_terminated_length": 188.17799377441406, "completions/min_length": 52.9, "completions/min_terminated_length": 52.9, "entropy": 0.12475171070545912, "epoch": 0.8407701363308396, "eval/gt_acc_batch": 0.8030555705229442, "frac_reward_zero_std": 1.0, "grad_norm": 0.29038041830062866, "kd/policy_loss": 0.007953320589149371, "kd/rkl_advantage_mean": 0.7325929601831983, "kd/rkl_advantage_p95": 4.893853304783503, "kd/rkl_advantage_std": 2.172626096010208, "kd/ssd_loss": 0.0, "learning_rate": 1.5926783883340296e-07, "loss": 0.03181328574816386, "num_tokens": 724237924.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555586020152, "rewards/gt_logging_reward/std": 0.3932329515616099, "sampling/importance_sampling_ratio/max": 2.0123003840446474, "sampling/importance_sampling_ratio/mean": 0.999665230512619, "sampling/importance_sampling_ratio/min": 0.47834560871124265, "sampling/sampling_logp_difference/max": 0.313453471660614, "sampling/sampling_logp_difference/mean": 0.0038195600577940543, "step": 22140, "step_time": 8.21171269856762, "student/entropy": 0.12475171070545912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445874541997, "completions/max_length": 502.2, "completions/max_terminated_length": 459.8666666666667, "completions/mean_length": 192.5791763305664, "completions/mean_terminated_length": 184.63370564778646, "completions/min_length": 57.5, "completions/min_terminated_length": 57.5, "entropy": 0.1325329078361392, "epoch": 0.8419093912581171, "eval/gt_acc_batch": 0.7763889054457347, "frac_reward_zero_std": 1.0, "grad_norm": 0.2585466206073761, "kd/policy_loss": 0.008656649163458496, "kd/rkl_advantage_mean": 0.781512791228791, "kd/rkl_advantage_p95": 4.956156925360362, "kd/rkl_advantage_std": 2.1873933414618176, "kd/ssd_loss": 0.0, "learning_rate": 1.5812858390612538e-07, "loss": 0.03462659517923991, "num_tokens": 725229337.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7763888875643412, "rewards/gt_logging_reward/std": 0.41076376338799797, "sampling/importance_sampling_ratio/max": 1.8384151816368104, "sampling/importance_sampling_ratio/mean": 0.9994127770264943, "sampling/importance_sampling_ratio/min": 0.4386747757593791, "sampling/sampling_logp_difference/max": 0.32744612693786623, "sampling/sampling_logp_difference/mean": 0.004014613844143848, "step": 22170, "step_time": 8.272946898335553, "student/entropy": 0.1325329078361392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02777777931963404, "completions/max_length": 494.8666666666667, "completions/max_terminated_length": 450.3, "completions/mean_length": 196.88778788248698, "completions/mean_terminated_length": 188.0404810587565, "completions/min_length": 55.13333333333333, "completions/min_terminated_length": 55.13333333333333, "entropy": 0.13313586711883546, "epoch": 0.8430486461853948, "eval/gt_acc_batch": 0.7691666861375173, "frac_reward_zero_std": 1.0, "grad_norm": 0.3742421269416809, "kd/policy_loss": 0.009073434649811437, "kd/rkl_advantage_mean": 0.7770038907726605, "kd/rkl_advantage_p95": 5.0364954829216, "kd/rkl_advantage_std": 2.2213787029186887, "kd/ssd_loss": 0.0, "learning_rate": 1.5698932897884782e-07, "loss": 0.036293745040893555, "num_tokens": 726245957.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7691666682561239, "rewards/gt_logging_reward/std": 0.41436497668425243, "sampling/importance_sampling_ratio/max": 1.9681978265444438, "sampling/importance_sampling_ratio/mean": 1.0085690359274546, "sampling/importance_sampling_ratio/min": 0.4166793256998062, "sampling/sampling_logp_difference/max": 0.35652629534403485, "sampling/sampling_logp_difference/mean": 0.004000342870131135, "step": 22200, "step_time": 8.35064946123748, "student/entropy": 0.13313586711883546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02416666808227698, "completions/max_length": 491.4, "completions/max_terminated_length": 468.96666666666664, "completions/mean_length": 186.5572311401367, "completions/mean_terminated_length": 178.61560567220053, "completions/min_length": 55.63333333333333, "completions/min_terminated_length": 55.63333333333333, "entropy": 0.13243679162114858, "epoch": 0.8441879011126723, "eval/gt_acc_batch": 0.7930555840333303, "frac_reward_zero_std": 1.0, "grad_norm": 0.31247201561927795, "kd/policy_loss": 0.008413689961889759, "kd/rkl_advantage_mean": 0.7919532576575875, "kd/rkl_advantage_p95": 5.088038432598114, "kd/rkl_advantage_std": 2.252047916253408, "kd/ssd_loss": 0.0, "learning_rate": 1.5585007405157026e-07, "loss": 0.03365476926167806, "num_tokens": 727207995.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.793055550257365, "rewards/gt_logging_reward/std": 0.3971247146526972, "sampling/importance_sampling_ratio/max": 1.8080207268397013, "sampling/importance_sampling_ratio/mean": 1.0013012290000916, "sampling/importance_sampling_ratio/min": 0.49271173278490704, "sampling/sampling_logp_difference/max": 0.3150181392828623, "sampling/sampling_logp_difference/mean": 0.004003947670571506, "step": 22230, "step_time": 7.908665397901981, "student/entropy": 0.13243679162114858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889051352938, "completions/max_length": 493.06666666666666, "completions/max_terminated_length": 460.2, "completions/mean_length": 186.25473175048828, "completions/mean_terminated_length": 177.5077865600586, "completions/min_length": 51.8, "completions/min_terminated_length": 51.8, "entropy": 0.13526720715065796, "epoch": 0.8453271560399499, "eval/gt_acc_batch": 0.8072222292423248, "frac_reward_zero_std": 1.0, "grad_norm": 0.2858476936817169, "kd/policy_loss": 0.008623088089128336, "kd/rkl_advantage_mean": 0.8256231498594085, "kd/rkl_advantage_p95": 5.08225781917572, "kd/rkl_advantage_std": 2.243401731053988, "kd/ssd_loss": 0.0, "learning_rate": 1.547108191242927e-07, "loss": 0.034492357571919756, "num_tokens": 728182680.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222252686818, "rewards/gt_logging_reward/std": 0.3881797323624293, "sampling/importance_sampling_ratio/max": 1.8032938877741496, "sampling/importance_sampling_ratio/mean": 0.9973913292090099, "sampling/importance_sampling_ratio/min": 0.44764808217684426, "sampling/sampling_logp_difference/max": 0.3074962337811788, "sampling/sampling_logp_difference/mean": 0.004107640784544249, "step": 22260, "step_time": 8.169953117502155, "student/entropy": 0.13526720715065796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777779214084148, "completions/max_length": 479.3, "completions/max_terminated_length": 441.73333333333335, "completions/mean_length": 192.5747309366862, "completions/mean_terminated_length": 185.19730580647786, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "entropy": 0.13079874627292157, "epoch": 0.8464664109672274, "eval/gt_acc_batch": 0.8036111255486806, "frac_reward_zero_std": 1.0, "grad_norm": 0.27309751510620117, "kd/policy_loss": 0.008394794488170494, "kd/rkl_advantage_mean": 0.7464213964218894, "kd/rkl_advantage_p95": 4.971682759126027, "kd/rkl_advantage_std": 2.182406407594681, "kd/ssd_loss": 0.0, "learning_rate": 1.5357156419701515e-07, "loss": 0.033579174677530924, "num_tokens": 729188253.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111116409301, "rewards/gt_logging_reward/std": 0.38983963926633197, "sampling/importance_sampling_ratio/max": 1.8818811535835267, "sampling/importance_sampling_ratio/mean": 0.9968519548575083, "sampling/importance_sampling_ratio/min": 0.42139291763305664, "sampling/sampling_logp_difference/max": 0.31740962465604144, "sampling/sampling_logp_difference/mean": 0.003916265691320101, "step": 22290, "step_time": 8.17872572777366, "student/entropy": 0.13079874627292157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020833334326744078, "completions/max_length": 480.9, "completions/max_terminated_length": 452.6, "completions/mean_length": 185.180007425944, "completions/mean_terminated_length": 178.4642593383789, "completions/min_length": 47.86666666666667, "completions/min_terminated_length": 47.86666666666667, "entropy": 0.13869922409454982, "epoch": 0.847605665894505, "eval/gt_acc_batch": 0.7888889014720917, "frac_reward_zero_std": 1.0, "grad_norm": 0.24101538956165314, "kd/policy_loss": 0.0088563627039548, "kd/rkl_advantage_mean": 0.7582301416744789, "kd/rkl_advantage_p95": 4.902921529610952, "kd/rkl_advantage_std": 2.1760416785875956, "kd/ssd_loss": 0.0, "learning_rate": 1.5243230926973757e-07, "loss": 0.035425448417663576, "num_tokens": 730153829.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7888888935248057, "rewards/gt_logging_reward/std": 0.39918265243371326, "sampling/importance_sampling_ratio/max": 1.8938840230305989, "sampling/importance_sampling_ratio/mean": 0.9947291672229767, "sampling/importance_sampling_ratio/min": 0.42905299812555314, "sampling/sampling_logp_difference/max": 0.3330888032913208, "sampling/sampling_logp_difference/mean": 0.004204424781103929, "step": 22320, "step_time": 8.082094586228292, "student/entropy": 0.13869922409454982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03388889074946443, "completions/max_length": 488.23333333333335, "completions/max_terminated_length": 452.1, "completions/mean_length": 199.21751098632814, "completions/mean_terminated_length": 188.52822926839193, "completions/min_length": 38.8, "completions/min_terminated_length": 38.8, "entropy": 0.13278496060520412, "epoch": 0.8487449208217825, "eval/gt_acc_batch": 0.7752777894337972, "frac_reward_zero_std": 1.0, "grad_norm": 0.2958698868751526, "kd/policy_loss": 0.008597005469103655, "kd/rkl_advantage_mean": 0.8313729441724718, "kd/rkl_advantage_p95": 5.160448896884918, "kd/rkl_advantage_std": 2.247417418162028, "kd/ssd_loss": 0.0, "learning_rate": 1.5129305434246004e-07, "loss": 0.03438801765441894, "num_tokens": 731174132.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7752777775128682, "rewards/gt_logging_reward/std": 0.41012552479902903, "sampling/importance_sampling_ratio/max": 1.9079638242721557, "sampling/importance_sampling_ratio/mean": 1.000334765513738, "sampling/importance_sampling_ratio/min": 0.446453199783961, "sampling/sampling_logp_difference/max": 0.3564078489939372, "sampling/sampling_logp_difference/mean": 0.004011299430082242, "step": 22350, "step_time": 8.183145063959333, "student/entropy": 0.13278496060520412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02250000142181913, "completions/max_length": 491.3333333333333, "completions/max_terminated_length": 449.6666666666667, "completions/mean_length": 189.1550094604492, "completions/mean_terminated_length": 181.88352813720704, "completions/min_length": 54.86666666666667, "completions/min_terminated_length": 54.86666666666667, "entropy": 0.13055445284893116, "epoch": 0.8498841757490602, "eval/gt_acc_batch": 0.8013889094193777, "frac_reward_zero_std": 1.0, "grad_norm": 0.3019527196884155, "kd/policy_loss": 0.008280803228262811, "kd/rkl_advantage_mean": 0.7814196186761061, "kd/rkl_advantage_p95": 5.035306944449743, "kd/rkl_advantage_std": 2.206218820810318, "kd/ssd_loss": 0.0, "learning_rate": 1.5015379941518246e-07, "loss": 0.03312321305274964, "num_tokens": 732156746.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888915379842, "rewards/gt_logging_reward/std": 0.38919543027877807, "sampling/importance_sampling_ratio/max": 1.9520423849423727, "sampling/importance_sampling_ratio/mean": 0.9978225847085317, "sampling/importance_sampling_ratio/min": 0.4314105858405431, "sampling/sampling_logp_difference/max": 0.354723991950353, "sampling/sampling_logp_difference/mean": 0.0039725443503508965, "step": 22380, "step_time": 8.111582698162723, "student/entropy": 0.13055445284893116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223722686372, "completions/max_length": 486.1666666666667, "completions/max_terminated_length": 449.73333333333335, "completions/mean_length": 190.33389943440756, "completions/mean_terminated_length": 183.07573699951172, "completions/min_length": 52.1, "completions/min_terminated_length": 52.1, "entropy": 0.12960996987919013, "epoch": 0.8510234306763377, "eval/gt_acc_batch": 0.80694446961085, "frac_reward_zero_std": 1.0, "grad_norm": 0.32027289271354675, "kd/policy_loss": 0.008315072030139465, "kd/rkl_advantage_mean": 0.6679072124883533, "kd/rkl_advantage_p95": 4.822523794571558, "kd/rkl_advantage_std": 2.181551867723465, "kd/ssd_loss": 0.0, "learning_rate": 1.490145444879049e-07, "loss": 0.033260289827982584, "num_tokens": 733144276.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8069444437821706, "rewards/gt_logging_reward/std": 0.38922295371691384, "sampling/importance_sampling_ratio/max": 1.888064738114675, "sampling/importance_sampling_ratio/mean": 1.0018999576568604, "sampling/importance_sampling_ratio/min": 0.4401904304822286, "sampling/sampling_logp_difference/max": 0.30855729182561237, "sampling/sampling_logp_difference/mean": 0.003941412440811595, "step": 22410, "step_time": 8.065161557405371, "student/entropy": 0.12960996987919013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02111111208796501, "completions/max_length": 484.06666666666666, "completions/max_terminated_length": 443.6666666666667, "completions/mean_length": 184.89056498209635, "completions/mean_terminated_length": 177.82989705403645, "completions/min_length": 53.56666666666667, "completions/min_terminated_length": 53.56666666666667, "entropy": 0.12675875040392082, "epoch": 0.8521626856036152, "eval/gt_acc_batch": 0.8036111454168956, "frac_reward_zero_std": 1.0, "grad_norm": 0.33963245153427124, "kd/policy_loss": 0.007787531719077379, "kd/rkl_advantage_mean": 0.7681762390925239, "kd/rkl_advantage_p95": 5.022990717490514, "kd/rkl_advantage_std": 2.2184844106435775, "kd/ssd_loss": 0.0, "learning_rate": 1.4787528956062735e-07, "loss": 0.03115013043085734, "num_tokens": 734114026.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111136277516, "rewards/gt_logging_reward/std": 0.3889392460385958, "sampling/importance_sampling_ratio/max": 1.9566850463549297, "sampling/importance_sampling_ratio/mean": 0.9954413255055745, "sampling/importance_sampling_ratio/min": 0.4417340432604154, "sampling/sampling_logp_difference/max": 0.33248944878578185, "sampling/sampling_logp_difference/mean": 0.00385664120161285, "step": 22440, "step_time": 8.128493446505551, "student/entropy": 0.12675875040392082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777779015402, "completions/max_length": 489.3, "completions/max_terminated_length": 460.53333333333336, "completions/mean_length": 187.16639963785806, "completions/mean_terminated_length": 181.50589955647786, "completions/min_length": 62.53333333333333, "completions/min_terminated_length": 62.53333333333333, "entropy": 0.13266368123392264, "epoch": 0.8533019405308928, "eval/gt_acc_batch": 0.7963889141877493, "frac_reward_zero_std": 1.0, "grad_norm": 0.3325611352920532, "kd/policy_loss": 0.008609746536239981, "kd/rkl_advantage_mean": 0.811208888143301, "kd/rkl_advantage_p95": 5.078963627417882, "kd/rkl_advantage_std": 2.246652549505234, "kd/ssd_loss": 0.0, "learning_rate": 1.4673603463334976e-07, "loss": 0.03443899154663086, "num_tokens": 735086849.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888923327128, "rewards/gt_logging_reward/std": 0.3939701954523722, "sampling/importance_sampling_ratio/max": 1.8926754236221313, "sampling/importance_sampling_ratio/mean": 0.9916149814923604, "sampling/importance_sampling_ratio/min": 0.47476595391829807, "sampling/sampling_logp_difference/max": 0.3210339307785034, "sampling/sampling_logp_difference/mean": 0.004034489711436133, "step": 22470, "step_time": 8.100266399757432, "student/entropy": 0.13266368123392264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029166668187826872, "completions/max_length": 485.56666666666666, "completions/max_terminated_length": 440.6666666666667, "completions/mean_length": 188.68528696695964, "completions/mean_terminated_length": 179.08577473958334, "completions/min_length": 57.46666666666667, "completions/min_terminated_length": 57.46666666666667, "entropy": 0.13335021746655304, "epoch": 0.8544411954581703, "eval/gt_acc_batch": 0.792222241560618, "frac_reward_zero_std": 1.0, "grad_norm": 0.28452086448669434, "kd/policy_loss": 0.008758650046850865, "kd/rkl_advantage_mean": 0.7873229287815169, "kd/rkl_advantage_p95": 5.027487107117971, "kd/rkl_advantage_std": 2.225318942467372, "kd/ssd_loss": 0.0, "learning_rate": 1.4559677970607223e-07, "loss": 0.03503460089365641, "num_tokens": 736064268.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222216924032, "rewards/gt_logging_reward/std": 0.3977049599091212, "sampling/importance_sampling_ratio/max": 1.9708264907201132, "sampling/importance_sampling_ratio/mean": 1.0023731311162314, "sampling/importance_sampling_ratio/min": 0.4492472931742668, "sampling/sampling_logp_difference/max": 0.3002629081408183, "sampling/sampling_logp_difference/mean": 0.004029122351979216, "step": 22500, "step_time": 8.042180602435838, "student/entropy": 0.13335021746655304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02111111239840587, "completions/max_length": 488.03333333333336, "completions/max_terminated_length": 448.1666666666667, "completions/mean_length": 187.7408437093099, "completions/mean_terminated_length": 180.81082763671876, "completions/min_length": 42.333333333333336, "completions/min_terminated_length": 42.333333333333336, "entropy": 0.12654674854129552, "epoch": 0.8555804503854479, "eval/gt_acc_batch": 0.8072222491105397, "frac_reward_zero_std": 1.0, "grad_norm": 0.23247447609901428, "kd/policy_loss": 0.008197744206214944, "kd/rkl_advantage_mean": 0.7917783304428061, "kd/rkl_advantage_p95": 5.043321806192398, "kd/rkl_advantage_std": 2.231949652234713, "kd/ssd_loss": 0.0, "learning_rate": 1.4445752477879465e-07, "loss": 0.0327909787495931, "num_tokens": 737058023.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8072222232818603, "rewards/gt_logging_reward/std": 0.3841112474600474, "sampling/importance_sampling_ratio/max": 1.8878981709480285, "sampling/importance_sampling_ratio/mean": 1.0032616257667542, "sampling/importance_sampling_ratio/min": 0.4674414202570915, "sampling/sampling_logp_difference/max": 0.379049026966095, "sampling/sampling_logp_difference/mean": 0.0038838873617351053, "step": 22530, "step_time": 8.24105469052544, "student/entropy": 0.12654674854129552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001111378273, "completions/max_length": 495.53333333333336, "completions/max_terminated_length": 452.8666666666667, "completions/mean_length": 191.7586222330729, "completions/mean_terminated_length": 184.6071782430013, "completions/min_length": 55.8, "completions/min_terminated_length": 55.8, "entropy": 0.13138593280067046, "epoch": 0.8567197053127255, "eval/gt_acc_batch": 0.7833333512147268, "frac_reward_zero_std": 1.0, "grad_norm": 0.2906091511249542, "kd/policy_loss": 0.008962466226269802, "kd/rkl_advantage_mean": 0.7732627461353938, "kd/rkl_advantage_p95": 5.078156393766403, "kd/rkl_advantage_std": 2.24686561524868, "kd/ssd_loss": 0.0, "learning_rate": 1.433182698515171e-07, "loss": 0.03584986925125122, "num_tokens": 738062018.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7833333313465118, "rewards/gt_logging_reward/std": 0.40062052607536314, "sampling/importance_sampling_ratio/max": 1.907651436328888, "sampling/importance_sampling_ratio/mean": 1.0028938094774882, "sampling/importance_sampling_ratio/min": 0.46810534099737805, "sampling/sampling_logp_difference/max": 0.3515829126040141, "sampling/sampling_logp_difference/mean": 0.0040045262081548575, "step": 22560, "step_time": 8.218773230231212, "student/entropy": 0.13138593280067046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030000001812974613, "completions/max_length": 492.5, "completions/max_terminated_length": 453.9, "completions/mean_length": 194.8225107828776, "completions/mean_terminated_length": 185.0391591389974, "completions/min_length": 57.53333333333333, "completions/min_terminated_length": 57.53333333333333, "entropy": 0.1269713355228305, "epoch": 0.8578589602400031, "eval/gt_acc_batch": 0.7925000210603078, "frac_reward_zero_std": 1.0, "grad_norm": 0.23916542530059814, "kd/policy_loss": 0.008523017290281132, "kd/rkl_advantage_mean": 0.7542879135658344, "kd/rkl_advantage_p95": 4.968574738502502, "kd/rkl_advantage_std": 2.204097118973732, "kd/ssd_loss": 0.0, "learning_rate": 1.4217901492423954e-07, "loss": 0.034092068672180176, "num_tokens": 739074187.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7924999992052714, "rewards/gt_logging_reward/std": 0.3998308499654134, "sampling/importance_sampling_ratio/max": 1.9216821233431498, "sampling/importance_sampling_ratio/mean": 1.0049819707870484, "sampling/importance_sampling_ratio/min": 0.4795647958914439, "sampling/sampling_logp_difference/max": 0.3324305931727091, "sampling/sampling_logp_difference/mean": 0.0038409030452991526, "step": 22590, "step_time": 8.170196252770257, "student/entropy": 0.1269713355228305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02388889032105605, "completions/max_length": 499.96666666666664, "completions/max_terminated_length": 448.9, "completions/mean_length": 193.6591781616211, "completions/mean_terminated_length": 185.92354329427084, "completions/min_length": 57.46666666666667, "completions/min_terminated_length": 57.46666666666667, "entropy": 0.12885711720834175, "epoch": 0.8589982151672806, "eval/gt_acc_batch": 0.8011111338933309, "frac_reward_zero_std": 1.0, "grad_norm": 0.23410849273204803, "kd/policy_loss": 0.00862525042030029, "kd/rkl_advantage_mean": 0.7329580425381815, "kd/rkl_advantage_p95": 4.884479556481043, "kd/rkl_advantage_std": 2.1987229426701864, "kd/ssd_loss": 0.0, "learning_rate": 1.4103975999696198e-07, "loss": 0.03450100421905518, "num_tokens": 740082288.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8011111080646515, "rewards/gt_logging_reward/std": 0.3924968302249908, "sampling/importance_sampling_ratio/max": 1.9062953511873881, "sampling/importance_sampling_ratio/mean": 0.999266900618871, "sampling/importance_sampling_ratio/min": 0.44493191738923393, "sampling/sampling_logp_difference/max": 0.31464346051216124, "sampling/sampling_logp_difference/mean": 0.003918656660243869, "step": 22620, "step_time": 8.181687529632473, "student/entropy": 0.12885711720834175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333454405268, "completions/max_length": 476.9, "completions/max_terminated_length": 444.6666666666667, "completions/mean_length": 192.02167663574218, "completions/mean_terminated_length": 185.1747589111328, "completions/min_length": 54.5, "completions/min_terminated_length": 54.5, "entropy": 0.12966063097119332, "epoch": 0.8601374700945582, "eval/gt_acc_batch": 0.7950000206629435, "frac_reward_zero_std": 1.0, "grad_norm": 0.3890187740325928, "kd/policy_loss": 0.008786219808583458, "kd/rkl_advantage_mean": 0.7296668699942529, "kd/rkl_advantage_p95": 4.898507034778595, "kd/rkl_advantage_std": 2.1949948330720264, "kd/ssd_loss": 0.0, "learning_rate": 1.3990050506968443e-07, "loss": 0.035144881407419844, "num_tokens": 741082526.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7949999948342641, "rewards/gt_logging_reward/std": 0.39559268653392793, "sampling/importance_sampling_ratio/max": 1.9263057708740234, "sampling/importance_sampling_ratio/mean": 1.0015836954116821, "sampling/importance_sampling_ratio/min": 0.47602372566858925, "sampling/sampling_logp_difference/max": 0.32400619188944496, "sampling/sampling_logp_difference/mean": 0.0039152637434502445, "step": 22650, "step_time": 8.323975195172048, "student/entropy": 0.12966063097119332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02388889032105605, "completions/max_length": 498.2, "completions/max_terminated_length": 463.2, "completions/mean_length": 188.03112030029297, "completions/mean_terminated_length": 180.14672292073567, "completions/min_length": 51.233333333333334, "completions/min_terminated_length": 51.233333333333334, "entropy": 0.13298477853337923, "epoch": 0.8612767250218357, "eval/gt_acc_batch": 0.782500022649765, "frac_reward_zero_std": 1.0, "grad_norm": 0.30567461252212524, "kd/policy_loss": 0.008825844885238136, "kd/rkl_advantage_mean": 0.8433149545143048, "kd/rkl_advantage_p95": 5.172426209847132, "kd/rkl_advantage_std": 2.2744755913813908, "kd/ssd_loss": 0.0, "learning_rate": 1.3876125014240687e-07, "loss": 0.035303378105163576, "num_tokens": 742051870.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7824999988079071, "rewards/gt_logging_reward/std": 0.4041748841603597, "sampling/importance_sampling_ratio/max": 1.9203033169110617, "sampling/importance_sampling_ratio/mean": 0.9997125784556071, "sampling/importance_sampling_ratio/min": 0.46459635297457375, "sampling/sampling_logp_difference/max": 0.31898285349210104, "sampling/sampling_logp_difference/mean": 0.004035418449590603, "step": 22680, "step_time": 8.099874800767672, "student/entropy": 0.13298477853337923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000001167257628, "completions/max_length": 492.76666666666665, "completions/max_terminated_length": 446.2, "completions/mean_length": 182.72862040201824, "completions/mean_terminated_length": 176.1713617960612, "completions/min_length": 50.96666666666667, "completions/min_terminated_length": 50.96666666666667, "entropy": 0.13534270289043585, "epoch": 0.8624159799491132, "eval/gt_acc_batch": 0.7983333468437195, "frac_reward_zero_std": 1.0, "grad_norm": 0.2659091055393219, "kd/policy_loss": 0.008991556313897793, "kd/rkl_advantage_mean": 0.7731174814204375, "kd/rkl_advantage_p95": 5.0061502655347185, "kd/rkl_advantage_std": 2.253721491495768, "kd/ssd_loss": 0.0, "learning_rate": 1.376219952151293e-07, "loss": 0.03596622149149577, "num_tokens": 743011909.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7983333388964335, "rewards/gt_logging_reward/std": 0.3950913369655609, "sampling/importance_sampling_ratio/max": 1.9593561172485352, "sampling/importance_sampling_ratio/mean": 1.004892881711324, "sampling/importance_sampling_ratio/min": 0.46151103178660074, "sampling/sampling_logp_difference/max": 0.3166197419166565, "sampling/sampling_logp_difference/mean": 0.004075832377808789, "step": 22710, "step_time": 8.161384889101221, "student/entropy": 0.13534270289043585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024722223666807017, "completions/max_length": 473.9, "completions/max_terminated_length": 451.1666666666667, "completions/mean_length": 190.69556630452473, "completions/mean_terminated_length": 182.83817291259766, "completions/min_length": 60.5, "completions/min_terminated_length": 60.5, "entropy": 0.13217275626957417, "epoch": 0.8635552348763909, "eval/gt_acc_batch": 0.7955555737018585, "frac_reward_zero_std": 1.0, "grad_norm": 0.3010310232639313, "kd/policy_loss": 0.008648102398728952, "kd/rkl_advantage_mean": 0.8061075311775009, "kd/rkl_advantage_p95": 5.063146662712097, "kd/rkl_advantage_std": 2.2115997771422067, "kd/ssd_loss": 0.0, "learning_rate": 1.3648274028785173e-07, "loss": 0.03459240992863973, "num_tokens": 743998573.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555578072866, "rewards/gt_logging_reward/std": 0.3938512533903122, "sampling/importance_sampling_ratio/max": 1.8555443167686463, "sampling/importance_sampling_ratio/mean": 1.0006631433963775, "sampling/importance_sampling_ratio/min": 0.4322515845298767, "sampling/sampling_logp_difference/max": 0.3007625142733256, "sampling/sampling_logp_difference/mean": 0.003977715449097256, "step": 22740, "step_time": 8.054252370401324, "student/entropy": 0.13217275626957417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02388889032105605, "completions/max_length": 479.9, "completions/max_terminated_length": 439.3, "completions/mean_length": 187.3636220296224, "completions/mean_terminated_length": 179.3752197265625, "completions/min_length": 54.96666666666667, "completions/min_terminated_length": 54.96666666666667, "entropy": 0.12611462449034055, "epoch": 0.8646944898036684, "eval/gt_acc_batch": 0.8094444751739502, "frac_reward_zero_std": 1.0, "grad_norm": 0.2863791584968567, "kd/policy_loss": 0.007816589332651346, "kd/rkl_advantage_mean": 0.7436263293027878, "kd/rkl_advantage_p95": 5.002106481790543, "kd/rkl_advantage_std": 2.2216884394486747, "kd/ssd_loss": 0.0, "learning_rate": 1.3534348536057418e-07, "loss": 0.03126635551452637, "num_tokens": 744965314.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8094444493452708, "rewards/gt_logging_reward/std": 0.385546279946963, "sampling/importance_sampling_ratio/max": 1.9446592330932617, "sampling/importance_sampling_ratio/mean": 1.0049631655216218, "sampling/importance_sampling_ratio/min": 0.4615677704413732, "sampling/sampling_logp_difference/max": 0.3129515280326208, "sampling/sampling_logp_difference/mean": 0.0038275106344372034, "step": 22770, "step_time": 8.027029435528675, "student/entropy": 0.12611462449034055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556491017341, "completions/max_length": 488.5, "completions/max_terminated_length": 462.3333333333333, "completions/mean_length": 187.83084411621093, "completions/mean_terminated_length": 182.75238342285155, "completions/min_length": 59.2, "completions/min_terminated_length": 59.2, "entropy": 0.13078331767270962, "epoch": 0.865833744730946, "eval/gt_acc_batch": 0.8041666825612386, "frac_reward_zero_std": 1.0, "grad_norm": 0.24576346576213837, "kd/policy_loss": 0.008285384303114067, "kd/rkl_advantage_mean": 0.7870866217340032, "kd/rkl_advantage_p95": 4.9916084210077925, "kd/rkl_advantage_std": 2.207761217157046, "kd/ssd_loss": 0.0, "learning_rate": 1.3420423043329662e-07, "loss": 0.03314153949419658, "num_tokens": 745947809.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8041666587193806, "rewards/gt_logging_reward/std": 0.3853556881348292, "sampling/importance_sampling_ratio/max": 1.9611984769503275, "sampling/importance_sampling_ratio/mean": 1.001955791314443, "sampling/importance_sampling_ratio/min": 0.4582179774840673, "sampling/sampling_logp_difference/max": 0.3205002297957738, "sampling/sampling_logp_difference/mean": 0.004002180183306336, "step": 22800, "step_time": 8.197869520298749, "student/entropy": 0.13078331767270962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030277779667327802, "completions/max_length": 485.6666666666667, "completions/max_terminated_length": 439.8333333333333, "completions/mean_length": 185.4747319539388, "completions/mean_terminated_length": 175.55718739827475, "completions/min_length": 51.53333333333333, "completions/min_terminated_length": 51.53333333333333, "entropy": 0.1318334046130379, "epoch": 0.8669729996582235, "eval/gt_acc_batch": 0.7816666801770528, "frac_reward_zero_std": 1.0, "grad_norm": 0.3620099425315857, "kd/policy_loss": 0.008636449290982758, "kd/rkl_advantage_mean": 0.749049085068206, "kd/rkl_advantage_p95": 4.94884881178538, "kd/rkl_advantage_std": 2.20999383131663, "kd/ssd_loss": 0.0, "learning_rate": 1.3306497550601907e-07, "loss": 0.03454579909642538, "num_tokens": 746915806.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7816666662693024, "rewards/gt_logging_reward/std": 0.40279511312643684, "sampling/importance_sampling_ratio/max": 1.8047226667404175, "sampling/importance_sampling_ratio/mean": 1.000716882944107, "sampling/importance_sampling_ratio/min": 0.46378561755021414, "sampling/sampling_logp_difference/max": 0.3257856428623199, "sampling/sampling_logp_difference/mean": 0.004005224024876952, "step": 22830, "step_time": 7.9810386973248875, "student/entropy": 0.1318334046130379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777778798093398, "completions/max_length": 473.1333333333333, "completions/max_terminated_length": 430.9, "completions/mean_length": 184.97862091064454, "completions/mean_terminated_length": 179.21016642252604, "completions/min_length": 47.666666666666664, "completions/min_terminated_length": 47.666666666666664, "entropy": 0.12774022612720728, "epoch": 0.8681122545855011, "eval/gt_acc_batch": 0.8233333349227905, "frac_reward_zero_std": 1.0, "grad_norm": 0.3894599676132202, "kd/policy_loss": 0.008211941768725714, "kd/rkl_advantage_mean": 0.6719420439951743, "kd/rkl_advantage_p95": 4.7874577383200325, "kd/rkl_advantage_std": 2.173890712857246, "kd/ssd_loss": 0.0, "learning_rate": 1.3192572057874148e-07, "loss": 0.032847766081492105, "num_tokens": 747884785.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8233333309491475, "rewards/gt_logging_reward/std": 0.3722184111674627, "sampling/importance_sampling_ratio/max": 1.9391035914421082, "sampling/importance_sampling_ratio/mean": 1.0083423793315887, "sampling/importance_sampling_ratio/min": 0.4645162343978882, "sampling/sampling_logp_difference/max": 0.3018129626909892, "sampling/sampling_logp_difference/mean": 0.003905597678385675, "step": 22860, "step_time": 8.10379835596638, "student/entropy": 0.12774022612720728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02861111263434092, "completions/max_length": 499.5, "completions/max_terminated_length": 456.8666666666667, "completions/mean_length": 189.402788289388, "completions/mean_terminated_length": 180.1240473429362, "completions/min_length": 53.6, "completions/min_terminated_length": 53.6, "entropy": 0.12710884287953378, "epoch": 0.8692515095127786, "eval/gt_acc_batch": 0.8027777969837189, "frac_reward_zero_std": 1.0, "grad_norm": 0.3038082420825958, "kd/policy_loss": 0.008547191692438597, "kd/rkl_advantage_mean": 0.7638656681713959, "kd/rkl_advantage_p95": 5.071392973264058, "kd/rkl_advantage_std": 2.2542886257171633, "kd/ssd_loss": 0.0, "learning_rate": 1.3078646565146393e-07, "loss": 0.03418877124786377, "num_tokens": 748874195.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8027777810891469, "rewards/gt_logging_reward/std": 0.3877773478627205, "sampling/importance_sampling_ratio/max": 1.9831329464912415, "sampling/importance_sampling_ratio/mean": 1.002388346195221, "sampling/importance_sampling_ratio/min": 0.4482503523429235, "sampling/sampling_logp_difference/max": 0.3296577463547389, "sampling/sampling_logp_difference/mean": 0.003844348209289213, "step": 22890, "step_time": 8.281567083243862, "student/entropy": 0.12710884287953378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333467443784, "completions/max_length": 478.46666666666664, "completions/max_terminated_length": 446.46666666666664, "completions/mean_length": 191.49806518554686, "completions/mean_terminated_length": 184.01276601155598, "completions/min_length": 55.03333333333333, "completions/min_terminated_length": 55.03333333333333, "entropy": 0.1259079256405433, "epoch": 0.8703907644400563, "eval/gt_acc_batch": 0.8033333579699199, "frac_reward_zero_std": 1.0, "grad_norm": 0.3195365369319916, "kd/policy_loss": 0.008189297669256726, "kd/rkl_advantage_mean": 0.6363264962720374, "kd/rkl_advantage_p95": 4.780127602815628, "kd/rkl_advantage_std": 2.1703596154848737, "kd/ssd_loss": 0.0, "learning_rate": 1.2964721072418637e-07, "loss": 0.032757192850112915, "num_tokens": 749875284.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8033333321412405, "rewards/gt_logging_reward/std": 0.3865125045180321, "sampling/importance_sampling_ratio/max": 1.95700706243515, "sampling/importance_sampling_ratio/mean": 1.0010215878486632, "sampling/importance_sampling_ratio/min": 0.4595142036676407, "sampling/sampling_logp_difference/max": 0.31296459833780926, "sampling/sampling_logp_difference/mean": 0.0038274982711300255, "step": 22920, "step_time": 8.123934472636513, "student/entropy": 0.1259079256405433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017777779015402, "completions/max_length": 466.1666666666667, "completions/max_terminated_length": 447.43333333333334, "completions/mean_length": 187.82612101236978, "completions/mean_terminated_length": 181.9598353068034, "completions/min_length": 57.36666666666667, "completions/min_terminated_length": 57.36666666666667, "entropy": 0.12668142033119997, "epoch": 0.8715300193673338, "eval/gt_acc_batch": 0.806944465637207, "frac_reward_zero_std": 1.0, "grad_norm": 0.26339268684387207, "kd/policy_loss": 0.008191281549322109, "kd/rkl_advantage_mean": 0.6666274428367615, "kd/rkl_advantage_p95": 4.831037294864655, "kd/rkl_advantage_std": 2.1502163181702296, "kd/ssd_loss": 0.0, "learning_rate": 1.2850795579690882e-07, "loss": 0.032765124241511026, "num_tokens": 750860850.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8069444437821706, "rewards/gt_logging_reward/std": 0.3854315310716629, "sampling/importance_sampling_ratio/max": 1.9626056631406148, "sampling/importance_sampling_ratio/mean": 0.996525130669276, "sampling/importance_sampling_ratio/min": 0.468292506535848, "sampling/sampling_logp_difference/max": 0.3021794279416402, "sampling/sampling_logp_difference/mean": 0.003850314075437685, "step": 22950, "step_time": 8.01469013816677, "student/entropy": 0.12668142033119997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888889874021212, "completions/max_length": 464.4, "completions/max_terminated_length": 431.2, "completions/mean_length": 182.20362040201823, "completions/mean_terminated_length": 176.02877807617188, "completions/min_length": 52.63333333333333, "completions/min_terminated_length": 52.63333333333333, "entropy": 0.12763134439786275, "epoch": 0.8726692742946113, "eval/gt_acc_batch": 0.8119444727897644, "frac_reward_zero_std": 1.0, "grad_norm": 0.3159853518009186, "kd/policy_loss": 0.008042981130226204, "kd/rkl_advantage_mean": 0.7860708992889461, "kd/rkl_advantage_p95": 4.968589395284653, "kd/rkl_advantage_std": 2.1899077316125233, "kd/ssd_loss": 0.0, "learning_rate": 1.2736870086963126e-07, "loss": 0.0321719229221344, "num_tokens": 751812639.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8119444429874421, "rewards/gt_logging_reward/std": 0.3829333007335663, "sampling/importance_sampling_ratio/max": 1.7145963072776795, "sampling/importance_sampling_ratio/mean": 0.9926984389623006, "sampling/importance_sampling_ratio/min": 0.4637375553448995, "sampling/sampling_logp_difference/max": 0.314682275056839, "sampling/sampling_logp_difference/mean": 0.0039014135332157214, "step": 22980, "step_time": 7.858233055607221, "student/entropy": 0.12763134439786275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02861111294478178, "completions/max_length": 508.6666666666667, "completions/max_terminated_length": 462.26666666666665, "completions/mean_length": 192.06723225911458, "completions/mean_terminated_length": 182.6823298136393, "completions/min_length": 52.266666666666666, "completions/min_terminated_length": 52.266666666666666, "entropy": 0.1379728484277924, "epoch": 0.8738085292218889, "eval/gt_acc_batch": 0.7794444620609283, "frac_reward_zero_std": 1.0, "grad_norm": 0.2683645188808441, "kd/policy_loss": 0.009489416866563261, "kd/rkl_advantage_mean": 0.9328331373631954, "kd/rkl_advantage_p95": 5.280851731697719, "kd/rkl_advantage_std": 2.303202109535535, "kd/ssd_loss": 0.0, "learning_rate": 1.2622944594235368e-07, "loss": 0.03795766830444336, "num_tokens": 752807569.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7794444402058919, "rewards/gt_logging_reward/std": 0.40996646583080293, "sampling/importance_sampling_ratio/max": 2.019237486521403, "sampling/importance_sampling_ratio/mean": 0.9977734486262003, "sampling/importance_sampling_ratio/min": 0.4443505257368088, "sampling/sampling_logp_difference/max": 0.3163887023925781, "sampling/sampling_logp_difference/mean": 0.004181384528055787, "step": 23010, "step_time": 8.375517830568908, "student/entropy": 0.1379728484277924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023055556726952395, "completions/max_length": 477.26666666666665, "completions/max_terminated_length": 445.1333333333333, "completions/mean_length": 186.9105662027995, "completions/mean_terminated_length": 179.39688873291016, "completions/min_length": 54.766666666666666, "completions/min_terminated_length": 54.766666666666666, "entropy": 0.13744439880053202, "epoch": 0.8749477841491664, "eval/gt_acc_batch": 0.7802778045336406, "frac_reward_zero_std": 1.0, "grad_norm": 0.2910434901714325, "kd/policy_loss": 0.008973856673886378, "kd/rkl_advantage_mean": 0.7343116017136102, "kd/rkl_advantage_p95": 4.899587333202362, "kd/rkl_advantage_std": 2.1812462468942004, "kd/ssd_loss": 0.0, "learning_rate": 1.2509019101507615e-07, "loss": 0.03589542706807455, "num_tokens": 753776535.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7802777767181397, "rewards/gt_logging_reward/std": 0.4041552593310674, "sampling/importance_sampling_ratio/max": 1.8749382932980856, "sampling/importance_sampling_ratio/mean": 0.9964870810508728, "sampling/importance_sampling_ratio/min": 0.44657853841781614, "sampling/sampling_logp_difference/max": 0.326977022488912, "sampling/sampling_logp_difference/mean": 0.0041608799869815504, "step": 23040, "step_time": 8.076094415956565, "student/entropy": 0.13744439880053202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667889803648, "completions/max_length": 494.5, "completions/max_terminated_length": 458.5, "completions/mean_length": 186.9386210123698, "completions/mean_terminated_length": 179.74625193277996, "completions/min_length": 49.13333333333333, "completions/min_terminated_length": 49.13333333333333, "entropy": 0.1315788966914018, "epoch": 0.876087039076444, "eval/gt_acc_batch": 0.795555571715037, "frac_reward_zero_std": 1.0, "grad_norm": 0.3713463544845581, "kd/policy_loss": 0.008503361391679695, "kd/rkl_advantage_mean": 0.8242284173145891, "kd/rkl_advantage_p95": 5.105216805140177, "kd/rkl_advantage_std": 2.2501171121994656, "kd/ssd_loss": 0.0, "learning_rate": 1.2395093608779857e-07, "loss": 0.03401344617207845, "num_tokens": 754750946.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555538336436, "rewards/gt_logging_reward/std": 0.39735410710175834, "sampling/importance_sampling_ratio/max": 1.9018062551816304, "sampling/importance_sampling_ratio/mean": 1.0000363051891328, "sampling/importance_sampling_ratio/min": 0.44314101040363313, "sampling/sampling_logp_difference/max": 0.3336210668087006, "sampling/sampling_logp_difference/mean": 0.004063480712162951, "step": 23070, "step_time": 8.120897257363929, "student/entropy": 0.1315788966914018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02472222379098336, "completions/max_length": 488.5, "completions/max_terminated_length": 461.76666666666665, "completions/mean_length": 191.97084350585936, "completions/mean_terminated_length": 184.0526372273763, "completions/min_length": 56.6, "completions/min_terminated_length": 56.6, "entropy": 0.1235576598594586, "epoch": 0.8772262940037215, "eval/gt_acc_batch": 0.8136111279328664, "frac_reward_zero_std": 1.0, "grad_norm": 0.31412848830223083, "kd/policy_loss": 0.00814084980908471, "kd/rkl_advantage_mean": 0.7211971155057351, "kd/rkl_advantage_p95": 4.931626149018606, "kd/rkl_advantage_std": 2.2074815442164737, "kd/ssd_loss": 0.0, "learning_rate": 1.22811681160521e-07, "loss": 0.03256339828173319, "num_tokens": 755750033.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813611102104187, "rewards/gt_logging_reward/std": 0.37787968864043553, "sampling/importance_sampling_ratio/max": 1.8633577783902486, "sampling/importance_sampling_ratio/mean": 0.9996126095453898, "sampling/importance_sampling_ratio/min": 0.4769335160652796, "sampling/sampling_logp_difference/max": 0.3282018502553304, "sampling/sampling_logp_difference/mean": 0.003831643184336523, "step": 23100, "step_time": 8.284613655266003, "student/entropy": 0.1235576598594586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277778835346302, "completions/max_length": 490.2, "completions/max_terminated_length": 434.3666666666667, "completions/mean_length": 184.14834340413412, "completions/mean_terminated_length": 177.4874725341797, "completions/min_length": 58.13333333333333, "completions/min_terminated_length": 58.13333333333333, "entropy": 0.13204672622183958, "epoch": 0.8783655489309992, "eval/gt_acc_batch": 0.8047222415606181, "frac_reward_zero_std": 1.0, "grad_norm": 0.3118095397949219, "kd/policy_loss": 0.008502556944343572, "kd/rkl_advantage_mean": 0.785019473110636, "kd/rkl_advantage_p95": 5.044319146871567, "kd/rkl_advantage_std": 2.221185443798701, "kd/ssd_loss": 0.0, "learning_rate": 1.2167242623324346e-07, "loss": 0.03401022752126058, "num_tokens": 756711111.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8047222236792246, "rewards/gt_logging_reward/std": 0.386617519458135, "sampling/importance_sampling_ratio/max": 1.8386635065078736, "sampling/importance_sampling_ratio/mean": 1.0054024199644724, "sampling/importance_sampling_ratio/min": 0.43596192797025046, "sampling/sampling_logp_difference/max": 0.32714031140009564, "sampling/sampling_logp_difference/mean": 0.003959918658559521, "step": 23130, "step_time": 8.142099202092504, "student/entropy": 0.13204672622183958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01750000109896064, "completions/max_length": 473.3, "completions/max_terminated_length": 442.53333333333336, "completions/mean_length": 188.0261205037435, "completions/mean_terminated_length": 182.4211196899414, "completions/min_length": 57.7, "completions/min_terminated_length": 57.7, "entropy": 0.12690939251333475, "epoch": 0.8795048038582767, "eval/gt_acc_batch": 0.8013889094193777, "frac_reward_zero_std": 1.0, "grad_norm": 0.3181326687335968, "kd/policy_loss": 0.007981227984419092, "kd/rkl_advantage_mean": 0.7134026215722163, "kd/rkl_advantage_p95": 4.92519794901212, "kd/rkl_advantage_std": 2.1877118150393167, "kd/ssd_loss": 0.0, "learning_rate": 1.205331713059659e-07, "loss": 0.03192491332689921, "num_tokens": 757689589.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888895511627, "rewards/gt_logging_reward/std": 0.388921661178271, "sampling/importance_sampling_ratio/max": 1.950731341044108, "sampling/importance_sampling_ratio/mean": 1.002543858687083, "sampling/importance_sampling_ratio/min": 0.4581389933824539, "sampling/sampling_logp_difference/max": 0.33347437580426537, "sampling/sampling_logp_difference/mean": 0.003855933124820391, "step": 23160, "step_time": 7.966261979668828, "student/entropy": 0.12690939251333475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333463718494, "completions/max_length": 479.2, "completions/max_terminated_length": 426.9, "completions/mean_length": 179.63973134358724, "completions/mean_terminated_length": 172.79292958577474, "completions/min_length": 59.96666666666667, "completions/min_terminated_length": 59.96666666666667, "entropy": 0.13109578931083282, "epoch": 0.8806440587855543, "eval/gt_acc_batch": 0.7986111303170522, "frac_reward_zero_std": 1.0, "grad_norm": 0.31850823760032654, "kd/policy_loss": 0.00883323613088578, "kd/rkl_advantage_mean": 0.7740820974732439, "kd/rkl_advantage_p95": 5.008342313766479, "kd/rkl_advantage_std": 2.2178747882445653, "kd/ssd_loss": 0.0, "learning_rate": 1.1939391637868832e-07, "loss": 0.03533295392990112, "num_tokens": 758633916.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7986111124356587, "rewards/gt_logging_reward/std": 0.3878565872708956, "sampling/importance_sampling_ratio/max": 1.929719050725301, "sampling/importance_sampling_ratio/mean": 0.995940633614858, "sampling/importance_sampling_ratio/min": 0.48022502263387046, "sampling/sampling_logp_difference/max": 0.32048014005025227, "sampling/sampling_logp_difference/mean": 0.00397167921376725, "step": 23190, "step_time": 7.953469023131765, "student/entropy": 0.13109578931083282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021944445713112753, "completions/max_length": 489.43333333333334, "completions/max_terminated_length": 442.5, "completions/mean_length": 195.04195556640624, "completions/mean_terminated_length": 187.94127349853517, "completions/min_length": 58.833333333333336, "completions/min_terminated_length": 58.833333333333336, "entropy": 0.12859247184048098, "epoch": 0.8817833137128318, "eval/gt_acc_batch": 0.7925000230471293, "frac_reward_zero_std": 1.0, "grad_norm": 0.29214024543762207, "kd/policy_loss": 0.008587343197238322, "kd/rkl_advantage_mean": 0.6732667512570818, "kd/rkl_advantage_p95": 4.850631990035375, "kd/rkl_advantage_std": 2.1883947253227234, "kd/ssd_loss": 0.0, "learning_rate": 1.1825466145141077e-07, "loss": 0.03434937795003255, "num_tokens": 759643675.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7924999972184499, "rewards/gt_logging_reward/std": 0.4015615314245224, "sampling/importance_sampling_ratio/max": 1.8821366906166077, "sampling/importance_sampling_ratio/mean": 1.0058015088240306, "sampling/importance_sampling_ratio/min": 0.4771183336774508, "sampling/sampling_logp_difference/max": 0.3369857887427012, "sampling/sampling_logp_difference/mean": 0.003912599470155934, "step": 23220, "step_time": 8.177910016001745, "student/entropy": 0.12859247184048098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014444445197780927, "completions/max_length": 474.06666666666666, "completions/max_terminated_length": 441.93333333333334, "completions/mean_length": 183.99584350585937, "completions/mean_terminated_length": 179.28905232747397, "completions/min_length": 52.666666666666664, "completions/min_terminated_length": 52.666666666666664, "entropy": 0.13549641085167727, "epoch": 0.8829225686401093, "eval/gt_acc_batch": 0.7916666865348816, "frac_reward_zero_std": 1.0, "grad_norm": 0.3782370984554291, "kd/policy_loss": 0.00875515874940902, "kd/rkl_advantage_mean": 0.7392922422538201, "kd/rkl_advantage_p95": 4.949899983406067, "kd/rkl_advantage_std": 2.201300186912219, "kd/ssd_loss": 0.0, "learning_rate": 1.171154065241332e-07, "loss": 0.03502063353856404, "num_tokens": 760604476.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7916666666666666, "rewards/gt_logging_reward/std": 0.39699329336484274, "sampling/importance_sampling_ratio/max": 1.763451588153839, "sampling/importance_sampling_ratio/mean": 0.994125618537267, "sampling/importance_sampling_ratio/min": 0.46525353839000066, "sampling/sampling_logp_difference/max": 0.3593585511048635, "sampling/sampling_logp_difference/mean": 0.004090516013093293, "step": 23250, "step_time": 7.89857203453236, "student/entropy": 0.13549641085167727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02500000180055698, "completions/max_length": 489.9, "completions/max_terminated_length": 461.23333333333335, "completions/mean_length": 193.51195526123047, "completions/mean_terminated_length": 185.50155080159504, "completions/min_length": 49.266666666666666, "completions/min_terminated_length": 49.266666666666666, "entropy": 0.1317541171486179, "epoch": 0.8840618235673869, "eval/gt_acc_batch": 0.7925000190734863, "frac_reward_zero_std": 1.0, "grad_norm": 0.307046502828598, "kd/policy_loss": 0.008795026332760851, "kd/rkl_advantage_mean": 0.7620453449587027, "kd/rkl_advantage_p95": 5.0891047755877175, "kd/rkl_advantage_std": 2.244570863246918, "kd/ssd_loss": 0.0, "learning_rate": 1.1597615159685566e-07, "loss": 0.03518010377883911, "num_tokens": 761612135.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7924999972184499, "rewards/gt_logging_reward/std": 0.39956229627132417, "sampling/importance_sampling_ratio/max": 1.9159420768419901, "sampling/importance_sampling_ratio/mean": 1.0006542305151622, "sampling/importance_sampling_ratio/min": 0.4180464789271355, "sampling/sampling_logp_difference/max": 0.3213007052739461, "sampling/sampling_logp_difference/mean": 0.00399587731808424, "step": 23280, "step_time": 8.168324610532727, "student/entropy": 0.1317541171486179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889051352938, "completions/max_length": 486.43333333333334, "completions/max_terminated_length": 454.06666666666666, "completions/mean_length": 186.83751017252604, "completions/mean_terminated_length": 178.27852579752604, "completions/min_length": 50.86666666666667, "completions/min_terminated_length": 50.86666666666667, "entropy": 0.12874856740236282, "epoch": 0.8852010784946645, "eval/gt_acc_batch": 0.8075000226497651, "frac_reward_zero_std": 1.0, "grad_norm": 0.32310038805007935, "kd/policy_loss": 0.008092415284287805, "kd/rkl_advantage_mean": 0.8169227125744025, "kd/rkl_advantage_p95": 5.1120341698328655, "kd/rkl_advantage_std": 2.2435388614734015, "kd/ssd_loss": 0.0, "learning_rate": 1.148368966695781e-07, "loss": 0.03236965934435527, "num_tokens": 762592470.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8074999968210856, "rewards/gt_logging_reward/std": 0.3868540346622467, "sampling/importance_sampling_ratio/max": 1.8689161856969199, "sampling/importance_sampling_ratio/mean": 0.995556092262268, "sampling/importance_sampling_ratio/min": 0.4691173583269119, "sampling/sampling_logp_difference/max": 0.32147066593170165, "sampling/sampling_logp_difference/mean": 0.003921871647859613, "step": 23310, "step_time": 8.214100531528432, "student/entropy": 0.12874856740236282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944445787618558, "completions/max_length": 490.03333333333336, "completions/max_terminated_length": 442.03333333333336, "completions/mean_length": 190.2230662027995, "completions/mean_terminated_length": 181.30670318603515, "completions/min_length": 48.96666666666667, "completions/min_terminated_length": 48.96666666666667, "entropy": 0.12794590033590794, "epoch": 0.8863403334219421, "eval/gt_acc_batch": 0.7816666821638744, "frac_reward_zero_std": 1.0, "grad_norm": 0.30589011311531067, "kd/policy_loss": 0.008202099396536747, "kd/rkl_advantage_mean": 0.7609225244726986, "kd/rkl_advantage_p95": 4.987123433748881, "kd/rkl_advantage_std": 2.208218417565028, "kd/ssd_loss": 0.0, "learning_rate": 1.1369764174230052e-07, "loss": 0.03280839721361796, "num_tokens": 763585753.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7816666642824809, "rewards/gt_logging_reward/std": 0.40750241180260977, "sampling/importance_sampling_ratio/max": 1.916506326198578, "sampling/importance_sampling_ratio/mean": 0.9932536959648133, "sampling/importance_sampling_ratio/min": 0.46115903158982596, "sampling/sampling_logp_difference/max": 0.30962676405906675, "sampling/sampling_logp_difference/mean": 0.0038949601196994386, "step": 23340, "step_time": 8.159490325363004, "student/entropy": 0.12794590033590794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890352100133, "completions/max_length": 494.76666666666665, "completions/max_terminated_length": 450.96666666666664, "completions/mean_length": 188.124733988444, "completions/mean_terminated_length": 180.25436197916667, "completions/min_length": 48.7, "completions/min_terminated_length": 48.7, "entropy": 0.1332431253666679, "epoch": 0.8874795883492196, "eval/gt_acc_batch": 0.8130555788675944, "frac_reward_zero_std": 1.0, "grad_norm": 0.2512551248073578, "kd/policy_loss": 0.00858370353622983, "kd/rkl_advantage_mean": 0.7310769268001119, "kd/rkl_advantage_p95": 4.896017819643021, "kd/rkl_advantage_std": 2.177577265103658, "kd/ssd_loss": 0.0, "learning_rate": 1.1255838681502297e-07, "loss": 0.034334818522135414, "num_tokens": 764573898.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813055553038915, "rewards/gt_logging_reward/std": 0.3828246742486954, "sampling/importance_sampling_ratio/max": 1.7741247375806173, "sampling/importance_sampling_ratio/mean": 0.9979776600996654, "sampling/importance_sampling_ratio/min": 0.44155739843845365, "sampling/sampling_logp_difference/max": 0.3160959601402283, "sampling/sampling_logp_difference/mean": 0.00409540346202751, "step": 23370, "step_time": 8.30296038139204, "student/entropy": 0.1332431253666679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024166668020188808, "completions/max_length": 492.53333333333336, "completions/max_terminated_length": 447.03333333333336, "completions/mean_length": 187.49362131754557, "completions/mean_terminated_length": 179.41869099934897, "completions/min_length": 54.9, "completions/min_terminated_length": 54.9, "entropy": 0.13305013366043567, "epoch": 0.8886188432764972, "eval/gt_acc_batch": 0.8011111319065094, "frac_reward_zero_std": 1.0, "grad_norm": 0.291963130235672, "kd/policy_loss": 0.008593492650349314, "kd/rkl_advantage_mean": 0.78143601672103, "kd/rkl_advantage_p95": 5.004545233647028, "kd/rkl_advantage_std": 2.194343473513921, "kd/ssd_loss": 0.0, "learning_rate": 1.114191318877454e-07, "loss": 0.03437397480010986, "num_tokens": 765552659.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.801111110051473, "rewards/gt_logging_reward/std": 0.3912470549345016, "sampling/importance_sampling_ratio/max": 1.893887428442637, "sampling/importance_sampling_ratio/mean": 0.9967490951220195, "sampling/importance_sampling_ratio/min": 0.45751746793588005, "sampling/sampling_logp_difference/max": 0.3309444099664688, "sampling/sampling_logp_difference/mean": 0.004061497103733321, "step": 23400, "step_time": 8.255273388834516, "student/entropy": 0.13305013366043567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016666667473812898, "completions/max_length": 472.6333333333333, "completions/max_terminated_length": 446.93333333333334, "completions/mean_length": 188.62334289550782, "completions/mean_terminated_length": 183.34205373128256, "completions/min_length": 59.46666666666667, "completions/min_terminated_length": 59.46666666666667, "entropy": 0.12337800289193789, "epoch": 0.8897580982037747, "eval/gt_acc_batch": 0.8155555744965871, "frac_reward_zero_std": 1.0, "grad_norm": 0.272050678730011, "kd/policy_loss": 0.008035271579865366, "kd/rkl_advantage_mean": 0.7515330073734124, "kd/rkl_advantage_p95": 4.978853589296341, "kd/rkl_advantage_std": 2.204276688893636, "kd/ssd_loss": 0.0, "learning_rate": 1.1027987696046786e-07, "loss": 0.03214108149210612, "num_tokens": 766540407.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8155555586020152, "rewards/gt_logging_reward/std": 0.3792866716782252, "sampling/importance_sampling_ratio/max": 1.8959392825762431, "sampling/importance_sampling_ratio/mean": 0.9992969592412313, "sampling/importance_sampling_ratio/min": 0.4805510888497035, "sampling/sampling_logp_difference/max": 0.32385849157969154, "sampling/sampling_logp_difference/mean": 0.0037831123685464264, "step": 23430, "step_time": 8.04136991656463, "student/entropy": 0.12337800289193789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779536942642, "completions/max_length": 493.8333333333333, "completions/max_terminated_length": 453.1333333333333, "completions/mean_length": 193.03028818766276, "completions/mean_terminated_length": 184.0612340291341, "completions/min_length": 50.56666666666667, "completions/min_terminated_length": 50.56666666666667, "entropy": 0.1322292722761631, "epoch": 0.8908973531310523, "eval/gt_acc_batch": 0.7825000246365865, "frac_reward_zero_std": 1.0, "grad_norm": 0.3117804229259491, "kd/policy_loss": 0.008922240906395018, "kd/rkl_advantage_mean": 0.7542698213209709, "kd/rkl_advantage_p95": 4.9807889521121975, "kd/rkl_advantage_std": 2.2299985736608505, "kd/ssd_loss": 0.0, "learning_rate": 1.0914062203319029e-07, "loss": 0.03568897247314453, "num_tokens": 767542484.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7824999988079071, "rewards/gt_logging_reward/std": 0.40217407047748566, "sampling/importance_sampling_ratio/max": 1.8568728605906168, "sampling/importance_sampling_ratio/mean": 1.0013722062110901, "sampling/importance_sampling_ratio/min": 0.4250192736585935, "sampling/sampling_logp_difference/max": 0.34292271931966145, "sampling/sampling_logp_difference/mean": 0.004014841327443719, "step": 23460, "step_time": 8.214302424768297, "student/entropy": 0.1322292722761631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016111112075547378, "completions/max_length": 481.23333333333335, "completions/max_terminated_length": 433.8333333333333, "completions/mean_length": 187.6958429972331, "completions/mean_terminated_length": 182.4075154622396, "completions/min_length": 53.833333333333336, "completions/min_terminated_length": 53.833333333333336, "entropy": 0.12617952469736338, "epoch": 0.8920366080583298, "eval/gt_acc_batch": 0.8058333535989125, "frac_reward_zero_std": 1.0, "grad_norm": 0.2548650801181793, "kd/policy_loss": 0.00812690524229159, "kd/rkl_advantage_mean": 0.7714449566633751, "kd/rkl_advantage_p95": 4.9145223518212635, "kd/rkl_advantage_std": 2.198826653758685, "kd/ssd_loss": 0.0, "learning_rate": 1.0800136710591273e-07, "loss": 0.03250762224197388, "num_tokens": 768525333.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8058333377043406, "rewards/gt_logging_reward/std": 0.38640056451161703, "sampling/importance_sampling_ratio/max": 1.9118595878283182, "sampling/importance_sampling_ratio/mean": 0.9879804313182831, "sampling/importance_sampling_ratio/min": 0.4411916196346283, "sampling/sampling_logp_difference/max": 0.30647833744684855, "sampling/sampling_logp_difference/mean": 0.003921381640248, "step": 23490, "step_time": 8.102866832898387, "student/entropy": 0.12617952469736338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944446098059415, "completions/max_length": 483.43333333333334, "completions/max_terminated_length": 446.53333333333336, "completions/mean_length": 191.19695332845052, "completions/mean_terminated_length": 182.44372965494793, "completions/min_length": 55.5, "completions/min_terminated_length": 55.5, "entropy": 0.12614054686079423, "epoch": 0.8931758629856074, "eval/gt_acc_batch": 0.8069444636503855, "frac_reward_zero_std": 1.0, "grad_norm": 0.25433751940727234, "kd/policy_loss": 0.008230253248863542, "kd/rkl_advantage_mean": 0.7323411053667466, "kd/rkl_advantage_p95": 4.915744103988012, "kd/rkl_advantage_std": 2.164852320154508, "kd/ssd_loss": 0.0, "learning_rate": 1.0686211217863516e-07, "loss": 0.03292101224263509, "num_tokens": 769516722.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8069444417953491, "rewards/gt_logging_reward/std": 0.38519238034884135, "sampling/importance_sampling_ratio/max": 1.845179311434428, "sampling/importance_sampling_ratio/mean": 1.001533458630244, "sampling/importance_sampling_ratio/min": 0.47524298826853434, "sampling/sampling_logp_difference/max": 0.36615239779154457, "sampling/sampling_logp_difference/mean": 0.003841124203366538, "step": 23520, "step_time": 8.132080155401491, "student/entropy": 0.12614054686079423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667790462572, "completions/max_length": 488.3, "completions/max_terminated_length": 456.46666666666664, "completions/mean_length": 189.2850118001302, "completions/mean_terminated_length": 183.15941823323567, "completions/min_length": 49.666666666666664, "completions/min_terminated_length": 49.666666666666664, "entropy": 0.1266453705728054, "epoch": 0.894315117912885, "eval/gt_acc_batch": 0.7933333655198415, "frac_reward_zero_std": 1.0, "grad_norm": 0.28384193778038025, "kd/policy_loss": 0.007978556378899763, "kd/rkl_advantage_mean": 0.7209117294599613, "kd/rkl_advantage_p95": 4.8692471047242485, "kd/rkl_advantage_std": 2.1787883301575977, "kd/ssd_loss": 0.0, "learning_rate": 1.0572285725135762e-07, "loss": 0.031914222240448, "num_tokens": 770505852.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7933333317438761, "rewards/gt_logging_reward/std": 0.3979853143294652, "sampling/importance_sampling_ratio/max": 1.7859151005744933, "sampling/importance_sampling_ratio/mean": 0.9967570801575979, "sampling/importance_sampling_ratio/min": 0.4357630848884583, "sampling/sampling_logp_difference/max": 0.32015870412190756, "sampling/sampling_logp_difference/mean": 0.00392954039076964, "step": 23550, "step_time": 8.050195835426955, "student/entropy": 0.1266453705728054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890445232392, "completions/max_length": 506.26666666666665, "completions/max_terminated_length": 463.76666666666665, "completions/mean_length": 189.72834320068358, "completions/mean_terminated_length": 181.82352650960286, "completions/min_length": 53.266666666666666, "completions/min_terminated_length": 53.266666666666666, "entropy": 0.13573348081360262, "epoch": 0.8954543728401625, "eval/gt_acc_batch": 0.7813889106114705, "frac_reward_zero_std": 1.0, "grad_norm": 0.3088338077068329, "kd/policy_loss": 0.00897150503199858, "kd/rkl_advantage_mean": 0.7853486250232284, "kd/rkl_advantage_p95": 5.014503516753515, "kd/rkl_advantage_std": 2.220570707321167, "kd/ssd_loss": 0.0, "learning_rate": 1.0458360232408005e-07, "loss": 0.035886017481486, "num_tokens": 771490210.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7813888867696126, "rewards/gt_logging_reward/std": 0.4043851484855016, "sampling/importance_sampling_ratio/max": 1.8822362860043844, "sampling/importance_sampling_ratio/mean": 1.0025106867154439, "sampling/importance_sampling_ratio/min": 0.46884009887774786, "sampling/sampling_logp_difference/max": 0.30804698864618935, "sampling/sampling_logp_difference/mean": 0.004097609128803015, "step": 23580, "step_time": 8.171456933397955, "student/entropy": 0.13573348081360262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027777779381722212, "completions/max_length": 480.8666666666667, "completions/max_terminated_length": 442.6333333333333, "completions/mean_length": 192.21417744954428, "completions/mean_terminated_length": 183.2458302815755, "completions/min_length": 47.93333333333333, "completions/min_terminated_length": 47.93333333333333, "entropy": 0.13117755074054002, "epoch": 0.8965936277674401, "eval/gt_acc_batch": 0.7872222463289896, "frac_reward_zero_std": 1.0, "grad_norm": 0.2844315469264984, "kd/policy_loss": 0.008805507244930292, "kd/rkl_advantage_mean": 0.7935517848003656, "kd/rkl_advantage_p95": 4.979940607150396, "kd/rkl_advantage_std": 2.178421734770139, "kd/ssd_loss": 0.0, "learning_rate": 1.0344434739680248e-07, "loss": 0.03522203763326009, "num_tokens": 772486829.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7872222145398458, "rewards/gt_logging_reward/std": 0.39905671576658885, "sampling/importance_sampling_ratio/max": 1.939840265115102, "sampling/importance_sampling_ratio/mean": 1.0023224731286366, "sampling/importance_sampling_ratio/min": 0.47162244419256844, "sampling/sampling_logp_difference/max": 0.33163593808809916, "sampling/sampling_logp_difference/mean": 0.003988621251968046, "step": 23610, "step_time": 8.102807867936402, "student/entropy": 0.13117755074054002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01611111229285598, "completions/max_length": 468.76666666666665, "completions/max_terminated_length": 434.96666666666664, "completions/mean_length": 186.86612091064453, "completions/mean_terminated_length": 181.6456502278646, "completions/min_length": 54.266666666666666, "completions/min_terminated_length": 54.266666666666666, "entropy": 0.12615315094590188, "epoch": 0.8977328826947176, "eval/gt_acc_batch": 0.8191666841506958, "frac_reward_zero_std": 1.0, "grad_norm": 0.27189311385154724, "kd/policy_loss": 0.007909120687205966, "kd/rkl_advantage_mean": 0.7094635979117204, "kd/rkl_advantage_p95": 4.767636255423228, "kd/rkl_advantage_std": 2.1316077013810477, "kd/ssd_loss": 0.0, "learning_rate": 1.0230509246952493e-07, "loss": 0.031636488437652585, "num_tokens": 773469195.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8191666682561238, "rewards/gt_logging_reward/std": 0.36976704249779385, "sampling/importance_sampling_ratio/max": 1.8388025005658468, "sampling/importance_sampling_ratio/mean": 0.9993334670861562, "sampling/importance_sampling_ratio/min": 0.4940034588177999, "sampling/sampling_logp_difference/max": 0.31555574138959247, "sampling/sampling_logp_difference/mean": 0.003834435824925701, "step": 23640, "step_time": 7.948519981171315, "student/entropy": 0.12615315094590188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019444445644815764, "completions/max_length": 476.0, "completions/max_terminated_length": 455.7, "completions/mean_length": 187.1372314453125, "completions/mean_terminated_length": 180.84486083984376, "completions/min_length": 49.5, "completions/min_terminated_length": 49.5, "entropy": 0.13058514626075823, "epoch": 0.8988721376219952, "eval/gt_acc_batch": 0.8119444688161214, "frac_reward_zero_std": 1.0, "grad_norm": 0.307647705078125, "kd/policy_loss": 0.008456412731902674, "kd/rkl_advantage_mean": 0.7763529910395542, "kd/rkl_advantage_p95": 5.081291961669922, "kd/rkl_advantage_std": 2.2426914711793264, "kd/ssd_loss": 0.0, "learning_rate": 1.0116583754224736e-07, "loss": 0.03382565577824911, "num_tokens": 774440297.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8119444449742635, "rewards/gt_logging_reward/std": 0.38603309293588, "sampling/importance_sampling_ratio/max": 1.90255552927653, "sampling/importance_sampling_ratio/mean": 0.9967171986897786, "sampling/importance_sampling_ratio/min": 0.45148248275121056, "sampling/sampling_logp_difference/max": 0.3350133736928304, "sampling/sampling_logp_difference/mean": 0.003976478373321394, "step": 23670, "step_time": 7.874063818599097, "student/entropy": 0.13058514626075823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019444445458551247, "completions/max_length": 488.1, "completions/max_terminated_length": 454.46666666666664, "completions/mean_length": 185.65945485432943, "completions/mean_terminated_length": 179.20720876057942, "completions/min_length": 61.63333333333333, "completions/min_terminated_length": 61.63333333333333, "entropy": 0.13355098962783812, "epoch": 0.9000113925492728, "eval/gt_acc_batch": 0.7922222375869751, "frac_reward_zero_std": 1.0, "grad_norm": 0.40698111057281494, "kd/policy_loss": 0.008877882072313999, "kd/rkl_advantage_mean": 0.7841199447711309, "kd/rkl_advantage_p95": 5.003775258858998, "kd/rkl_advantage_std": 2.22268219490846, "kd/ssd_loss": 0.0, "learning_rate": 1.0002658261496981e-07, "loss": 0.035511533419291176, "num_tokens": 775409743.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7922222236792247, "rewards/gt_logging_reward/std": 0.3983675668636958, "sampling/importance_sampling_ratio/max": 1.9016271432240803, "sampling/importance_sampling_ratio/mean": 1.001073694229126, "sampling/importance_sampling_ratio/min": 0.4980588068564733, "sampling/sampling_logp_difference/max": 0.3543634951114655, "sampling/sampling_logp_difference/mean": 0.0040837232333918415, "step": 23700, "step_time": 7.94702851936648, "student/entropy": 0.13355098962783812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022500001142422357, "completions/max_length": 484.4, "completions/max_terminated_length": 449.3333333333333, "completions/mean_length": 188.2966761271159, "completions/mean_terminated_length": 180.95999755859376, "completions/min_length": 54.56666666666667, "completions/min_terminated_length": 54.56666666666667, "entropy": 0.1267381079494953, "epoch": 0.9011506474765504, "eval/gt_acc_batch": 0.7991666853427887, "frac_reward_zero_std": 1.0, "grad_norm": 0.2846234440803528, "kd/policy_loss": 0.00822012421558611, "kd/rkl_advantage_mean": 0.6475600599505318, "kd/rkl_advantage_p95": 4.8337732712427774, "kd/rkl_advantage_std": 2.1716086586316425, "kd/ssd_loss": 0.0, "learning_rate": 9.888732768769224e-08, "loss": 0.03288049300511678, "num_tokens": 776388731.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7991666654745738, "rewards/gt_logging_reward/std": 0.39278576771418255, "sampling/importance_sampling_ratio/max": 1.873330811659495, "sampling/importance_sampling_ratio/mean": 0.999523264169693, "sampling/importance_sampling_ratio/min": 0.42483115941286087, "sampling/sampling_logp_difference/max": 0.30087687969207766, "sampling/sampling_logp_difference/mean": 0.003892188884007434, "step": 23730, "step_time": 7.9158067875629055, "student/entropy": 0.1267381079494953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020277779114743075, "completions/max_length": 480.9, "completions/max_terminated_length": 449.26666666666665, "completions/mean_length": 182.67278747558595, "completions/mean_terminated_length": 175.970654296875, "completions/min_length": 50.1, "completions/min_terminated_length": 50.1, "entropy": 0.1270023458947738, "epoch": 0.9022899024038279, "eval/gt_acc_batch": 0.8150000194708507, "frac_reward_zero_std": 1.0, "grad_norm": 0.3233932852745056, "kd/policy_loss": 0.008069769232921924, "kd/rkl_advantage_mean": 0.6946959936370453, "kd/rkl_advantage_p95": 4.885032192866007, "kd/rkl_advantage_std": 2.22254022359848, "kd/ssd_loss": 0.0, "learning_rate": 9.774807276041469e-08, "loss": 0.03227907419204712, "num_tokens": 777344753.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8149999996026357, "rewards/gt_logging_reward/std": 0.3778323481480281, "sampling/importance_sampling_ratio/max": 1.8723241567611695, "sampling/importance_sampling_ratio/mean": 0.9986918608347575, "sampling/importance_sampling_ratio/min": 0.4546799709399541, "sampling/sampling_logp_difference/max": 0.3340025693178177, "sampling/sampling_logp_difference/mean": 0.003962777298875153, "step": 23760, "step_time": 8.002034998271847, "student/entropy": 0.1270023458947738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01944444573794802, "completions/max_length": 484.0, "completions/max_terminated_length": 446.46666666666664, "completions/mean_length": 183.0741760253906, "completions/mean_terminated_length": 176.61975708007813, "completions/min_length": 56.333333333333336, "completions/min_terminated_length": 56.333333333333336, "entropy": 0.1321736787756284, "epoch": 0.9034291573311054, "eval/gt_acc_batch": 0.8013889014720916, "frac_reward_zero_std": 1.0, "grad_norm": 0.3071274757385254, "kd/policy_loss": 0.008758024085545912, "kd/rkl_advantage_mean": 0.7820529286051169, "kd/rkl_advantage_p95": 4.968200701475143, "kd/rkl_advantage_std": 2.2192766745885213, "kd/ssd_loss": 0.0, "learning_rate": 9.660881783313712e-08, "loss": 0.035032097498575845, "num_tokens": 778305316.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8013888875643412, "rewards/gt_logging_reward/std": 0.3920508270462354, "sampling/importance_sampling_ratio/max": 1.8845593492190043, "sampling/importance_sampling_ratio/mean": 0.9999983290831248, "sampling/importance_sampling_ratio/min": 0.4751937607924143, "sampling/sampling_logp_difference/max": 0.3381643603245417, "sampling/sampling_logp_difference/mean": 0.003992102846192817, "step": 23790, "step_time": 8.029779204102427, "student/entropy": 0.1321736787756284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223803400994, "completions/max_length": 500.8666666666667, "completions/max_terminated_length": 465.7, "completions/mean_length": 195.7247314453125, "completions/mean_terminated_length": 186.19615936279297, "completions/min_length": 55.36666666666667, "completions/min_terminated_length": 55.36666666666667, "entropy": 0.12782016259928544, "epoch": 0.904568412258383, "eval/gt_acc_batch": 0.7794444600741068, "frac_reward_zero_std": 1.0, "grad_norm": 0.24723078310489655, "kd/policy_loss": 0.008521928042561437, "kd/rkl_advantage_mean": 0.8118948372701804, "kd/rkl_advantage_p95": 5.034744377930959, "kd/rkl_advantage_std": 2.2229801625013352, "kd/ssd_loss": 0.0, "learning_rate": 9.546956290585956e-08, "loss": 0.03408771355946859, "num_tokens": 779307973.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7794444382190704, "rewards/gt_logging_reward/std": 0.40750878949960073, "sampling/importance_sampling_ratio/max": 1.8676459948221842, "sampling/importance_sampling_ratio/mean": 0.999921691417694, "sampling/importance_sampling_ratio/min": 0.44719278216362, "sampling/sampling_logp_difference/max": 0.33270096182823183, "sampling/sampling_logp_difference/mean": 0.0038990817808856565, "step": 23820, "step_time": 8.200439462598297, "student/entropy": 0.12782016259928544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021666667827715477, "completions/max_length": 475.1, "completions/max_terminated_length": 426.1333333333333, "completions/mean_length": 187.86112162272136, "completions/mean_terminated_length": 180.65626831054686, "completions/min_length": 57.166666666666664, "completions/min_terminated_length": 57.166666666666664, "entropy": 0.12658199910074472, "epoch": 0.9057076671856605, "eval/gt_acc_batch": 0.7969444692134857, "frac_reward_zero_std": 1.0, "grad_norm": 0.2314251959323883, "kd/policy_loss": 0.008088951010722666, "kd/rkl_advantage_mean": 0.8036596624801556, "kd/rkl_advantage_p95": 5.028963228066762, "kd/rkl_advantage_std": 2.1908066819111505, "kd/ssd_loss": 0.0, "learning_rate": 9.433030797858201e-08, "loss": 0.032355807224909466, "num_tokens": 780282809.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7969444413979848, "rewards/gt_logging_reward/std": 0.3957925816377004, "sampling/importance_sampling_ratio/max": 1.866747999191284, "sampling/importance_sampling_ratio/mean": 0.9975006441275279, "sampling/importance_sampling_ratio/min": 0.4563299208879471, "sampling/sampling_logp_difference/max": 0.3290259838104248, "sampling/sampling_logp_difference/mean": 0.0038924424288173517, "step": 23850, "step_time": 7.912624507861135, "student/entropy": 0.12658199910074472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026944446222235758, "completions/max_length": 495.5, "completions/max_terminated_length": 453.6666666666667, "completions/mean_length": 193.49584096272787, "completions/mean_terminated_length": 184.64491424560546, "completions/min_length": 51.4, "completions/min_terminated_length": 51.4, "entropy": 0.12901725495855013, "epoch": 0.9068469221129382, "eval/gt_acc_batch": 0.7897222439448038, "frac_reward_zero_std": 1.0, "grad_norm": 0.20007796585559845, "kd/policy_loss": 0.00850366265901054, "kd/rkl_advantage_mean": 0.8614704355597496, "kd/rkl_advantage_p95": 5.234601287047068, "kd/rkl_advantage_std": 2.2819776187340417, "kd/ssd_loss": 0.0, "learning_rate": 9.319105305130444e-08, "loss": 0.03401464621225993, "num_tokens": 781284706.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.789722220102946, "rewards/gt_logging_reward/std": 0.39822153747081757, "sampling/importance_sampling_ratio/max": 1.9900052467981975, "sampling/importance_sampling_ratio/mean": 1.0023556172847747, "sampling/importance_sampling_ratio/min": 0.4394782910744349, "sampling/sampling_logp_difference/max": 0.3486208478609721, "sampling/sampling_logp_difference/mean": 0.0039587310825785005, "step": 23880, "step_time": 8.201442767296491, "student/entropy": 0.12901725495855013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016388889836768308, "completions/max_length": 456.6666666666667, "completions/max_terminated_length": 425.9, "completions/mean_length": 181.38001022338867, "completions/mean_terminated_length": 176.0780382792155, "completions/min_length": 47.6, "completions/min_terminated_length": 47.6, "entropy": 0.12604821442315975, "epoch": 0.9079861770402157, "eval/gt_acc_batch": 0.8305555721124013, "frac_reward_zero_std": 1.0, "grad_norm": 0.304845929145813, "kd/policy_loss": 0.007890726769498238, "kd/rkl_advantage_mean": 0.74743870832026, "kd/rkl_advantage_p95": 4.945240527391434, "kd/rkl_advantage_std": 2.1709065407514574, "kd/ssd_loss": 0.0, "learning_rate": 9.205179812402688e-08, "loss": 0.031562906503677365, "num_tokens": 782236338.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8305555621782938, "rewards/gt_logging_reward/std": 0.3652407129605611, "sampling/importance_sampling_ratio/max": 1.860983177026113, "sampling/importance_sampling_ratio/mean": 1.0001076638698578, "sampling/importance_sampling_ratio/min": 0.44710733691851295, "sampling/sampling_logp_difference/max": 0.33841455380121865, "sampling/sampling_logp_difference/mean": 0.003875758087572952, "step": 23910, "step_time": 7.709823411332521, "student/entropy": 0.12604821442315975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444565102458, "completions/max_length": 494.03333333333336, "completions/max_terminated_length": 448.26666666666665, "completions/mean_length": 188.48056335449218, "completions/mean_terminated_length": 181.38579559326172, "completions/min_length": 49.36666666666667, "completions/min_terminated_length": 49.36666666666667, "entropy": 0.12924063143630823, "epoch": 0.9091254319674933, "eval/gt_acc_batch": 0.8175000250339508, "frac_reward_zero_std": 1.0, "grad_norm": 0.3764802813529968, "kd/policy_loss": 0.008610068874744078, "kd/rkl_advantage_mean": 0.6941043509015192, "kd/rkl_advantage_p95": 4.899414589007695, "kd/rkl_advantage_std": 2.2093613783518475, "kd/ssd_loss": 0.0, "learning_rate": 9.091254319674931e-08, "loss": 0.03444027900695801, "num_tokens": 783231996.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8174999992052714, "rewards/gt_logging_reward/std": 0.3803617646296819, "sampling/importance_sampling_ratio/max": 1.9767000118891398, "sampling/importance_sampling_ratio/mean": 1.0027135570844015, "sampling/importance_sampling_ratio/min": 0.4536391923824946, "sampling/sampling_logp_difference/max": 0.31942854324976605, "sampling/sampling_logp_difference/mean": 0.003973771608434618, "step": 23940, "step_time": 8.185593083463027, "student/entropy": 0.12924063143630823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03055555736646056, "completions/max_length": 491.3, "completions/max_terminated_length": 451.96666666666664, "completions/mean_length": 190.96973368326823, "completions/mean_terminated_length": 181.04071756998698, "completions/min_length": 54.233333333333334, "completions/min_terminated_length": 54.233333333333334, "entropy": 0.1316263790552815, "epoch": 0.9102646868947708, "eval/gt_acc_batch": 0.8011111338933309, "frac_reward_zero_std": 1.0, "grad_norm": 0.26349952816963196, "kd/policy_loss": 0.008730001186874385, "kd/rkl_advantage_mean": 0.7578324956819416, "kd/rkl_advantage_p95": 4.910652361313502, "kd/rkl_advantage_std": 2.1888424227635066, "kd/ssd_loss": 0.0, "learning_rate": 8.977328826947177e-08, "loss": 0.03492000500361125, "num_tokens": 784229687.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8011111140251159, "rewards/gt_logging_reward/std": 0.39369708200295767, "sampling/importance_sampling_ratio/max": 1.9972583691279093, "sampling/importance_sampling_ratio/mean": 1.000782020886739, "sampling/importance_sampling_ratio/min": 0.45396350224812826, "sampling/sampling_logp_difference/max": 0.33780250549316404, "sampling/sampling_logp_difference/mean": 0.004032314824871719, "step": 23970, "step_time": 8.221964917665657, "student/entropy": 0.1316263790552815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029722223927577337, "completions/max_length": 490.43333333333334, "completions/max_terminated_length": 464.5, "completions/mean_length": 193.19862111409506, "completions/mean_terminated_length": 183.675026957194, "completions/min_length": 47.96666666666667, "completions/min_terminated_length": 47.96666666666667, "entropy": 0.14080434776842593, "epoch": 0.9114039418220484, "eval/gt_acc_batch": 0.7633333464463552, "frac_reward_zero_std": 1.0, "grad_norm": 0.27079567313194275, "kd/policy_loss": 0.009601093892706558, "kd/rkl_advantage_mean": 0.7961800189261946, "kd/rkl_advantage_p95": 5.007323930660884, "kd/rkl_advantage_std": 2.2040095210075377, "kd/ssd_loss": 0.0, "learning_rate": 8.86340333421942e-08, "loss": 0.038404377301534016, "num_tokens": 785228378.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7633333384990693, "rewards/gt_logging_reward/std": 0.4196274613340696, "sampling/importance_sampling_ratio/max": 1.9754541635513305, "sampling/importance_sampling_ratio/mean": 0.9967398901780447, "sampling/importance_sampling_ratio/min": 0.4341379741827647, "sampling/sampling_logp_difference/max": 0.32603402932484943, "sampling/sampling_logp_difference/mean": 0.004281433423360189, "step": 24000, "step_time": 8.131207675193824, "student/entropy": 0.14080434776842593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025000001521160206, "completions/max_length": 506.03333333333336, "completions/max_terminated_length": 469.03333333333336, "completions/mean_length": 189.5555648803711, "completions/mean_terminated_length": 181.2795440673828, "completions/min_length": 54.733333333333334, "completions/min_terminated_length": 54.733333333333334, "entropy": 0.13616626200576623, "epoch": 0.9125431967493259, "eval/gt_acc_batch": 0.7833333611488342, "frac_reward_zero_std": 1.0, "grad_norm": 0.24687303602695465, "kd/policy_loss": 0.009002959602124367, "kd/rkl_advantage_mean": 0.8708430830389261, "kd/rkl_advantage_p95": 5.202428833643595, "kd/rkl_advantage_std": 2.2701585680246352, "kd/ssd_loss": 0.0, "learning_rate": 8.749477841491663e-08, "loss": 0.03601183891296387, "num_tokens": 786211626.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7833333353201548, "rewards/gt_logging_reward/std": 0.40647223591804504, "sampling/importance_sampling_ratio/max": 1.9809089223543803, "sampling/importance_sampling_ratio/mean": 0.9978289703528086, "sampling/importance_sampling_ratio/min": 0.465154571334521, "sampling/sampling_logp_difference/max": 0.3328208009401957, "sampling/sampling_logp_difference/mean": 0.004184119077399373, "step": 24030, "step_time": 8.137740776697562, "student/entropy": 0.13616626200576623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223499168953, "completions/max_length": 501.06666666666666, "completions/max_terminated_length": 453.03333333333336, "completions/mean_length": 191.9058415730794, "completions/mean_terminated_length": 185.41827290852865, "completions/min_length": 54.96666666666667, "completions/min_terminated_length": 54.96666666666667, "entropy": 0.1336612952252229, "epoch": 0.9136824516766034, "eval/gt_acc_batch": 0.7777778029441833, "frac_reward_zero_std": 1.0, "grad_norm": 0.37164852023124695, "kd/policy_loss": 0.00917275834751005, "kd/rkl_advantage_mean": 0.8903593073288599, "kd/rkl_advantage_p95": 5.203114833434423, "kd/rkl_advantage_std": 2.2665322492520015, "kd/ssd_loss": 0.0, "learning_rate": 8.635552348763908e-08, "loss": 0.036691033840179445, "num_tokens": 787200151.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7777777830759685, "rewards/gt_logging_reward/std": 0.41261003812154134, "sampling/importance_sampling_ratio/max": 1.868987508614858, "sampling/importance_sampling_ratio/mean": 0.9985127210617065, "sampling/importance_sampling_ratio/min": 0.44862388173739115, "sampling/sampling_logp_difference/max": 0.3474954187870026, "sampling/sampling_logp_difference/mean": 0.004056721576489508, "step": 24060, "step_time": 8.0661915777028, "student/entropy": 0.1336612952252229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02194444558893641, "completions/max_length": 488.53333333333336, "completions/max_terminated_length": 445.06666666666666, "completions/mean_length": 188.54584248860678, "completions/mean_terminated_length": 181.3757746378581, "completions/min_length": 58.93333333333333, "completions/min_terminated_length": 58.93333333333333, "entropy": 0.12421761391063531, "epoch": 0.9148217066038811, "eval/gt_acc_batch": 0.8116666932900747, "frac_reward_zero_std": 1.0, "grad_norm": 0.23088796436786652, "kd/policy_loss": 0.008355669075778374, "kd/rkl_advantage_mean": 0.8125200080374877, "kd/rkl_advantage_p95": 5.07315554022789, "kd/rkl_advantage_std": 2.2190392752488455, "kd/ssd_loss": 0.0, "learning_rate": 8.521626856036152e-08, "loss": 0.03342267672220866, "num_tokens": 788181548.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8116666674613953, "rewards/gt_logging_reward/std": 0.38566667040189107, "sampling/importance_sampling_ratio/max": 1.8316002130508422, "sampling/importance_sampling_ratio/mean": 0.9992101530234019, "sampling/importance_sampling_ratio/min": 0.478206401069959, "sampling/sampling_logp_difference/max": 0.3250868101914724, "sampling/sampling_logp_difference/mean": 0.003849405624593298, "step": 24090, "step_time": 7.9537887388287345, "student/entropy": 0.12421761391063531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017222223151475192, "completions/max_length": 484.53333333333336, "completions/max_terminated_length": 450.93333333333334, "completions/mean_length": 183.47723032633465, "completions/mean_terminated_length": 177.88633473714194, "completions/min_length": 58.86666666666667, "completions/min_terminated_length": 58.86666666666667, "entropy": 0.13513108311841884, "epoch": 0.9159609615311586, "eval/gt_acc_batch": 0.8008333583672841, "frac_reward_zero_std": 1.0, "grad_norm": 0.29545730352401733, "kd/policy_loss": 0.008764328039251267, "kd/rkl_advantage_mean": 0.7693331571916739, "kd/rkl_advantage_p95": 4.984845026334127, "kd/rkl_advantage_std": 2.2300729433695476, "kd/ssd_loss": 0.0, "learning_rate": 8.407701363308397e-08, "loss": 0.03505731423695882, "num_tokens": 789146722.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333345254263, "rewards/gt_logging_reward/std": 0.3928915242354075, "sampling/importance_sampling_ratio/max": 1.8894257426261902, "sampling/importance_sampling_ratio/mean": 0.9994841953118642, "sampling/importance_sampling_ratio/min": 0.44878021130959195, "sampling/sampling_logp_difference/max": 0.3135238945484161, "sampling/sampling_logp_difference/mean": 0.004165143558445076, "step": 24120, "step_time": 7.88628006609603, "student/entropy": 0.13513108311841884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022222223381201427, "completions/max_length": 487.1333333333333, "completions/max_terminated_length": 440.23333333333335, "completions/mean_length": 182.37417500813802, "completions/mean_terminated_length": 174.99422454833984, "completions/min_length": 53.2, "completions/min_terminated_length": 53.2, "entropy": 0.12878218721598386, "epoch": 0.9171002164584362, "eval/gt_acc_batch": 0.8025000194708506, "frac_reward_zero_std": 1.0, "grad_norm": 0.2513884902000427, "kd/policy_loss": 0.007957524136872961, "kd/rkl_advantage_mean": 0.7734642741580804, "kd/rkl_advantage_p95": 4.935596162080765, "kd/rkl_advantage_std": 2.1918361455202104, "kd/ssd_loss": 0.0, "learning_rate": 8.29377587058064e-08, "loss": 0.031830100218455, "num_tokens": 790096925.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8024999996026357, "rewards/gt_logging_reward/std": 0.3855873907605807, "sampling/importance_sampling_ratio/max": 1.9429988543192545, "sampling/importance_sampling_ratio/mean": 0.9985186477502187, "sampling/importance_sampling_ratio/min": 0.449700129032135, "sampling/sampling_logp_difference/max": 0.35033594568570453, "sampling/sampling_logp_difference/mean": 0.003937622462399304, "step": 24150, "step_time": 7.92726465529607, "student/entropy": 0.12878218721598386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890389353038, "completions/max_length": 497.3, "completions/max_terminated_length": 453.1666666666667, "completions/mean_length": 189.63139750162762, "completions/mean_terminated_length": 180.96622670491536, "completions/min_length": 47.333333333333336, "completions/min_terminated_length": 47.333333333333336, "entropy": 0.13250616223861775, "epoch": 0.9182394713857137, "eval/gt_acc_batch": 0.8036111374696095, "frac_reward_zero_std": 1.0, "grad_norm": 0.34810641407966614, "kd/policy_loss": 0.008769661973929033, "kd/rkl_advantage_mean": 0.7697878775497278, "kd/rkl_advantage_p95": 4.981754980484644, "kd/rkl_advantage_std": 2.1975827028354007, "kd/ssd_loss": 0.0, "learning_rate": 8.179850377852884e-08, "loss": 0.035078648726145426, "num_tokens": 791091054.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111116409301, "rewards/gt_logging_reward/std": 0.39145358502864835, "sampling/importance_sampling_ratio/max": 1.8775548895200094, "sampling/importance_sampling_ratio/mean": 0.9950630307197571, "sampling/importance_sampling_ratio/min": 0.3956416661540667, "sampling/sampling_logp_difference/max": 0.348774724205335, "sampling/sampling_logp_difference/mean": 0.004057976448287566, "step": 24180, "step_time": 8.19715348695075, "student/entropy": 0.13250616223861775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055557142943145, "completions/max_length": 486.1333333333333, "completions/max_terminated_length": 444.8, "completions/mean_length": 193.95528717041014, "completions/mean_terminated_length": 185.0251261393229, "completions/min_length": 53.0, "completions/min_terminated_length": 53.0, "entropy": 0.1290957521647215, "epoch": 0.9193787263129913, "eval/gt_acc_batch": 0.7950000206629435, "frac_reward_zero_std": 1.0, "grad_norm": 0.28387215733528137, "kd/policy_loss": 0.00836574508381697, "kd/rkl_advantage_mean": 0.7482448396272957, "kd/rkl_advantage_p95": 5.060183203220367, "kd/rkl_advantage_std": 2.261568104227384, "kd/ssd_loss": 0.0, "learning_rate": 8.065924885125127e-08, "loss": 0.03346298535664876, "num_tokens": 792099293.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7949999988079071, "rewards/gt_logging_reward/std": 0.3953895553946495, "sampling/importance_sampling_ratio/max": 1.8498003482818604, "sampling/importance_sampling_ratio/mean": 0.9954717953999838, "sampling/importance_sampling_ratio/min": 0.4513934224843979, "sampling/sampling_logp_difference/max": 0.3205001493295034, "sampling/sampling_logp_difference/mean": 0.003914256067946553, "step": 24210, "step_time": 8.059627126530783, "student/entropy": 0.1290957521647215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02833333509042859, "completions/max_length": 480.3333333333333, "completions/max_terminated_length": 442.9, "completions/mean_length": 186.40056584676105, "completions/mean_terminated_length": 177.15359217325846, "completions/min_length": 58.266666666666666, "completions/min_terminated_length": 58.266666666666666, "entropy": 0.13411923591047525, "epoch": 0.9205179812402688, "eval/gt_acc_batch": 0.7952778041362762, "frac_reward_zero_std": 1.0, "grad_norm": 0.28909939527511597, "kd/policy_loss": 0.008853009977610782, "kd/rkl_advantage_mean": 0.7594434167879324, "kd/rkl_advantage_p95": 5.03018672267596, "kd/rkl_advantage_std": 2.2243890474239985, "kd/ssd_loss": 0.0, "learning_rate": 7.951999392397373e-08, "loss": 0.03541203737258911, "num_tokens": 793068439.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777763207753, "rewards/gt_logging_reward/std": 0.3914156893889109, "sampling/importance_sampling_ratio/max": 1.9108066836992899, "sampling/importance_sampling_ratio/mean": 0.9997287193934122, "sampling/importance_sampling_ratio/min": 0.4419751390814781, "sampling/sampling_logp_difference/max": 0.3472080171108246, "sampling/sampling_logp_difference/mean": 0.004082925726349155, "step": 24240, "step_time": 8.025126761002078, "student/entropy": 0.13411923591047525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444578140974, "completions/max_length": 489.96666666666664, "completions/max_terminated_length": 444.6, "completions/mean_length": 190.00223083496093, "completions/mean_terminated_length": 182.16716664632162, "completions/min_length": 54.766666666666666, "completions/min_terminated_length": 54.766666666666666, "entropy": 0.1296947694073121, "epoch": 0.9216572361675465, "eval/gt_acc_batch": 0.7955555895964305, "frac_reward_zero_std": 1.0, "grad_norm": 0.310287207365036, "kd/policy_loss": 0.008348648560543855, "kd/rkl_advantage_mean": 0.7641405857012917, "kd/rkl_advantage_p95": 4.962390444676081, "kd/rkl_advantage_std": 2.2000066488981247, "kd/ssd_loss": 0.0, "learning_rate": 7.838073899669616e-08, "loss": 0.033394598960876466, "num_tokens": 794058743.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555518468221, "rewards/gt_logging_reward/std": 0.3976501002907753, "sampling/importance_sampling_ratio/max": 1.9544238805770875, "sampling/importance_sampling_ratio/mean": 1.0007420202096304, "sampling/importance_sampling_ratio/min": 0.44488295416037243, "sampling/sampling_logp_difference/max": 0.340520699818929, "sampling/sampling_logp_difference/mean": 0.0040046232752501965, "step": 24270, "step_time": 8.173836630471245, "student/entropy": 0.1296947694073121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223890324432, "completions/max_length": 491.8, "completions/max_terminated_length": 461.26666666666665, "completions/mean_length": 193.03889973958334, "completions/mean_terminated_length": 184.26748708089193, "completions/min_length": 56.56666666666667, "completions/min_terminated_length": 56.56666666666667, "entropy": 0.12809086559961239, "epoch": 0.922796491094824, "eval/gt_acc_batch": 0.7877778053283692, "frac_reward_zero_std": 1.0, "grad_norm": 0.23859140276908875, "kd/policy_loss": 0.008671386803810795, "kd/rkl_advantage_mean": 0.7436593965239202, "kd/rkl_advantage_p95": 5.017655656735102, "kd/rkl_advantage_std": 2.227001033226649, "kd/ssd_loss": 0.0, "learning_rate": 7.724148406941859e-08, "loss": 0.03468554019927979, "num_tokens": 795058347.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7877777755260468, "rewards/gt_logging_reward/std": 0.39831776122252144, "sampling/importance_sampling_ratio/max": 1.9549832741419475, "sampling/importance_sampling_ratio/mean": 1.0001146833101908, "sampling/importance_sampling_ratio/min": 0.41914865548412006, "sampling/sampling_logp_difference/max": 0.39626035690307615, "sampling/sampling_logp_difference/mean": 0.003894400356026987, "step": 24300, "step_time": 8.110658858840663, "student/entropy": 0.12809086559961239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444562618931, "completions/max_length": 471.46666666666664, "completions/max_terminated_length": 440.8, "completions/mean_length": 180.58639882405598, "completions/mean_terminated_length": 172.3366673787435, "completions/min_length": 52.2, "completions/min_terminated_length": 52.2, "entropy": 0.13257531964530547, "epoch": 0.9239357460221016, "eval/gt_acc_batch": 0.7947222371896108, "frac_reward_zero_std": 1.0, "grad_norm": 0.3346957266330719, "kd/policy_loss": 0.009037291871694227, "kd/rkl_advantage_mean": 0.7096733149606734, "kd/rkl_advantage_p95": 4.84236698547999, "kd/rkl_advantage_std": 2.1794380893309913, "kd/ssd_loss": 0.0, "learning_rate": 7.610222914214103e-08, "loss": 0.03614916801452637, "num_tokens": 796004642.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222212950389, "rewards/gt_logging_reward/std": 0.39697710374991096, "sampling/importance_sampling_ratio/max": 1.8644489924112955, "sampling/importance_sampling_ratio/mean": 1.0002525786558787, "sampling/importance_sampling_ratio/min": 0.43266439735889434, "sampling/sampling_logp_difference/max": 0.3183053692181905, "sampling/sampling_logp_difference/mean": 0.004057357312800983, "step": 24330, "step_time": 7.837908811730449, "student/entropy": 0.13257531964530547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112559835116, "completions/max_length": 480.23333333333335, "completions/max_terminated_length": 441.06666666666666, "completions/mean_length": 184.64278818766277, "completions/mean_terminated_length": 176.7641143798828, "completions/min_length": 53.266666666666666, "completions/min_terminated_length": 53.266666666666666, "entropy": 0.13107618348052105, "epoch": 0.9250750009493791, "eval/gt_acc_batch": 0.8094444632530212, "frac_reward_zero_std": 1.0, "grad_norm": 0.3053552508354187, "kd/policy_loss": 0.00838627337361686, "kd/rkl_advantage_mean": 0.780356620127956, "kd/rkl_advantage_p95": 5.0758395314216616, "kd/rkl_advantage_std": 2.253603415687879, "kd/ssd_loss": 0.0, "learning_rate": 7.496297421486348e-08, "loss": 0.03354509671529134, "num_tokens": 796974292.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8094444433848064, "rewards/gt_logging_reward/std": 0.37785479923089343, "sampling/importance_sampling_ratio/max": 1.9051557938257853, "sampling/importance_sampling_ratio/mean": 1.0008688906828562, "sampling/importance_sampling_ratio/min": 0.454911079009374, "sampling/sampling_logp_difference/max": 0.3558944980303446, "sampling/sampling_logp_difference/mean": 0.0039773758578424655, "step": 24360, "step_time": 8.201096722832881, "student/entropy": 0.13107618348052105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020555556875964005, "completions/max_length": 497.2, "completions/max_terminated_length": 463.4, "completions/mean_length": 190.2702875773112, "completions/mean_terminated_length": 183.46165924072267, "completions/min_length": 60.4, "completions/min_terminated_length": 60.4, "entropy": 0.13213521111756563, "epoch": 0.9262142558766566, "eval/gt_acc_batch": 0.7897222439448038, "frac_reward_zero_std": 1.0, "grad_norm": 0.2506757080554962, "kd/policy_loss": 0.008648138827023406, "kd/rkl_advantage_mean": 0.7827513482421636, "kd/rkl_advantage_p95": 5.07906840244929, "kd/rkl_advantage_std": 2.245951982339223, "kd/ssd_loss": 0.0, "learning_rate": 7.382371928758592e-08, "loss": 0.03459256092707316, "num_tokens": 797977865.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7897222260634105, "rewards/gt_logging_reward/std": 0.40194974541664125, "sampling/importance_sampling_ratio/max": 1.9685458461443583, "sampling/importance_sampling_ratio/mean": 0.997860982020696, "sampling/importance_sampling_ratio/min": 0.4405864412585894, "sampling/sampling_logp_difference/max": 0.3488954842090607, "sampling/sampling_logp_difference/mean": 0.004046107153408229, "step": 24390, "step_time": 8.335301085035704, "student/entropy": 0.13213521111756563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777778965731462, "completions/max_length": 493.96666666666664, "completions/max_terminated_length": 456.76666666666665, "completions/mean_length": 184.8872314453125, "completions/mean_terminated_length": 177.47720692952473, "completions/min_length": 52.733333333333334, "completions/min_terminated_length": 52.733333333333334, "entropy": 0.13279480542987585, "epoch": 0.9273535108039342, "eval/gt_acc_batch": 0.8005555828412374, "frac_reward_zero_std": 1.0, "grad_norm": 0.2746477723121643, "kd/policy_loss": 0.008811842844200632, "kd/rkl_advantage_mean": 0.7819192149986823, "kd/rkl_advantage_p95": 4.989268231391907, "kd/rkl_advantage_std": 2.2203943908214567, "kd/ssd_loss": 0.0, "learning_rate": 7.268446436030835e-08, "loss": 0.035247377554575604, "num_tokens": 798944491.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.800555557012558, "rewards/gt_logging_reward/std": 0.38707745969295504, "sampling/importance_sampling_ratio/max": 1.9298817992210389, "sampling/importance_sampling_ratio/mean": 0.9952809711297353, "sampling/importance_sampling_ratio/min": 0.46735452065865196, "sampling/sampling_logp_difference/max": 0.3331974546114604, "sampling/sampling_logp_difference/mean": 0.004064155369997024, "step": 24420, "step_time": 7.994101880199741, "student/entropy": 0.13279480542987585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015833334407458703, "completions/max_length": 478.6666666666667, "completions/max_terminated_length": 445.93333333333334, "completions/mean_length": 182.7969543457031, "completions/mean_terminated_length": 177.5429911295573, "completions/min_length": 50.96666666666667, "completions/min_terminated_length": 50.96666666666667, "entropy": 0.1280438815553983, "epoch": 0.9284927657312118, "eval/gt_acc_batch": 0.805833347638448, "frac_reward_zero_std": 1.0, "grad_norm": 0.2484748214483261, "kd/policy_loss": 0.007990691585776706, "kd/rkl_advantage_mean": 0.7109651803039014, "kd/rkl_advantage_p95": 4.87907306154569, "kd/rkl_advantage_std": 2.206076489885648, "kd/ssd_loss": 0.0, "learning_rate": 7.15452094330308e-08, "loss": 0.031962768236796064, "num_tokens": 799909648.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8058333337306977, "rewards/gt_logging_reward/std": 0.38797997385263444, "sampling/importance_sampling_ratio/max": 1.890613353252411, "sampling/importance_sampling_ratio/mean": 0.9921106934547425, "sampling/importance_sampling_ratio/min": 0.44995856086413066, "sampling/sampling_logp_difference/max": 0.3203708529472351, "sampling/sampling_logp_difference/mean": 0.00391869165468961, "step": 24450, "step_time": 7.935221141927953, "student/entropy": 0.1280438815553983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0191666677283744, "completions/max_length": 480.26666666666665, "completions/max_terminated_length": 446.26666666666665, "completions/mean_length": 189.16750895182292, "completions/mean_terminated_length": 182.8718490600586, "completions/min_length": 61.166666666666664, "completions/min_terminated_length": 61.166666666666664, "entropy": 0.12428132109344006, "epoch": 0.9296320206584894, "eval/gt_acc_batch": 0.7972222407658894, "frac_reward_zero_std": 1.0, "grad_norm": 0.27417561411857605, "kd/policy_loss": 0.007864566820596034, "kd/rkl_advantage_mean": 0.7553389204666019, "kd/rkl_advantage_p95": 5.055232191085816, "kd/rkl_advantage_std": 2.2313181350628537, "kd/ssd_loss": 0.0, "learning_rate": 7.040595450575323e-08, "loss": 0.03145826657613118, "num_tokens": 800899635.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7972222208976746, "rewards/gt_logging_reward/std": 0.3851306393742561, "sampling/importance_sampling_ratio/max": 1.9357712348302205, "sampling/importance_sampling_ratio/mean": 0.9947208841641744, "sampling/importance_sampling_ratio/min": 0.4542262921730677, "sampling/sampling_logp_difference/max": 0.33078117767969767, "sampling/sampling_logp_difference/mean": 0.003824336190397541, "step": 24480, "step_time": 8.041672167632107, "student/entropy": 0.12428132109344006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.027222223424663145, "completions/max_length": 481.0, "completions/max_terminated_length": 447.96666666666664, "completions/mean_length": 186.6791753133138, "completions/mean_terminated_length": 177.6014205932617, "completions/min_length": 43.96666666666667, "completions/min_terminated_length": 43.96666666666667, "entropy": 0.13201345540583134, "epoch": 0.9307712755857669, "eval/gt_acc_batch": 0.7900000194708506, "frac_reward_zero_std": 1.0, "grad_norm": 0.2628467381000519, "kd/policy_loss": 0.009080128222315883, "kd/rkl_advantage_mean": 0.7401744809933006, "kd/rkl_advantage_p95": 4.918387589852015, "kd/rkl_advantage_std": 2.206055647134781, "kd/ssd_loss": 0.0, "learning_rate": 6.926669957847567e-08, "loss": 0.03632051150004069, "num_tokens": 801887408.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7899999996026357, "rewards/gt_logging_reward/std": 0.396776453157266, "sampling/importance_sampling_ratio/max": 1.9390531778335571, "sampling/importance_sampling_ratio/mean": 0.9906928956508636, "sampling/importance_sampling_ratio/min": 0.422007887562116, "sampling/sampling_logp_difference/max": 0.30706318815549216, "sampling/sampling_logp_difference/mean": 0.004028386529535055, "step": 24510, "step_time": 8.23317424923104, "student/entropy": 0.13201345540583134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02500000090027849, "completions/max_length": 483.93333333333334, "completions/max_terminated_length": 449.6, "completions/mean_length": 184.25084330240887, "completions/mean_terminated_length": 176.05179951985676, "completions/min_length": 55.53333333333333, "completions/min_terminated_length": 55.53333333333333, "entropy": 0.1298482804869612, "epoch": 0.9319105305130445, "eval/gt_acc_batch": 0.7872222463289896, "frac_reward_zero_std": 1.0, "grad_norm": 0.2328183799982071, "kd/policy_loss": 0.008557815102782721, "kd/rkl_advantage_mean": 0.8083499699831009, "kd/rkl_advantage_p95": 5.007997518777847, "kd/rkl_advantage_std": 2.1932487616936367, "kd/ssd_loss": 0.0, "learning_rate": 6.812744465119812e-08, "loss": 0.034231261412302656, "num_tokens": 802849951.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7872222244739533, "rewards/gt_logging_reward/std": 0.4008211006720861, "sampling/importance_sampling_ratio/max": 1.9704442779223124, "sampling/importance_sampling_ratio/mean": 1.006645546356837, "sampling/importance_sampling_ratio/min": 0.4473637893795967, "sampling/sampling_logp_difference/max": 0.3264566500981649, "sampling/sampling_logp_difference/mean": 0.004021703801117838, "step": 24540, "step_time": 7.893545758256611, "student/entropy": 0.1298482804869612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02638889035830895, "completions/max_length": 476.23333333333335, "completions/max_terminated_length": 444.3333333333333, "completions/mean_length": 194.45945434570314, "completions/mean_terminated_length": 186.00371602376302, "completions/min_length": 58.166666666666664, "completions/min_terminated_length": 58.166666666666664, "entropy": 0.12729665028552214, "epoch": 0.933049785440322, "eval/gt_acc_batch": 0.7997222443421682, "frac_reward_zero_std": 1.0, "grad_norm": 0.24804744124412537, "kd/policy_loss": 0.008204941420505445, "kd/rkl_advantage_mean": 0.7460974522633478, "kd/rkl_advantage_p95": 4.990056389570237, "kd/rkl_advantage_std": 2.21130576133728, "kd/ssd_loss": 0.0, "learning_rate": 6.698818972392055e-08, "loss": 0.03281976977984111, "num_tokens": 803857381.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7997222145398458, "rewards/gt_logging_reward/std": 0.39104114919900895, "sampling/importance_sampling_ratio/max": 1.9348055322964985, "sampling/importance_sampling_ratio/mean": 1.0016352593898774, "sampling/importance_sampling_ratio/min": 0.46754954059918724, "sampling/sampling_logp_difference/max": 0.31488598783810934, "sampling/sampling_logp_difference/mean": 0.003885683223294715, "step": 24570, "step_time": 8.026657565836407, "student/entropy": 0.12729665028552214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01694444560756286, "completions/max_length": 476.06666666666666, "completions/max_terminated_length": 433.0, "completions/mean_length": 183.76834360758463, "completions/mean_terminated_length": 178.11383666992188, "completions/min_length": 62.2, "completions/min_terminated_length": 62.2, "entropy": 0.12456653509289026, "epoch": 0.9341890403675996, "eval/gt_acc_batch": 0.8050000250339509, "frac_reward_zero_std": 1.0, "grad_norm": 0.2625117599964142, "kd/policy_loss": 0.008142147552765286, "kd/rkl_advantage_mean": 0.6941774322961768, "kd/rkl_advantage_p95": 4.922849386930466, "kd/rkl_advantage_std": 2.1999955624341965, "kd/ssd_loss": 0.0, "learning_rate": 6.584893479664299e-08, "loss": 0.03256859381993612, "num_tokens": 804824363.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.804999993244807, "rewards/gt_logging_reward/std": 0.38858784834543864, "sampling/importance_sampling_ratio/max": 1.8702336192131042, "sampling/importance_sampling_ratio/mean": 0.9999607602755228, "sampling/importance_sampling_ratio/min": 0.48782880008220675, "sampling/sampling_logp_difference/max": 0.2991174499193827, "sampling/sampling_logp_difference/mean": 0.003878850140608847, "step": 24600, "step_time": 7.896493811063313, "student/entropy": 0.12456653509289026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333467443784, "completions/max_length": 497.9, "completions/max_terminated_length": 449.96666666666664, "completions/mean_length": 186.63112131754556, "completions/mean_terminated_length": 178.8535919189453, "completions/min_length": 56.333333333333336, "completions/min_terminated_length": 56.333333333333336, "entropy": 0.13612669066836436, "epoch": 0.9353282952948772, "eval/gt_acc_batch": 0.7925000250339508, "frac_reward_zero_std": 1.0, "grad_norm": 0.28351742029190063, "kd/policy_loss": 0.008878160048819457, "kd/rkl_advantage_mean": 0.754980098331968, "kd/rkl_advantage_p95": 5.0162323117256165, "kd/rkl_advantage_std": 2.2303546160459518, "kd/ssd_loss": 0.0, "learning_rate": 6.470967986936544e-08, "loss": 0.03551264603932699, "num_tokens": 805793251.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7924999992052714, "rewards/gt_logging_reward/std": 0.39557243287563326, "sampling/importance_sampling_ratio/max": 1.9094221790631611, "sampling/importance_sampling_ratio/mean": 1.0015060504277546, "sampling/importance_sampling_ratio/min": 0.4497531841198603, "sampling/sampling_logp_difference/max": 0.3101345539093018, "sampling/sampling_logp_difference/mean": 0.004115506967840095, "step": 24630, "step_time": 8.077667048366857, "student/entropy": 0.13612669066836436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02611111244186759, "completions/max_length": 490.3666666666667, "completions/max_terminated_length": 462.6666666666667, "completions/mean_length": 187.8416763305664, "completions/mean_terminated_length": 179.04737040201823, "completions/min_length": 46.63333333333333, "completions/min_terminated_length": 46.63333333333333, "entropy": 0.14009424888839325, "epoch": 0.9364675502221547, "eval/gt_acc_batch": 0.784722238779068, "frac_reward_zero_std": 1.0, "grad_norm": 0.3100256621837616, "kd/policy_loss": 0.009220946064063658, "kd/rkl_advantage_mean": 0.8778209338677698, "kd/rkl_advantage_p95": 5.231694370508194, "kd/rkl_advantage_std": 2.2863646397988, "kd/ssd_loss": 0.0, "learning_rate": 6.357042494208788e-08, "loss": 0.0368837833404541, "num_tokens": 806773673.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7847222208976745, "rewards/gt_logging_reward/std": 0.4038927664359411, "sampling/importance_sampling_ratio/max": 1.991897968451182, "sampling/importance_sampling_ratio/mean": 1.0021692295869191, "sampling/importance_sampling_ratio/min": 0.41651338438193003, "sampling/sampling_logp_difference/max": 0.35691334704558053, "sampling/sampling_logp_difference/mean": 0.004245945120540758, "step": 24660, "step_time": 8.091892351696151, "student/entropy": 0.14009424888839325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02250000151495139, "completions/max_length": 487.96666666666664, "completions/max_terminated_length": 450.56666666666666, "completions/mean_length": 191.33167622884113, "completions/mean_terminated_length": 184.0648956298828, "completions/min_length": 54.1, "completions/min_terminated_length": 54.1, "entropy": 0.1305841570099195, "epoch": 0.9376068051494323, "eval/gt_acc_batch": 0.7786111275355021, "frac_reward_zero_std": 1.0, "grad_norm": 0.26745399832725525, "kd/policy_loss": 0.008719816821394489, "kd/rkl_advantage_mean": 0.7728871301437418, "kd/rkl_advantage_p95": 5.0111926635106405, "kd/rkl_advantage_std": 2.2147573113441466, "kd/ssd_loss": 0.0, "learning_rate": 6.243117001481031e-08, "loss": 0.034879271189371744, "num_tokens": 807761579.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7786111116409302, "rewards/gt_logging_reward/std": 0.4103176732858022, "sampling/importance_sampling_ratio/max": 1.933432682355245, "sampling/importance_sampling_ratio/mean": 0.9951293845971425, "sampling/importance_sampling_ratio/min": 0.45215099354585014, "sampling/sampling_logp_difference/max": 0.3550235907236735, "sampling/sampling_logp_difference/mean": 0.003992706226805846, "step": 24690, "step_time": 8.086319020169322, "student/entropy": 0.1305841570099195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030277779481063288, "completions/max_length": 476.0, "completions/max_terminated_length": 434.6333333333333, "completions/mean_length": 185.2533432006836, "completions/mean_terminated_length": 175.2404993693034, "completions/min_length": 50.63333333333333, "completions/min_terminated_length": 50.63333333333333, "entropy": 0.13155985604971648, "epoch": 0.9387460600767098, "eval/gt_acc_batch": 0.7991666793823242, "frac_reward_zero_std": 1.0, "grad_norm": 0.29941579699516296, "kd/policy_loss": 0.008949914601786683, "kd/rkl_advantage_mean": 0.7581661449434857, "kd/rkl_advantage_p95": 4.904439270496368, "kd/rkl_advantage_std": 2.1815010209878287, "kd/ssd_loss": 0.0, "learning_rate": 6.129191508753276e-08, "loss": 0.035799654324849446, "num_tokens": 808732651.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7991666674613953, "rewards/gt_logging_reward/std": 0.3921600530544917, "sampling/importance_sampling_ratio/max": 1.894372038046519, "sampling/importance_sampling_ratio/mean": 0.9984656731287639, "sampling/importance_sampling_ratio/min": 0.48466932972272236, "sampling/sampling_logp_difference/max": 0.3439579923947652, "sampling/sampling_logp_difference/mean": 0.004039739662160476, "step": 24720, "step_time": 8.047826841437685, "student/entropy": 0.13155985604971648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018888889998197555, "completions/max_length": 486.76666666666665, "completions/max_terminated_length": 451.23333333333335, "completions/mean_length": 191.569455464681, "completions/mean_terminated_length": 185.39664052327473, "completions/min_length": 55.9, "completions/min_terminated_length": 55.9, "entropy": 0.13240692317485808, "epoch": 0.9398853150039874, "eval/gt_acc_batch": 0.7844444632530212, "frac_reward_zero_std": 1.0, "grad_norm": 0.24432946741580963, "kd/policy_loss": 0.008980188850546255, "kd/rkl_advantage_mean": 0.6877846585586667, "kd/rkl_advantage_p95": 4.842800623178482, "kd/rkl_advantage_std": 2.1546165466308596, "kd/ssd_loss": 0.0, "learning_rate": 6.015266016025519e-08, "loss": 0.03592075109481811, "num_tokens": 809728637.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7844444413979849, "rewards/gt_logging_reward/std": 0.40016779899597166, "sampling/importance_sampling_ratio/max": 1.8770415385564168, "sampling/importance_sampling_ratio/mean": 0.9985061764717102, "sampling/importance_sampling_ratio/min": 0.42059174130360283, "sampling/sampling_logp_difference/max": 0.29994409481684364, "sampling/sampling_logp_difference/mean": 0.004023049461344878, "step": 24750, "step_time": 8.199638664635131, "student/entropy": 0.13240692317485808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016944445421298346, "completions/max_length": 479.1, "completions/max_terminated_length": 450.8333333333333, "completions/mean_length": 181.13556569417318, "completions/mean_terminated_length": 175.58120930989583, "completions/min_length": 53.166666666666664, "completions/min_terminated_length": 53.166666666666664, "entropy": 0.1324826224396626, "epoch": 0.9410245699312649, "eval/gt_acc_batch": 0.8036111334959666, "frac_reward_zero_std": 1.0, "grad_norm": 0.24316681921482086, "kd/policy_loss": 0.008656927667713414, "kd/rkl_advantage_mean": 0.7674268210927645, "kd/rkl_advantage_p95": 4.956534141302109, "kd/rkl_advantage_std": 2.216938504576683, "kd/ssd_loss": 0.0, "learning_rate": 5.901340523297763e-08, "loss": 0.034627715746561684, "num_tokens": 810685925.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111096541086, "rewards/gt_logging_reward/std": 0.38776003271341325, "sampling/importance_sampling_ratio/max": 1.8172779560089112, "sampling/importance_sampling_ratio/mean": 0.9998158633708953, "sampling/importance_sampling_ratio/min": 0.5027601917584738, "sampling/sampling_logp_difference/max": 0.3442122578620911, "sampling/sampling_logp_difference/mean": 0.0040316622781877715, "step": 24780, "step_time": 7.979967062174304, "student/entropy": 0.1324826224396626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025555556857337555, "completions/max_length": 489.6333333333333, "completions/max_terminated_length": 448.4, "completions/mean_length": 185.99362080891927, "completions/mean_terminated_length": 177.4325932820638, "completions/min_length": 56.666666666666664, "completions/min_terminated_length": 56.666666666666664, "entropy": 0.1282394488652547, "epoch": 0.9421638248585426, "eval/gt_acc_batch": 0.8033333599567414, "frac_reward_zero_std": 1.0, "grad_norm": 0.3410654664039612, "kd/policy_loss": 0.008083765969301264, "kd/rkl_advantage_mean": 0.8538721030888458, "kd/rkl_advantage_p95": 5.138617493708929, "kd/rkl_advantage_std": 2.259693310658137, "kd/ssd_loss": 0.0, "learning_rate": 5.7874150305700074e-08, "loss": 0.03233507076899211, "num_tokens": 811658334.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8033333361148834, "rewards/gt_logging_reward/std": 0.3898633097608884, "sampling/importance_sampling_ratio/max": 1.8815387845039369, "sampling/importance_sampling_ratio/mean": 0.9928783456484477, "sampling/importance_sampling_ratio/min": 0.42269068559010825, "sampling/sampling_logp_difference/max": 0.3339910884698232, "sampling/sampling_logp_difference/mean": 0.003935288397284845, "step": 24810, "step_time": 8.061000793993783, "student/entropy": 0.1282394488652547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029444446259488662, "completions/max_length": 493.6333333333333, "completions/max_terminated_length": 457.3666666666667, "completions/mean_length": 197.34334360758464, "completions/mean_terminated_length": 187.9322280883789, "completions/min_length": 58.5, "completions/min_terminated_length": 58.5, "entropy": 0.12375564215083917, "epoch": 0.9433030797858201, "eval/gt_acc_batch": 0.7741667052110036, "frac_reward_zero_std": 1.0, "grad_norm": 0.2412082552909851, "kd/policy_loss": 0.008412272333710764, "kd/rkl_advantage_mean": 0.7543401495243113, "kd/rkl_advantage_p95": 4.991121147076289, "kd/rkl_advantage_std": 2.1901863773663837, "kd/ssd_loss": 0.0, "learning_rate": 5.6734895378422505e-08, "loss": 0.03364909092585246, "num_tokens": 812677898.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7741666634877523, "rewards/gt_logging_reward/std": 0.4098821312189102, "sampling/importance_sampling_ratio/max": 1.9714237928390503, "sampling/importance_sampling_ratio/mean": 1.0045884509881338, "sampling/importance_sampling_ratio/min": 0.4727432663242022, "sampling/sampling_logp_difference/max": 0.3472247004508972, "sampling/sampling_logp_difference/mean": 0.003791413743359347, "step": 24840, "step_time": 8.12551270206944, "student/entropy": 0.12375564215083917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.022777779214084148, "completions/max_length": 493.0, "completions/max_terminated_length": 464.2, "completions/mean_length": 186.17389882405598, "completions/mean_terminated_length": 178.67349650065105, "completions/min_length": 51.06666666666667, "completions/min_terminated_length": 51.06666666666667, "entropy": 0.13023078398158153, "epoch": 0.9444423347130977, "eval/gt_acc_batch": 0.7883333504199982, "frac_reward_zero_std": 1.0, "grad_norm": 0.26948243379592896, "kd/policy_loss": 0.008595509852360314, "kd/rkl_advantage_mean": 0.7534908684281011, "kd/rkl_advantage_p95": 4.9672191619873045, "kd/rkl_advantage_std": 2.2364750812451044, "kd/ssd_loss": 0.0, "learning_rate": 5.559564045114495e-08, "loss": 0.03438203732172648, "num_tokens": 813662620.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7883333305517832, "rewards/gt_logging_reward/std": 0.40187323788801826, "sampling/importance_sampling_ratio/max": 1.9222272356351218, "sampling/importance_sampling_ratio/mean": 0.9968758583068847, "sampling/importance_sampling_ratio/min": 0.432826758424441, "sampling/sampling_logp_difference/max": 0.33258210817972816, "sampling/sampling_logp_difference/mean": 0.003977641579695046, "step": 24870, "step_time": 8.219752277301934, "student/entropy": 0.13023078398158153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.030000001781930526, "completions/max_length": 501.7, "completions/max_terminated_length": 462.23333333333335, "completions/mean_length": 193.41695505777994, "completions/mean_terminated_length": 183.54995422363282, "completions/min_length": 49.0, "completions/min_terminated_length": 49.0, "entropy": 0.1269750672702988, "epoch": 0.9455815896403752, "eval/gt_acc_batch": 0.7952778001626333, "frac_reward_zero_std": 1.0, "grad_norm": 0.28251707553863525, "kd/policy_loss": 0.008518616273067891, "kd/rkl_advantage_mean": 0.7720505397766828, "kd/rkl_advantage_p95": 5.044822671016058, "kd/rkl_advantage_std": 2.236423121889432, "kd/ssd_loss": 0.0, "learning_rate": 5.445638552386739e-08, "loss": 0.03407446543375651, "num_tokens": 814676041.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7952777842680613, "rewards/gt_logging_reward/std": 0.3983636399110158, "sampling/importance_sampling_ratio/max": 1.8320323626200359, "sampling/importance_sampling_ratio/mean": 0.9946711719036102, "sampling/importance_sampling_ratio/min": 0.4405620962381363, "sampling/sampling_logp_difference/max": 0.329513156414032, "sampling/sampling_logp_difference/mean": 0.0038699347836275897, "step": 24900, "step_time": 8.296199215034722, "student/entropy": 0.1269750672702988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334866911174, "completions/max_length": 488.6, "completions/max_terminated_length": 446.4, "completions/mean_length": 186.58250986735027, "completions/mean_terminated_length": 178.02739562988282, "completions/min_length": 48.7, "completions/min_terminated_length": 48.7, "entropy": 0.13645621774097283, "epoch": 0.9467208445676527, "eval/gt_acc_batch": 0.7936111291249593, "frac_reward_zero_std": 1.0, "grad_norm": 0.3241773545742035, "kd/policy_loss": 0.008837635333960255, "kd/rkl_advantage_mean": 0.7753033785149455, "kd/rkl_advantage_p95": 4.997669806083043, "kd/rkl_advantage_std": 2.206843830148379, "kd/ssd_loss": 0.0, "learning_rate": 5.331713059658983e-08, "loss": 0.03535054524739583, "num_tokens": 815636402.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7936111112435659, "rewards/gt_logging_reward/std": 0.3980326294898987, "sampling/importance_sampling_ratio/max": 1.8778317093849182, "sampling/importance_sampling_ratio/mean": 0.9986012697219848, "sampling/importance_sampling_ratio/min": 0.44899581720431647, "sampling/sampling_logp_difference/max": 0.34055788318316144, "sampling/sampling_logp_difference/mean": 0.004154442483559251, "step": 24930, "step_time": 7.944392096399679, "student/entropy": 0.13645621774097283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03055555730437239, "completions/max_length": 488.43333333333334, "completions/max_terminated_length": 452.03333333333336, "completions/mean_length": 192.2047322591146, "completions/mean_terminated_length": 182.10979665120442, "completions/min_length": 56.56666666666667, "completions/min_terminated_length": 56.56666666666667, "entropy": 0.12518393745025, "epoch": 0.9478600994949303, "eval/gt_acc_batch": 0.7805555800596873, "frac_reward_zero_std": 1.0, "grad_norm": 0.2518783211708069, "kd/policy_loss": 0.008176267067513739, "kd/rkl_advantage_mean": 0.8226165916770697, "kd/rkl_advantage_p95": 5.113885047038396, "kd/rkl_advantage_std": 2.25910314420859, "kd/ssd_loss": 0.0, "learning_rate": 5.217787566931227e-08, "loss": 0.03270507454872131, "num_tokens": 816632515.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7805555542310079, "rewards/gt_logging_reward/std": 0.40695204635461174, "sampling/importance_sampling_ratio/max": 1.8491510192553202, "sampling/importance_sampling_ratio/mean": 1.0000239650408427, "sampling/importance_sampling_ratio/min": 0.5042829831441243, "sampling/sampling_logp_difference/max": 0.335065354903539, "sampling/sampling_logp_difference/mean": 0.0038080062872419755, "step": 24960, "step_time": 8.069969138635981, "student/entropy": 0.12518393745025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02055555668969949, "completions/max_length": 470.8333333333333, "completions/max_terminated_length": 433.8333333333333, "completions/mean_length": 182.3908426920573, "completions/mean_terminated_length": 175.6163365681966, "completions/min_length": 55.333333333333336, "completions/min_terminated_length": 55.333333333333336, "entropy": 0.13099185538788635, "epoch": 0.9489993544222078, "eval/gt_acc_batch": 0.8030555586020152, "frac_reward_zero_std": 1.0, "grad_norm": 0.23619896173477173, "kd/policy_loss": 0.008292750992889826, "kd/rkl_advantage_mean": 0.7920535263294975, "kd/rkl_advantage_p95": 4.982867064078649, "kd/rkl_advantage_std": 2.2105432470639546, "kd/ssd_loss": 0.0, "learning_rate": 5.1038620742034706e-08, "loss": 0.033171008030573525, "num_tokens": 817585266.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555566151937, "rewards/gt_logging_reward/std": 0.3934993237257004, "sampling/importance_sampling_ratio/max": 1.842803911368052, "sampling/importance_sampling_ratio/mean": 0.9949578245480856, "sampling/importance_sampling_ratio/min": 0.42378868559996286, "sampling/sampling_logp_difference/max": 0.3309706022342046, "sampling/sampling_logp_difference/mean": 0.004035165944757561, "step": 24990, "step_time": 7.845307249266383, "student/entropy": 0.13099185538788635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02777777916441361, "completions/max_length": 482.6, "completions/max_terminated_length": 450.6, "completions/mean_length": 188.91500956217448, "completions/mean_terminated_length": 179.74742584228517, "completions/min_length": 53.93333333333333, "completions/min_terminated_length": 53.93333333333333, "entropy": 0.13674042597413064, "epoch": 0.9501386093494855, "eval/gt_acc_batch": 0.7983333607514699, "frac_reward_zero_std": 1.0, "grad_norm": 0.2626497149467468, "kd/policy_loss": 0.00898749147503016, "kd/rkl_advantage_mean": 0.7908262362703681, "kd/rkl_advantage_p95": 5.016065907478333, "kd/rkl_advantage_std": 2.201545310020447, "kd/ssd_loss": 0.0, "learning_rate": 4.989936581475715e-08, "loss": 0.03594997326533, "num_tokens": 818566872.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7983333309491475, "rewards/gt_logging_reward/std": 0.3888216023643812, "sampling/importance_sampling_ratio/max": 2.020621585845947, "sampling/importance_sampling_ratio/mean": 0.9963590323925018, "sampling/importance_sampling_ratio/min": 0.39448228627443316, "sampling/sampling_logp_difference/max": 0.3104137102762858, "sampling/sampling_logp_difference/mean": 0.004176092993778487, "step": 25020, "step_time": 8.060562322795159, "student/entropy": 0.13674042597413064 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01861111239219705, "completions/max_length": 474.6333333333333, "completions/max_terminated_length": 433.46666666666664, "completions/mean_length": 184.36528727213542, "completions/mean_terminated_length": 178.17527669270834, "completions/min_length": 58.03333333333333, "completions/min_terminated_length": 58.03333333333333, "entropy": 0.12982334128270548, "epoch": 0.951277864276763, "eval/gt_acc_batch": 0.8075000226497651, "frac_reward_zero_std": 1.0, "grad_norm": 0.29888150095939636, "kd/policy_loss": 0.008432998314189415, "kd/rkl_advantage_mean": 0.695499346529444, "kd/rkl_advantage_p95": 4.821553150812785, "kd/rkl_advantage_std": 2.1806140055259067, "kd/ssd_loss": 0.0, "learning_rate": 4.876011088747958e-08, "loss": 0.03373199701309204, "num_tokens": 819525531.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8074999948342642, "rewards/gt_logging_reward/std": 0.3828940689563751, "sampling/importance_sampling_ratio/max": 1.8876761198043823, "sampling/importance_sampling_ratio/mean": 1.0072312851746876, "sampling/importance_sampling_ratio/min": 0.4735039174556732, "sampling/sampling_logp_difference/max": 0.3200324167807897, "sampling/sampling_logp_difference/mean": 0.0040294544693703456, "step": 25050, "step_time": 7.889603491432111, "student/entropy": 0.12982334128270548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023888890352100133, "completions/max_length": 490.2, "completions/max_terminated_length": 450.6333333333333, "completions/mean_length": 182.591677347819, "completions/mean_terminated_length": 174.49136606852213, "completions/min_length": 54.666666666666664, "completions/min_terminated_length": 54.666666666666664, "entropy": 0.1327617097645998, "epoch": 0.9524171192040406, "eval/gt_acc_batch": 0.7944444576899211, "frac_reward_zero_std": 1.0, "grad_norm": 0.31507647037506104, "kd/policy_loss": 0.00863617088762112, "kd/rkl_advantage_mean": 0.8222860315193733, "kd/rkl_advantage_p95": 5.05715768734614, "kd/rkl_advantage_std": 2.2318135609229404, "kd/ssd_loss": 0.0, "learning_rate": 4.7620855960202025e-08, "loss": 0.034544686476389565, "num_tokens": 820485069.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7944444417953491, "rewards/gt_logging_reward/std": 0.39885089844465255, "sampling/importance_sampling_ratio/max": 1.9525042494138083, "sampling/importance_sampling_ratio/mean": 0.9930332481861115, "sampling/importance_sampling_ratio/min": 0.46541177332401273, "sampling/sampling_logp_difference/max": 0.3345559388399124, "sampling/sampling_logp_difference/mean": 0.004076984811884662, "step": 25080, "step_time": 8.028514647929114, "student/entropy": 0.1327617097645998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03583333548158407, "completions/max_length": 485.56666666666666, "completions/max_terminated_length": 454.96666666666664, "completions/mean_length": 189.4269541422526, "completions/mean_terminated_length": 177.6274429321289, "completions/min_length": 53.8, "completions/min_terminated_length": 53.8, "entropy": 0.13320758274445932, "epoch": 0.9535563741313181, "eval/gt_acc_batch": 0.7816666901111603, "frac_reward_zero_std": 1.0, "grad_norm": 0.3119816482067108, "kd/policy_loss": 0.008925532423503076, "kd/rkl_advantage_mean": 0.7891461253787081, "kd/rkl_advantage_p95": 5.03370398680369, "kd/rkl_advantage_std": 2.22294635673364, "kd/ssd_loss": 0.0, "learning_rate": 4.648160103292446e-08, "loss": 0.035702129205067955, "num_tokens": 821467950.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7816666662693024, "rewards/gt_logging_reward/std": 0.3988654578725497, "sampling/importance_sampling_ratio/max": 1.9532179633776348, "sampling/importance_sampling_ratio/mean": 0.9995966076850891, "sampling/importance_sampling_ratio/min": 0.45515121122201285, "sampling/sampling_logp_difference/max": 0.3302091062068939, "sampling/sampling_logp_difference/mean": 0.004063619168785711, "step": 25110, "step_time": 8.038658925229296, "student/entropy": 0.13320758274445932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.012222222797572613, "completions/max_length": 458.3666666666667, "completions/max_terminated_length": 430.3333333333333, "completions/mean_length": 176.2438969930013, "completions/mean_terminated_length": 172.21841430664062, "completions/min_length": 55.4, "completions/min_terminated_length": 55.4, "entropy": 0.13083066393931705, "epoch": 0.9546956290585957, "eval/gt_acc_batch": 0.8113888998826345, "frac_reward_zero_std": 1.0, "grad_norm": 0.2782667279243469, "kd/policy_loss": 0.008640762492238234, "kd/rkl_advantage_mean": 0.7172781199454524, "kd/rkl_advantage_p95": 4.959921995798747, "kd/rkl_advantage_std": 2.221266954143842, "kd/ssd_loss": 0.0, "learning_rate": 4.534234610564691e-08, "loss": 0.034563056627909344, "num_tokens": 822398100.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8113888919353485, "rewards/gt_logging_reward/std": 0.38282817900180816, "sampling/importance_sampling_ratio/max": 1.9554167111714682, "sampling/importance_sampling_ratio/mean": 1.0003062903881073, "sampling/importance_sampling_ratio/min": 0.4829064577817917, "sampling/sampling_logp_difference/max": 0.3442732453346252, "sampling/sampling_logp_difference/mean": 0.004070508464549979, "step": 25140, "step_time": 7.744701725930402, "student/entropy": 0.13083066393931705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444571932157, "completions/max_length": 484.3, "completions/max_terminated_length": 447.46666666666664, "completions/mean_length": 188.47973225911457, "completions/mean_terminated_length": 180.49605611165364, "completions/min_length": 50.46666666666667, "completions/min_terminated_length": 50.46666666666667, "entropy": 0.13268559388816356, "epoch": 0.9558348839858732, "eval/gt_acc_batch": 0.7955555657545725, "frac_reward_zero_std": 1.0, "grad_norm": 0.3911365568637848, "kd/policy_loss": 0.00865779813223829, "kd/rkl_advantage_mean": 0.7199710711836815, "kd/rkl_advantage_p95": 4.960261215766271, "kd/rkl_advantage_std": 2.200184987982114, "kd/ssd_loss": 0.0, "learning_rate": 4.4203091178369344e-08, "loss": 0.0346311887105306, "num_tokens": 823377787.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7955555498600007, "rewards/gt_logging_reward/std": 0.39419225255648294, "sampling/importance_sampling_ratio/max": 1.997536019484202, "sampling/importance_sampling_ratio/mean": 1.0060322244962057, "sampling/importance_sampling_ratio/min": 0.4423445905248324, "sampling/sampling_logp_difference/max": 0.3111525237560272, "sampling/sampling_logp_difference/mean": 0.004049931885674596, "step": 25170, "step_time": 8.153587773335554, "student/entropy": 0.13268559388816356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01972222337499261, "completions/max_length": 475.26666666666665, "completions/max_terminated_length": 437.2, "completions/mean_length": 184.54000854492188, "completions/mean_terminated_length": 177.9655314127604, "completions/min_length": 59.233333333333334, "completions/min_terminated_length": 59.233333333333334, "entropy": 0.12393961225946744, "epoch": 0.9569741389131508, "eval/gt_acc_batch": 0.8052777926127116, "frac_reward_zero_std": 1.0, "grad_norm": 0.25958582758903503, "kd/policy_loss": 0.007926155956617246, "kd/rkl_advantage_mean": 0.5950914551193516, "kd/rkl_advantage_p95": 4.647977377971014, "kd/rkl_advantage_std": 2.1287116328875224, "kd/ssd_loss": 0.0, "learning_rate": 4.306383625109179e-08, "loss": 0.03170462449391683, "num_tokens": 824337491.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8052777787049611, "rewards/gt_logging_reward/std": 0.39072629312674206, "sampling/importance_sampling_ratio/max": 1.8277850111325582, "sampling/importance_sampling_ratio/mean": 1.0026935358842215, "sampling/importance_sampling_ratio/min": 0.47143253187338513, "sampling/sampling_logp_difference/max": 0.3106773793697357, "sampling/sampling_logp_difference/mean": 0.0038217910410215457, "step": 25200, "step_time": 7.909526073031399, "student/entropy": 0.12393961225946744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03333333497866988, "completions/max_length": 492.2, "completions/max_terminated_length": 463.06666666666666, "completions/mean_length": 188.28334248860676, "completions/mean_terminated_length": 177.1316380818685, "completions/min_length": 50.266666666666666, "completions/min_terminated_length": 50.266666666666666, "entropy": 0.12831711278607447, "epoch": 0.9581133938404284, "eval/gt_acc_batch": 0.8036111315091451, "frac_reward_zero_std": 1.0, "grad_norm": 0.27893489599227905, "kd/policy_loss": 0.008534739015158267, "kd/rkl_advantage_mean": 0.7615919126042475, "kd/rkl_advantage_p95": 5.017473479111989, "kd/rkl_advantage_std": 2.225096133351326, "kd/ssd_loss": 0.0, "learning_rate": 4.1924581323814226e-08, "loss": 0.034138961633046465, "num_tokens": 825321175.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8036111076672872, "rewards/gt_logging_reward/std": 0.39063728402058284, "sampling/importance_sampling_ratio/max": 2.056057361761729, "sampling/importance_sampling_ratio/mean": 0.9954114337762197, "sampling/importance_sampling_ratio/min": 0.4280686954657237, "sampling/sampling_logp_difference/max": 0.3311657468477885, "sampling/sampling_logp_difference/mean": 0.0039273877395316955, "step": 25230, "step_time": 8.189894308732862, "student/entropy": 0.12831711278607447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02694444600492716, "completions/max_length": 497.73333333333335, "completions/max_terminated_length": 458.3, "completions/mean_length": 193.13223164876302, "completions/mean_terminated_length": 184.48110911051432, "completions/min_length": 60.03333333333333, "completions/min_terminated_length": 60.03333333333333, "entropy": 0.12899899029483397, "epoch": 0.9592526487677059, "eval/gt_acc_batch": 0.7972222288449605, "frac_reward_zero_std": 1.0, "grad_norm": 0.29795926809310913, "kd/policy_loss": 0.008532306456860777, "kd/rkl_advantage_mean": 0.7856274671852589, "kd/rkl_advantage_p95": 5.033992226918539, "kd/rkl_advantage_std": 2.245484550793966, "kd/ssd_loss": 0.0, "learning_rate": 4.0785326396536663e-08, "loss": 0.034129226207733156, "num_tokens": 826317707.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7972222248713176, "rewards/gt_logging_reward/std": 0.39417805075645446, "sampling/importance_sampling_ratio/max": 1.9255762497584026, "sampling/importance_sampling_ratio/mean": 1.00436971783638, "sampling/importance_sampling_ratio/min": 0.46487816870212556, "sampling/sampling_logp_difference/max": 0.32222875754038494, "sampling/sampling_logp_difference/mean": 0.0039301298403491575, "step": 25260, "step_time": 8.08619044929801, "student/entropy": 0.12899899029483397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024444445905586084, "completions/max_length": 488.2, "completions/max_terminated_length": 441.9, "completions/mean_length": 186.44417724609374, "completions/mean_terminated_length": 178.3998041788737, "completions/min_length": 51.86666666666667, "completions/min_terminated_length": 51.86666666666667, "entropy": 0.13008198930571477, "epoch": 0.9603919036949835, "eval/gt_acc_batch": 0.8005555709203084, "frac_reward_zero_std": 1.0, "grad_norm": 0.30398818850517273, "kd/policy_loss": 0.008552576545237873, "kd/rkl_advantage_mean": 0.7159406668215524, "kd/rkl_advantage_p95": 4.859252285957337, "kd/rkl_advantage_std": 2.1790125489234926, "kd/ssd_loss": 0.0, "learning_rate": 3.96460714692591e-08, "loss": 0.03421030441919962, "num_tokens": 827289914.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8005555629730224, "rewards/gt_logging_reward/std": 0.39188215136528015, "sampling/importance_sampling_ratio/max": 1.9239445328712463, "sampling/importance_sampling_ratio/mean": 0.9947705447673798, "sampling/importance_sampling_ratio/min": 0.45737740596135457, "sampling/sampling_logp_difference/max": 0.3281169652938843, "sampling/sampling_logp_difference/mean": 0.003963267725581924, "step": 25290, "step_time": 8.110874351026723, "student/entropy": 0.13008198930571477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021388890190670886, "completions/max_length": 477.8333333333333, "completions/max_terminated_length": 441.8, "completions/mean_length": 193.0925099690755, "completions/mean_terminated_length": 186.2031723022461, "completions/min_length": 55.56666666666667, "completions/min_terminated_length": 55.56666666666667, "entropy": 0.1319371560588479, "epoch": 0.961531158622261, "eval/gt_acc_batch": 0.7838889122009277, "frac_reward_zero_std": 1.0, "grad_norm": 0.3222583532333374, "kd/policy_loss": 0.008579969646719594, "kd/rkl_advantage_mean": 0.7255859084427356, "kd/rkl_advantage_p95": 4.888201304276785, "kd/rkl_advantage_std": 2.2006726801395415, "kd/ssd_loss": 0.0, "learning_rate": 3.850681654198154e-08, "loss": 0.0343198815981547, "num_tokens": 828281751.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7838888903458913, "rewards/gt_logging_reward/std": 0.4039016664028168, "sampling/importance_sampling_ratio/max": 1.921486492951711, "sampling/importance_sampling_ratio/mean": 1.001198556025823, "sampling/importance_sampling_ratio/min": 0.42929480771223705, "sampling/sampling_logp_difference/max": 0.31789360344409945, "sampling/sampling_logp_difference/mean": 0.004033027302163343, "step": 25320, "step_time": 8.043627736403142, "student/entropy": 0.1319371560588479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014722223145266373, "completions/max_length": 463.73333333333335, "completions/max_terminated_length": 430.1, "completions/mean_length": 181.66695404052734, "completions/mean_terminated_length": 176.85006052652994, "completions/min_length": 53.766666666666666, "completions/min_terminated_length": 53.766666666666666, "entropy": 0.12740204868217309, "epoch": 0.9626704135495386, "eval/gt_acc_batch": 0.8152778009573619, "frac_reward_zero_std": 1.0, "grad_norm": 0.29385823011398315, "kd/policy_loss": 0.008285001137604315, "kd/rkl_advantage_mean": 0.7118590155150741, "kd/rkl_advantage_p95": 4.8768149276574455, "kd/rkl_advantage_std": 2.1778985530138018, "kd/ssd_loss": 0.0, "learning_rate": 3.736756161470398e-08, "loss": 0.033140007654825845, "num_tokens": 829244280.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.815277781089147, "rewards/gt_logging_reward/std": 0.37563092559576033, "sampling/importance_sampling_ratio/max": 1.8216176788012186, "sampling/importance_sampling_ratio/mean": 0.9986369848251343, "sampling/importance_sampling_ratio/min": 0.4820076088110606, "sampling/sampling_logp_difference/max": 0.3144509792327881, "sampling/sampling_logp_difference/mean": 0.0039407208639507495, "step": 25350, "step_time": 7.98829766047808, "student/entropy": 0.12740204868217309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028611112479120494, "completions/max_length": 483.3666666666667, "completions/max_terminated_length": 452.06666666666666, "completions/mean_length": 195.40528869628906, "completions/mean_terminated_length": 186.26628519694012, "completions/min_length": 53.53333333333333, "completions/min_terminated_length": 53.53333333333333, "entropy": 0.130682162878414, "epoch": 0.9638096684768162, "eval/gt_acc_batch": 0.7750000278155009, "frac_reward_zero_std": 1.0, "grad_norm": 0.32080820202827454, "kd/policy_loss": 0.008627666622245064, "kd/rkl_advantage_mean": 0.7892380439986785, "kd/rkl_advantage_p95": 5.063161144653956, "kd/rkl_advantage_std": 2.223318849007289, "kd/ssd_loss": 0.0, "learning_rate": 3.622830668742642e-08, "loss": 0.03451066811879476, "num_tokens": 830254403.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.774999996026357, "rewards/gt_logging_reward/std": 0.40774583717187246, "sampling/importance_sampling_ratio/max": 2.0273783206939697, "sampling/importance_sampling_ratio/mean": 0.9962488373120626, "sampling/importance_sampling_ratio/min": 0.4330617904663086, "sampling/sampling_logp_difference/max": 0.3336852014064789, "sampling/sampling_logp_difference/mean": 0.003982524806633592, "step": 25380, "step_time": 8.16838131583063, "student/entropy": 0.130682162878414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016388889929900567, "completions/max_length": 492.03333333333336, "completions/max_terminated_length": 451.1666666666667, "completions/mean_length": 183.53389943440754, "completions/mean_terminated_length": 178.07000172932942, "completions/min_length": 56.6, "completions/min_terminated_length": 56.6, "entropy": 0.12952319358785946, "epoch": 0.9649489234040938, "eval/gt_acc_batch": 0.7872222363948822, "frac_reward_zero_std": 1.0, "grad_norm": 0.2743333578109741, "kd/policy_loss": 0.008752003345095242, "kd/rkl_advantage_mean": 0.7564835985501607, "kd/rkl_advantage_p95": 5.04572499593099, "kd/rkl_advantage_std": 2.228487279017766, "kd/ssd_loss": 0.0, "learning_rate": 3.5089051760148864e-08, "loss": 0.03500801722208659, "num_tokens": 831211197.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7872222264607748, "rewards/gt_logging_reward/std": 0.40619795421759286, "sampling/importance_sampling_ratio/max": 1.9462404052416484, "sampling/importance_sampling_ratio/mean": 1.0059245884418488, "sampling/importance_sampling_ratio/min": 0.4865587999423345, "sampling/sampling_logp_difference/max": 0.32678637504577634, "sampling/sampling_logp_difference/mean": 0.003986581368371845, "step": 25410, "step_time": 8.071607213292737, "student/entropy": 0.12952319358785946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019722223499168953, "completions/max_length": 479.3, "completions/max_terminated_length": 453.26666666666665, "completions/mean_length": 185.22278747558593, "completions/mean_terminated_length": 178.74498392740887, "completions/min_length": 53.3, "completions/min_terminated_length": 53.3, "entropy": 0.133568988678356, "epoch": 0.9660881783313713, "eval/gt_acc_batch": 0.7963889042536417, "frac_reward_zero_std": 1.0, "grad_norm": 0.36205175518989563, "kd/policy_loss": 0.008795742188037062, "kd/rkl_advantage_mean": 0.7411377023284634, "kd/rkl_advantage_p95": 4.95534702539444, "kd/rkl_advantage_std": 2.20900960067908, "kd/ssd_loss": 0.0, "learning_rate": 3.39497968328713e-08, "loss": 0.03518296877543132, "num_tokens": 832184031.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7963888903458913, "rewards/gt_logging_reward/std": 0.3977812141180038, "sampling/importance_sampling_ratio/max": 1.859949823220571, "sampling/importance_sampling_ratio/mean": 1.0003622849782308, "sampling/importance_sampling_ratio/min": 0.4547459751367569, "sampling/sampling_logp_difference/max": 0.3435240606466929, "sampling/sampling_logp_difference/mean": 0.004102508203747372, "step": 25440, "step_time": 7.96726013943262, "student/entropy": 0.133568988678356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02944444582487146, "completions/max_length": 492.73333333333335, "completions/max_terminated_length": 442.1333333333333, "completions/mean_length": 192.36167704264324, "completions/mean_terminated_length": 182.72003580729168, "completions/min_length": 45.1, "completions/min_terminated_length": 45.1, "entropy": 0.12862639868011078, "epoch": 0.9672274332586488, "eval/gt_acc_batch": 0.7769444723924, "frac_reward_zero_std": 1.0, "grad_norm": 0.19486892223358154, "kd/policy_loss": 0.008372719252171616, "kd/rkl_advantage_mean": 0.7333988219499588, "kd/rkl_advantage_p95": 4.972723340988159, "kd/rkl_advantage_std": 2.1998937080303826, "kd/ssd_loss": 0.0, "learning_rate": 3.2810541905593746e-08, "loss": 0.03349088033040364, "num_tokens": 833191845.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7769444386164347, "rewards/gt_logging_reward/std": 0.41019166161616644, "sampling/importance_sampling_ratio/max": 1.9767670075098673, "sampling/importance_sampling_ratio/mean": 0.9956629653771718, "sampling/importance_sampling_ratio/min": 0.4242442155877749, "sampling/sampling_logp_difference/max": 0.3230234483877818, "sampling/sampling_logp_difference/mean": 0.003906546401170393, "step": 25470, "step_time": 8.332011923235648, "student/entropy": 0.12862639868011078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02500000170742472, "completions/max_length": 486.73333333333335, "completions/max_terminated_length": 456.03333333333336, "completions/mean_length": 193.88306579589843, "completions/mean_terminated_length": 185.86634775797526, "completions/min_length": 48.63333333333333, "completions/min_terminated_length": 48.63333333333333, "entropy": 0.1330043929318587, "epoch": 0.9683666881859264, "eval/gt_acc_batch": 0.7825000087420145, "frac_reward_zero_std": 1.0, "grad_norm": 0.3081846833229065, "kd/policy_loss": 0.008917039235044891, "kd/rkl_advantage_mean": 0.7315323878700535, "kd/rkl_advantage_p95": 4.918702753384908, "kd/rkl_advantage_std": 2.196933067838351, "kd/ssd_loss": 0.0, "learning_rate": 3.1671286978316177e-08, "loss": 0.035668158531188966, "num_tokens": 834209000.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7824999968210856, "rewards/gt_logging_reward/std": 0.40117308497428894, "sampling/importance_sampling_ratio/max": 1.9461123148600261, "sampling/importance_sampling_ratio/mean": 0.9954127887884776, "sampling/importance_sampling_ratio/min": 0.47072348892688753, "sampling/sampling_logp_difference/max": 0.3070799251397451, "sampling/sampling_logp_difference/mean": 0.0040574379575749235, "step": 25500, "step_time": 8.458601801834691, "student/entropy": 0.1330043929318587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013055556360632182, "completions/max_length": 487.6, "completions/max_terminated_length": 455.06666666666666, "completions/mean_length": 183.0130645751953, "completions/mean_terminated_length": 178.71727040608724, "completions/min_length": 49.96666666666667, "completions/min_terminated_length": 49.96666666666667, "entropy": 0.13192408047616483, "epoch": 0.9695059431132039, "eval/gt_acc_batch": 0.7925000170866648, "frac_reward_zero_std": 1.0, "grad_norm": 0.3157636523246765, "kd/policy_loss": 0.008837189377906422, "kd/rkl_advantage_mean": 0.7337217430273691, "kd/rkl_advantage_p95": 4.9609897593657175, "kd/rkl_advantage_std": 2.20098641316096, "kd/ssd_loss": 0.0, "learning_rate": 3.053203205103862e-08, "loss": 0.03534875710805257, "num_tokens": 835169567.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7925000011920929, "rewards/gt_logging_reward/std": 0.3976173902551333, "sampling/importance_sampling_ratio/max": 1.8620247681935629, "sampling/importance_sampling_ratio/mean": 0.999866267045339, "sampling/importance_sampling_ratio/min": 0.4585867941379547, "sampling/sampling_logp_difference/max": 0.31798043648401897, "sampling/sampling_logp_difference/mean": 0.004014422923016052, "step": 25530, "step_time": 8.037151841366237, "student/entropy": 0.13192408047616483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03611111296340823, "completions/max_length": 484.73333333333335, "completions/max_terminated_length": 444.03333333333336, "completions/mean_length": 198.80834197998047, "completions/mean_terminated_length": 187.2748026529948, "completions/min_length": 55.56666666666667, "completions/min_terminated_length": 55.56666666666667, "entropy": 0.1325644062831998, "epoch": 0.9706451980404815, "eval/gt_acc_batch": 0.7744444708029429, "frac_reward_zero_std": 1.0, "grad_norm": 0.22444459795951843, "kd/policy_loss": 0.009424723112412417, "kd/rkl_advantage_mean": 0.7562341613695025, "kd/rkl_advantage_p95": 4.920807061592738, "kd/rkl_advantage_std": 2.183630327383677, "kd/ssd_loss": 0.0, "learning_rate": 2.9392777123761058e-08, "loss": 0.037698888778686525, "num_tokens": 836186213.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.774444446961085, "rewards/gt_logging_reward/std": 0.40797437777121864, "sampling/importance_sampling_ratio/max": 1.9879270156224569, "sampling/importance_sampling_ratio/mean": 1.0029697219530742, "sampling/importance_sampling_ratio/min": 0.4610103170077006, "sampling/sampling_logp_difference/max": 0.30439485708872477, "sampling/sampling_logp_difference/mean": 0.004002917751980324, "step": 25560, "step_time": 8.161123877001227, "student/entropy": 0.1325644062831998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01972222328186035, "completions/max_length": 476.53333333333336, "completions/max_terminated_length": 447.76666666666665, "completions/mean_length": 184.72723236083985, "completions/mean_terminated_length": 178.18252207438152, "completions/min_length": 49.6, "completions/min_terminated_length": 49.6, "entropy": 0.12961186543107034, "epoch": 0.9717844529677591, "eval/gt_acc_batch": 0.7983333508173625, "frac_reward_zero_std": 1.0, "grad_norm": 0.43864524364471436, "kd/policy_loss": 0.00867119050041462, "kd/rkl_advantage_mean": 0.6454908416916927, "kd/rkl_advantage_p95": 4.6812934339046475, "kd/rkl_advantage_std": 2.138492211699486, "kd/ssd_loss": 0.0, "learning_rate": 2.82535221964835e-08, "loss": 0.03468476533889771, "num_tokens": 837159487.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7983333349227906, "rewards/gt_logging_reward/std": 0.39216124365727106, "sampling/importance_sampling_ratio/max": 1.9744460344314576, "sampling/importance_sampling_ratio/mean": 1.001758627096812, "sampling/importance_sampling_ratio/min": 0.4741469740867615, "sampling/sampling_logp_difference/max": 0.3270647287368774, "sampling/sampling_logp_difference/mean": 0.003954213089309632, "step": 25590, "step_time": 8.148778123563776, "student/entropy": 0.12961186543107034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.023611112621923287, "completions/max_length": 490.46666666666664, "completions/max_terminated_length": 445.6, "completions/mean_length": 187.34612325032552, "completions/mean_terminated_length": 179.43053741455077, "completions/min_length": 51.6, "completions/min_terminated_length": 51.6, "entropy": 0.13513201499978703, "epoch": 0.9729237078950367, "eval/gt_acc_batch": 0.7983333468437195, "frac_reward_zero_std": 1.0, "grad_norm": 0.2579764425754547, "kd/policy_loss": 0.008943776964830856, "kd/rkl_advantage_mean": 0.7586506355553866, "kd/rkl_advantage_p95": 5.039712723096212, "kd/rkl_advantage_std": 2.2496157546838123, "kd/ssd_loss": 0.0, "learning_rate": 2.711426726920594e-08, "loss": 0.03577511310577393, "num_tokens": 838140149.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7983333349227906, "rewards/gt_logging_reward/std": 0.3941521962483724, "sampling/importance_sampling_ratio/max": 1.8266638278961183, "sampling/importance_sampling_ratio/mean": 0.9985974629720052, "sampling/importance_sampling_ratio/min": 0.456720499197642, "sampling/sampling_logp_difference/max": 0.3177338182926178, "sampling/sampling_logp_difference/mean": 0.004120384125659863, "step": 25620, "step_time": 8.228294184467329, "student/entropy": 0.13513201499978703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025833334897955258, "completions/max_length": 493.96666666666664, "completions/max_terminated_length": 449.5, "completions/mean_length": 193.77862294514975, "completions/mean_terminated_length": 185.498393758138, "completions/min_length": 58.1, "completions/min_terminated_length": 58.1, "entropy": 0.13006400186568498, "epoch": 0.9740629628223142, "eval/gt_acc_batch": 0.7797222415606181, "frac_reward_zero_std": 1.0, "grad_norm": 0.2566074728965759, "kd/policy_loss": 0.009084195614559577, "kd/rkl_advantage_mean": 0.8185616782555978, "kd/rkl_advantage_p95": 5.181704910596212, "kd/rkl_advantage_std": 2.278538206219673, "kd/ssd_loss": 0.0, "learning_rate": 2.5975012341928377e-08, "loss": 0.03633677959442139, "num_tokens": 839150816.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7797222256660461, "rewards/gt_logging_reward/std": 0.41056076685587567, "sampling/importance_sampling_ratio/max": 1.9577651778856913, "sampling/importance_sampling_ratio/mean": 1.0018564105033874, "sampling/importance_sampling_ratio/min": 0.45205502609411874, "sampling/sampling_logp_difference/max": 0.30457824071248374, "sampling/sampling_logp_difference/mean": 0.003944680692317585, "step": 25650, "step_time": 8.40865607560554, "student/entropy": 0.13006400186568498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016388889929900567, "completions/max_length": 489.5, "completions/max_terminated_length": 462.6, "completions/mean_length": 193.68973134358723, "completions/mean_terminated_length": 188.46128946940104, "completions/min_length": 64.33333333333333, "completions/min_terminated_length": 64.33333333333333, "entropy": 0.12831056204934915, "epoch": 0.9752022177495918, "eval/gt_acc_batch": 0.7897222379843394, "frac_reward_zero_std": 1.0, "grad_norm": 0.34627944231033325, "kd/policy_loss": 0.008487316220998764, "kd/rkl_advantage_mean": 0.6922600186429918, "kd/rkl_advantage_p95": 4.841612490018209, "kd/rkl_advantage_std": 2.179883915185928, "kd/ssd_loss": 0.0, "learning_rate": 2.4835757414650815e-08, "loss": 0.03394926389058431, "num_tokens": 840167579.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.789722228050232, "rewards/gt_logging_reward/std": 0.4006514991323153, "sampling/importance_sampling_ratio/max": 1.9565821528434753, "sampling/importance_sampling_ratio/mean": 1.0069037179152172, "sampling/importance_sampling_ratio/min": 0.4590534955263138, "sampling/sampling_logp_difference/max": 0.33016475041707355, "sampling/sampling_logp_difference/mean": 0.003921854930619399, "step": 25680, "step_time": 8.31306609630119, "student/entropy": 0.12831056204934915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.020000000980993114, "completions/max_length": 486.26666666666665, "completions/max_terminated_length": 452.06666666666666, "completions/mean_length": 186.80973256429036, "completions/mean_terminated_length": 180.45391693115235, "completions/min_length": 50.46666666666667, "completions/min_terminated_length": 50.46666666666667, "entropy": 0.13245037912080684, "epoch": 0.9763414726768693, "eval/gt_acc_batch": 0.7847222367922465, "frac_reward_zero_std": 1.0, "grad_norm": 0.31581375002861023, "kd/policy_loss": 0.009109217122507592, "kd/rkl_advantage_mean": 0.767375441469873, "kd/rkl_advantage_p95": 5.05412505865097, "kd/rkl_advantage_std": 2.2235380431016285, "kd/ssd_loss": 0.0, "learning_rate": 2.3696502487373256e-08, "loss": 0.03643686771392822, "num_tokens": 841159334.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7847222268581391, "rewards/gt_logging_reward/std": 0.3974231307705243, "sampling/importance_sampling_ratio/max": 1.9175263007481893, "sampling/importance_sampling_ratio/mean": 0.9995357354482015, "sampling/importance_sampling_ratio/min": 0.4588465323050817, "sampling/sampling_logp_difference/max": 0.33755595088005064, "sampling/sampling_logp_difference/mean": 0.004001803308104475, "step": 25710, "step_time": 8.36937265713544, "student/entropy": 0.13245037912080684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01666666744276881, "completions/max_length": 473.6666666666667, "completions/max_terminated_length": 447.8333333333333, "completions/mean_length": 185.72973175048827, "completions/mean_terminated_length": 180.29439697265624, "completions/min_length": 49.6, "completions/min_terminated_length": 49.6, "entropy": 0.12360023887207111, "epoch": 0.9774807276041468, "eval/gt_acc_batch": 0.8169444640477498, "frac_reward_zero_std": 1.0, "grad_norm": 0.2651395797729492, "kd/policy_loss": 0.007767474247763554, "kd/rkl_advantage_mean": 0.6748367723698417, "kd/rkl_advantage_p95": 4.840807024637858, "kd/rkl_advantage_std": 2.170597208539645, "kd/ssd_loss": 0.0, "learning_rate": 2.2557247560095696e-08, "loss": 0.031069904565811157, "num_tokens": 842131425.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8169444521268209, "rewards/gt_logging_reward/std": 0.3762322689096133, "sampling/importance_sampling_ratio/max": 1.8097192168235778, "sampling/importance_sampling_ratio/mean": 0.9995728572209676, "sampling/importance_sampling_ratio/min": 0.4578324576218923, "sampling/sampling_logp_difference/max": 0.36437430381774905, "sampling/sampling_logp_difference/mean": 0.0038115835981443525, "step": 25740, "step_time": 8.140447488966553, "student/entropy": 0.12360023887207111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019166667790462572, "completions/max_length": 489.06666666666666, "completions/max_terminated_length": 455.7, "completions/mean_length": 182.92362060546876, "completions/mean_terminated_length": 176.5881825764974, "completions/min_length": 56.266666666666666, "completions/min_terminated_length": 56.266666666666666, "entropy": 0.1334075702354312, "epoch": 0.9786199825314245, "eval/gt_acc_batch": 0.7947222371896108, "frac_reward_zero_std": 1.0, "grad_norm": 0.289486289024353, "kd/policy_loss": 0.008670657714052748, "kd/rkl_advantage_mean": 0.7988404126216968, "kd/rkl_advantage_p95": 5.024180165926615, "kd/rkl_advantage_std": 2.2157385806242624, "kd/ssd_loss": 0.0, "learning_rate": 2.1417992632818137e-08, "loss": 0.034682631492614746, "num_tokens": 843087006.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222173213959, "rewards/gt_logging_reward/std": 0.3954265962044398, "sampling/importance_sampling_ratio/max": 1.8596965670585632, "sampling/importance_sampling_ratio/mean": 1.0011861622333527, "sampling/importance_sampling_ratio/min": 0.4236236706376076, "sampling/sampling_logp_difference/max": 0.3128378947575887, "sampling/sampling_logp_difference/mean": 0.004080408268297712, "step": 25770, "step_time": 8.09093707347056, "student/entropy": 0.1334075702354312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02861111266538501, "completions/max_length": 495.3666666666667, "completions/max_terminated_length": 442.8333333333333, "completions/mean_length": 188.39889831542968, "completions/mean_terminated_length": 178.9198262532552, "completions/min_length": 55.5, "completions/min_terminated_length": 55.5, "entropy": 0.1299333170677225, "epoch": 0.979759237458702, "eval/gt_acc_batch": 0.8044444759686787, "frac_reward_zero_std": 1.0, "grad_norm": 0.31937655806541443, "kd/policy_loss": 0.00869324939752308, "kd/rkl_advantage_mean": 0.7774871803199251, "kd/rkl_advantage_p95": 5.026236832141876, "kd/rkl_advantage_std": 2.2389445920785267, "kd/ssd_loss": 0.0, "learning_rate": 2.0278737705540575e-08, "loss": 0.034773000081380206, "num_tokens": 844065122.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8044444402058919, "rewards/gt_logging_reward/std": 0.38964470227559406, "sampling/importance_sampling_ratio/max": 1.9585163593292236, "sampling/importance_sampling_ratio/mean": 1.0083916326363882, "sampling/importance_sampling_ratio/min": 0.44392294536034266, "sampling/sampling_logp_difference/max": 0.3110794146855672, "sampling/sampling_logp_difference/mean": 0.003963031976794203, "step": 25800, "step_time": 8.15649747736558, "student/entropy": 0.1299333170677225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02416666786496838, "completions/max_length": 477.8666666666667, "completions/max_terminated_length": 452.9, "completions/mean_length": 189.24445444742838, "completions/mean_terminated_length": 181.42300618489583, "completions/min_length": 52.233333333333334, "completions/min_terminated_length": 52.233333333333334, "entropy": 0.12997254468500613, "epoch": 0.9808984923859796, "eval/gt_acc_batch": 0.7947222411632537, "frac_reward_zero_std": 1.0, "grad_norm": 0.24728579819202423, "kd/policy_loss": 0.008445959183154628, "kd/rkl_advantage_mean": 0.7119853398452203, "kd/rkl_advantage_p95": 4.874627381563187, "kd/rkl_advantage_std": 2.174870640039444, "kd/ssd_loss": 0.0, "learning_rate": 1.9139482778263016e-08, "loss": 0.03378383715947469, "num_tokens": 845049298.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7947222252686819, "rewards/gt_logging_reward/std": 0.39616652329762775, "sampling/importance_sampling_ratio/max": 1.912439469496409, "sampling/importance_sampling_ratio/mean": 0.9956080496311188, "sampling/importance_sampling_ratio/min": 0.4217689990997314, "sampling/sampling_logp_difference/max": 0.3144365211327871, "sampling/sampling_logp_difference/mean": 0.003977785104264816, "step": 25830, "step_time": 8.105376634336427, "student/entropy": 0.12997254468500613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.026388890327264866, "completions/max_length": 480.6333333333333, "completions/max_terminated_length": 448.53333333333336, "completions/mean_length": 189.8400095621745, "completions/mean_terminated_length": 181.2978775024414, "completions/min_length": 52.96666666666667, "completions/min_terminated_length": 52.96666666666667, "entropy": 0.13036849020669858, "epoch": 0.9820377473132571, "eval/gt_acc_batch": 0.8008333563804626, "frac_reward_zero_std": 1.0, "grad_norm": 0.30574876070022583, "kd/policy_loss": 0.0084376962215174, "kd/rkl_advantage_mean": 0.6649701997327307, "kd/rkl_advantage_p95": 4.843249678611755, "kd/rkl_advantage_std": 2.1995602627595265, "kd/ssd_loss": 0.0, "learning_rate": 1.8000227850985456e-08, "loss": 0.03375078439712524, "num_tokens": 846051050.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8008333365122478, "rewards/gt_logging_reward/std": 0.3911197200417519, "sampling/importance_sampling_ratio/max": 1.8602129141489665, "sampling/importance_sampling_ratio/mean": 0.9930752774079641, "sampling/importance_sampling_ratio/min": 0.46860237618287404, "sampling/sampling_logp_difference/max": 0.32267380952835084, "sampling/sampling_logp_difference/mean": 0.0040006229964395365, "step": 25860, "step_time": 8.43520971473578, "student/entropy": 0.13036849020669858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02444444578140974, "completions/max_length": 488.9, "completions/max_terminated_length": 459.1333333333333, "completions/mean_length": 191.5130650838216, "completions/mean_terminated_length": 183.56522572835286, "completions/min_length": 56.333333333333336, "completions/min_terminated_length": 56.333333333333336, "entropy": 0.13120677849898735, "epoch": 0.9831770022405347, "eval/gt_acc_batch": 0.7836111346880595, "frac_reward_zero_std": 1.0, "grad_norm": 0.315908282995224, "kd/policy_loss": 0.008970911831905444, "kd/rkl_advantage_mean": 0.7408635417620341, "kd/rkl_advantage_p95": 4.904705486694971, "kd/rkl_advantage_std": 2.1858988871177036, "kd/ssd_loss": 0.0, "learning_rate": 1.6860972923707897e-08, "loss": 0.03588364521662394, "num_tokens": 847032721.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7836111108462016, "rewards/gt_logging_reward/std": 0.40685406227906545, "sampling/importance_sampling_ratio/max": 1.9558974186579385, "sampling/importance_sampling_ratio/mean": 1.0013761301835378, "sampling/importance_sampling_ratio/min": 0.48268304268519086, "sampling/sampling_logp_difference/max": 0.32509916126728056, "sampling/sampling_logp_difference/mean": 0.004024115313465397, "step": 25890, "step_time": 8.13915358936841, "student/entropy": 0.13120677849898735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018611112236976625, "completions/max_length": 473.5, "completions/max_terminated_length": 430.4, "completions/mean_length": 179.9275115966797, "completions/mean_terminated_length": 173.85843251546223, "completions/min_length": 52.266666666666666, "completions/min_terminated_length": 52.266666666666666, "entropy": 0.1267509805659453, "epoch": 0.9843162571678122, "eval/gt_acc_batch": 0.8350000222524007, "frac_reward_zero_std": 1.0, "grad_norm": 0.3241766691207886, "kd/policy_loss": 0.007837413342591996, "kd/rkl_advantage_mean": 0.715189221004645, "kd/rkl_advantage_p95": 4.838110377391179, "kd/rkl_advantage_std": 2.1672684381405514, "kd/ssd_loss": 0.0, "learning_rate": 1.572171799643033e-08, "loss": 0.031349651018778485, "num_tokens": 847983404.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8349999984105428, "rewards/gt_logging_reward/std": 0.35999320149421693, "sampling/importance_sampling_ratio/max": 1.9078854282697042, "sampling/importance_sampling_ratio/mean": 0.9985920468966166, "sampling/importance_sampling_ratio/min": 0.47632539172967275, "sampling/sampling_logp_difference/max": 0.3207974314689636, "sampling/sampling_logp_difference/mean": 0.003906527161598205, "step": 25920, "step_time": 7.95295362896189, "student/entropy": 0.1267509805659453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015555556335796913, "completions/max_length": 481.6333333333333, "completions/max_terminated_length": 448.4, "completions/mean_length": 183.0652877807617, "completions/mean_terminated_length": 177.93433024088543, "completions/min_length": 51.03333333333333, "completions/min_terminated_length": 51.03333333333333, "entropy": 0.13137134729574124, "epoch": 0.9854555120950899, "eval/gt_acc_batch": 0.8027777969837189, "frac_reward_zero_std": 1.0, "grad_norm": 0.30164381861686707, "kd/policy_loss": 0.008437655022135004, "kd/rkl_advantage_mean": 0.7230048411215345, "kd/rkl_advantage_p95": 4.961972419420878, "kd/rkl_advantage_std": 2.2239978432655336, "kd/ssd_loss": 0.0, "learning_rate": 1.4582463069152774e-08, "loss": 0.03375062147776286, "num_tokens": 848958231.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8027777731418609, "rewards/gt_logging_reward/std": 0.3892322599887848, "sampling/importance_sampling_ratio/max": 1.9103217482566834, "sampling/importance_sampling_ratio/mean": 0.9940680921077728, "sampling/importance_sampling_ratio/min": 0.43847170571486155, "sampling/sampling_logp_difference/max": 0.3145290851593018, "sampling/sampling_logp_difference/mean": 0.004032742255367339, "step": 25950, "step_time": 8.174920318300913, "student/entropy": 0.13137134729574124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021111112367361785, "completions/max_length": 491.5, "completions/max_terminated_length": 449.8, "completions/mean_length": 187.0666753133138, "completions/mean_terminated_length": 180.1588348388672, "completions/min_length": 54.43333333333333, "completions/min_terminated_length": 54.43333333333333, "entropy": 0.1301615135744214, "epoch": 0.9865947670223674, "eval/gt_acc_batch": 0.7886111418406169, "frac_reward_zero_std": 1.0, "grad_norm": 0.2689487338066101, "kd/policy_loss": 0.008552788170830657, "kd/rkl_advantage_mean": 0.7168316151636343, "kd/rkl_advantage_p95": 4.809187984466552, "kd/rkl_advantage_std": 2.1598029802242915, "kd/ssd_loss": 0.0, "learning_rate": 1.3443208141875213e-08, "loss": 0.03421115477879842, "num_tokens": 849925999.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7886111120382945, "rewards/gt_logging_reward/std": 0.39979895850022634, "sampling/importance_sampling_ratio/max": 1.8350385069847106, "sampling/importance_sampling_ratio/mean": 0.992460717757543, "sampling/importance_sampling_ratio/min": 0.46694736878077187, "sampling/sampling_logp_difference/max": 0.34262816508611044, "sampling/sampling_logp_difference/mean": 0.004015247283192972, "step": 25980, "step_time": 8.111343131936156, "student/entropy": 0.1301615135744214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019444445768992107, "completions/max_length": 476.26666666666665, "completions/max_terminated_length": 446.43333333333334, "completions/mean_length": 182.17639872233073, "completions/mean_terminated_length": 175.74290110270184, "completions/min_length": 48.36666666666667, "completions/min_terminated_length": 48.36666666666667, "entropy": 0.12609328714509804, "epoch": 0.9877340219496449, "eval/gt_acc_batch": 0.8136111219724019, "frac_reward_zero_std": 1.0, "grad_norm": 0.29485341906547546, "kd/policy_loss": 0.008153392117431697, "kd/rkl_advantage_mean": 0.72918068356812, "kd/rkl_advantage_p95": 4.981149832407634, "kd/rkl_advantage_std": 2.223110869526863, "kd/ssd_loss": 0.0, "learning_rate": 1.2303953214597652e-08, "loss": 0.03261356751124064, "num_tokens": 850885322.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813611114025116, "rewards/gt_logging_reward/std": 0.383774197101593, "sampling/importance_sampling_ratio/max": 1.8540683786074321, "sampling/importance_sampling_ratio/mean": 0.9976613938808441, "sampling/importance_sampling_ratio/min": 0.4809098000327746, "sampling/sampling_logp_difference/max": 0.3186839004357656, "sampling/sampling_logp_difference/mean": 0.003838386681551735, "step": 26010, "step_time": 8.073742181133518, "student/entropy": 0.12609328714509804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0169444455144306, "completions/max_length": 455.1333333333333, "completions/max_terminated_length": 426.1, "completions/mean_length": 183.83584238688152, "completions/mean_terminated_length": 178.28578389485676, "completions/min_length": 58.4, "completions/min_terminated_length": 58.4, "entropy": 0.1264139175415039, "epoch": 0.9888732768769225, "eval/gt_acc_batch": 0.7883333484331767, "frac_reward_zero_std": 1.0, "grad_norm": 0.2913793921470642, "kd/policy_loss": 0.0083192681777291, "kd/rkl_advantage_mean": 0.6752849956443242, "kd/rkl_advantage_p95": 4.787274132172267, "kd/rkl_advantage_std": 2.1445546338955563, "kd/ssd_loss": 0.0, "learning_rate": 1.1164698287320091e-08, "loss": 0.033277078469594316, "num_tokens": 851850435.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7883333285649617, "rewards/gt_logging_reward/std": 0.3997439170877139, "sampling/importance_sampling_ratio/max": 1.859102157751719, "sampling/importance_sampling_ratio/mean": 0.9983133157094319, "sampling/importance_sampling_ratio/min": 0.4347180128097534, "sampling/sampling_logp_difference/max": 0.31340410709381106, "sampling/sampling_logp_difference/mean": 0.003875847921396295, "step": 26040, "step_time": 7.882410497805298, "student/entropy": 0.1264139175415039 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0188888899050653, "completions/max_length": 469.8, "completions/max_terminated_length": 435.6666666666667, "completions/mean_length": 183.3408447265625, "completions/mean_terminated_length": 177.20124359130858, "completions/min_length": 53.3, "completions/min_terminated_length": 53.3, "entropy": 0.1300936117147406, "epoch": 0.9900125318042, "eval/gt_acc_batch": 0.7925000190734863, "frac_reward_zero_std": 1.0, "grad_norm": 0.2765967845916748, "kd/policy_loss": 0.00893237089427809, "kd/rkl_advantage_mean": 0.6858150751019517, "kd/rkl_advantage_p95": 4.789360505342484, "kd/rkl_advantage_std": 2.137502234180768, "kd/ssd_loss": 0.0, "learning_rate": 1.0025443360042532e-08, "loss": 0.035729479789733884, "num_tokens": 852813422.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7924999992052714, "rewards/gt_logging_reward/std": 0.3983038117488225, "sampling/importance_sampling_ratio/max": 1.8832035700480143, "sampling/importance_sampling_ratio/mean": 1.0029024620850882, "sampling/importance_sampling_ratio/min": 0.47241399983565013, "sampling/sampling_logp_difference/max": 0.3255273868640264, "sampling/sampling_logp_difference/mean": 0.003997542647023996, "step": 26070, "step_time": 7.918748533410447, "student/entropy": 0.1300936117147406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02333333473652601, "completions/max_length": 469.73333333333335, "completions/max_terminated_length": 434.2, "completions/mean_length": 186.2983428955078, "completions/mean_terminated_length": 178.56804453531902, "completions/min_length": 57.766666666666666, "completions/min_terminated_length": 57.766666666666666, "entropy": 0.12867186795920132, "epoch": 0.9911517867314776, "eval/gt_acc_batch": 0.8083333472410837, "frac_reward_zero_std": 1.0, "grad_norm": 0.390809029340744, "kd/policy_loss": 0.008630790755463143, "kd/rkl_advantage_mean": 0.7488794637843966, "kd/rkl_advantage_p95": 4.955602953831355, "kd/rkl_advantage_std": 2.202052347858747, "kd/ssd_loss": 0.0, "learning_rate": 8.886188432764971e-09, "loss": 0.034523165225982665, "num_tokens": 853786024.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8083333392937978, "rewards/gt_logging_reward/std": 0.38258339365323385, "sampling/importance_sampling_ratio/max": 1.8806252559026082, "sampling/importance_sampling_ratio/mean": 0.9985775391260783, "sampling/importance_sampling_ratio/min": 0.39658739368120827, "sampling/sampling_logp_difference/max": 0.34525205691655475, "sampling/sampling_logp_difference/mean": 0.003973177308216691, "step": 26100, "step_time": 7.850210814535967, "student/entropy": 0.12867186795920132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0202777790526549, "completions/max_length": 470.56666666666666, "completions/max_terminated_length": 446.53333333333336, "completions/mean_length": 192.93084208170572, "completions/mean_terminated_length": 186.38890991210937, "completions/min_length": 55.46666666666667, "completions/min_terminated_length": 55.46666666666667, "entropy": 0.12089140995716055, "epoch": 0.9922910416587551, "eval/gt_acc_batch": 0.809166685740153, "frac_reward_zero_std": 1.0, "grad_norm": 0.21763113141059875, "kd/policy_loss": 0.00786741889314726, "kd/rkl_advantage_mean": 0.7188729146495462, "kd/rkl_advantage_p95": 4.885275481144587, "kd/rkl_advantage_std": 2.158831504980723, "kd/ssd_loss": 0.0, "learning_rate": 7.74693350548741e-09, "loss": 0.031469676891962686, "num_tokens": 854787423.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8091666718324025, "rewards/gt_logging_reward/std": 0.3849528248111407, "sampling/importance_sampling_ratio/max": 1.8665737191836038, "sampling/importance_sampling_ratio/mean": 1.0004808187484742, "sampling/importance_sampling_ratio/min": 0.45080735931793847, "sampling/sampling_logp_difference/max": 0.35430472493171694, "sampling/sampling_logp_difference/mean": 0.0037062507200365264, "step": 26130, "step_time": 7.993042539756668, "student/entropy": 0.12089140995716055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01750000085060795, "completions/max_length": 460.6666666666667, "completions/max_terminated_length": 431.3333333333333, "completions/mean_length": 180.97000834147136, "completions/mean_terminated_length": 175.1681889851888, "completions/min_length": 55.96666666666667, "completions/min_terminated_length": 55.96666666666667, "entropy": 0.12800014751652877, "epoch": 0.9934302965860328, "eval/gt_acc_batch": 0.813055564959844, "frac_reward_zero_std": 1.0, "grad_norm": 0.27454864978790283, "kd/policy_loss": 0.008255690512790655, "kd/rkl_advantage_mean": 0.7758802431325118, "kd/rkl_advantage_p95": 4.995607282718023, "kd/rkl_advantage_std": 2.202400025725365, "kd/ssd_loss": 0.0, "learning_rate": 6.607678578209851e-09, "loss": 0.03302276134490967, "num_tokens": 855747827.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.813055553038915, "rewards/gt_logging_reward/std": 0.3846996436516444, "sampling/importance_sampling_ratio/max": 1.9079003930091858, "sampling/importance_sampling_ratio/mean": 1.0022391974925995, "sampling/importance_sampling_ratio/min": 0.4842998261253039, "sampling/sampling_logp_difference/max": 0.32255615790685016, "sampling/sampling_logp_difference/mean": 0.003932675944330792, "step": 26160, "step_time": 7.902928676539644, "student/entropy": 0.12800014751652877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02166666785875956, "completions/max_length": 476.76666666666665, "completions/max_terminated_length": 438.23333333333335, "completions/mean_length": 185.26834208170573, "completions/mean_terminated_length": 178.12603454589845, "completions/min_length": 52.6, "completions/min_terminated_length": 52.6, "entropy": 0.1309782262891531, "epoch": 0.9945695515133103, "eval/gt_acc_batch": 0.7983333468437195, "frac_reward_zero_std": 1.0, "grad_norm": 0.24374276399612427, "kd/policy_loss": 0.008675073692575096, "kd/rkl_advantage_mean": 0.8372010676811139, "kd/rkl_advantage_p95": 5.139206876357396, "kd/rkl_advantage_std": 2.2301428854465484, "kd/ssd_loss": 0.0, "learning_rate": 5.4684236509322896e-09, "loss": 0.034700290362040205, "num_tokens": 856714585.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.798333328962326, "rewards/gt_logging_reward/std": 0.3951241299510002, "sampling/importance_sampling_ratio/max": 1.8785622477531434, "sampling/importance_sampling_ratio/mean": 1.0008698006470997, "sampling/importance_sampling_ratio/min": 0.45275761783123014, "sampling/sampling_logp_difference/max": 0.3333398203055064, "sampling/sampling_logp_difference/mean": 0.00402181150081257, "step": 26190, "step_time": 7.946380482030993, "student/entropy": 0.1309782262891531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.028055557142943145, "completions/max_length": 486.4, "completions/max_terminated_length": 452.56666666666666, "completions/mean_length": 193.12167612711588, "completions/mean_terminated_length": 183.9206756591797, "completions/min_length": 54.53333333333333, "completions/min_terminated_length": 54.53333333333333, "entropy": 0.13113699809958537, "epoch": 0.9957088064405879, "eval/gt_acc_batch": 0.7805555780728658, "frac_reward_zero_std": 1.0, "grad_norm": 0.28879261016845703, "kd/policy_loss": 0.008728436174957702, "kd/rkl_advantage_mean": 0.7380654470374187, "kd/rkl_advantage_p95": 4.967583119869232, "kd/rkl_advantage_std": 2.2158943941195806, "kd/ssd_loss": 0.0, "learning_rate": 4.3291687236547296e-09, "loss": 0.03491374254226685, "num_tokens": 857716311.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7805555522441864, "rewards/gt_logging_reward/std": 0.40181585351626076, "sampling/importance_sampling_ratio/max": 1.884137252966563, "sampling/importance_sampling_ratio/mean": 1.000909827152888, "sampling/importance_sampling_ratio/min": 0.43389966239531835, "sampling/sampling_logp_difference/max": 0.3700602054595947, "sampling/sampling_logp_difference/mean": 0.003988236056951185, "step": 26220, "step_time": 8.093219568100176, "student/entropy": 0.13113699809958537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013333334090809027, "completions/max_length": 471.96666666666664, "completions/max_terminated_length": 433.6333333333333, "completions/mean_length": 178.25500946044923, "completions/mean_terminated_length": 173.84633229573566, "completions/min_length": 56.2, "completions/min_terminated_length": 56.2, "entropy": 0.13539091056833666, "epoch": 0.9968480613678654, "eval/gt_acc_batch": 0.8030555764834086, "frac_reward_zero_std": 1.0, "grad_norm": 0.35537031292915344, "kd/policy_loss": 0.008484880897837381, "kd/rkl_advantage_mean": 0.8018611467909068, "kd/rkl_advantage_p95": 5.036621918280919, "kd/rkl_advantage_std": 2.2405777255694073, "kd/ssd_loss": 0.0, "learning_rate": 3.189913796377169e-09, "loss": 0.033939528465271, "num_tokens": 858665373.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.8030555526415507, "rewards/gt_logging_reward/std": 0.39192101260026296, "sampling/importance_sampling_ratio/max": 1.876450788974762, "sampling/importance_sampling_ratio/mean": 0.998519241809845, "sampling/importance_sampling_ratio/min": 0.45547755261262257, "sampling/sampling_logp_difference/max": 0.30508578817049664, "sampling/sampling_logp_difference/mean": 0.0040684374592577415, "step": 26250, "step_time": 8.077394647765322, "student/entropy": 0.13539091056833666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.018333334165314834, "completions/max_length": 485.0, "completions/max_terminated_length": 460.3666666666667, "completions/mean_length": 184.33417612711588, "completions/mean_terminated_length": 178.0758814493815, "completions/min_length": 51.13333333333333, "completions/min_terminated_length": 51.13333333333333, "entropy": 0.13116911152998606, "epoch": 0.9979873162951429, "eval/gt_acc_batch": 0.7977778057257334, "frac_reward_zero_std": 1.0, "grad_norm": 0.3382938802242279, "kd/policy_loss": 0.008836072588261837, "kd/rkl_advantage_mean": 0.786857272281001, "kd/rkl_advantage_p95": 4.997472576300303, "kd/rkl_advantage_std": 2.2221710364023846, "kd/ssd_loss": 0.0, "learning_rate": 2.050658869099609e-09, "loss": 0.03534429868062337, "num_tokens": 859624080.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.797777779897054, "rewards/gt_logging_reward/std": 0.3975469579299291, "sampling/importance_sampling_ratio/max": 1.8823233803113302, "sampling/importance_sampling_ratio/mean": 1.0015786170959473, "sampling/importance_sampling_ratio/min": 0.49061890244483947, "sampling/sampling_logp_difference/max": 0.3357905725638072, "sampling/sampling_logp_difference/mean": 0.004006391111761332, "step": 26280, "step_time": 8.099503362265144, "student/entropy": 0.13116911152998606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02083333448196451, "completions/max_length": 484.03333333333336, "completions/max_terminated_length": 442.8333333333333, "completions/mean_length": 189.11084391276043, "completions/mean_terminated_length": 182.35631612141927, "completions/min_length": 52.733333333333334, "completions/min_terminated_length": 52.733333333333334, "entropy": 0.13059442571053903, "epoch": 0.9991265712224205, "eval/gt_acc_batch": 0.7827777981758117, "frac_reward_zero_std": 1.0, "grad_norm": 0.30764850974082947, "kd/policy_loss": 0.008549473979898418, "kd/rkl_advantage_mean": 0.7253896731262406, "kd/rkl_advantage_p95": 4.902711906035742, "kd/rkl_advantage_std": 2.189149084687233, "kd/ssd_loss": 0.0, "learning_rate": 9.114039418220484e-10, "loss": 0.034197898705800374, "num_tokens": 860620687.0, "reward": 0.0, "reward_std": 0.0, "rewards/gt_logging_reward/mean": 0.7827777802944184, "rewards/gt_logging_reward/std": 0.40299503107865653, "sampling/importance_sampling_ratio/max": 1.898701306184133, "sampling/importance_sampling_ratio/mean": 0.9932234068711598, "sampling/importance_sampling_ratio/min": 0.41570016046365105, "sampling/sampling_logp_difference/max": 0.3144602735837301, "sampling/sampling_logp_difference/mean": 0.003965286064582566, "step": 26310, "step_time": 8.335269565691124, "student/entropy": 0.13059442571053903 } ], "logging_steps": 30, "max_steps": 26333, "num_input_tokens_seen": 861353169, "num_train_epochs": 1, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }