{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8191126279863481,
  "eval_steps": 60,
  "global_step": 120,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4126.0,
      "completions/max_terminated_length": 3978.0,
      "completions/mean_length": 2078.7084350585938,
      "completions/mean_terminated_length": 1958.6099243164062,
      "completions/min_length": 962.5,
      "completions/min_terminated_length": 962.5,
      "entropy": 0.19557882845401764,
      "epoch": 0.006825938566552901,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.03708447515964508,
      "learning_rate": 0.0,
      "loss": 0.07332071661949158,
      "num_tokens": 86135.0,
      "reward": -0.4166666865348816,
      "reward_std": 0.8109081089496613,
      "rewards/correctness_reward/mean": -0.4166666567325592,
      "rewards/correctness_reward/std": 0.8109080791473389,
      "sampling/importance_sampling_ratio/max": 1.7469300627708435,
      "sampling/importance_sampling_ratio/mean": 0.6238087564706802,
      "sampling/importance_sampling_ratio/min": 0.02465523686259985,
      "sampling/sampling_logp_difference/max": 0.5807536840438843,
      "sampling/sampling_logp_difference/mean": 0.009548636619001627,
      "step": 1,
      "step_time": 143.7903330239933
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3109.5,
      "completions/max_terminated_length": 3109.5,
      "completions/mean_length": 1972.4584350585938,
      "completions/mean_terminated_length": 1972.4584350585938,
      "completions/min_length": 942.0,
      "completions/min_terminated_length": 942.0,
      "entropy": 0.1991811003535986,
      "epoch": 0.013651877133105802,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2e-08,
      "loss": 0.0,
      "num_tokens": 164872.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.464595675468445,
      "sampling/importance_sampling_ratio/mean": 0.7223336100578308,
      "sampling/importance_sampling_ratio/min": 0.04568341374397278,
      "sampling/sampling_logp_difference/max": 1.6412757635116577,
      "sampling/sampling_logp_difference/mean": 0.009283150546252728,
      "step": 2,
      "step_time": 103.18303876400023
    },
    {
      "clip_ratio/high_max": 0.00012525788770290092,
      "clip_ratio/high_mean": 0.00012525788770290092,
      "clip_ratio/low_mean": 0.000619696089415811,
      "clip_ratio/low_min": 0.000619696089415811,
      "clip_ratio/region_mean": 0.0007449539843946695,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 4050.5,
      "completions/max_terminated_length": 3202.0,
      "completions/mean_length": 2460.9583740234375,
      "completions/mean_terminated_length": 2179.125,
      "completions/min_length": 1478.5,
      "completions/min_terminated_length": 1478.5,
      "entropy": 0.2095671109855175,
      "epoch": 0.020477815699658702,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.12216407060623169,
      "learning_rate": 4e-08,
      "loss": -0.18665893375873566,
      "num_tokens": 263763.0,
      "reward": -0.6666666865348816,
      "reward_std": 0.7409421801567078,
      "rewards/correctness_reward/mean": -0.6666666567325592,
      "rewards/correctness_reward/std": 0.7409421503543854,
      "sampling/importance_sampling_ratio/max": 2.2802764177322388,
      "sampling/importance_sampling_ratio/mean": 0.6775670647621155,
      "sampling/importance_sampling_ratio/min": 0.009539297316223383,
      "sampling/sampling_logp_difference/max": 0.8306642770767212,
      "sampling/sampling_logp_difference/mean": 0.00935443863272667,
      "step": 3,
      "step_time": 135.05255382400355
    },
    {
      "clip_ratio/high_max": 0.00012446496657503303,
      "clip_ratio/high_mean": 0.00012446496657503303,
      "clip_ratio/low_mean": 0.00017372676666127518,
      "clip_ratio/low_min": 0.00017372676666127518,
      "clip_ratio/region_mean": 0.0002981917332363082,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3878.0,
      "completions/max_terminated_length": 3628.0,
      "completions/mean_length": 2135.791748046875,
      "completions/mean_terminated_length": 2011.3409423828125,
      "completions/min_length": 860.5,
      "completions/min_terminated_length": 860.5,
      "entropy": 0.1768759936094284,
      "epoch": 0.027303754266211604,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.04616802930831909,
      "learning_rate": 6e-08,
      "loss": 0.047608934342861176,
      "num_tokens": 354190.0,
      "reward": -0.3333333358168602,
      "reward_std": 0.9671956598758698,
      "rewards/correctness_reward/mean": -0.3333333358168602,
      "rewards/correctness_reward/std": 0.9671956598758698,
      "sampling/importance_sampling_ratio/max": 1.2191235721111298,
      "sampling/importance_sampling_ratio/mean": 0.41029319167137146,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1395794749259949,
      "sampling/sampling_logp_difference/mean": 0.008533329702913761,
      "step": 4,
      "step_time": 129.13586523798585
    },
    {
      "clip_ratio/high_max": 0.00011122030264232308,
      "clip_ratio/high_mean": 0.00011122030264232308,
      "clip_ratio/low_mean": 0.0007960342863952974,
      "clip_ratio/low_min": 0.0007960342863952974,
      "clip_ratio/region_mean": 0.0009072545872186311,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4103.0,
      "completions/max_terminated_length": 4103.0,
      "completions/mean_length": 2328.8333740234375,
      "completions/mean_terminated_length": 2328.8333740234375,
      "completions/min_length": 1169.5,
      "completions/min_terminated_length": 1169.5,
      "entropy": 0.18773097731173038,
      "epoch": 0.034129692832764506,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.05336736515164375,
      "learning_rate": 8e-08,
      "loss": -0.271327942609787,
      "num_tokens": 444492.0,
      "reward": -0.5,
      "reward_std": 0.9045340418815613,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.9045340418815613,
      "sampling/importance_sampling_ratio/max": 2.003176987171173,
      "sampling/importance_sampling_ratio/mean": 0.4816400110721588,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.6100131273269653,
      "sampling/sampling_logp_difference/mean": 0.008833470288664103,
      "step": 5,
      "step_time": 133.57865831398522
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1452.0,
      "completions/max_terminated_length": 1452.0,
      "completions/mean_length": 986.5833740234375,
      "completions/mean_terminated_length": 986.5833740234375,
      "completions/min_length": 662.5,
      "completions/min_terminated_length": 662.5,
      "entropy": 0.1458668615669012,
      "epoch": 0.040955631399317405,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 1e-07,
      "loss": 0.0,
      "num_tokens": 503954.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5658745169639587,
      "sampling/importance_sampling_ratio/mean": 0.6347561478614807,
      "sampling/importance_sampling_ratio/min": 0.18051937222480774,
      "sampling/sampling_logp_difference/max": 0.647778332233429,
      "sampling/sampling_logp_difference/mean": 0.008760722819715738,
      "step": 6,
      "step_time": 57.31431425399933
    },
    {
      "clip_ratio/high_max": 4.898119004792534e-05,
      "clip_ratio/high_mean": 4.898119004792534e-05,
      "clip_ratio/low_mean": 0.0005076292945886962,
      "clip_ratio/low_min": 0.0005076292945886962,
      "clip_ratio/region_mean": 0.0005566104846366215,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3297.5,
      "completions/max_terminated_length": 3297.5,
      "completions/mean_length": 1440.4166870117188,
      "completions/mean_terminated_length": 1440.4166870117188,
      "completions/min_length": 855.0,
      "completions/min_terminated_length": 855.0,
      "entropy": 0.1631603855639696,
      "epoch": 0.04778156996587031,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.10335278511047363,
      "learning_rate": 1.2e-07,
      "loss": -0.6942333579063416,
      "num_tokens": 574770.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.28867512941360474,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.8089405596256256,
      "sampling/importance_sampling_ratio/mean": 0.5232424139976501,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.5933345556259155,
      "sampling/sampling_logp_difference/mean": 0.009010098408907652,
      "step": 7,
      "step_time": 115.51082664800197
    },
    {
      "clip_ratio/high_max": 2.7079722713097e-05,
      "clip_ratio/high_mean": 2.7079722713097e-05,
      "clip_ratio/low_mean": 0.00011799410276580602,
      "clip_ratio/low_min": 0.00011799410276580602,
      "clip_ratio/region_mean": 0.00014507382547890302,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3793.5,
      "completions/max_terminated_length": 3278.5,
      "completions/mean_length": 1910.75,
      "completions/mean_terminated_length": 1647.050048828125,
      "completions/min_length": 912.5,
      "completions/min_terminated_length": 912.5,
      "entropy": 0.18195712938904762,
      "epoch": 0.05460750853242321,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.003391283331438899,
      "learning_rate": 1.3999999999999998e-07,
      "loss": 0.019645797088742256,
      "num_tokens": 657492.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.3892494738101959,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.332252025604248,
      "sampling/importance_sampling_ratio/mean": 0.46315374970436096,
      "sampling/importance_sampling_ratio/min": 0.03832871373742819,
      "sampling/sampling_logp_difference/max": 0.6777635812759399,
      "sampling/sampling_logp_difference/mean": 0.00894957734271884,
      "step": 8,
      "step_time": 124.02444752900192
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.00037109364347998053,
      "clip_ratio/low_min": 0.00037109364347998053,
      "clip_ratio/region_mean": 0.00037109364347998053,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3602.0,
      "completions/max_terminated_length": 2851.5,
      "completions/mean_length": 1949.0,
      "completions/mean_terminated_length": 1830.397705078125,
      "completions/min_length": 1087.0,
      "completions/min_terminated_length": 1087.0,
      "entropy": 0.2037021517753601,
      "epoch": 0.06143344709897611,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.029773741960525513,
      "learning_rate": 1.6e-07,
      "loss": -0.05949871987104416,
      "num_tokens": 749352.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 2.4376940727233887,
      "sampling/importance_sampling_ratio/mean": 0.5821068584918976,
      "sampling/importance_sampling_ratio/min": 0.005494819954037666,
      "sampling/sampling_logp_difference/max": 0.8441054224967957,
      "sampling/sampling_logp_difference/mean": 0.00960251223295927,
      "step": 9,
      "step_time": 122.19422086099075
    },
    {
      "clip_ratio/high_max": 0.0002809852703649085,
      "clip_ratio/high_mean": 0.0002809852703649085,
      "clip_ratio/low_mean": 0.0008867749929777347,
      "clip_ratio/low_min": 0.0008867749929777347,
      "clip_ratio/region_mean": 0.0011677602742565796,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3277.0,
      "completions/max_terminated_length": 3277.0,
      "completions/mean_length": 1764.25,
      "completions/mean_terminated_length": 1764.25,
      "completions/min_length": 786.5,
      "completions/min_terminated_length": 786.5,
      "entropy": 0.18435939773917198,
      "epoch": 0.06825938566552901,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.032720599323511124,
      "learning_rate": 1.8e-07,
      "loss": 0.12537115812301636,
      "num_tokens": 820326.0,
      "reward": -0.583333358168602,
      "reward_std": 0.7810410857200623,
      "rewards/correctness_reward/mean": -0.5833333283662796,
      "rewards/correctness_reward/std": 0.7810411155223846,
      "sampling/importance_sampling_ratio/max": 1.6888806819915771,
      "sampling/importance_sampling_ratio/mean": 0.5177919268608093,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.6338849067687988,
      "sampling/sampling_logp_difference/mean": 0.00877432944253087,
      "step": 10,
      "step_time": 105.65176495799096
    },
    {
      "clip_ratio/high_max": 0.00015119444287847728,
      "clip_ratio/high_mean": 0.00015119444287847728,
      "clip_ratio/low_mean": 0.0004296297502150992,
      "clip_ratio/low_min": 0.0004296297502150992,
      "clip_ratio/region_mean": 0.00058082418217964,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3852.5,
      "completions/max_terminated_length": 3593.0,
      "completions/mean_length": 2337.9166870117188,
      "completions/mean_terminated_length": 2242.7689819335938,
      "completions/min_length": 1315.5,
      "completions/min_terminated_length": 1315.5,
      "entropy": 0.18981880322098732,
      "epoch": 0.07508532423208192,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.016524910926818848,
      "learning_rate": 2e-07,
      "loss": 0.023730065673589706,
      "num_tokens": 916900.0,
      "reward": -0.1666666641831398,
      "reward_std": 0.9671956598758698,
      "rewards/correctness_reward/mean": -0.1666666641831398,
      "rewards/correctness_reward/std": 0.9671956598758698,
      "sampling/importance_sampling_ratio/max": 2.6923059225082397,
      "sampling/importance_sampling_ratio/mean": 0.6565994620323181,
      "sampling/importance_sampling_ratio/min": 0.0022264549043029547,
      "sampling/sampling_logp_difference/max": 0.5726124048233032,
      "sampling/sampling_logp_difference/mean": 0.009137257002294064,
      "step": 11,
      "step_time": 129.848471055986
    },
    {
      "clip_ratio/high_max": 9.70824057731079e-05,
      "clip_ratio/high_mean": 9.70824057731079e-05,
      "clip_ratio/low_mean": 0.0009453010425204411,
      "clip_ratio/low_min": 0.0009453010425204411,
      "clip_ratio/region_mean": 0.0010423834464745596,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4029.5,
      "completions/max_terminated_length": 4029.5,
      "completions/mean_length": 2146.25,
      "completions/mean_terminated_length": 2146.25,
      "completions/min_length": 590.5,
      "completions/min_terminated_length": 590.5,
      "entropy": 0.20860109478235245,
      "epoch": 0.08191126279863481,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.06532567739486694,
      "learning_rate": 2.1999999999999998e-07,
      "loss": -0.1271137297153473,
      "num_tokens": 1000354.0,
      "reward": -0.5833333432674408,
      "reward_std": 0.841516524553299,
      "rewards/correctness_reward/mean": -0.5833333432674408,
      "rewards/correctness_reward/std": 0.841516524553299,
      "sampling/importance_sampling_ratio/max": 1.4567700028419495,
      "sampling/importance_sampling_ratio/mean": 0.5064111053943634,
      "sampling/importance_sampling_ratio/min": 0.006421342492103577,
      "sampling/sampling_logp_difference/max": 0.7267114520072937,
      "sampling/sampling_logp_difference/mean": 0.009367370512336493,
      "step": 12,
      "step_time": 130.4215078309935
    },
    {
      "clip_ratio/high_max": 3.9846989238867536e-05,
      "clip_ratio/high_mean": 3.9846989238867536e-05,
      "clip_ratio/low_mean": 0.0003410607132536825,
      "clip_ratio/low_min": 0.0003410607132536825,
      "clip_ratio/region_mean": 0.00038090770249255,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2148.0,
      "completions/max_terminated_length": 2148.0,
      "completions/mean_length": 1350.7916870117188,
      "completions/mean_terminated_length": 1350.7916870117188,
      "completions/min_length": 779.5,
      "completions/min_terminated_length": 779.5,
      "entropy": 0.14979218784719706,
      "epoch": 0.08873720136518772,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.03845459595322609,
      "learning_rate": 2.4e-07,
      "loss": -0.04120948165655136,
      "num_tokens": 1064921.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 2.167421817779541,
      "sampling/importance_sampling_ratio/mean": 0.7132795751094818,
      "sampling/importance_sampling_ratio/min": 0.04248122125864029,
      "sampling/sampling_logp_difference/max": 0.6888407468795776,
      "sampling/sampling_logp_difference/mean": 0.008195365080609918,
      "step": 13,
      "step_time": 75.68014991000382
    },
    {
      "clip_ratio/high_max": 0.0002453310153214261,
      "clip_ratio/high_mean": 0.0002453310153214261,
      "clip_ratio/low_mean": 0.00010895619925577193,
      "clip_ratio/low_min": 0.00010895619925577193,
      "clip_ratio/region_mean": 0.00035428721457719803,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3994.5,
      "completions/max_terminated_length": 3507.5,
      "completions/mean_length": 1930.166748046875,
      "completions/mean_terminated_length": 1651.4500122070312,
      "completions/min_length": 822.5,
      "completions/min_terminated_length": 822.5,
      "entropy": 0.19265830144286156,
      "epoch": 0.09556313993174062,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.03448331356048584,
      "learning_rate": 2.6e-07,
      "loss": -0.12286396324634552,
      "num_tokens": 1147695.0,
      "reward": -0.5833333358168602,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.5833333358168602,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 2.6719439029693604,
      "sampling/importance_sampling_ratio/mean": 0.6675399243831635,
      "sampling/importance_sampling_ratio/min": 0.0005254613934084773,
      "sampling/sampling_logp_difference/max": 0.6318219900131226,
      "sampling/sampling_logp_difference/mean": 0.009593375492841005,
      "step": 14,
      "step_time": 127.86682878998545
    },
    {
      "clip_ratio/high_max": 0.00015420510499097873,
      "clip_ratio/high_mean": 0.00015420510499097873,
      "clip_ratio/low_mean": 0.000891726536792703,
      "clip_ratio/low_min": 0.000891726536792703,
      "clip_ratio/region_mean": 0.0010459316545166075,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3989.0,
      "completions/max_terminated_length": 3116.0,
      "completions/mean_length": 2372.041748046875,
      "completions/mean_terminated_length": 2264.75,
      "completions/min_length": 1013.0,
      "completions/min_terminated_length": 1013.0,
      "entropy": 0.2017787005752325,
      "epoch": 0.10238907849829351,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07834500819444656,
      "learning_rate": 2.7999999999999997e-07,
      "loss": -0.19363535940647125,
      "num_tokens": 1237786.0,
      "reward": -0.5000000149011612,
      "reward_std": 0.8816154301166534,
      "rewards/correctness_reward/mean": -0.5000000149011612,
      "rewards/correctness_reward/std": 0.8816154599189758,
      "sampling/importance_sampling_ratio/max": 1.4357083439826965,
      "sampling/importance_sampling_ratio/mean": 0.4262894541025162,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.798983097076416,
      "sampling/sampling_logp_difference/mean": 0.009441147092729807,
      "step": 15,
      "step_time": 131.49785820600664
    },
    {
      "clip_ratio/high_max": 0.00020652647253882606,
      "clip_ratio/high_mean": 0.00020652647253882606,
      "clip_ratio/low_mean": 0.00024393484818574507,
      "clip_ratio/low_min": 0.00024393484818574507,
      "clip_ratio/region_mean": 0.0004504613189055817,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3986.0,
      "completions/max_terminated_length": 3296.0,
      "completions/mean_length": 1959.8333740234375,
      "completions/mean_terminated_length": 1826.901611328125,
      "completions/min_length": 1002.0,
      "completions/min_terminated_length": 1002.0,
      "entropy": 0.17979588359594345,
      "epoch": 0.10921501706484642,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.06618808954954147,
      "learning_rate": 3e-07,
      "loss": -0.1892964094877243,
      "num_tokens": 1315212.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.3892494738101959,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.7599316835403442,
      "sampling/importance_sampling_ratio/mean": 0.5774809271097183,
      "sampling/importance_sampling_ratio/min": 0.07288849353790283,
      "sampling/sampling_logp_difference/max": 0.7448713779449463,
      "sampling/sampling_logp_difference/mean": 0.008799537550657988,
      "step": 16,
      "step_time": 125.46948220199556
    },
    {
      "clip_ratio/high_max": 9.079917072085664e-05,
      "clip_ratio/high_mean": 9.079917072085664e-05,
      "clip_ratio/low_mean": 7.84781514084898e-05,
      "clip_ratio/low_min": 7.84781514084898e-05,
      "clip_ratio/region_mean": 0.00016927732212934643,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4361.0,
      "completions/max_terminated_length": 4361.0,
      "completions/mean_length": 2464.291748046875,
      "completions/mean_terminated_length": 2464.291748046875,
      "completions/min_length": 1042.5,
      "completions/min_terminated_length": 1042.5,
      "entropy": 0.23505781777203083,
      "epoch": 0.11604095563139932,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.052851345390081406,
      "learning_rate": 2.999904907117206e-07,
      "loss": 0.24631723761558533,
      "num_tokens": 1409605.0,
      "reward": -0.5833333358168602,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.5833333358168602,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 1.705892026424408,
      "sampling/importance_sampling_ratio/mean": 0.5010480284690857,
      "sampling/importance_sampling_ratio/min": 0.06038624048233032,
      "sampling/sampling_logp_difference/max": 0.6204233765602112,
      "sampling/sampling_logp_difference/mean": 0.01033627800643444,
      "step": 17,
      "step_time": 141.54468511500454
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4423.5,
      "completions/max_terminated_length": 4423.5,
      "completions/mean_length": 2031.4166870117188,
      "completions/mean_terminated_length": 2031.4166870117188,
      "completions/min_length": 844.5,
      "completions/min_terminated_length": 844.5,
      "entropy": 0.20310819149017334,
      "epoch": 0.12286689419795221,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.9996196405256996e-07,
      "loss": 0.0,
      "num_tokens": 1491989.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 1.893693208694458,
      "sampling/importance_sampling_ratio/mean": 0.5588868856430054,
      "sampling/importance_sampling_ratio/min": 0.013609365560114384,
      "sampling/sampling_logp_difference/max": 0.5993608236312866,
      "sampling/sampling_logp_difference/mean": 0.009505682159215212,
      "step": 18,
      "step_time": 138.96269857199513
    },
    {
      "clip_ratio/high_max": 3.729117088369094e-05,
      "clip_ratio/high_mean": 3.729117088369094e-05,
      "clip_ratio/low_mean": 0.0003078869340242818,
      "clip_ratio/low_min": 0.0003078869340242818,
      "clip_ratio/region_mean": 0.00034517810490797274,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3841.5,
      "completions/max_terminated_length": 3414.5,
      "completions/mean_length": 2340.541748046875,
      "completions/mean_terminated_length": 2127.9500732421875,
      "completions/min_length": 1127.0,
      "completions/min_terminated_length": 1127.0,
      "entropy": 0.20708073489367962,
      "epoch": 0.1296928327645051,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.006704013328999281,
      "learning_rate": 2.9991442363945767e-07,
      "loss": 0.05776930972933769,
      "num_tokens": 1581882.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.3786442875862122,
      "sampling/importance_sampling_ratio/mean": 0.4760366827249527,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.523478627204895,
      "sampling/sampling_logp_difference/mean": 0.009245706256479025,
      "step": 19,
      "step_time": 127.39751178900042
    },
    {
      "clip_ratio/high_max": 8.210180385503918e-05,
      "clip_ratio/high_mean": 8.210180385503918e-05,
      "clip_ratio/low_mean": 0.0006445428680308396,
      "clip_ratio/low_min": 0.0006445428680308396,
      "clip_ratio/region_mean": 0.0007266446718858788,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4055.0,
      "completions/max_terminated_length": 4055.0,
      "completions/mean_length": 1693.2083740234375,
      "completions/mean_terminated_length": 1693.2083740234375,
      "completions/min_length": 865.5,
      "completions/min_terminated_length": 865.5,
      "entropy": 0.16844954527914524,
      "epoch": 0.13651877133105803,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.07481791824102402,
      "learning_rate": 2.998478755000571e-07,
      "loss": -0.2300712913274765,
      "num_tokens": 1653749.0,
      "reward": -0.7500000298023224,
      "reward_std": 0.6779246032238007,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.6779246032238007,
      "sampling/importance_sampling_ratio/max": 1.5487863421440125,
      "sampling/importance_sampling_ratio/mean": 0.6358051598072052,
      "sampling/importance_sampling_ratio/min": 0.04564919415861368,
      "sampling/sampling_logp_difference/max": 0.6578903198242188,
      "sampling/sampling_logp_difference/mean": 0.008329455275088549,
      "step": 20,
      "step_time": 125.59296355299011
    },
    {
      "clip_ratio/high_max": 0.0003053993041248759,
      "clip_ratio/high_mean": 0.0003053993041248759,
      "clip_ratio/low_mean": 0.0005765805062765139,
      "clip_ratio/low_min": 0.0005765805062765139,
      "clip_ratio/region_mean": 0.0008819798167678528,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 4515.5,
      "completions/max_terminated_length": 4067.5,
      "completions/mean_length": 2374.875,
      "completions/mean_terminated_length": 2117.5750732421875,
      "completions/min_length": 941.0,
      "completions/min_terminated_length": 941.0,
      "entropy": 0.18689643032848835,
      "epoch": 0.14334470989761092,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.03002380020916462,
      "learning_rate": 2.997623280720407e-07,
      "loss": -0.08155997842550278,
      "num_tokens": 1746350.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446330070495605,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 1.3880415558815002,
      "sampling/importance_sampling_ratio/mean": 0.5663973093032837,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.7681431770324707,
      "sampling/sampling_logp_difference/mean": 0.008763373363763094,
      "step": 21,
      "step_time": 147.82532634199015
    },
    {
      "clip_ratio/high_max": 0.0001005809517664602,
      "clip_ratio/high_mean": 0.0001005809517664602,
      "clip_ratio/low_mean": 0.00033674173755571246,
      "clip_ratio/low_min": 0.00033674173755571246,
      "clip_ratio/region_mean": 0.00043732268932217266,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3619.0,
      "completions/max_terminated_length": 3619.0,
      "completions/mean_length": 1774.0833740234375,
      "completions/mean_terminated_length": 1774.0833740234375,
      "completions/min_length": 923.5,
      "completions/min_terminated_length": 923.5,
      "entropy": 0.1736101247370243,
      "epoch": 0.15017064846416384,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.03494279459118843,
      "learning_rate": 2.996577922020106e-07,
      "loss": 0.08088157325983047,
      "num_tokens": 1820956.0,
      "reward": -0.6666666865348816,
      "reward_std": 0.7409421503543854,
      "rewards/correctness_reward/mean": -0.6666666567325592,
      "rewards/correctness_reward/std": 0.7409421503543854,
      "sampling/importance_sampling_ratio/max": 2.3706082105636597,
      "sampling/importance_sampling_ratio/mean": 0.6451850235462189,
      "sampling/importance_sampling_ratio/min": 0.02104969136416912,
      "sampling/sampling_logp_difference/max": 0.6389624774456024,
      "sampling/sampling_logp_difference/mean": 0.008609656244516373,
      "step": 22,
      "step_time": 116.07473812201351
    },
    {
      "clip_ratio/high_max": 0.00021113662296556868,
      "clip_ratio/high_mean": 0.00021113662296556868,
      "clip_ratio/low_mean": 0.00011071744665969163,
      "clip_ratio/low_min": 0.00011071744665969163,
      "clip_ratio/region_mean": 0.0003218540696252603,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 4188.0,
      "completions/max_terminated_length": 3987.0,
      "completions/mean_length": 2107.0,
      "completions/mean_terminated_length": 1845.0,
      "completions/min_length": 903.5,
      "completions/min_terminated_length": 903.5,
      "entropy": 0.17278275545686483,
      "epoch": 0.15699658703071673,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.040954675525426865,
      "learning_rate": 2.9953428114412303e-07,
      "loss": 0.018079154193401337,
      "num_tokens": 1911322.0,
      "reward": -0.6666666716337204,
      "reward_std": 0.4923659861087799,
      "rewards/correctness_reward/mean": -0.6666666716337204,
      "rewards/correctness_reward/std": 0.4923659861087799,
      "sampling/importance_sampling_ratio/max": 1.6411811113357544,
      "sampling/importance_sampling_ratio/mean": 0.565078467130661,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.6274092197418213,
      "sampling/sampling_logp_difference/mean": 0.00899354089051485,
      "step": 23,
      "step_time": 135.8156637239881
    },
    {
      "clip_ratio/high_max": 0.00011917552455997793,
      "clip_ratio/high_mean": 0.00011917552455997793,
      "clip_ratio/low_mean": 0.00062593947222922,
      "clip_ratio/low_min": 0.00062593947222922,
      "clip_ratio/region_mean": 0.0007451150013366714,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3819.0,
      "completions/max_terminated_length": 3004.5,
      "completions/mean_length": 2192.166748046875,
      "completions/mean_terminated_length": 1937.783447265625,
      "completions/min_length": 772.5,
      "completions/min_terminated_length": 772.5,
      "entropy": 0.19274314120411873,
      "epoch": 0.16382252559726962,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.04705621302127838,
      "learning_rate": 2.993918105584082e-07,
      "loss": -0.0761164128780365,
      "num_tokens": 1998752.0,
      "reward": -0.7500000298023224,
      "reward_std": 0.677924633026123,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.677924633026123,
      "sampling/importance_sampling_ratio/max": 2.2044373750686646,
      "sampling/importance_sampling_ratio/mean": 0.5416325330734253,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.7398251295089722,
      "sampling/sampling_logp_difference/mean": 0.009043822530657053,
      "step": 24,
      "step_time": 125.81809424500534
    },
    {
      "clip_ratio/high_max": 0.00017605540779186413,
      "clip_ratio/high_mean": 0.00017605540779186413,
      "clip_ratio/low_mean": 0.00029181257104937686,
      "clip_ratio/low_min": 0.00029181257104937686,
      "clip_ratio/region_mean": 0.0004678679906646721,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 4321.0,
      "completions/mean_length": 2522.541748046875,
      "completions/mean_terminated_length": 2274.772705078125,
      "completions/min_length": 658.0,
      "completions/min_terminated_length": 658.0,
      "entropy": 0.22828248515725136,
      "epoch": 0.17064846416382254,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.02641363814473152,
      "learning_rate": 2.9923039850878424e-07,
      "loss": -0.0734773501753807,
      "num_tokens": 2098737.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 2.215272545814514,
      "sampling/importance_sampling_ratio/mean": 0.5476150363683701,
      "sampling/importance_sampling_ratio/min": 0.08045727759599686,
      "sampling/sampling_logp_difference/max": 0.6184170246124268,
      "sampling/sampling_logp_difference/mean": 0.010321022942662239,
      "step": 25,
      "step_time": 170.05788060501072
    },
    {
      "clip_ratio/high_max": 0.00031355412284028716,
      "clip_ratio/high_mean": 0.00031355412284028716,
      "clip_ratio/low_mean": 0.000488045678139315,
      "clip_ratio/low_min": 0.000488045678139315,
      "clip_ratio/region_mean": 0.0008015997955226339,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3655.0,
      "completions/max_terminated_length": 3551.5,
      "completions/mean_length": 2256.416748046875,
      "completions/mean_terminated_length": 2062.0250244140625,
      "completions/min_length": 962.5,
      "completions/min_terminated_length": 962.5,
      "entropy": 0.19479426182806492,
      "epoch": 0.17747440273037543,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.03679096698760986,
      "learning_rate": 2.9905006546076744e-07,
      "loss": 0.04023721441626549,
      "num_tokens": 2185249.0,
      "reward": -0.1666666641831398,
      "reward_std": 0.9671956598758698,
      "rewards/correctness_reward/mean": -0.1666666641831398,
      "rewards/correctness_reward/std": 0.9671956598758698,
      "sampling/importance_sampling_ratio/max": 2.40914648771286,
      "sampling/importance_sampling_ratio/mean": 0.6711709499359131,
      "sampling/importance_sampling_ratio/min": 0.08630403692950495,
      "sampling/sampling_logp_difference/max": 0.9873652458190918,
      "sampling/sampling_logp_difference/mean": 0.009495267178863287,
      "step": 26,
      "step_time": 121.49674113400397
    },
    {
      "clip_ratio/high_max": 0.0003286330756964162,
      "clip_ratio/high_mean": 0.0003286330756964162,
      "clip_ratio/low_mean": 0.0003672421880764887,
      "clip_ratio/low_min": 0.0003672421880764887,
      "clip_ratio/region_mean": 0.0006958752710488625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3712.0,
      "completions/max_terminated_length": 3712.0,
      "completions/mean_length": 1608.9166870117188,
      "completions/mean_terminated_length": 1608.9166870117188,
      "completions/min_length": 881.0,
      "completions/min_terminated_length": 881.0,
      "entropy": 0.19669272750616074,
      "epoch": 0.18430034129692832,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.048227936029434204,
      "learning_rate": 2.988508342788768e-07,
      "loss": 0.15509483218193054,
      "num_tokens": 2252771.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 1.7127284407615662,
      "sampling/importance_sampling_ratio/mean": 0.5389788150787354,
      "sampling/importance_sampling_ratio/min": 0.028196483850479126,
      "sampling/sampling_logp_difference/max": 0.8035677671432495,
      "sampling/sampling_logp_difference/mean": 0.009410608559846878,
      "step": 27,
      "step_time": 116.56851660798566
    },
    {
      "clip_ratio/high_max": 2.1826435840921476e-05,
      "clip_ratio/high_mean": 2.1826435840921476e-05,
      "clip_ratio/low_mean": 0.00039388730510836467,
      "clip_ratio/low_min": 0.00039388730510836467,
      "clip_ratio/region_mean": 0.00041571374094928615,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2754.0,
      "completions/max_terminated_length": 2754.0,
      "completions/mean_length": 1630.9166870117188,
      "completions/mean_terminated_length": 1630.9166870117188,
      "completions/min_length": 925.0,
      "completions/min_terminated_length": 925.0,
      "entropy": 0.16313775815069675,
      "epoch": 0.19112627986348124,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.006842807866632938,
      "learning_rate": 2.9863273022373584e-07,
      "loss": 0.07145501673221588,
      "num_tokens": 2321181.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 2.4500311613082886,
      "sampling/importance_sampling_ratio/mean": 0.8633750975131989,
      "sampling/importance_sampling_ratio/min": 0.02915837988257408,
      "sampling/sampling_logp_difference/max": 0.6632177233695984,
      "sampling/sampling_logp_difference/mean": 0.008668553549796343,
      "step": 28,
      "step_time": 91.4905828859919
    },
    {
      "clip_ratio/high_max": 0.0002642529752847622,
      "clip_ratio/high_mean": 0.0002642529752847622,
      "clip_ratio/low_mean": 0.0005223052976361942,
      "clip_ratio/low_min": 0.0005223052976361942,
      "clip_ratio/region_mean": 0.0007865582774684299,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4784.5,
      "completions/max_terminated_length": 4765.0,
      "completions/mean_length": 2501.666748046875,
      "completions/mean_terminated_length": 2392.621337890625,
      "completions/min_length": 888.0,
      "completions/min_terminated_length": 888.0,
      "entropy": 0.22441293857991695,
      "epoch": 0.19795221843003413,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.05232253298163414,
      "learning_rate": 2.9839578094886877e-07,
      "loss": 0.29416829347610474,
      "num_tokens": 2419909.0,
      "reward": -0.3333333507180214,
      "reward_std": 0.8036037981510162,
      "rewards/correctness_reward/mean": -0.333333320915699,
      "rewards/correctness_reward/std": 0.8036037683486938,
      "sampling/importance_sampling_ratio/max": 1.3574910461902618,
      "sampling/importance_sampling_ratio/mean": 0.46175582706928253,
      "sampling/importance_sampling_ratio/min": 0.023056588135659695,
      "sampling/sampling_logp_difference/max": 0.7084431648254395,
      "sampling/sampling_logp_difference/mean": 0.009676191490143538,
      "step": 29,
      "step_time": 154.80130539400125
    },
    {
      "clip_ratio/high_max": 0.00027174162278242875,
      "clip_ratio/high_mean": 0.00027174162278242875,
      "clip_ratio/low_mean": 0.0007876567906350829,
      "clip_ratio/low_min": 0.0007876567906350829,
      "clip_ratio/region_mean": 0.0010593984261504374,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3716.0,
      "completions/max_terminated_length": 2443.0,
      "completions/mean_length": 1590.0833740234375,
      "completions/mean_terminated_length": 1435.5568237304688,
      "completions/min_length": 855.5,
      "completions/min_terminated_length": 855.5,
      "entropy": 0.16478881239891052,
      "epoch": 0.20477815699658702,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.08924312144517899,
      "learning_rate": 2.981400164971952e-07,
      "loss": -0.43429163098335266,
      "num_tokens": 2485449.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446329772472382,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 1.8077207803726196,
      "sampling/importance_sampling_ratio/mean": 0.641688883304596,
      "sampling/importance_sampling_ratio/min": 0.07364495098590851,
      "sampling/sampling_logp_difference/max": 0.6166839599609375,
      "sampling/sampling_logp_difference/mean": 0.00862132990732789,
      "step": 30,
      "step_time": 115.32614517699403
    },
    {
      "clip_ratio/high_max": 0.00017621346887608524,
      "clip_ratio/high_mean": 0.00017621346887608524,
      "clip_ratio/low_mean": 0.0005795853066956624,
      "clip_ratio/low_min": 0.0005795853066956624,
      "clip_ratio/region_mean": 0.0007557987719337689,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3910.5,
      "completions/max_terminated_length": 3090.0,
      "completions/mean_length": 2015.8333740234375,
      "completions/mean_terminated_length": 1889.776611328125,
      "completions/min_length": 889.0,
      "completions/min_terminated_length": 889.0,
      "entropy": 0.2223886251449585,
      "epoch": 0.21160409556313994,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.06942225992679596,
      "learning_rate": 2.9786546929722056e-07,
      "loss": 0.3086526393890381,
      "num_tokens": 2564933.0,
      "reward": -0.1666666865348816,
      "reward_std": 0.7409421503543854,
      "rewards/correctness_reward/mean": -0.1666666567325592,
      "rewards/correctness_reward/std": 0.7409421503543854,
      "sampling/importance_sampling_ratio/max": 2.386189043521881,
      "sampling/importance_sampling_ratio/mean": 0.6213788092136383,
      "sampling/importance_sampling_ratio/min": 0.015281436033546925,
      "sampling/sampling_logp_difference/max": 0.7943930625915527,
      "sampling/sampling_logp_difference/mean": 0.00985921872779727,
      "step": 31,
      "step_time": 125.07035979099601
    },
    {
      "clip_ratio/high_max": 0.0002656124333952903,
      "clip_ratio/high_mean": 0.0002656124333952903,
      "clip_ratio/low_mean": 0.00017724649842421059,
      "clip_ratio/low_min": 0.00017724649842421059,
      "clip_ratio/region_mean": 0.0004428589454619214,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3942.0,
      "completions/max_terminated_length": 3820.5,
      "completions/mean_length": 1951.7500610351562,
      "completions/mean_terminated_length": 1836.6439819335938,
      "completions/min_length": 996.5,
      "completions/min_terminated_length": 996.5,
      "entropy": 0.17561497911810875,
      "epoch": 0.21843003412969283,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.01926896534860134,
      "learning_rate": 2.9757217415892433e-07,
      "loss": 0.052177101373672485,
      "num_tokens": 2646665.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 2.242147386074066,
      "sampling/importance_sampling_ratio/mean": 0.6626814007759094,
      "sampling/importance_sampling_ratio/min": 0.031560447067022324,
      "sampling/sampling_logp_difference/max": 0.6647169589996338,
      "sampling/sampling_logp_difference/mean": 0.00904133077710867,
      "step": 32,
      "step_time": 127.47206454698608
    },
    {
      "clip_ratio/high_max": 0.00018125995120499283,
      "clip_ratio/high_mean": 0.00018125995120499283,
      "clip_ratio/low_mean": 0.0005849039262102451,
      "clip_ratio/low_min": 0.0005849039262102451,
      "clip_ratio/region_mean": 0.000766163877415238,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3630.5,
      "completions/max_terminated_length": 3218.5,
      "completions/mean_length": 1927.5416870117188,
      "completions/mean_terminated_length": 1798.4393920898438,
      "completions/min_length": 866.0,
      "completions/min_terminated_length": 866.0,
      "entropy": 0.17528906650841236,
      "epoch": 0.22525597269624573,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.03713307902216911,
      "learning_rate": 2.9726016826934687e-07,
      "loss": -0.05819889158010483,
      "num_tokens": 2732742.0,
      "reward": -0.5,
      "reward_std": 0.9045340418815613,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.9045340418815613,
      "sampling/importance_sampling_ratio/max": 1.9077945947647095,
      "sampling/importance_sampling_ratio/mean": 0.5301178991794586,
      "sampling/importance_sampling_ratio/min": 0.037661814130842686,
      "sampling/sampling_logp_difference/max": 0.8656803965568542,
      "sampling/sampling_logp_difference/mean": 0.008524105418473482,
      "step": 33,
      "step_time": 124.27680982700258
    },
    {
      "clip_ratio/high_max": 0.0005821565191581612,
      "clip_ratio/high_mean": 0.0005821565191581612,
      "clip_ratio/low_mean": 0.0001287642153329216,
      "clip_ratio/low_min": 0.0001287642153329216,
      "clip_ratio/region_mean": 0.0007109207344910828,
      "completions/clipped_ratio": 0.1250000037252903,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 3240.5,
      "completions/mean_length": 1932.5,
      "completions/mean_terminated_length": 1463.3091430664062,
      "completions/min_length": 811.5,
      "completions/min_terminated_length": 811.5,
      "entropy": 0.16712841391563416,
      "epoch": 0.23208191126279865,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.05192067101597786,
      "learning_rate": 2.9692949118787414e-07,
      "loss": 0.12746219336986542,
      "num_tokens": 2813184.0,
      "reward": -0.3333333358168602,
      "reward_std": 0.9671956598758698,
      "rewards/correctness_reward/mean": -0.3333333358168602,
      "rewards/correctness_reward/std": 0.9671956598758698,
      "sampling/importance_sampling_ratio/max": 1.7298161387443542,
      "sampling/importance_sampling_ratio/mean": 0.6548172235488892,
      "sampling/importance_sampling_ratio/min": 0.0035946578718721867,
      "sampling/sampling_logp_difference/max": 0.6443392038345337,
      "sampling/sampling_logp_difference/mean": 0.008729535154998302,
      "step": 34,
      "step_time": 159.94666257500648
    },
    {
      "clip_ratio/high_max": 9.326366489403881e-05,
      "clip_ratio/high_mean": 9.326366489403881e-05,
      "clip_ratio/low_mean": 0.0005161476583452895,
      "clip_ratio/low_min": 0.0005161476583452895,
      "clip_ratio/region_mean": 0.0006094113232393283,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 4532.0,
      "completions/max_terminated_length": 3245.0,
      "completions/mean_length": 1874.7500610351562,
      "completions/mean_terminated_length": 1559.3416748046875,
      "completions/min_length": 875.0,
      "completions/min_terminated_length": 875.0,
      "entropy": 0.1692411694675684,
      "epoch": 0.23890784982935154,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.04213385283946991,
      "learning_rate": 2.965801848412221e-07,
      "loss": 0.07331521809101105,
      "num_tokens": 2888670.0,
      "reward": -0.6666666865348816,
      "reward_std": 0.7409421503543854,
      "rewards/correctness_reward/mean": -0.6666666567325592,
      "rewards/correctness_reward/std": 0.7409421503543854,
      "sampling/importance_sampling_ratio/max": 2.3725807666778564,
      "sampling/importance_sampling_ratio/mean": 0.8309193253517151,
      "sampling/importance_sampling_ratio/min": 0.0595044381916523,
      "sampling/sampling_logp_difference/max": 0.5672608017921448,
      "sampling/sampling_logp_difference/mean": 0.008656369056552649,
      "step": 35,
      "step_time": 139.26266696700623
    },
    {
      "clip_ratio/high_max": 0.00012051208250341006,
      "clip_ratio/high_mean": 0.00012051208250341006,
      "clip_ratio/low_mean": 0.0003169747687934432,
      "clip_ratio/low_min": 0.0003169747687934432,
      "clip_ratio/region_mean": 0.0004374868512968533,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3681.5,
      "completions/max_terminated_length": 3509.5,
      "completions/mean_length": 2362.6250610351562,
      "completions/mean_terminated_length": 2181.7666625976562,
      "completions/min_length": 1057.0,
      "completions/min_terminated_length": 1057.0,
      "entropy": 0.17967960238456726,
      "epoch": 0.24573378839590443,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.008928037248551846,
      "learning_rate": 2.9621229351812067e-07,
      "loss": 0.05752938240766525,
      "num_tokens": 2981865.0,
      "reward": -0.25,
      "reward_std": 0.9744999706745148,
      "rewards/correctness_reward/mean": -0.25,
      "rewards/correctness_reward/std": 0.9744999706745148,
      "sampling/importance_sampling_ratio/max": 2.1388869285583496,
      "sampling/importance_sampling_ratio/mean": 0.5870790481567383,
      "sampling/importance_sampling_ratio/min": 0.001988642383366823,
      "sampling/sampling_logp_difference/max": 0.9233172535896301,
      "sampling/sampling_logp_difference/mean": 0.009207553695887327,
      "step": 36,
      "step_time": 125.00127266600612
    },
    {
      "clip_ratio/high_max": 0.00010507144907023758,
      "clip_ratio/high_mean": 0.00010507144907023758,
      "clip_ratio/low_mean": 0.00033121939486591145,
      "clip_ratio/low_min": 0.00033121939486591145,
      "clip_ratio/region_mean": 0.00043629084393614903,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2232.0,
      "completions/max_terminated_length": 2232.0,
      "completions/mean_length": 1288.7500610351562,
      "completions/mean_terminated_length": 1288.7500610351562,
      "completions/min_length": 793.0,
      "completions/min_terminated_length": 793.0,
      "entropy": 0.1518058469519019,
      "epoch": 0.2525597269624573,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.11457372456789017,
      "learning_rate": 2.958258638636986e-07,
      "loss": -0.41082119941711426,
      "num_tokens": 3040839.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.28867512941360474,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 2.3129096031188965,
      "sampling/importance_sampling_ratio/mean": 0.7068502902984619,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.5720200538635254,
      "sampling/sampling_logp_difference/mean": 0.008375945966690779,
      "step": 37,
      "step_time": 74.90589488900878
    },
    {
      "clip_ratio/high_max": 4.4680284190690145e-05,
      "clip_ratio/high_mean": 4.4680284190690145e-05,
      "clip_ratio/low_mean": 0.0001608589809620753,
      "clip_ratio/low_min": 0.0001608589809620753,
      "clip_ratio/region_mean": 0.00020553926515276544,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3663.5,
      "completions/max_terminated_length": 3535.0,
      "completions/mean_length": 2338.7083740234375,
      "completions/mean_terminated_length": 2058.736083984375,
      "completions/min_length": 748.5,
      "completions/min_terminated_length": 748.5,
      "entropy": 0.2067808359861374,
      "epoch": 0.2593856655290102,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.019327981397509575,
      "learning_rate": 2.9542094487356887e-07,
      "loss": 0.0506199486553669,
      "num_tokens": 3132146.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 1.9883459210395813,
      "sampling/importance_sampling_ratio/mean": 0.6568911969661713,
      "sampling/importance_sampling_ratio/min": 0.0012566318036988378,
      "sampling/sampling_logp_difference/max": 0.622053861618042,
      "sampling/sampling_logp_difference/mean": 0.009283999912440777,
      "step": 38,
      "step_time": 124.15157590300805
    },
    {
      "clip_ratio/high_max": 0.0002599538038339233,
      "clip_ratio/high_mean": 0.0002599538038339233,
      "clip_ratio/low_mean": 0.0009598340366210323,
      "clip_ratio/low_min": 0.0009598340366210323,
      "clip_ratio/region_mean": 0.0012197878386359662,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4883.0,
      "completions/max_terminated_length": 4883.0,
      "completions/mean_length": 2541.3333740234375,
      "completions/mean_terminated_length": 2541.3333740234375,
      "completions/min_length": 1086.0,
      "completions/min_terminated_length": 1086.0,
      "entropy": 0.20533941686153412,
      "epoch": 0.26621160409556316,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.03861364349722862,
      "learning_rate": 2.949975878876169e-07,
      "loss": 0.37017422914505005,
      "num_tokens": 3228928.0,
      "reward": -0.3333333358168602,
      "reward_std": 0.9671956598758698,
      "rewards/correctness_reward/mean": -0.3333333358168602,
      "rewards/correctness_reward/std": 0.9671956598758698,
      "sampling/importance_sampling_ratio/max": 1.7460360527038574,
      "sampling/importance_sampling_ratio/mean": 0.4507356733083725,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.7737934589385986,
      "sampling/sampling_logp_difference/mean": 0.009104832075536251,
      "step": 39,
      "step_time": 157.81988949599327
    },
    {
      "clip_ratio/high_max": 0.00011135237582493573,
      "clip_ratio/high_mean": 0.00011135237582493573,
      "clip_ratio/low_mean": 0.0005540200945688412,
      "clip_ratio/low_min": 0.0005540200945688412,
      "clip_ratio/region_mean": 0.0006653724703937769,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 4377.0,
      "completions/max_terminated_length": 4269.0,
      "completions/mean_length": 2209.0416870117188,
      "completions/mean_terminated_length": 1929.1917114257812,
      "completions/min_length": 875.5,
      "completions/min_terminated_length": 875.5,
      "entropy": 0.1815284639596939,
      "epoch": 0.27303754266211605,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.06392155587673187,
      "learning_rate": 2.945558465834911e-07,
      "loss": -0.38024067878723145,
      "num_tokens": 3317567.0,
      "reward": -0.6666666865348816,
      "reward_std": 0.7409421503543854,
      "rewards/correctness_reward/mean": -0.6666666567325592,
      "rewards/correctness_reward/std": 0.7409421503543854,
      "sampling/importance_sampling_ratio/max": 2.0949626564979553,
      "sampling/importance_sampling_ratio/mean": 0.5830715447664261,
      "sampling/importance_sampling_ratio/min": 0.007053101202473044,
      "sampling/sampling_logp_difference/max": 0.4775382876396179,
      "sampling/sampling_logp_difference/mean": 0.008772795088589191,
      "step": 40,
      "step_time": 140.69885612401413
    },
    {
      "clip_ratio/high_max": 8.099789556581527e-05,
      "clip_ratio/high_mean": 8.099789556581527e-05,
      "clip_ratio/low_mean": 0.0003992624915554188,
      "clip_ratio/low_min": 0.0003992624915554188,
      "clip_ratio/region_mean": 0.0004802603871212341,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3610.0,
      "completions/max_terminated_length": 3610.0,
      "completions/mean_length": 1637.6250610351562,
      "completions/mean_terminated_length": 1637.6250610351562,
      "completions/min_length": 712.5,
      "completions/min_terminated_length": 712.5,
      "entropy": 0.18336657993495464,
      "epoch": 0.27986348122866894,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.04287489131093025,
      "learning_rate": 2.9409577696979687e-07,
      "loss": -0.1049296110868454,
      "num_tokens": 3391028.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.28867512941360474,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.7601441740989685,
      "sampling/importance_sampling_ratio/mean": 0.5774812400341034,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.6337925791740417,
      "sampling/sampling_logp_difference/mean": 0.008997163735330105,
      "step": 41,
      "step_time": 116.36959670999931
    },
    {
      "clip_ratio/high_max": 0.0003851416477118619,
      "clip_ratio/high_mean": 0.0003851416477118619,
      "clip_ratio/low_mean": 0.0006896271115692798,
      "clip_ratio/low_min": 0.0006896271115692798,
      "clip_ratio/region_mean": 0.0010747687556431629,
      "completions/clipped_ratio": 0.1250000037252903,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 4576.5,
      "completions/mean_length": 2773.1251220703125,
      "completions/mean_terminated_length": 2403.9500732421875,
      "completions/min_length": 867.0,
      "completions/min_terminated_length": 867.0,
      "entropy": 0.2158426120877266,
      "epoch": 0.28668941979522183,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.054242201149463654,
      "learning_rate": 2.9361743737899534e-07,
      "loss": 0.15749189257621765,
      "num_tokens": 3488897.0,
      "reward": 0.0,
      "reward_std": 1.0444658994674683,
      "rewards/correctness_reward/mean": 0.0,
      "rewards/correctness_reward/std": 1.0444658994674683,
      "sampling/importance_sampling_ratio/max": 2.1604199409484863,
      "sampling/importance_sampling_ratio/mean": 0.6884078681468964,
      "sampling/importance_sampling_ratio/min": 0.034998440532945096,
      "sampling/sampling_logp_difference/max": 0.7063878774642944,
      "sampling/sampling_logp_difference/mean": 0.009720813482999802,
      "step": 42,
      "step_time": 165.69028340099612
    },
    {
      "clip_ratio/high_max": 2.61752702499507e-05,
      "clip_ratio/high_mean": 2.61752702499507e-05,
      "clip_ratio/low_mean": 0.00038996605871943757,
      "clip_ratio/low_min": 0.00038996605871943757,
      "clip_ratio/region_mean": 0.00041614132896938827,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4010.5,
      "completions/max_terminated_length": 4010.5,
      "completions/mean_length": 1930.5000610351562,
      "completions/mean_terminated_length": 1930.5000610351562,
      "completions/min_length": 913.0,
      "completions/min_terminated_length": 913.0,
      "entropy": 0.18189280107617378,
      "epoch": 0.2935153583617747,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.016736026853322983,
      "learning_rate": 2.931208884600073e-07,
      "loss": 0.052612289786338806,
      "num_tokens": 3567983.0,
      "reward": -0.75,
      "reward_std": 0.45226702094078064,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.45226702094078064,
      "sampling/importance_sampling_ratio/max": 1.9738622903823853,
      "sampling/importance_sampling_ratio/mean": 0.554217055439949,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.6788408756256104,
      "sampling/sampling_logp_difference/mean": 0.008132760412991047,
      "step": 43,
      "step_time": 127.23670873800438
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 4686.5,
      "completions/max_terminated_length": 4259.0,
      "completions/mean_length": 2431.666748046875,
      "completions/mean_terminated_length": 2103.15283203125,
      "completions/min_length": 1169.5,
      "completions/min_terminated_length": 1169.5,
      "entropy": 0.1827030796557665,
      "epoch": 0.3003412969283277,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.926061931705237e-07,
      "loss": 0.0,
      "num_tokens": 3667377.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.0915364623069763,
      "sampling/importance_sampling_ratio/mean": 0.481503888964653,
      "sampling/importance_sampling_ratio/min": 0.01448526605963707,
      "sampling/sampling_logp_difference/max": 0.601245641708374,
      "sampling/sampling_logp_difference/mean": 0.008691030088812113,
      "step": 44,
      "step_time": 151.4489465440056
    },
    {
      "clip_ratio/high_max": 4.348203037807252e-05,
      "clip_ratio/high_mean": 4.348203037807252e-05,
      "clip_ratio/low_mean": 0.00013300800128490664,
      "clip_ratio/low_min": 0.00013300800128490664,
      "clip_ratio/region_mean": 0.00017649003166297916,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3959.5,
      "completions/max_terminated_length": 3033.0,
      "completions/mean_length": 2206.7500610351562,
      "completions/mean_terminated_length": 1852.5695190429688,
      "completions/min_length": 1081.0,
      "completions/min_terminated_length": 1081.0,
      "entropy": 0.17536703869700432,
      "epoch": 0.30716723549488056,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.019536586478352547,
      "learning_rate": 2.92073416769023e-07,
      "loss": -0.05542347952723503,
      "num_tokens": 3756813.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.38924944400787354,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.5530624985694885,
      "sampling/importance_sampling_ratio/mean": 0.5366989821195602,
      "sampling/importance_sampling_ratio/min": 0.04268880607560277,
      "sampling/sampling_logp_difference/max": 0.5965178608894348,
      "sampling/sampling_logp_difference/mean": 0.008516309317201376,
      "step": 45,
      "step_time": 131.10619422599848
    },
    {
      "clip_ratio/high_max": 0.00021378630663093645,
      "clip_ratio/high_mean": 0.00021378630663093645,
      "clip_ratio/low_mean": 0.0004329298644734081,
      "clip_ratio/low_min": 0.0004329298644734081,
      "clip_ratio/region_mean": 0.0006467161674663657,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2647.0,
      "completions/max_terminated_length": 2647.0,
      "completions/mean_length": 1851.291748046875,
      "completions/mean_terminated_length": 1851.291748046875,
      "completions/min_length": 1014.5,
      "completions/min_terminated_length": 1014.5,
      "entropy": 0.17232570238411427,
      "epoch": 0.31399317406143346,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.04990917444229126,
      "learning_rate": 2.91522626806497e-07,
      "loss": -0.06765934079885483,
      "num_tokens": 3830830.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446329772472382,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 2.2500372529029846,
      "sampling/importance_sampling_ratio/mean": 0.6319310963153839,
      "sampling/importance_sampling_ratio/min": 0.02974645048379898,
      "sampling/sampling_logp_difference/max": 0.6080000102519989,
      "sampling/sampling_logp_difference/mean": 0.008410447742789984,
      "step": 46,
      "step_time": 90.71643152000615
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4392.5,
      "completions/max_terminated_length": 3531.5,
      "completions/mean_length": 1753.2083740234375,
      "completions/mean_terminated_length": 1598.6931762695312,
      "completions/min_length": 749.5,
      "completions/min_terminated_length": 749.5,
      "entropy": 0.18032599054276943,
      "epoch": 0.32081911262798635,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.9095389311788627e-07,
      "loss": 0.0,
      "num_tokens": 3907401.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.022447347640991,
      "sampling/importance_sampling_ratio/mean": 0.5649960935115814,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.661316990852356,
      "sampling/sampling_logp_difference/mean": 0.008854995481669903,
      "step": 47,
      "step_time": 136.0928902579908
    },
    {
      "clip_ratio/high_max": 0.0002303535029568593,
      "clip_ratio/high_mean": 0.0002303535029568593,
      "clip_ratio/low_mean": 0.0008515775716659846,
      "clip_ratio/low_min": 0.0008515775716659846,
      "clip_ratio/region_mean": 0.0010819310809893068,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 4282.0,
      "completions/max_terminated_length": 4095.0,
      "completions/mean_length": 2293.2083740234375,
      "completions/mean_terminated_length": 2041.425048828125,
      "completions/min_length": 985.5,
      "completions/min_terminated_length": 985.5,
      "entropy": 0.16349471919238567,
      "epoch": 0.32764505119453924,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.029812565073370934,
      "learning_rate": 2.9036728781322535e-07,
      "loss": 0.2133912295103073,
      "num_tokens": 4000670.0,
      "reward": -0.6666666865348816,
      "reward_std": 0.7409421503543854,
      "rewards/correctness_reward/mean": -0.6666666567325592,
      "rewards/correctness_reward/std": 0.7409421503543854,
      "sampling/importance_sampling_ratio/max": 1.9735003113746643,
      "sampling/importance_sampling_ratio/mean": 0.5840648710727692,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8899977207183838,
      "sampling/sampling_logp_difference/mean": 0.008015737868845463,
      "step": 48,
      "step_time": 139.79937548300222
    },
    {
      "clip_ratio/high_max": 0.0002933151881734375,
      "clip_ratio/high_mean": 0.0002933151881734375,
      "clip_ratio/low_mean": 0.0008847473072819412,
      "clip_ratio/low_min": 0.0008847473072819412,
      "clip_ratio/region_mean": 0.001178062506369315,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5097.5,
      "completions/max_terminated_length": 5097.5,
      "completions/mean_length": 2149.375,
      "completions/mean_terminated_length": 2149.375,
      "completions/min_length": 979.0,
      "completions/min_terminated_length": 979.0,
      "entropy": 0.17316051665693521,
      "epoch": 0.33447098976109213,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.06560072302818298,
      "learning_rate": 2.8976288526850024e-07,
      "loss": -0.19419705867767334,
      "num_tokens": 4081457.0,
      "reward": -0.0833333432674408,
      "reward_std": 0.8415164947509766,
      "rewards/correctness_reward/mean": -0.0833333432674408,
      "rewards/correctness_reward/std": 0.8415164947509766,
      "sampling/importance_sampling_ratio/max": 2.216562867164612,
      "sampling/importance_sampling_ratio/mean": 0.6668431162834167,
      "sampling/importance_sampling_ratio/min": 0.0018459694692865014,
      "sampling/sampling_logp_difference/max": 1.8778375387191772,
      "sampling/sampling_logp_difference/mean": 0.008765012491494417,
      "step": 49,
      "step_time": 156.69416434800223
    },
    {
      "clip_ratio/high_max": 3.402286165510304e-05,
      "clip_ratio/high_mean": 3.402286165510304e-05,
      "clip_ratio/low_mean": 0.0005146802941453643,
      "clip_ratio/low_min": 0.0005146802941453643,
      "clip_ratio/region_mean": 0.0005487031558004674,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3714.0,
      "completions/max_terminated_length": 3508.5,
      "completions/mean_length": 1895.25,
      "completions/mean_terminated_length": 1630.2249755859375,
      "completions/min_length": 710.5,
      "completions/min_terminated_length": 710.5,
      "entropy": 0.17622262984514236,
      "epoch": 0.3412969283276451,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.027309048920869827,
      "learning_rate": 2.8914076211621813e-07,
      "loss": -0.07097186893224716,
      "num_tokens": 4160171.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.38924944400787354,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.059992790222168,
      "sampling/importance_sampling_ratio/mean": 0.4521847665309906,
      "sampling/importance_sampling_ratio/min": 0.01352127268910408,
      "sampling/sampling_logp_difference/max": 0.6476984620094299,
      "sampling/sampling_logp_difference/mean": 0.008724214509129524,
      "step": 50,
      "step_time": 121.86180935701123
    },
    {
      "clip_ratio/high_max": 0.0002972058136947453,
      "clip_ratio/high_mean": 0.0002972058136947453,
      "clip_ratio/low_mean": 0.000536448962520808,
      "clip_ratio/low_min": 0.000536448962520808,
      "clip_ratio/region_mean": 0.0008336547798535321,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4687.0,
      "completions/max_terminated_length": 4029.5,
      "completions/mean_length": 2090.8333740234375,
      "completions/mean_terminated_length": 1952.7614135742188,
      "completions/min_length": 814.5,
      "completions/min_terminated_length": 814.5,
      "entropy": 0.19055325351655483,
      "epoch": 0.34812286689419797,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.02999180555343628,
      "learning_rate": 2.8850099723569103e-07,
      "loss": -0.10701270401477814,
      "num_tokens": 4253803.0,
      "reward": -0.4166666865348816,
      "reward_std": 0.8109080791473389,
      "rewards/correctness_reward/mean": -0.4166666567325592,
      "rewards/correctness_reward/std": 0.8109080791473389,
      "sampling/importance_sampling_ratio/max": 1.5244572162628174,
      "sampling/importance_sampling_ratio/mean": 0.48259805142879486,
      "sampling/importance_sampling_ratio/min": 0.005122458096593618,
      "sampling/sampling_logp_difference/max": 0.5854331254959106,
      "sampling/sampling_logp_difference/mean": 0.009422262199223042,
      "step": 51,
      "step_time": 151.26289811401512
    },
    {
      "clip_ratio/high_max": 0.00013200865942053497,
      "clip_ratio/high_mean": 0.00013200865942053497,
      "clip_ratio/low_mean": 0.0007060257175908191,
      "clip_ratio/low_min": 0.0007060257175908191,
      "clip_ratio/region_mean": 0.0008380343770113541,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4348.5,
      "completions/max_terminated_length": 3834.5,
      "completions/mean_length": 2119.6251220703125,
      "completions/mean_terminated_length": 2002.0606689453125,
      "completions/min_length": 1073.5,
      "completions/min_terminated_length": 1073.5,
      "entropy": 0.16607994306832552,
      "epoch": 0.35494880546075086,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.04837702214717865,
      "learning_rate": 2.878436717430346e-07,
      "loss": -0.04106302559375763,
      "num_tokens": 4342324.0,
      "reward": -0.7500000298023224,
      "reward_std": 0.677924633026123,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.677924633026123,
      "sampling/importance_sampling_ratio/max": 1.6572447419166565,
      "sampling/importance_sampling_ratio/mean": 0.5404750406742096,
      "sampling/importance_sampling_ratio/min": 0.019457601010799408,
      "sampling/sampling_logp_difference/max": 0.7372019290924072,
      "sampling/sampling_logp_difference/mean": 0.007984871277585626,
      "step": 52,
      "step_time": 140.17531206600688
    },
    {
      "clip_ratio/high_max": 0.0001694170387054328,
      "clip_ratio/high_mean": 0.0001694170387054328,
      "clip_ratio/low_mean": 0.0002777307599899359,
      "clip_ratio/low_min": 0.0002777307599899359,
      "clip_ratio/region_mean": 0.0004471477986953687,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 3680.0,
      "completions/mean_length": 2798.2501220703125,
      "completions/mean_terminated_length": 1981.6666870117188,
      "completions/min_length": 879.0,
      "completions/min_terminated_length": 879.0,
      "entropy": 0.19165829755365849,
      "epoch": 0.36177474402730375,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.01472468115389347,
      "learning_rate": 2.8716886898088355e-07,
      "loss": -0.03431159257888794,
      "num_tokens": 4451944.0,
      "reward": -0.5833333358168602,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.5833333358168602,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 1.4794809222221375,
      "sampling/importance_sampling_ratio/mean": 0.6550164222717285,
      "sampling/importance_sampling_ratio/min": 0.03906719759106636,
      "sampling/sampling_logp_difference/max": 0.6392964124679565,
      "sampling/sampling_logp_difference/mean": 0.009628844913095236,
      "step": 53,
      "step_time": 170.42672173799656
    },
    {
      "clip_ratio/high_max": 0.00013047829088463914,
      "clip_ratio/high_mean": 0.00013047829088463914,
      "clip_ratio/low_mean": 0.0011631374727585353,
      "clip_ratio/low_min": 0.0011631374727585353,
      "clip_ratio/region_mean": 0.0012936157727381214,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 4837.5,
      "completions/mean_length": 2533.08349609375,
      "completions/mean_terminated_length": 2286.2727661132812,
      "completions/min_length": 865.0,
      "completions/min_terminated_length": 865.0,
      "entropy": 0.206689840182662,
      "epoch": 0.36860068259385664,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.034136850386857986,
      "learning_rate": 2.864766745078246e-07,
      "loss": 0.03963364660739899,
      "num_tokens": 4551342.0,
      "reward": -0.6666666865348816,
      "reward_std": 0.7784989774227142,
      "rewards/correctness_reward/mean": -0.6666666865348816,
      "rewards/correctness_reward/std": 0.7784989774227142,
      "sampling/importance_sampling_ratio/max": 2.0958030223846436,
      "sampling/importance_sampling_ratio/mean": 0.6207669377326965,
      "sampling/importance_sampling_ratio/min": 0.010971732437610626,
      "sampling/sampling_logp_difference/max": 0.5716677308082581,
      "sampling/sampling_logp_difference/mean": 0.009138945490121841,
      "step": 54,
      "step_time": 169.02359899500152
    },
    {
      "clip_ratio/high_max": 0.00012305460040806793,
      "clip_ratio/high_mean": 0.00012305460040806793,
      "clip_ratio/low_mean": 0.0003682131282403134,
      "clip_ratio/low_min": 0.0003682131282403134,
      "clip_ratio/region_mean": 0.0004912677286483813,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3839.0,
      "completions/max_terminated_length": 3335.5,
      "completions/mean_length": 1927.2083740234375,
      "completions/mean_terminated_length": 1801.401611328125,
      "completions/min_length": 784.0,
      "completions/min_terminated_length": 784.0,
      "entropy": 0.19140255451202393,
      "epoch": 0.37542662116040953,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.0412936732172966,
      "learning_rate": 2.857671760875482e-07,
      "loss": 0.2316821962594986,
      "num_tokens": 4634279.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.38924944400787354,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.9013534784317017,
      "sampling/importance_sampling_ratio/mean": 0.7683794945478439,
      "sampling/importance_sampling_ratio/min": 0.052524975035339594,
      "sampling/sampling_logp_difference/max": 0.5893399715423584,
      "sampling/sampling_logp_difference/mean": 0.009064876940101385,
      "step": 55,
      "step_time": 126.5851471220085
    },
    {
      "clip_ratio/high_max": 6.827863762737252e-05,
      "clip_ratio/high_mean": 6.827863762737252e-05,
      "clip_ratio/low_mean": 0.00023662939202040434,
      "clip_ratio/low_min": 0.00023662939202040434,
      "clip_ratio/region_mean": 0.00030490803328575566,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 4025.0,
      "completions/mean_length": 2566.3751220703125,
      "completions/mean_terminated_length": 2322.5909423828125,
      "completions/min_length": 634.0,
      "completions/min_terminated_length": 634.0,
      "entropy": 0.22426036186516285,
      "epoch": 0.3822525597269625,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.017772767692804337,
      "learning_rate": 2.850404636777212e-07,
      "loss": 0.10956238210201263,
      "num_tokens": 4733990.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.3892495036125183,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.8248947858810425,
      "sampling/importance_sampling_ratio/mean": 0.4554801285266876,
      "sampling/importance_sampling_ratio/min": 0.000969524378888309,
      "sampling/sampling_logp_difference/max": 0.7188509702682495,
      "sampling/sampling_logp_difference/mean": 0.009816299192607403,
      "step": 56,
      "step_time": 167.3950643390126
    },
    {
      "clip_ratio/high_max": 0.00019467264428385533,
      "clip_ratio/high_mean": 0.00019467264428385533,
      "clip_ratio/low_mean": 0.0008429501613136381,
      "clip_ratio/low_min": 0.0008429501613136381,
      "clip_ratio/region_mean": 0.0010376228055974934,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4506.5,
      "completions/max_terminated_length": 3469.5,
      "completions/mean_length": 2122.791748046875,
      "completions/mean_terminated_length": 1973.5531616210938,
      "completions/min_length": 829.0,
      "completions/min_terminated_length": 829.0,
      "entropy": 0.2076687552034855,
      "epoch": 0.3890784982935154,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.056109074503183365,
      "learning_rate": 2.842966294185809e-07,
      "loss": -0.13481320440769196,
      "num_tokens": 4833657.0,
      "reward": -0.583333358168602,
      "reward_std": 0.781041145324707,
      "rewards/correctness_reward/mean": -0.5833333283662796,
      "rewards/correctness_reward/std": 0.7810411155223846,
      "sampling/importance_sampling_ratio/max": 1.5594306886196136,
      "sampling/importance_sampling_ratio/mean": 0.45633988082408905,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.6542015075683594,
      "sampling/sampling_logp_difference/mean": 0.009512615855783224,
      "step": 57,
      "step_time": 149.88567502400838
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3770.5,
      "completions/max_terminated_length": 3770.5,
      "completions/mean_length": 1786.1250610351562,
      "completions/mean_terminated_length": 1786.1250610351562,
      "completions/min_length": 765.5,
      "completions/min_terminated_length": 765.5,
      "entropy": 0.18106183130294085,
      "epoch": 0.39590443686006827,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.835357676212527e-07,
      "loss": 0.0,
      "num_tokens": 4909302.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.9227526783943176,
      "sampling/importance_sampling_ratio/mean": 0.3771863132715225,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8851069211959839,
      "sampling/sampling_logp_difference/mean": 0.009029736276715994,
      "step": 58,
      "step_time": 120.19218669401016
    },
    {
      "clip_ratio/high_max": 0.00031629430486646015,
      "clip_ratio/high_mean": 0.00031629430486646015,
      "clip_ratio/low_mean": 0.0007711920734436717,
      "clip_ratio/low_min": 0.0007711920734436717,
      "clip_ratio/region_mean": 0.0010874863874050789,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3482.5,
      "completions/max_terminated_length": 3482.5,
      "completions/mean_length": 1716.0000610351562,
      "completions/mean_terminated_length": 1716.0000610351562,
      "completions/min_length": 797.0,
      "completions/min_terminated_length": 797.0,
      "entropy": 0.1720494432374835,
      "epoch": 0.40273037542662116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.04583508521318436,
      "learning_rate": 2.827579747557923e-07,
      "loss": 0.1589364856481552,
      "num_tokens": 4978320.0,
      "reward": -0.5,
      "reward_std": 0.9045339822769165,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.9045340418815613,
      "sampling/importance_sampling_ratio/max": 1.5915037989616394,
      "sampling/importance_sampling_ratio/mean": 0.5548273622989655,
      "sampling/importance_sampling_ratio/min": 0.009591552894562483,
      "sampling/sampling_logp_difference/max": 0.9339792132377625,
      "sampling/sampling_logp_difference/mean": 0.00888055469840765,
      "step": 59,
      "step_time": 110.96668717899593
    },
    {
      "clip_ratio/high_max": 0.0003037179012608249,
      "clip_ratio/high_mean": 0.0003037179012608249,
      "clip_ratio/low_mean": 0.0006711918067594524,
      "clip_ratio/low_min": 0.0006711918067594524,
      "clip_ratio/region_mean": 0.0009749097152962349,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4913.0,
      "completions/max_terminated_length": 4913.0,
      "completions/mean_length": 1996.2083740234375,
      "completions/mean_terminated_length": 1996.2083740234375,
      "completions/min_length": 917.5,
      "completions/min_terminated_length": 917.5,
      "entropy": 0.1703935917466879,
      "epoch": 0.40955631399317405,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.029283197596669197,
      "learning_rate": 2.81963349438954e-07,
      "loss": -0.008943863213062286,
      "num_tokens": 5062631.0,
      "reward": -0.583333358168602,
      "reward_std": 0.7810411155223846,
      "rewards/correctness_reward/mean": -0.5833333283662796,
      "rewards/correctness_reward/std": 0.7810411155223846,
      "sampling/importance_sampling_ratio/max": 1.8522260189056396,
      "sampling/importance_sampling_ratio/mean": 0.4740467965602875,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0869746208190918,
      "sampling/sampling_logp_difference/mean": 0.007830763701349497,
      "step": 60,
      "step_time": 154.14188899399596
    },
    {
      "clip_ratio/high_max": 5.544539453694597e-05,
      "clip_ratio/high_mean": 5.544539453694597e-05,
      "clip_ratio/low_mean": 0.00029268264552229084,
      "clip_ratio/low_min": 0.00029268264552229084,
      "clip_ratio/region_mean": 0.0003481280436972156,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3764.5,
      "completions/max_terminated_length": 3764.5,
      "completions/mean_length": 1855.75,
      "completions/mean_terminated_length": 1855.75,
      "completions/min_length": 918.5,
      "completions/min_terminated_length": 918.5,
      "entropy": 0.16261542588472366,
      "epoch": 0.41638225255972694,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.01945802941918373,
      "learning_rate": 2.811519924216873e-07,
      "loss": 0.05384628474712372,
      "num_tokens": 5140289.0,
      "reward": -0.3333333432674408,
      "reward_std": 0.91148242354393,
      "rewards/correctness_reward/mean": -0.3333333432674408,
      "rewards/correctness_reward/std": 0.91148242354393,
      "sampling/importance_sampling_ratio/max": 1.528772234916687,
      "sampling/importance_sampling_ratio/mean": 0.4670892059803009,
      "sampling/importance_sampling_ratio/min": 0.05911402031779289,
      "sampling/sampling_logp_difference/max": 0.6520828008651733,
      "sampling/sampling_logp_difference/mean": 0.007892156019806862,
      "step": 61,
      "step_time": 121.0787803270141
    },
    {
      "clip_ratio/high_max": 4.087170600541867e-05,
      "clip_ratio/high_mean": 4.087170600541867e-05,
      "clip_ratio/low_mean": 0.0002310327981831506,
      "clip_ratio/low_min": 0.0002310327981831506,
      "clip_ratio/region_mean": 0.00027190449873160105,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3292.5,
      "completions/max_terminated_length": 3292.5,
      "completions/mean_length": 1349.1666870117188,
      "completions/mean_terminated_length": 1349.1666870117188,
      "completions/min_length": 692.5,
      "completions/min_terminated_length": 692.5,
      "entropy": 0.16260085813701153,
      "epoch": 0.4232081911262799,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.03200114518404007,
      "learning_rate": 2.8032400657636255e-07,
      "loss": -0.13627246022224426,
      "num_tokens": 5202531.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.38924944400787354,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 0.8998820185661316,
      "sampling/importance_sampling_ratio/mean": 0.38338422775268555,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.6040194034576416,
      "sampling/sampling_logp_difference/mean": 0.008680101484060287,
      "step": 62,
      "step_time": 103.78884234600264
    },
    {
      "clip_ratio/high_max": 8.822122617857531e-05,
      "clip_ratio/high_mean": 8.822122617857531e-05,
      "clip_ratio/low_mean": 2.095557465509046e-05,
      "clip_ratio/low_min": 2.095557465509046e-05,
      "clip_ratio/region_mean": 0.00010917680083366577,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3387.0,
      "completions/max_terminated_length": 3387.0,
      "completions/mean_length": 1841.0833740234375,
      "completions/mean_terminated_length": 1841.0833740234375,
      "completions/min_length": 961.5,
      "completions/min_terminated_length": 961.5,
      "entropy": 0.18213766440749168,
      "epoch": 0.4300341296928328,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.012850549072027206,
      "learning_rate": 2.7947949688372767e-07,
      "loss": 0.007055250927805901,
      "num_tokens": 5286647.0,
      "reward": -0.6666666716337204,
      "reward_std": 0.4923659861087799,
      "rewards/correctness_reward/mean": -0.6666666716337204,
      "rewards/correctness_reward/std": 0.4923659861087799,
      "sampling/importance_sampling_ratio/max": 1.8162547945976257,
      "sampling/importance_sampling_ratio/mean": 0.5006676018238068,
      "sampling/importance_sampling_ratio/min": 0.015822945162653923,
      "sampling/sampling_logp_difference/max": 0.787606954574585,
      "sampling/sampling_logp_difference/mean": 0.008724508807063103,
      "step": 63,
      "step_time": 115.6498547349911
    },
    {
      "clip_ratio/high_max": 8.47601259010844e-05,
      "clip_ratio/high_mean": 8.47601259010844e-05,
      "clip_ratio/low_mean": 0.0001956572632479947,
      "clip_ratio/low_min": 0.0001956572632479947,
      "clip_ratio/region_mean": 0.0002804173818731215,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 3345.0,
      "completions/mean_length": 2251.666748046875,
      "completions/mean_terminated_length": 1979.2727661132812,
      "completions/min_length": 803.0,
      "completions/min_terminated_length": 803.0,
      "entropy": 0.19157697446644306,
      "epoch": 0.43686006825938567,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.022400015965104103,
      "learning_rate": 2.786185704195976e-07,
      "loss": 0.15593445301055908,
      "num_tokens": 5381205.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.956946074962616,
      "sampling/importance_sampling_ratio/mean": 0.652755469083786,
      "sampling/importance_sampling_ratio/min": 0.0161164328455925,
      "sampling/sampling_logp_difference/max": 0.4603617191314697,
      "sampling/sampling_logp_difference/mean": 0.009298257529735565,
      "step": 64,
      "step_time": 166.16724363300455
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3447.5,
      "completions/max_terminated_length": 3447.5,
      "completions/mean_length": 1786.75,
      "completions/mean_terminated_length": 1786.75,
      "completions/min_length": 725.5,
      "completions/min_terminated_length": 725.5,
      "entropy": 0.18305392190814018,
      "epoch": 0.44368600682593856,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.7774133634127823e-07,
      "loss": 0.0,
      "num_tokens": 5453997.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5538569688796997,
      "sampling/importance_sampling_ratio/mean": 0.5863878726959229,
      "sampling/importance_sampling_ratio/min": 0.018695535138249397,
      "sampling/sampling_logp_difference/max": 0.466518759727478,
      "sampling/sampling_logp_difference/mean": 0.00866021541878581,
      "step": 65,
      "step_time": 110.4639795779949
    },
    {
      "clip_ratio/high_max": 0.0001480766040913295,
      "clip_ratio/high_mean": 0.0001480766040913295,
      "clip_ratio/low_mean": 0.0002666572872840334,
      "clip_ratio/low_min": 0.0002666572872840334,
      "clip_ratio/region_mean": 0.0004147338913753629,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4059.0,
      "completions/max_terminated_length": 4059.0,
      "completions/mean_length": 2139.2501220703125,
      "completions/mean_terminated_length": 2139.2501220703125,
      "completions/min_length": 799.0,
      "completions/min_terminated_length": 799.0,
      "entropy": 0.18904976546764374,
      "epoch": 0.45051194539249145,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.04415519908070564,
      "learning_rate": 2.7684790587372594e-07,
      "loss": -0.35014355182647705,
      "num_tokens": 5532975.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446330070495605,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 1.7579538822174072,
      "sampling/importance_sampling_ratio/mean": 0.5573211014270782,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.5224248170852661,
      "sampling/sampling_logp_difference/mean": 0.00864137476310134,
      "step": 66,
      "step_time": 128.31755204999354
    },
    {
      "clip_ratio/high_max": 0.0003504481737763854,
      "clip_ratio/high_mean": 0.0003504481737763854,
      "clip_ratio/low_mean": 0.00035990733886137605,
      "clip_ratio/low_min": 0.00035990733886137605,
      "clip_ratio/region_mean": 0.0007103555126377614,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 4231.5,
      "completions/max_terminated_length": 4021.5,
      "completions/mean_length": 2462.5416870117188,
      "completions/mean_terminated_length": 2111.1666870117188,
      "completions/min_length": 1038.0,
      "completions/min_terminated_length": 1038.0,
      "entropy": 0.17603121791034937,
      "epoch": 0.45733788395904434,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.043682895600795746,
      "learning_rate": 2.7593839229544584e-07,
      "loss": -0.15456409752368927,
      "num_tokens": 5631004.0,
      "reward": -0.5833333432674408,
      "reward_std": 0.8415164649486542,
      "rewards/correctness_reward/mean": -0.5833333432674408,
      "rewards/correctness_reward/std": 0.841516524553299,
      "sampling/importance_sampling_ratio/max": 2.3461341857910156,
      "sampling/importance_sampling_ratio/mean": 0.5733579099178314,
      "sampling/importance_sampling_ratio/min": 0.04074254631996155,
      "sampling/sampling_logp_difference/max": 0.6068271398544312,
      "sampling/sampling_logp_difference/mean": 0.008984747808426619,
      "step": 67,
      "step_time": 141.14267089701025
    },
    {
      "clip_ratio/high_max": 0.0001881080479506636,
      "clip_ratio/high_mean": 0.0001881080479506636,
      "clip_ratio/low_mean": 0.00042036333616124466,
      "clip_ratio/low_min": 0.00042036333616124466,
      "clip_ratio/region_mean": 0.0006084713841119083,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4071.5,
      "completions/max_terminated_length": 2543.5,
      "completions/mean_length": 1812.375,
      "completions/mean_terminated_length": 1657.6931762695312,
      "completions/min_length": 895.0,
      "completions/min_terminated_length": 895.0,
      "entropy": 0.16869750246405602,
      "epoch": 0.4641638225255973,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.06864888221025467,
      "learning_rate": 2.750129109241286e-07,
      "loss": -0.16236957907676697,
      "num_tokens": 5704339.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446329772472382,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 2.2440454363822937,
      "sampling/importance_sampling_ratio/mean": 0.7120912075042725,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.5913791656494141,
      "sampling/sampling_logp_difference/mean": 0.008479172829538584,
      "step": 68,
      "step_time": 127.05132082500495
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4289.0,
      "completions/max_terminated_length": 3739.0,
      "completions/mean_length": 2125.7500610351562,
      "completions/mean_terminated_length": 1998.4622192382812,
      "completions/min_length": 785.5,
      "completions/min_terminated_length": 785.5,
      "entropy": 0.2187929842621088,
      "epoch": 0.4709897610921502,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.740715791020298e-07,
      "loss": 0.0,
      "num_tokens": 5796139.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.015613377094269,
      "sampling/importance_sampling_ratio/mean": 0.6376230865716934,
      "sampling/importance_sampling_ratio/min": 0.016034055734053254,
      "sampling/sampling_logp_difference/max": 0.5144100189208984,
      "sampling/sampling_logp_difference/mean": 0.009602179750800133,
      "step": 69,
      "step_time": 140.69199190499057
    },
    {
      "clip_ratio/high_max": 7.757430648780428e-05,
      "clip_ratio/high_mean": 7.757430648780428e-05,
      "clip_ratio/low_mean": 0.0004446139955689432,
      "clip_ratio/low_min": 0.0004446139955689432,
      "clip_ratio/region_mean": 0.0005221882984187687,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4236.5,
      "completions/max_terminated_length": 3391.5,
      "completions/mean_length": 2219.5000610351562,
      "completions/mean_terminated_length": 2107.5530395507812,
      "completions/min_length": 1279.5,
      "completions/min_terminated_length": 1279.5,
      "entropy": 0.1923739816993475,
      "epoch": 0.4778156996587031,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.0228436142206192,
      "learning_rate": 2.731145161810915e-07,
      "loss": -0.00633201003074646,
      "num_tokens": 5885617.0,
      "reward": -0.7500000298023224,
      "reward_std": 0.677924633026123,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.6779246032238007,
      "sampling/importance_sampling_ratio/max": 1.157751977443695,
      "sampling/importance_sampling_ratio/mean": 0.45236852765083313,
      "sampling/importance_sampling_ratio/min": 0.06210612959694117,
      "sampling/sampling_logp_difference/max": 1.0103888511657715,
      "sampling/sampling_logp_difference/mean": 0.009076605550944805,
      "step": 70,
      "step_time": 136.6656262119941
    },
    {
      "clip_ratio/high_max": 0.0002730607448029332,
      "clip_ratio/high_mean": 0.0002730607448029332,
      "clip_ratio/low_mean": 0.0010750940673460718,
      "clip_ratio/low_min": 0.0010750940673460718,
      "clip_ratio/region_mean": 0.0013481548157869838,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3523.0,
      "completions/max_terminated_length": 3138.0,
      "completions/mean_length": 2051.291748046875,
      "completions/mean_terminated_length": 1816.800048828125,
      "completions/min_length": 989.5,
      "completions/min_terminated_length": 989.5,
      "entropy": 0.19321701675653458,
      "epoch": 0.48464163822525597,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07435967028141022,
      "learning_rate": 2.7214184350780987e-07,
      "loss": -0.22458279132843018,
      "num_tokens": 5969330.0,
      "reward": -0.1666666716337204,
      "reward_std": 1.0298572778701782,
      "rewards/correctness_reward/mean": -0.1666666716337204,
      "rewards/correctness_reward/std": 1.0298572778701782,
      "sampling/importance_sampling_ratio/max": 2.1131256818771362,
      "sampling/importance_sampling_ratio/mean": 0.6783826649188995,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.5828158259391785,
      "sampling/sampling_logp_difference/mean": 0.00917751993983984,
      "step": 71,
      "step_time": 117.55696539898781
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.00040589617856312543,
      "clip_ratio/low_min": 0.00040589617856312543,
      "clip_ratio/region_mean": 0.00040589617856312543,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3371.5,
      "completions/max_terminated_length": 3371.5,
      "completions/mean_length": 1673.375,
      "completions/mean_terminated_length": 1673.375,
      "completions/min_length": 821.5,
      "completions/min_terminated_length": 821.5,
      "entropy": 0.18138444982469082,
      "epoch": 0.49146757679180886,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.07799665629863739,
      "learning_rate": 2.711536844078497e-07,
      "loss": -0.00800640881061554,
      "num_tokens": 6043151.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.28867512941360474,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.7861913442611694,
      "sampling/importance_sampling_ratio/mean": 0.6217601448297501,
      "sampling/importance_sampling_ratio/min": 0.031388456001877785,
      "sampling/sampling_logp_difference/max": 0.6879630088806152,
      "sampling/sampling_logp_difference/mean": 0.008580193389207125,
      "step": 72,
      "step_time": 109.68210411000473
    },
    {
      "clip_ratio/high_max": 0.00011866039858432487,
      "clip_ratio/high_mean": 0.00011866039858432487,
      "clip_ratio/low_mean": 0.0003503719690343132,
      "clip_ratio/low_min": 0.0003503719690343132,
      "clip_ratio/region_mean": 0.0004690323676186381,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 5204.5,
      "completions/max_terminated_length": 4409.5,
      "completions/mean_length": 2428.5001220703125,
      "completions/mean_terminated_length": 2314.700927734375,
      "completions/min_length": 777.5,
      "completions/min_terminated_length": 777.5,
      "entropy": 0.22688594833016396,
      "epoch": 0.49829351535836175,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.028718864545226097,
      "learning_rate": 2.7015016417040766e-07,
      "loss": 0.02665596641600132,
      "num_tokens": 6135209.0,
      "reward": -0.7500000298023224,
      "reward_std": 0.6779246032238007,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.6779246032238007,
      "sampling/importance_sampling_ratio/max": 0.949919581413269,
      "sampling/importance_sampling_ratio/mean": 0.30701562762260437,
      "sampling/importance_sampling_ratio/min": 0.004413304850459099,
      "sampling/sampling_logp_difference/max": 0.6100250482559204,
      "sampling/sampling_logp_difference/mean": 0.009902372024953365,
      "step": 73,
      "step_time": 162.6344251230039
    },
    {
      "clip_ratio/high_max": 0.0002700084096431965,
      "clip_ratio/high_mean": 0.0002700084096431965,
      "clip_ratio/low_mean": 0.0005516534420166863,
      "clip_ratio/low_min": 0.0005516534420166863,
      "clip_ratio/region_mean": 0.0008216618552978616,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3201.0,
      "completions/max_terminated_length": 3201.0,
      "completions/mean_length": 1547.125,
      "completions/mean_terminated_length": 1547.125,
      "completions/min_length": 702.0,
      "completions/min_terminated_length": 702.0,
      "entropy": 0.1647388245910406,
      "epoch": 0.5051194539249146,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.05796932429075241,
      "learning_rate": 2.691314100323268e-07,
      "loss": -0.05011383071541786,
      "num_tokens": 6201476.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446330070495605,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 2.1914119720458984,
      "sampling/importance_sampling_ratio/mean": 0.5255703777074814,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.827777624130249,
      "sampling/sampling_logp_difference/mean": 0.008731815032660961,
      "step": 74,
      "step_time": 102.39683082199917
    },
    {
      "clip_ratio/high_max": 0.0002842457906808704,
      "clip_ratio/high_mean": 0.0002842457906808704,
      "clip_ratio/low_mean": 0.0002800585643853992,
      "clip_ratio/low_min": 0.0002800585643853992,
      "clip_ratio/region_mean": 0.0005643043550662696,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4044.5,
      "completions/max_terminated_length": 4044.5,
      "completions/mean_length": 1764.3750610351562,
      "completions/mean_terminated_length": 1764.3750610351562,
      "completions/min_length": 800.0,
      "completions/min_terminated_length": 800.0,
      "entropy": 0.19630672596395016,
      "epoch": 0.5119453924914675,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.04505336284637451,
      "learning_rate": 2.680975511619642e-07,
      "loss": -0.05365559831261635,
      "num_tokens": 6276863.0,
      "reward": -0.5000000149011612,
      "reward_std": 0.8816154599189758,
      "rewards/correctness_reward/mean": -0.5000000149011612,
      "rewards/correctness_reward/std": 0.8816154599189758,
      "sampling/importance_sampling_ratio/max": 2.4316436648368835,
      "sampling/importance_sampling_ratio/mean": 0.6821621358394623,
      "sampling/importance_sampling_ratio/min": 0.060118645429611206,
      "sampling/sampling_logp_difference/max": 0.9917401075363159,
      "sampling/sampling_logp_difference/mean": 0.009168979246169329,
      "step": 75,
      "step_time": 128.50791397599096
    },
    {
      "clip_ratio/high_max": 8.07665528554935e-05,
      "clip_ratio/high_mean": 8.07665528554935e-05,
      "clip_ratio/low_mean": 0.0001678744811215438,
      "clip_ratio/low_min": 0.0001678744811215438,
      "clip_ratio/region_mean": 0.0002486410339770373,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 4977.0,
      "completions/max_terminated_length": 3904.5,
      "completions/mean_length": 2161.7501220703125,
      "completions/mean_terminated_length": 1859.7833862304688,
      "completions/min_length": 882.0,
      "completions/min_terminated_length": 882.0,
      "entropy": 0.16465575248003006,
      "epoch": 0.5187713310580204,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.0189689751714468,
      "learning_rate": 2.670487186428138e-07,
      "loss": 0.09658992290496826,
      "num_tokens": 6362699.0,
      "reward": -0.75,
      "reward_std": 0.45226702094078064,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.45226702094078064,
      "sampling/importance_sampling_ratio/max": 1.0848638117313385,
      "sampling/importance_sampling_ratio/mean": 0.5145403891801834,
      "sampling/importance_sampling_ratio/min": 0.04004676640033722,
      "sampling/sampling_logp_difference/max": 0.589957058429718,
      "sampling/sampling_logp_difference/mean": 0.007860165322199464,
      "step": 76,
      "step_time": 154.08570145599515
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.2083333432674408,
      "completions/max_length": 4109.0,
      "completions/max_terminated_length": 3585.5,
      "completions/mean_length": 2682.041748046875,
      "completions/mean_terminated_length": 2152.9049072265625,
      "completions/min_length": 1140.0,
      "completions/min_terminated_length": 1140.0,
      "entropy": 0.18603627104312181,
      "epoch": 0.5255972696245734,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.659850454568859e-07,
      "loss": 0.0,
      "num_tokens": 6463632.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/correctness_reward/mean": -1.0,
      "rewards/correctness_reward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0550328493118286,
      "sampling/importance_sampling_ratio/mean": 0.7448717653751373,
      "sampling/importance_sampling_ratio/min": 0.004171021748334169,
      "sampling/sampling_logp_difference/max": 0.5743434429168701,
      "sampling/sampling_logp_difference/mean": 0.008833530824631453,
      "step": 77,
      "step_time": 138.94735664299515
    },
    {
      "clip_ratio/high_max": 0.00019208645971957594,
      "clip_ratio/high_mean": 0.00019208645971957594,
      "clip_ratio/low_mean": 0.0003050619870919036,
      "clip_ratio/low_min": 0.0003050619870919036,
      "clip_ratio/region_mean": 0.0004971484468114795,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3626.0,
      "completions/max_terminated_length": 3626.0,
      "completions/mean_length": 1688.7500610351562,
      "completions/mean_terminated_length": 1688.7500610351562,
      "completions/min_length": 717.0,
      "completions/min_terminated_length": 717.0,
      "entropy": 0.19578155875205994,
      "epoch": 0.5324232081911263,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.08791548758745193,
      "learning_rate": 2.6490666646784667e-07,
      "loss": -0.6725838780403137,
      "num_tokens": 6537786.0,
      "reward": -0.416666679084301,
      "reward_std": 0.904178112745285,
      "rewards/correctness_reward/mean": -0.416666679084301,
      "rewards/correctness_reward/std": 0.9041781425476074,
      "sampling/importance_sampling_ratio/max": 1.8620446026325226,
      "sampling/importance_sampling_ratio/mean": 0.5930303931236267,
      "sampling/importance_sampling_ratio/min": 0.021932126954197884,
      "sampling/sampling_logp_difference/max": 0.49617767333984375,
      "sampling/sampling_logp_difference/mean": 0.00903140613809228,
      "step": 78,
      "step_time": 116.53287122999609
    },
    {
      "clip_ratio/high_max": 0.00013660525837622117,
      "clip_ratio/high_mean": 0.00013660525837622117,
      "clip_ratio/low_mean": 0.00034289806717424653,
      "clip_ratio/low_min": 0.00034289806717424653,
      "clip_ratio/region_mean": 0.0004795033182745101,
      "completions/clipped_ratio": 0.1250000037252903,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 4759.0,
      "completions/mean_length": 2415.4583740234375,
      "completions/mean_terminated_length": 2004.5682373046875,
      "completions/min_length": 896.5,
      "completions/min_terminated_length": 896.5,
      "entropy": 0.19903319235891104,
      "epoch": 0.5392491467576792,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.012782029807567596,
      "learning_rate": 2.6381371840391864e-07,
      "loss": 0.03598427399992943,
      "num_tokens": 6634505.0,
      "reward": -0.583333358168602,
      "reward_std": 0.7810410857200623,
      "rewards/correctness_reward/mean": -0.5833333283662796,
      "rewards/correctness_reward/std": 0.7810411155223846,
      "sampling/importance_sampling_ratio/max": 1.7369353771209717,
      "sampling/importance_sampling_ratio/mean": 0.5799808204174042,
      "sampling/importance_sampling_ratio/min": 0.027315656188875437,
      "sampling/sampling_logp_difference/max": 0.6335808634757996,
      "sampling/sampling_logp_difference/mean": 0.009694334119558334,
      "step": 79,
      "step_time": 165.85728075699444
    },
    {
      "clip_ratio/high_max": 0.00034807420161087066,
      "clip_ratio/high_mean": 0.00034807420161087066,
      "clip_ratio/low_mean": 0.001164372471976094,
      "clip_ratio/low_min": 0.001164372471976094,
      "clip_ratio/region_mean": 0.0015124466590350494,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2498.5,
      "completions/max_terminated_length": 2498.5,
      "completions/mean_length": 1373.4583740234375,
      "completions/mean_terminated_length": 1373.4583740234375,
      "completions/min_length": 766.5,
      "completions/min_terminated_length": 766.5,
      "entropy": 0.16406875103712082,
      "epoch": 0.5460750853242321,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.09411022067070007,
      "learning_rate": 2.627063398405445e-07,
      "loss": -0.2403782606124878,
      "num_tokens": 6699142.0,
      "reward": -0.5833333432674408,
      "reward_std": 0.841516524553299,
      "rewards/correctness_reward/mean": -0.5833333432674408,
      "rewards/correctness_reward/std": 0.841516524553299,
      "sampling/importance_sampling_ratio/max": 1.8231101036071777,
      "sampling/importance_sampling_ratio/mean": 0.6702441573143005,
      "sampling/importance_sampling_ratio/min": 0.01660044677555561,
      "sampling/sampling_logp_difference/max": 0.6353662014007568,
      "sampling/sampling_logp_difference/mean": 0.0088970260694623,
      "step": 80,
      "step_time": 84.66325323101773
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.291666679084301,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 2788.5,
      "completions/mean_length": 2848.166748046875,
      "completions/mean_terminated_length": 1871.357177734375,
      "completions/min_length": 853.0,
      "completions/min_terminated_length": 853.0,
      "entropy": 0.14244998339563608,
      "epoch": 0.552901023890785,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.6158467118281766e-07,
      "loss": 0.0,
      "num_tokens": 6799250.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 1.8679000735282898,
      "sampling/importance_sampling_ratio/mean": 0.7701037526130676,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.7022671699523926,
      "sampling/sampling_logp_difference/mean": 0.008580916561186314,
      "step": 81,
      "step_time": 165.6021267400065
    },
    {
      "clip_ratio/high_max": 1.6419282474089414e-05,
      "clip_ratio/high_mean": 1.6419282474089414e-05,
      "clip_ratio/low_mean": 0.00016853575289133005,
      "clip_ratio/low_min": 0.00016853575289133005,
      "clip_ratio/region_mean": 0.00018495503536541946,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2679.5,
      "completions/max_terminated_length": 2679.5,
      "completions/mean_length": 1440.25,
      "completions/mean_terminated_length": 1440.25,
      "completions/min_length": 905.5,
      "completions/min_terminated_length": 905.5,
      "entropy": 0.15635963436216116,
      "epoch": 0.5597269624573379,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.009361106902360916,
      "learning_rate": 2.6044885464767957e-07,
      "loss": 0.08625303953886032,
      "num_tokens": 6865784.0,
      "reward": -0.4166666641831398,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.4166666641831398,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 1.5225063860416412,
      "sampling/importance_sampling_ratio/mean": 0.482978492975235,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0508806705474854,
      "sampling/sampling_logp_difference/mean": 0.008569143246859312,
      "step": 82,
      "step_time": 88.88227481501235
    },
    {
      "clip_ratio/high_max": 0.00012391141262924066,
      "clip_ratio/high_mean": 0.00012391141262924066,
      "clip_ratio/low_mean": 2.3360120394499972e-05,
      "clip_ratio/low_min": 2.3360120394499972e-05,
      "clip_ratio/region_mean": 0.00014727153302374063,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4927.0,
      "completions/max_terminated_length": 4851.0,
      "completions/mean_length": 2604.541748046875,
      "completions/mean_terminated_length": 2506.9129638671875,
      "completions/min_length": 1156.0,
      "completions/min_terminated_length": 1156.0,
      "entropy": 0.222346480935812,
      "epoch": 0.5665529010238908,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.009380086325109005,
      "learning_rate": 2.5929903424588857e-07,
      "loss": -0.015079451724886894,
      "num_tokens": 6957447.0,
      "reward": -0.6666666716337204,
      "reward_std": 0.4923659563064575,
      "rewards/correctness_reward/mean": -0.6666666716337204,
      "rewards/correctness_reward/std": 0.4923659861087799,
      "sampling/importance_sampling_ratio/max": 1.3343388438224792,
      "sampling/importance_sampling_ratio/mean": 0.4296713322401047,
      "sampling/importance_sampling_ratio/min": 0.014223325997591019,
      "sampling/sampling_logp_difference/max": 0.621342658996582,
      "sampling/sampling_logp_difference/mean": 0.009445840958505869,
      "step": 83,
      "step_time": 154.10503394300758
    },
    {
      "clip_ratio/high_max": 3.4239539672853425e-05,
      "clip_ratio/high_mean": 3.4239539672853425e-05,
      "clip_ratio/low_mean": 0.0003129054239252582,
      "clip_ratio/low_min": 0.0003129054239252582,
      "clip_ratio/region_mean": 0.00034714496359811164,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3495.0,
      "completions/max_terminated_length": 2862.0,
      "completions/mean_length": 1840.041748046875,
      "completions/mean_terminated_length": 1710.3182373046875,
      "completions/min_length": 768.0,
      "completions/min_terminated_length": 768.0,
      "entropy": 0.20816332660615444,
      "epoch": 0.5733788395904437,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.06777017563581467,
      "learning_rate": 2.581353557637602e-07,
      "loss": -0.5427741408348083,
      "num_tokens": 7043392.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.3892494738101959,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 2.5436028242111206,
      "sampling/importance_sampling_ratio/mean": 0.7241137623786926,
      "sampling/importance_sampling_ratio/min": 0.014458982273936272,
      "sampling/sampling_logp_difference/max": 0.6590601205825806,
      "sampling/sampling_logp_difference/mean": 0.00995600363239646,
      "step": 84,
      "step_time": 117.45463798900164
    },
    {
      "clip_ratio/high_max": 0.00032514029953745194,
      "clip_ratio/high_mean": 0.00032514029953745194,
      "clip_ratio/low_mean": 5.560498175327666e-05,
      "clip_ratio/low_min": 5.560498175327666e-05,
      "clip_ratio/region_mean": 0.0003807452812907286,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 3557.5,
      "completions/mean_length": 2173.166748046875,
      "completions/mean_terminated_length": 1893.6364135742188,
      "completions/min_length": 872.0,
      "completions/min_terminated_length": 872.0,
      "entropy": 0.21564749628305435,
      "epoch": 0.5802047781569966,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.009651628322899342,
      "learning_rate": 2.56957966744683e-07,
      "loss": 0.011977184563875198,
      "num_tokens": 7132724.0,
      "reward": -0.5833333358168602,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.5833333358168602,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 1.0573405623435974,
      "sampling/importance_sampling_ratio/mean": 0.36841432750225067,
      "sampling/importance_sampling_ratio/min": 0.052347431890666485,
      "sampling/sampling_logp_difference/max": 0.855263888835907,
      "sampling/sampling_logp_difference/mean": 0.009965576231479645,
      "step": 85,
      "step_time": 162.9061557780151
    },
    {
      "clip_ratio/high_max": 4.3267566070426255e-05,
      "clip_ratio/high_mean": 4.3267566070426255e-05,
      "clip_ratio/low_mean": 0.00032204676244873554,
      "clip_ratio/low_min": 0.00032204676244873554,
      "clip_ratio/region_mean": 0.0003653143357951194,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3607.0,
      "completions/max_terminated_length": 3038.5,
      "completions/mean_length": 2108.3334350585938,
      "completions/mean_terminated_length": 1769.8472290039062,
      "completions/min_length": 1082.0,
      "completions/min_terminated_length": 1082.0,
      "entropy": 0.19981952756643295,
      "epoch": 0.5870307167235495,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.04601443558931351,
      "learning_rate": 2.557670164704119e-07,
      "loss": 0.07306678593158722,
      "num_tokens": 7226698.0,
      "reward": -0.4166666641831398,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.4166666641831398,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 1.9394845962524414,
      "sampling/importance_sampling_ratio/mean": 0.650184690952301,
      "sampling/importance_sampling_ratio/min": 0.03433588892221451,
      "sampling/sampling_logp_difference/max": 0.633596658706665,
      "sampling/sampling_logp_difference/mean": 0.009664933197200298,
      "step": 86,
      "step_time": 123.40111500900093
    },
    {
      "clip_ratio/high_max": 0.00013916854732087813,
      "clip_ratio/high_mean": 0.00013916854732087813,
      "clip_ratio/low_mean": 0.0005129784458404174,
      "clip_ratio/low_min": 0.0005129784458404174,
      "clip_ratio/region_mean": 0.0006521469986182638,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2699.0,
      "completions/max_terminated_length": 2699.0,
      "completions/mean_length": 1503.5000610351562,
      "completions/mean_terminated_length": 1503.5000610351562,
      "completions/min_length": 725.0,
      "completions/min_terminated_length": 725.0,
      "entropy": 0.17778978310525417,
      "epoch": 0.5938566552901023,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.06781456619501114,
      "learning_rate": 2.5456265594213984e-07,
      "loss": -0.1386936753988266,
      "num_tokens": 7291168.0,
      "reward": -0.5833333432674408,
      "reward_std": 0.8415164947509766,
      "rewards/correctness_reward/mean": -0.5833333432674408,
      "rewards/correctness_reward/std": 0.8415164947509766,
      "sampling/importance_sampling_ratio/max": 1.674412488937378,
      "sampling/importance_sampling_ratio/mean": 0.516161248087883,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.7146804332733154,
      "sampling/sampling_logp_difference/mean": 0.00882220035418868,
      "step": 87,
      "step_time": 89.02695259900065
    },
    {
      "clip_ratio/high_max": 4.265726238372736e-05,
      "clip_ratio/high_mean": 4.265726238372736e-05,
      "clip_ratio/low_mean": 0.0002642589679453522,
      "clip_ratio/low_min": 0.0002642589679453522,
      "clip_ratio/region_mean": 0.00030691623032907955,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3686.5,
      "completions/max_terminated_length": 3596.0,
      "completions/mean_length": 1874.291748046875,
      "completions/mean_terminated_length": 1750.0682373046875,
      "completions/min_length": 1003.5,
      "completions/min_terminated_length": 1003.5,
      "entropy": 0.1829511933028698,
      "epoch": 0.6006825938566553,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.012510834261775017,
      "learning_rate": 2.5334503786135296e-07,
      "loss": 0.06788667291402817,
      "num_tokens": 7379951.0,
      "reward": -0.4166666865348816,
      "reward_std": 0.8109080791473389,
      "rewards/correctness_reward/mean": -0.4166666567325592,
      "rewards/correctness_reward/std": 0.8109080791473389,
      "sampling/importance_sampling_ratio/max": 1.515598714351654,
      "sampling/importance_sampling_ratio/mean": 0.47678041458129883,
      "sampling/importance_sampling_ratio/min": 0.011095719411969185,
      "sampling/sampling_logp_difference/max": 0.580707848072052,
      "sampling/sampling_logp_difference/mean": 0.00906964810565114,
      "step": 88,
      "step_time": 123.02009241499036
    },
    {
      "clip_ratio/high_max": 9.884079736366402e-05,
      "clip_ratio/high_mean": 9.884079736366402e-05,
      "clip_ratio/low_mean": 0.00012933203470311128,
      "clip_ratio/low_min": 0.00012933203470311128,
      "clip_ratio/region_mean": 0.0002281728320667753,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3009.5,
      "completions/max_terminated_length": 3009.5,
      "completions/mean_length": 1605.9166870117188,
      "completions/mean_terminated_length": 1605.9166870117188,
      "completions/min_length": 824.5,
      "completions/min_terminated_length": 824.5,
      "entropy": 0.16808602958917618,
      "epoch": 0.6075085324232082,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.03892357274889946,
      "learning_rate": 2.521143166104692e-07,
      "loss": -0.304046094417572,
      "num_tokens": 7451169.0,
      "reward": -0.75,
      "reward_std": 0.45226702094078064,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.45226702094078064,
      "sampling/importance_sampling_ratio/max": 1.7958517670631409,
      "sampling/importance_sampling_ratio/mean": 0.7084388136863708,
      "sampling/importance_sampling_ratio/min": 0.08083060570061207,
      "sampling/sampling_logp_difference/max": 0.5716913342475891,
      "sampling/sampling_logp_difference/mean": 0.008326051756739616,
      "step": 89,
      "step_time": 100.39591565699811
    },
    {
      "clip_ratio/high_max": 0.0003322830416436773,
      "clip_ratio/high_mean": 0.0003322830416436773,
      "clip_ratio/low_mean": 0.0004085483597009443,
      "clip_ratio/low_min": 0.0004085483597009443,
      "clip_ratio/region_mean": 0.0007408313977066427,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4037.5,
      "completions/max_terminated_length": 2841.0,
      "completions/mean_length": 1738.7500610351562,
      "completions/mean_terminated_length": 1585.0606689453125,
      "completions/min_length": 826.5,
      "completions/min_terminated_length": 826.5,
      "entropy": 0.1912026945501566,
      "epoch": 0.6143344709897611,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.02759329229593277,
      "learning_rate": 2.50870648233264e-07,
      "loss": 0.1086440235376358,
      "num_tokens": 7526013.0,
      "reward": -0.5000000149011612,
      "reward_std": 0.8816154599189758,
      "rewards/correctness_reward/mean": -0.5000000149011612,
      "rewards/correctness_reward/std": 0.8816154301166534,
      "sampling/importance_sampling_ratio/max": 2.0824206471443176,
      "sampling/importance_sampling_ratio/mean": 0.7261651158332825,
      "sampling/importance_sampling_ratio/min": 0.018040256574749947,
      "sampling/sampling_logp_difference/max": 0.7164096236228943,
      "sampling/sampling_logp_difference/mean": 0.009469267446547747,
      "step": 90,
      "step_time": 128.0492773010119
    },
    {
      "clip_ratio/high_max": 0.0001939719804795459,
      "clip_ratio/high_mean": 0.0001939719804795459,
      "clip_ratio/low_mean": 0.0007475265156244859,
      "clip_ratio/low_min": 0.0007475265156244859,
      "clip_ratio/region_mean": 0.0009414984961040318,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3000.0,
      "completions/max_terminated_length": 3000.0,
      "completions/mean_length": 1316.6250610351562,
      "completions/mean_terminated_length": 1316.6250610351562,
      "completions/min_length": 819.5,
      "completions/min_terminated_length": 819.5,
      "entropy": 0.1598931821063161,
      "epoch": 0.621160409556314,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.08526404947042465,
      "learning_rate": 2.496141904150859e-07,
      "loss": -0.23110927641391754,
      "num_tokens": 7589220.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.3892494738101959,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.29092675447464,
      "sampling/importance_sampling_ratio/mean": 0.47361135482788086,
      "sampling/importance_sampling_ratio/min": 0.030142774805426598,
      "sampling/sampling_logp_difference/max": 0.7877267003059387,
      "sampling/sampling_logp_difference/mean": 0.008897948078811169,
      "step": 91,
      "step_time": 97.83714397101721
    },
    {
      "clip_ratio/high_max": 0.0002154113790311385,
      "clip_ratio/high_mean": 0.0002154113790311385,
      "clip_ratio/low_mean": 0.0007493217526644003,
      "clip_ratio/low_min": 0.0007493217526644003,
      "clip_ratio/region_mean": 0.000964733137152507,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4933.5,
      "completions/max_terminated_length": 3842.5,
      "completions/mean_length": 1734.9583740234375,
      "completions/mean_terminated_length": 1582.9356689453125,
      "completions/min_length": 715.5,
      "completions/min_terminated_length": 715.5,
      "entropy": 0.18329482711851597,
      "epoch": 0.6279863481228669,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.10777463763952255,
      "learning_rate": 2.4834510246286263e-07,
      "loss": -0.3767480254173279,
      "num_tokens": 7665203.0,
      "reward": -0.4166666865348816,
      "reward_std": 0.8109081089496613,
      "rewards/correctness_reward/mean": -0.4166666567325592,
      "rewards/correctness_reward/std": 0.8109080791473389,
      "sampling/importance_sampling_ratio/max": 2.762686014175415,
      "sampling/importance_sampling_ratio/mean": 0.737210601568222,
      "sampling/importance_sampling_ratio/min": 0.0013398262672126293,
      "sampling/sampling_logp_difference/max": 0.8068320751190186,
      "sampling/sampling_logp_difference/mean": 0.009431012906134129,
      "step": 92,
      "step_time": 158.21941642699676
    },
    {
      "clip_ratio/high_max": 9.0035306129721e-05,
      "clip_ratio/high_mean": 9.0035306129721e-05,
      "clip_ratio/low_mean": 7.182532135630026e-05,
      "clip_ratio/low_min": 7.182532135630026e-05,
      "clip_ratio/region_mean": 0.00016186062748602126,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4526.0,
      "completions/max_terminated_length": 3674.5,
      "completions/mean_length": 1940.5833740234375,
      "completions/mean_terminated_length": 1800.1136474609375,
      "completions/min_length": 866.0,
      "completions/min_terminated_length": 866.0,
      "entropy": 0.2036778088659048,
      "epoch": 0.6348122866894198,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.019959157332777977,
      "learning_rate": 2.4706354528490365e-07,
      "loss": -0.13522621989250183,
      "num_tokens": 7746607.0,
      "reward": -0.5833333358168602,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.5833333358168602,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 1.7632864117622375,
      "sampling/importance_sampling_ratio/mean": 0.6962140649557114,
      "sampling/importance_sampling_ratio/min": 0.03561893478035927,
      "sampling/sampling_logp_difference/max": 0.5904067754745483,
      "sampling/sampling_logp_difference/mean": 0.009093993343412876,
      "step": 93,
      "step_time": 143.2192148319955
    },
    {
      "clip_ratio/high_max": 0.0002615741323097609,
      "clip_ratio/high_mean": 0.0002615741323097609,
      "clip_ratio/low_mean": 0.00030253883960540406,
      "clip_ratio/low_min": 0.00030253883960540406,
      "clip_ratio/region_mean": 0.0005641129682771862,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2530.5,
      "completions/max_terminated_length": 2530.5,
      "completions/mean_length": 1570.5,
      "completions/mean_terminated_length": 1570.5,
      "completions/min_length": 689.0,
      "completions/min_terminated_length": 689.0,
      "entropy": 0.18163765780627728,
      "epoch": 0.6416382252559727,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.03428397327661514,
      "learning_rate": 2.457696813704975e-07,
      "loss": 0.15175065398216248,
      "num_tokens": 7813495.0,
      "reward": -0.5833333358168602,
      "reward_std": 0.5149286389350891,
      "rewards/correctness_reward/mean": -0.5833333358168602,
      "rewards/correctness_reward/std": 0.5149286389350891,
      "sampling/importance_sampling_ratio/max": 1.7397316694259644,
      "sampling/importance_sampling_ratio/mean": 0.5945777893066406,
      "sampling/importance_sampling_ratio/min": 0.020148619078099728,
      "sampling/sampling_logp_difference/max": 0.7906639575958252,
      "sampling/sampling_logp_difference/mean": 0.009107701480388641,
      "step": 94,
      "step_time": 85.37202533299569
    },
    {
      "clip_ratio/high_max": 0.0002010115931625478,
      "clip_ratio/high_mean": 0.0002010115931625478,
      "clip_ratio/low_mean": 0.00027532511012395844,
      "clip_ratio/low_min": 0.00027532511012395844,
      "clip_ratio/region_mean": 0.00047633669964852743,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3140.5,
      "completions/max_terminated_length": 3140.5,
      "completions/mean_length": 1883.5,
      "completions/mean_terminated_length": 1883.5,
      "completions/min_length": 952.0,
      "completions/min_terminated_length": 952.0,
      "entropy": 0.19250520505011082,
      "epoch": 0.6484641638225256,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.04572583734989166,
      "learning_rate": 2.4446367476931036e-07,
      "loss": 0.1977643370628357,
      "num_tokens": 7889431.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446330070495605,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 2.768944263458252,
      "sampling/importance_sampling_ratio/mean": 0.7922740876674652,
      "sampling/importance_sampling_ratio/min": 0.029824020341038704,
      "sampling/sampling_logp_difference/max": 0.5023660659790039,
      "sampling/sampling_logp_difference/mean": 0.008869106881320477,
      "step": 95,
      "step_time": 103.88041676799912
    },
    {
      "clip_ratio/high_max": 3.110226316493936e-05,
      "clip_ratio/high_mean": 3.110226316493936e-05,
      "clip_ratio/low_mean": 0.00014555339657817967,
      "clip_ratio/low_min": 0.00014555339657817967,
      "clip_ratio/region_mean": 0.00017665565610514022,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4723.0,
      "completions/max_terminated_length": 4356.0,
      "completions/mean_length": 2388.125,
      "completions/mean_terminated_length": 2270.897705078125,
      "completions/min_length": 867.5,
      "completions/min_terminated_length": 867.5,
      "entropy": 0.2005954124033451,
      "epoch": 0.6552901023890785,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.009689918719232082,
      "learning_rate": 2.4314569107058573e-07,
      "loss": 0.007732723373919725,
      "num_tokens": 7979722.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.38924944400787354,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 1.914838969707489,
      "sampling/importance_sampling_ratio/mean": 0.43378978967666626,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.5895647406578064,
      "sampling/sampling_logp_difference/mean": 0.008874210994690657,
      "step": 96,
      "step_time": 149.69903900801
    },
    {
      "clip_ratio/high_max": 8.717645869182888e-05,
      "clip_ratio/high_mean": 8.717645869182888e-05,
      "clip_ratio/low_mean": 0.00030598924422520213,
      "clip_ratio/low_min": 0.00030598924422520213,
      "clip_ratio/region_mean": 0.0003931656974600628,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4195.5,
      "completions/max_terminated_length": 2847.5,
      "completions/mean_length": 1947.416748046875,
      "completions/mean_terminated_length": 1795.2197875976562,
      "completions/min_length": 882.5,
      "completions/min_terminated_length": 882.5,
      "entropy": 0.15587482787668705,
      "epoch": 0.6621160409556314,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.022968977689743042,
      "learning_rate": 2.418158973821494e-07,
      "loss": -0.006974846124649048,
      "num_tokens": 8060606.0,
      "reward": -0.6666666716337204,
      "reward_std": 0.4923659861087799,
      "rewards/correctness_reward/mean": -0.6666666716337204,
      "rewards/correctness_reward/std": 0.4923659563064575,
      "sampling/importance_sampling_ratio/max": 1.4343634843826294,
      "sampling/importance_sampling_ratio/mean": 0.5058686882257462,
      "sampling/importance_sampling_ratio/min": 0.018607817590236664,
      "sampling/sampling_logp_difference/max": 0.613651305437088,
      "sampling/sampling_logp_difference/mean": 0.007983496179804206,
      "step": 97,
      "step_time": 133.11254283601738
    },
    {
      "clip_ratio/high_max": 5.8293175243306905e-05,
      "clip_ratio/high_mean": 5.8293175243306905e-05,
      "clip_ratio/low_mean": 0.00016594905901001766,
      "clip_ratio/low_min": 0.00016594905901001766,
      "clip_ratio/region_mean": 0.00022424223425332457,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4004.0,
      "completions/max_terminated_length": 4002.5,
      "completions/mean_length": 2277.5834350585938,
      "completions/mean_terminated_length": 2169.5416870117188,
      "completions/min_length": 847.0,
      "completions/min_terminated_length": 847.0,
      "entropy": 0.16203098744153976,
      "epoch": 0.6689419795221843,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.013039804995059967,
      "learning_rate": 2.4047446230922195e-07,
      "loss": 0.12956568598747253,
      "num_tokens": 8148760.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.7119688391685486,
      "sampling/importance_sampling_ratio/mean": 0.514645516872406,
      "sampling/importance_sampling_ratio/min": 0.017148795071989298,
      "sampling/sampling_logp_difference/max": 0.690927267074585,
      "sampling/sampling_logp_difference/mean": 0.007868173997849226,
      "step": 98,
      "step_time": 130.74494820101245
    },
    {
      "clip_ratio/high_max": 7.856693991925567e-05,
      "clip_ratio/high_mean": 7.856693991925567e-05,
      "clip_ratio/low_mean": 0.0005291322886478156,
      "clip_ratio/low_min": 0.0005291322886478156,
      "clip_ratio/region_mean": 0.0006076992212911136,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1717.5,
      "completions/max_terminated_length": 1717.5,
      "completions/mean_length": 1047.8333740234375,
      "completions/mean_terminated_length": 1047.8333740234375,
      "completions/min_length": 759.0,
      "completions/min_terminated_length": 759.0,
      "entropy": 0.14885542448610067,
      "epoch": 0.6757679180887372,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.09916958212852478,
      "learning_rate": 2.391215559330409e-07,
      "loss": -0.1368696540594101,
      "num_tokens": 8202576.0,
      "reward": -0.75,
      "reward_std": 0.452267050743103,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.45226702094078064,
      "sampling/importance_sampling_ratio/max": 2.7344236373901367,
      "sampling/importance_sampling_ratio/mean": 0.7819750905036926,
      "sampling/importance_sampling_ratio/min": 0.033862002193927765,
      "sampling/sampling_logp_difference/max": 0.9169849455356598,
      "sampling/sampling_logp_difference/mean": 0.009436240419745445,
      "step": 99,
      "step_time": 60.29736274699826
    },
    {
      "clip_ratio/high_max": 6.98723870300455e-05,
      "clip_ratio/high_mean": 6.98723870300455e-05,
      "clip_ratio/low_mean": 0.0011807848823082168,
      "clip_ratio/low_min": 0.0011807848823082168,
      "clip_ratio/region_mean": 0.0012506572711572517,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3254.0,
      "completions/max_terminated_length": 3254.0,
      "completions/mean_length": 1745.5416870117188,
      "completions/mean_terminated_length": 1745.5416870117188,
      "completions/min_length": 971.0,
      "completions/min_terminated_length": 971.0,
      "entropy": 0.19362994097173214,
      "epoch": 0.6825938566552902,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.0453786700963974,
      "learning_rate": 2.3775734978929614e-07,
      "loss": 0.11239898949861526,
      "num_tokens": 8280883.0,
      "reward": -0.3333333507180214,
      "reward_std": 0.8036037981510162,
      "rewards/correctness_reward/mean": -0.333333320915699,
      "rewards/correctness_reward/std": 0.8036037683486938,
      "sampling/importance_sampling_ratio/max": 1.9463191032409668,
      "sampling/importance_sampling_ratio/mean": 0.6716760843992233,
      "sampling/importance_sampling_ratio/min": 0.09884109068661928,
      "sampling/sampling_logp_difference/max": 0.6764633655548096,
      "sampling/sampling_logp_difference/mean": 0.009771426673978567,
      "step": 100,
      "step_time": 108.78550151999661
    },
    {
      "clip_ratio/high_max": 4.9233836762141436e-05,
      "clip_ratio/high_mean": 4.9233836762141436e-05,
      "clip_ratio/low_mean": 0.0006928291040821932,
      "clip_ratio/low_min": 0.0006928291040821932,
      "clip_ratio/region_mean": 0.0007420629408443347,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 5040.5,
      "completions/max_terminated_length": 4454.0,
      "completions/mean_length": 2518.7501220703125,
      "completions/mean_terminated_length": 2111.8612060546875,
      "completions/min_length": 920.5,
      "completions/min_terminated_length": 920.5,
      "entropy": 0.1930021233856678,
      "epoch": 0.689419795221843,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.03779743239283562,
      "learning_rate": 2.3638201684638098e-07,
      "loss": -0.3188192844390869,
      "num_tokens": 8373763.0,
      "reward": -0.7500000298023224,
      "reward_std": 0.6779245734214783,
      "rewards/correctness_reward/mean": -0.75,
      "rewards/correctness_reward/std": 0.677924633026123,
      "sampling/importance_sampling_ratio/max": 1.8769155144691467,
      "sampling/importance_sampling_ratio/mean": 0.6328006386756897,
      "sampling/importance_sampling_ratio/min": 0.0016347093041986227,
      "sampling/sampling_logp_difference/max": 1.0652188658714294,
      "sampling/sampling_logp_difference/mean": 0.008982821833342314,
      "step": 101,
      "step_time": 157.72760972299875
    },
    {
      "clip_ratio/high_max": 0.0002456316269672243,
      "clip_ratio/high_mean": 0.0002456316269672243,
      "clip_ratio/low_mean": 0.0006088513764552772,
      "clip_ratio/low_min": 0.0006088513764552772,
      "clip_ratio/region_mean": 0.0008544830034225015,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 3448.0,
      "completions/max_terminated_length": 3441.5,
      "completions/mean_length": 1830.3333740234375,
      "completions/mean_terminated_length": 1571.3250732421875,
      "completions/min_length": 923.0,
      "completions/min_terminated_length": 923.0,
      "entropy": 0.17076147999614477,
      "epoch": 0.6962457337883959,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.041363637894392014,
      "learning_rate": 2.3499573148346115e-07,
      "loss": 0.0010967161506414413,
      "num_tokens": 8450487.0,
      "reward": -0.6666666865348816,
      "reward_std": 0.7409421503543854,
      "rewards/correctness_reward/mean": -0.6666666567325592,
      "rewards/correctness_reward/std": 0.7409421503543854,
      "sampling/importance_sampling_ratio/max": 1.3017855882644653,
      "sampling/importance_sampling_ratio/mean": 0.47573068737983704,
      "sampling/importance_sampling_ratio/min": 0.011353067122399807,
      "sampling/sampling_logp_difference/max": 0.5613093376159668,
      "sampling/sampling_logp_difference/mean": 0.008331953082233667,
      "step": 102,
      "step_time": 113.68557855300605
    },
    {
      "clip_ratio/high_max": 0.00023342900021816604,
      "clip_ratio/high_mean": 0.00023342900021816604,
      "clip_ratio/low_mean": 0.0005807926718262024,
      "clip_ratio/low_min": 0.0005807926718262024,
      "clip_ratio/region_mean": 0.0008142216684063897,
      "completions/clipped_ratio": 0.1250000037252903,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 3850.5,
      "completions/mean_length": 2237.1666870117188,
      "completions/mean_terminated_length": 1812.9091186523438,
      "completions/min_length": 826.0,
      "completions/min_terminated_length": 826.0,
      "entropy": 0.20056550204753876,
      "epoch": 0.7030716723549488,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.04666848108172417,
      "learning_rate": 2.3359866946836543e-07,
      "loss": -0.10311692208051682,
      "num_tokens": 8540965.0,
      "reward": -0.583333358168602,
      "reward_std": 0.7810411155223846,
      "rewards/correctness_reward/mean": -0.5833333283662796,
      "rewards/correctness_reward/std": 0.7810411155223846,
      "sampling/importance_sampling_ratio/max": 1.8114920854568481,
      "sampling/importance_sampling_ratio/mean": 0.6123473793268204,
      "sampling/importance_sampling_ratio/min": 0.04999285563826561,
      "sampling/sampling_logp_difference/max": 0.6254404187202454,
      "sampling/sampling_logp_difference/mean": 0.009581032674759626,
      "step": 103,
      "step_time": 162.55260506199556
    },
    {
      "clip_ratio/high_max": 0.00010359214502386749,
      "clip_ratio/high_mean": 0.00010359214502386749,
      "clip_ratio/low_mean": 8.616077320766635e-05,
      "clip_ratio/low_min": 8.616077320766635e-05,
      "clip_ratio/region_mean": 0.00018975292186951265,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 5049.5,
      "completions/max_terminated_length": 4768.0,
      "completions/mean_length": 2366.416748046875,
      "completions/mean_terminated_length": 2242.166748046875,
      "completions/min_length": 899.0,
      "completions/min_terminated_length": 899.0,
      "entropy": 0.19171350076794624,
      "epoch": 0.7098976109215017,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.032151710242033005,
      "learning_rate": 2.3219100793529975e-07,
      "loss": 0.07015686482191086,
      "num_tokens": 8629127.0,
      "reward": -0.25,
      "reward_std": 0.9744999706745148,
      "rewards/correctness_reward/mean": -0.25,
      "rewards/correctness_reward/std": 0.9744999706745148,
      "sampling/importance_sampling_ratio/max": 2.378260016441345,
      "sampling/importance_sampling_ratio/mean": 0.6469814479351044,
      "sampling/importance_sampling_ratio/min": 0.00819753110408783,
      "sampling/sampling_logp_difference/max": 0.730820894241333,
      "sampling/sampling_logp_difference/mean": 0.008754108101129532,
      "step": 104,
      "step_time": 156.84601822699187
    },
    {
      "clip_ratio/high_max": 2.9079910746077076e-05,
      "clip_ratio/high_mean": 2.9079910746077076e-05,
      "clip_ratio/low_mean": 0.00017557437968207523,
      "clip_ratio/low_min": 0.00017557437968207523,
      "clip_ratio/region_mean": 0.0002046542867901735,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2998.0,
      "completions/max_terminated_length": 2998.0,
      "completions/mean_length": 1943.7916870117188,
      "completions/mean_terminated_length": 1943.7916870117188,
      "completions/min_length": 969.5,
      "completions/min_terminated_length": 969.5,
      "entropy": 0.1940737683326006,
      "epoch": 0.7167235494880546,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.02384648658335209,
      "learning_rate": 2.3077292536238835e-07,
      "loss": -0.05765508860349655,
      "num_tokens": 8708640.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.28867512941360474,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.6162993907928467,
      "sampling/importance_sampling_ratio/mean": 0.42033903300762177,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.9300663471221924,
      "sampling/sampling_logp_difference/mean": 0.008964573498815298,
      "step": 105,
      "step_time": 101.75178025999776
    },
    {
      "clip_ratio/high_max": 0.00018588699094834737,
      "clip_ratio/high_mean": 0.00018588699094834737,
      "clip_ratio/low_mean": 0.0007062332515488379,
      "clip_ratio/low_min": 0.0007062332515488379,
      "clip_ratio/region_mean": 0.000892120246135164,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3964.0,
      "completions/max_terminated_length": 3964.0,
      "completions/mean_length": 2023.166748046875,
      "completions/mean_terminated_length": 2023.166748046875,
      "completions/min_length": 870.0,
      "completions/min_terminated_length": 870.0,
      "entropy": 0.19571753405034542,
      "epoch": 0.7235494880546075,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.04298916086554527,
      "learning_rate": 2.2934460154904435e-07,
      "loss": -0.20362494885921478,
      "num_tokens": 8785792.0,
      "reward": -0.5000000223517418,
      "reward_std": 0.8036037981510162,
      "rewards/correctness_reward/mean": -0.4999999925494194,
      "rewards/correctness_reward/std": 0.8036037683486938,
      "sampling/importance_sampling_ratio/max": 1.5308769345283508,
      "sampling/importance_sampling_ratio/mean": 0.4027136564254761,
      "sampling/importance_sampling_ratio/min": 0.05640080850571394,
      "sampling/sampling_logp_difference/max": 0.7667943835258484,
      "sampling/sampling_logp_difference/mean": 0.00927647016942501,
      "step": 106,
      "step_time": 125.02470587198331
    },
    {
      "clip_ratio/high_max": 0.00026201896253041923,
      "clip_ratio/high_mean": 0.00026201896253041923,
      "clip_ratio/low_mean": 0.0005470625947054941,
      "clip_ratio/low_min": 0.0005470625947054941,
      "clip_ratio/region_mean": 0.0008090815663308604,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4992.5,
      "completions/max_terminated_length": 3967.5,
      "completions/mean_length": 1849.5,
      "completions/mean_terminated_length": 1689.727294921875,
      "completions/min_length": 721.0,
      "completions/min_terminated_length": 721.0,
      "entropy": 0.17433863878250122,
      "epoch": 0.7303754266211604,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.06949550658464432,
      "learning_rate": 2.2790621759317312e-07,
      "loss": -0.28901875019073486,
      "num_tokens": 8857900.0,
      "reward": -0.2500000074505806,
      "reward_std": 1.0072945952415466,
      "rewards/correctness_reward/mean": -0.2500000074505806,
      "rewards/correctness_reward/std": 1.007294625043869,
      "sampling/importance_sampling_ratio/max": 1.8129876852035522,
      "sampling/importance_sampling_ratio/mean": 0.8787103593349457,
      "sampling/importance_sampling_ratio/min": 0.06362062506377697,
      "sampling/sampling_logp_difference/max": 0.5592628717422485,
      "sampling/sampling_logp_difference/mean": 0.008585838600993156,
      "step": 107,
      "step_time": 149.98139648498181
    },
    {
      "clip_ratio/high_max": 0.0001897049278341001,
      "clip_ratio/high_mean": 0.0001897049278341001,
      "clip_ratio/low_mean": 0.0005273691203910857,
      "clip_ratio/low_min": 0.0005273691203910857,
      "clip_ratio/region_mean": 0.0007170740500441752,
      "completions/clipped_ratio": 0.2500000074505806,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 3779.5,
      "completions/mean_length": 2909.416748046875,
      "completions/mean_terminated_length": 2244.6500244140625,
      "completions/min_length": 1466.5,
      "completions/min_terminated_length": 1466.5,
      "entropy": 0.2094256915152073,
      "epoch": 0.7372013651877133,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.041358582675457,
      "learning_rate": 2.264579558682106e-07,
      "loss": -0.10254944115877151,
      "num_tokens": 8968760.0,
      "reward": -0.5833333432674408,
      "reward_std": 0.8415164947509766,
      "rewards/correctness_reward/mean": -0.5833333432674408,
      "rewards/correctness_reward/std": 0.8415164947509766,
      "sampling/importance_sampling_ratio/max": 1.5969599485397339,
      "sampling/importance_sampling_ratio/mean": 0.7714826762676239,
      "sampling/importance_sampling_ratio/min": 0.02449956108466722,
      "sampling/sampling_logp_difference/max": 0.5438909530639648,
      "sampling/sampling_logp_difference/mean": 0.009290697518736124,
      "step": 108,
      "step_time": 169.33149299799697
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0002111037993017817,
      "clip_ratio/low_min": 0.0002111037993017817,
      "clip_ratio/region_mean": 0.0002111037993017817,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 4061.5,
      "completions/max_terminated_length": 2890.5,
      "completions/mean_length": 1793.4583740234375,
      "completions/mean_terminated_length": 1470.6000366210938,
      "completions/min_length": 679.0,
      "completions/min_terminated_length": 679.0,
      "entropy": 0.15238594356924295,
      "epoch": 0.7440273037542662,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.004517465829849243,
      "learning_rate": 2.25e-07,
      "loss": 0.024714305996894836,
      "num_tokens": 9052189.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 1.6460945010185242,
      "sampling/importance_sampling_ratio/mean": 0.6706594079732895,
      "sampling/importance_sampling_ratio/min": 0.05606750026345253,
      "sampling/sampling_logp_difference/max": 0.6999292373657227,
      "sampling/sampling_logp_difference/mean": 0.008071616757661104,
      "step": 109,
      "step_time": 130.72493847899023
    },
    {
      "clip_ratio/high_max": 6.980315811233595e-05,
      "clip_ratio/high_mean": 6.980315811233595e-05,
      "clip_ratio/low_mean": 0.0002183696378779132,
      "clip_ratio/low_min": 0.0002183696378779132,
      "clip_ratio/region_mean": 0.00028817279235227033,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3514.5,
      "completions/max_terminated_length": 3514.5,
      "completions/mean_length": 1805.0000610351562,
      "completions/mean_terminated_length": 1805.0000610351562,
      "completions/min_length": 947.0,
      "completions/min_terminated_length": 947.0,
      "entropy": 0.16883638687431812,
      "epoch": 0.7508532423208191,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.01729542203247547,
      "learning_rate": 2.2353253484351e-07,
      "loss": -0.04131258651614189,
      "num_tokens": 9124381.0,
      "reward": -0.8333333432674408,
      "reward_std": 0.3892494738101959,
      "rewards/correctness_reward/mean": -0.8333333432674408,
      "rewards/correctness_reward/std": 0.3892495036125183,
      "sampling/importance_sampling_ratio/max": 2.4297763109207153,
      "sampling/importance_sampling_ratio/mean": 0.6028274744749069,
      "sampling/importance_sampling_ratio/min": 0.022295957431197166,
      "sampling/sampling_logp_difference/max": 0.6852223873138428,
      "sampling/sampling_logp_difference/mean": 0.008053431054577231,
      "step": 110,
      "step_time": 112.09678150700347
    },
    {
      "clip_ratio/high_max": 2.134927490260452e-05,
      "clip_ratio/high_mean": 2.134927490260452e-05,
      "clip_ratio/low_mean": 0.000559045140107628,
      "clip_ratio/low_min": 0.000559045140107628,
      "clip_ratio/region_mean": 0.0005803944150102325,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4755.0,
      "completions/max_terminated_length": 3513.0,
      "completions/mean_length": 1957.75,
      "completions/mean_terminated_length": 1804.0,
      "completions/min_length": 946.5,
      "completions/min_terminated_length": 946.5,
      "entropy": 0.16786883305758238,
      "epoch": 0.757679180887372,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.021987807005643845,
      "learning_rate": 2.220557464593968e-07,
      "loss": 0.15698328614234924,
      "num_tokens": 9208027.0,
      "reward": -0.8333333730697632,
      "reward_std": 0.5773502588272095,
      "rewards/correctness_reward/mean": -0.8333333134651184,
      "rewards/correctness_reward/std": 0.5773502588272095,
      "sampling/importance_sampling_ratio/max": 2.5532615184783936,
      "sampling/importance_sampling_ratio/mean": 0.5662012547254562,
      "sampling/importance_sampling_ratio/min": 0.01069966983050108,
      "sampling/sampling_logp_difference/max": 0.802309513092041,
      "sampling/sampling_logp_difference/mean": 0.008713751565665007,
      "step": 111,
      "step_time": 149.50963626700832
    },
    {
      "clip_ratio/high_max": 9.413296356797218e-05,
      "clip_ratio/high_mean": 9.413296356797218e-05,
      "clip_ratio/low_mean": 0.00085310242138803,
      "clip_ratio/low_min": 0.00085310242138803,
      "clip_ratio/region_mean": 0.0009472353849560022,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3690.0,
      "completions/max_terminated_length": 3690.0,
      "completions/mean_length": 1725.5000610351562,
      "completions/mean_terminated_length": 1725.5000610351562,
      "completions/min_length": 748.0,
      "completions/min_terminated_length": 748.0,
      "entropy": 0.19234910048544407,
      "epoch": 0.764505119453925,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.08581659942865372,
      "learning_rate": 2.2056982209041334e-07,
      "loss": -0.2922076880931854,
      "num_tokens": 9286429.0,
      "reward": -0.5000000149011612,
      "reward_std": 0.8816154599189758,
      "rewards/correctness_reward/mean": -0.5000000149011612,
      "rewards/correctness_reward/std": 0.8816154599189758,
      "sampling/importance_sampling_ratio/max": 2.0683520436286926,
      "sampling/importance_sampling_ratio/mean": 0.5630929172039032,
      "sampling/importance_sampling_ratio/min": 0.025849847123026848,
      "sampling/sampling_logp_difference/max": 0.6896476745605469,
      "sampling/sampling_logp_difference/mean": 0.009252662304788828,
      "step": 112,
      "step_time": 122.5797243149791
    },
    {
      "clip_ratio/high_max": 5.361355215427466e-05,
      "clip_ratio/high_mean": 5.361355215427466e-05,
      "clip_ratio/low_mean": 0.0003404093768040184,
      "clip_ratio/low_min": 0.0003404093768040184,
      "clip_ratio/region_mean": 0.00039402292895829305,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 4042.0,
      "completions/max_terminated_length": 3283.5,
      "completions/mean_length": 1776.7083740234375,
      "completions/mean_terminated_length": 1627.439453125,
      "completions/min_length": 730.5,
      "completions/min_terminated_length": 730.5,
      "entropy": 0.1821483038365841,
      "epoch": 0.7713310580204779,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.023123860359191895,
      "learning_rate": 2.1907495013766874e-07,
      "loss": -0.03904452547430992,
      "num_tokens": 9365520.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.28867512941360474,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 0.9768067002296448,
      "sampling/importance_sampling_ratio/mean": 0.32466551661491394,
      "sampling/importance_sampling_ratio/min": 0.0039460924454033375,
      "sampling/sampling_logp_difference/max": 0.5433354377746582,
      "sampling/sampling_logp_difference/mean": 0.009037391748279333,
      "step": 113,
      "step_time": 131.63761309701658
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 2787.5,
      "completions/mean_length": 2246.4583740234375,
      "completions/mean_terminated_length": 1973.5909423828125,
      "completions/min_length": 1119.0,
      "completions/min_terminated_length": 1119.0,
      "entropy": 0.19245805777609348,
      "epoch": 0.7781569965870307,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "learning_rate": 2.1757132013674093e-07,
      "loss": 0.0,
      "num_tokens": 9457013.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 2.2650750875473022,
      "sampling/importance_sampling_ratio/mean": 0.6504296064376831,
      "sampling/importance_sampling_ratio/min": 0.011082793585956097,
      "sampling/sampling_logp_difference/max": 0.6627502739429474,
      "sampling/sampling_logp_difference/mean": 0.009212277363985777,
      "step": 114,
      "step_time": 164.27334200701443
    },
    {
      "clip_ratio/high_max": 0.00012562813935801387,
      "clip_ratio/high_mean": 0.00012562813935801387,
      "clip_ratio/low_mean": 0.0004480607240111567,
      "clip_ratio/low_min": 0.0004480607240111567,
      "clip_ratio/region_mean": 0.0005736888633691706,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2268.0,
      "completions/max_terminated_length": 2268.0,
      "completions/mean_length": 1129.8333740234375,
      "completions/mean_terminated_length": 1129.8333740234375,
      "completions/min_length": 767.5,
      "completions/min_terminated_length": 767.5,
      "entropy": 0.1530211679637432,
      "epoch": 0.7849829351535836,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.17116226255893707,
      "learning_rate": 2.1605912273364515e-07,
      "loss": -0.6148641705513,
      "num_tokens": 9514183.0,
      "reward": -0.9166666865348816,
      "reward_std": 0.2886751592159271,
      "rewards/correctness_reward/mean": -0.9166666567325592,
      "rewards/correctness_reward/std": 0.28867512941360474,
      "sampling/importance_sampling_ratio/max": 2.204539656639099,
      "sampling/importance_sampling_ratio/mean": 0.6949987411499023,
      "sampling/importance_sampling_ratio/min": 0.050576210021972656,
      "sampling/sampling_logp_difference/max": 0.9088044762611389,
      "sampling/sampling_logp_difference/mean": 0.008726058527827263,
      "step": 115,
      "step_time": 75.91544955001154
    },
    {
      "clip_ratio/high_max": 0.0002909997583628865,
      "clip_ratio/high_mean": 0.0002909997583628865,
      "clip_ratio/low_mean": 0.0008542009891243652,
      "clip_ratio/low_min": 0.0008542009891243652,
      "clip_ratio/region_mean": 0.0011452007420302834,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2597.5,
      "completions/max_terminated_length": 2597.5,
      "completions/mean_length": 1472.3333740234375,
      "completions/mean_terminated_length": 1472.3333740234375,
      "completions/min_length": 778.5,
      "completions/min_terminated_length": 778.5,
      "entropy": 0.15164860244840384,
      "epoch": 0.7918088737201365,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.052857693284749985,
      "learning_rate": 2.145385496606619e-07,
      "loss": 0.2539215087890625,
      "num_tokens": 9578811.0,
      "reward": -0.5833333432674408,
      "reward_std": 0.8415164947509766,
      "rewards/correctness_reward/mean": -0.5833333432674408,
      "rewards/correctness_reward/std": 0.8415164947509766,
      "sampling/importance_sampling_ratio/max": 2.0988221168518066,
      "sampling/importance_sampling_ratio/mean": 0.6562928557395935,
      "sampling/importance_sampling_ratio/min": 0.020620742812752724,
      "sampling/sampling_logp_difference/max": 0.5115723609924316,
      "sampling/sampling_logp_difference/mean": 0.008505056146532297,
      "step": 116,
      "step_time": 86.72707642504247
    },
    {
      "clip_ratio/high_max": 0.0003095987376582343,
      "clip_ratio/high_mean": 0.0003095987376582343,
      "clip_ratio/low_mean": 0.0005687712473445572,
      "clip_ratio/low_min": 0.0005687712473445572,
      "clip_ratio/region_mean": 0.0008783699813648127,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4099.0,
      "completions/max_terminated_length": 4099.0,
      "completions/mean_length": 2552.791748046875,
      "completions/mean_terminated_length": 2552.791748046875,
      "completions/min_length": 804.5,
      "completions/min_terminated_length": 804.5,
      "entropy": 0.23589052446186543,
      "epoch": 0.7986348122866894,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.03518350422382355,
      "learning_rate": 2.1300979371202724e-07,
      "loss": 0.08621601015329361,
      "num_tokens": 9672556.0,
      "reward": -0.4166666716337204,
      "reward_std": 0.9446329772472382,
      "rewards/correctness_reward/mean": -0.4166666716337204,
      "rewards/correctness_reward/std": 0.9446330070495605,
      "sampling/importance_sampling_ratio/max": 1.4925788640975952,
      "sampling/importance_sampling_ratio/mean": 0.4850645363330841,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.696826696395874,
      "sampling/sampling_logp_difference/mean": 0.009785029105842113,
      "step": 117,
      "step_time": 134.44173820799915
    },
    {
      "clip_ratio/high_max": 0.00024063924229267286,
      "clip_ratio/high_mean": 0.00024063924229267286,
      "clip_ratio/low_mean": 0.0004523810584942112,
      "clip_ratio/low_min": 0.0004523810584942112,
      "clip_ratio/region_mean": 0.0006930202853254741,
      "completions/clipped_ratio": 0.0833333358168602,
      "completions/max_length": 5248.0,
      "completions/max_terminated_length": 4326.0,
      "completions/mean_length": 3003.916748046875,
      "completions/mean_terminated_length": 2799.9091796875,
      "completions/min_length": 1060.0,
      "completions/min_terminated_length": 1060.0,
      "entropy": 0.22496706247329712,
      "epoch": 0.8054607508532423,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.04817913472652435,
      "learning_rate": 2.1147304871948812e-07,
      "loss": -0.20130401849746704,
      "num_tokens": 9779246.0,
      "reward": -0.3333333358168602,
      "reward_std": 0.9671956598758698,
      "rewards/correctness_reward/mean": -0.3333333358168602,
      "rewards/correctness_reward/std": 0.9671956598758698,
      "sampling/importance_sampling_ratio/max": 2.08801132440567,
      "sampling/importance_sampling_ratio/mean": 0.5505248308181763,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.7242177128791809,
      "sampling/sampling_logp_difference/mean": 0.009443401359021664,
      "step": 118,
      "step_time": 168.65139187700697
    },
    {
      "clip_ratio/high_max": 5.621346281259321e-05,
      "clip_ratio/high_mean": 5.621346281259321e-05,
      "clip_ratio/low_mean": 0.0005924530651100213,
      "clip_ratio/low_min": 0.0005924530651100213,
      "clip_ratio/region_mean": 0.0006486665315605933,
      "completions/clipped_ratio": 0.0416666679084301,
      "completions/max_length": 3854.5,
      "completions/max_terminated_length": 3206.5,
      "completions/mean_length": 1865.7500610351562,
      "completions/mean_terminated_length": 1732.3485107421875,
      "completions/min_length": 898.0,
      "completions/min_terminated_length": 898.0,
      "entropy": 0.18109257612377405,
      "epoch": 0.8122866894197952,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.02632739767432213,
      "learning_rate": 2.0992850952772656e-07,
      "loss": -0.015607806853950024,
      "num_tokens": 9859526.0,
      "reward": -0.8333333730697632,
      "reward_std": 0.5773503184318542,
      "rewards/correctness_reward/mean": -0.8333333134651184,
      "rewards/correctness_reward/std": 0.5773502588272095,
      "sampling/importance_sampling_ratio/max": 1.622683048248291,
      "sampling/importance_sampling_ratio/mean": 0.5376181304454803,
      "sampling/importance_sampling_ratio/min": 0.06714884378015995,
      "sampling/sampling_logp_difference/max": 0.8372238874435425,
      "sampling/sampling_logp_difference/mean": 0.008855800610035658,
      "step": 119,
      "step_time": 125.9446228260058
    },
    {
      "clip_ratio/high_max": 0.00019927212451875675,
      "clip_ratio/high_mean": 0.00019927212451875675,
      "clip_ratio/low_mean": 0.0003339088070788421,
      "clip_ratio/low_min": 0.0003339088070788421,
      "clip_ratio/region_mean": 0.0005331809225026518,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3373.0,
      "completions/max_terminated_length": 3373.0,
      "completions/mean_length": 1848.75,
      "completions/mean_terminated_length": 1848.75,
      "completions/min_length": 1006.0,
      "completions/min_terminated_length": 1006.0,
      "entropy": 0.185127517208457,
      "epoch": 0.8191126279863481,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.03174300491809845,
      "learning_rate": 2.083763719696549e-07,
      "loss": 0.12570519745349884,
      "num_tokens": 9934550.0,
      "reward": -0.5,
      "reward_std": 0.5222329497337341,
      "rewards/correctness_reward/mean": -0.5,
      "rewards/correctness_reward/std": 0.5222329497337341,
      "sampling/importance_sampling_ratio/max": 1.1787127256393433,
      "sampling/importance_sampling_ratio/mean": 0.4706980586051941,
      "sampling/importance_sampling_ratio/min": 0.03853647783398628,
      "sampling/sampling_logp_difference/max": 0.7632927894592285,
      "sampling/sampling_logp_difference/mean": 0.00860992493107915,
      "step": 120,
      "step_time": 109.8505892399844
    }
  ],
  "logging_steps": 1,
  "max_steps": 294,
  "num_input_tokens_seen": 9934550,
  "num_train_epochs": 2,
  "save_steps": 60,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 3,
  "trial_name": null,
  "trial_params": null
}