diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,9234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.39564787339268054, + "eval_steps": 500, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 67.40625, + "completions/mean_terminated_length": 67.40625, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.19049232598626986, + "epoch": 0.0009891196834817012, + "grad_norm": 28.529056549072266, + "kl_approx": 0.3928680419921875, + "learning_rate": 0.0, + "loss": 0.6768, + "num_tokens": 22373.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.6535420417785645, + "sampling/importance_sampling_ratio/mean": 1.0012898445129395, + "sampling/importance_sampling_ratio/min": 0.6950725317001343, + "sampling/sampling_logp_difference/max": 0.5029196739196777, + "sampling/sampling_logp_difference/mean": 0.011248193681240082, + "step": 1 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 86.0, + "completions/max_terminated_length": 86.0, + "completions/mean_length": 54.59375, + "completions/mean_terminated_length": 54.59375, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "entropy": 0.17993419412096046, + "epoch": 0.0019782393669634025, + "grad_norm": 32.43034362792969, + "kl_approx": 0.3604269027709961, + "learning_rate": 1.9607843137254904e-07, + "loss": 0.5903, + "num_tokens": 46384.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.957443356513977, + "sampling/importance_sampling_ratio/mean": 1.0004465579986572, + "sampling/importance_sampling_ratio/min": 0.5866379737854004, + "sampling/sampling_logp_difference/max": 0.6716392040252686, + "sampling/sampling_logp_difference/mean": 0.012668934650719166, + "step": 2 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/max_terminated_length": 216.0, + "completions/mean_length": 83.3125, + "completions/mean_terminated_length": 83.3125, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "entropy": 0.2114392985822633, + "epoch": 0.002967359050445104, + "grad_norm": 22.401935577392578, + "kl_approx": 0.2788887023925781, + "learning_rate": 3.921568627450981e-07, + "loss": 0.3177, + "num_tokens": 73674.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.5837955474853516, + "sampling/importance_sampling_ratio/mean": 0.9998602867126465, + "sampling/importance_sampling_ratio/min": 0.4914727509021759, + "sampling/sampling_logp_difference/max": 0.7103487253189087, + "sampling/sampling_logp_difference/mean": 0.012666420079767704, + "step": 3 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/max_terminated_length": 197.0, + "completions/mean_length": 67.25, + "completions/mean_terminated_length": 67.25, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "entropy": 0.16613194230012596, + "epoch": 0.003956478733926805, + "grad_norm": 47.69809341430664, + "kl_approx": 0.32745361328125, + "learning_rate": 5.882352941176471e-07, + "loss": 0.3548, + "num_tokens": 96210.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2800475358963013, + "sampling/importance_sampling_ratio/mean": 0.9995731711387634, + "sampling/importance_sampling_ratio/min": 0.7826456427574158, + "sampling/sampling_logp_difference/max": 0.24689722061157227, + "sampling/sampling_logp_difference/mean": 0.009745625779032707, + "step": 4 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/max_terminated_length": 182.0, + "completions/mean_length": 87.5625, + "completions/mean_terminated_length": 87.5625, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.3289017961360514, + "epoch": 0.004945598417408506, + "grad_norm": 30.9542293548584, + "kl_approx": 0.433685302734375, + "learning_rate": 7.843137254901962e-07, + "loss": 0.4406, + "num_tokens": 124812.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.471364974975586, + "sampling/importance_sampling_ratio/mean": 0.9996128082275391, + "sampling/importance_sampling_ratio/min": 0.6268442273139954, + "sampling/sampling_logp_difference/max": 0.4670572280883789, + "sampling/sampling_logp_difference/mean": 0.014928525313735008, + "step": 5 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 52.1875, + "completions/mean_terminated_length": 52.1875, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "entropy": 0.2653167946264148, + "epoch": 0.005934718100890208, + "grad_norm": 35.00068283081055, + "kl_approx": 0.328765869140625, + "learning_rate": 9.80392156862745e-07, + "loss": 0.4772, + "num_tokens": 149818.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.4770218133926392, + "sampling/importance_sampling_ratio/mean": 1.0003488063812256, + "sampling/importance_sampling_ratio/min": 0.4372307062149048, + "sampling/sampling_logp_difference/max": 0.8272943496704102, + "sampling/sampling_logp_difference/mean": 0.015758465975522995, + "step": 6 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 132.0, + "completions/max_terminated_length": 132.0, + "completions/mean_length": 63.46875, + "completions/mean_terminated_length": 63.46875, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.20784346293658018, + "epoch": 0.006923837784371909, + "grad_norm": 27.148252487182617, + "kl_approx": 0.32172393798828125, + "learning_rate": 1.1764705882352942e-06, + "loss": 0.3827, + "num_tokens": 167953.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.380575180053711, + "sampling/importance_sampling_ratio/mean": 1.0021108388900757, + "sampling/importance_sampling_ratio/min": 0.6953052282333374, + "sampling/sampling_logp_difference/max": 0.3634042739868164, + "sampling/sampling_logp_difference/mean": 0.012451596558094025, + "step": 7 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/max_terminated_length": 149.0, + "completions/mean_length": 63.21875, + "completions/mean_terminated_length": 63.21875, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.2678709211759269, + "epoch": 0.00791295746785361, + "grad_norm": 22.741029739379883, + "kl_approx": 0.245819091796875, + "learning_rate": 1.3725490196078434e-06, + "loss": 0.3534, + "num_tokens": 186672.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3052524328231812, + "sampling/importance_sampling_ratio/mean": 0.9987507462501526, + "sampling/importance_sampling_ratio/min": 0.6951146721839905, + "sampling/sampling_logp_difference/max": 0.3636784553527832, + "sampling/sampling_logp_difference/mean": 0.014225856401026249, + "step": 8 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 125.0, + "completions/max_terminated_length": 125.0, + "completions/mean_length": 83.90625, + "completions/mean_terminated_length": 83.90625, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.2498362367041409, + "epoch": 0.008902077151335312, + "grad_norm": 15.692536354064941, + "kl_approx": 0.2193756103515625, + "learning_rate": 1.5686274509803923e-06, + "loss": 0.2845, + "num_tokens": 212293.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.407726764678955, + "sampling/importance_sampling_ratio/mean": 1.0001542568206787, + "sampling/importance_sampling_ratio/min": 0.6554859280586243, + "sampling/sampling_logp_difference/max": 0.42237842082977295, + "sampling/sampling_logp_difference/mean": 0.012924418784677982, + "step": 9 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/max_terminated_length": 215.0, + "completions/mean_length": 69.0, + "completions/mean_terminated_length": 69.0, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "entropy": 0.40094609512016177, + "epoch": 0.009891196834817012, + "grad_norm": 12.470873832702637, + "kl_approx": 0.2755889892578125, + "learning_rate": 1.7647058823529414e-06, + "loss": 0.249, + "num_tokens": 237045.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2497589588165283, + "sampling/importance_sampling_ratio/mean": 1.0003273487091064, + "sampling/importance_sampling_ratio/min": 0.7366809844970703, + "sampling/sampling_logp_difference/max": 0.3056004047393799, + "sampling/sampling_logp_difference/mean": 0.015060663223266602, + "step": 10 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 249.0, + "completions/max_terminated_length": 249.0, + "completions/mean_length": 84.5, + "completions/mean_terminated_length": 84.5, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "entropy": 0.44598684390075505, + "epoch": 0.010880316518298714, + "grad_norm": 10.184185028076172, + "kl_approx": 0.19950103759765625, + "learning_rate": 1.96078431372549e-06, + "loss": 0.1355, + "num_tokens": 261333.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2790429592132568, + "sampling/importance_sampling_ratio/mean": 1.0003671646118164, + "sampling/importance_sampling_ratio/min": 0.7788791656494141, + "sampling/sampling_logp_difference/max": 0.24989932775497437, + "sampling/sampling_logp_difference/mean": 0.01756957359611988, + "step": 11 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 871.0, + "completions/max_terminated_length": 871.0, + "completions/mean_length": 263.1875, + "completions/mean_terminated_length": 263.1875, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.712527384981513, + "epoch": 0.011869436201780416, + "grad_norm": 5.928795337677002, + "kl_approx": 0.1987152099609375, + "learning_rate": 2.1568627450980393e-06, + "loss": 0.1364, + "num_tokens": 292675.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.7851656675338745, + "sampling/importance_sampling_ratio/mean": 1.0001658201217651, + "sampling/importance_sampling_ratio/min": 0.5906986594200134, + "sampling/sampling_logp_difference/max": 0.5795111656188965, + "sampling/sampling_logp_difference/mean": 0.01856686733663082, + "step": 12 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 176.625, + "completions/mean_terminated_length": 176.625, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "entropy": 0.6713496631709859, + "epoch": 0.012858555885262116, + "grad_norm": 8.06400203704834, + "kl_approx": 0.2364654541015625, + "learning_rate": 2.3529411764705885e-06, + "loss": 0.1389, + "num_tokens": 316655.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.5149643421173096, + "sampling/importance_sampling_ratio/mean": 0.9990436434745789, + "sampling/importance_sampling_ratio/min": 0.5753442049026489, + "sampling/sampling_logp_difference/max": 0.5527868270874023, + "sampling/sampling_logp_difference/mean": 0.019171396270394325, + "step": 13 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 801.0, + "completions/max_terminated_length": 801.0, + "completions/mean_length": 230.34375, + "completions/mean_terminated_length": 230.34375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.6651557786390185, + "epoch": 0.013847675568743818, + "grad_norm": 6.31292724609375, + "kl_approx": 0.2437744140625, + "learning_rate": 2.549019607843137e-06, + "loss": 0.1763, + "num_tokens": 345250.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3279917240142822, + "sampling/importance_sampling_ratio/mean": 0.9989785552024841, + "sampling/importance_sampling_ratio/min": 0.40005072951316833, + "sampling/sampling_logp_difference/max": 0.9161639213562012, + "sampling/sampling_logp_difference/mean": 0.01682412624359131, + "step": 14 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 197.0, + "completions/mean_terminated_length": 170.32257080078125, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.742519180290401, + "epoch": 0.01483679525222552, + "grad_norm": 6.316934585571289, + "kl_approx": 0.2740325927734375, + "learning_rate": 2.7450980392156867e-06, + "loss": 0.1793, + "num_tokens": 376546.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.529558539390564, + "sampling/importance_sampling_ratio/mean": 1.000232458114624, + "sampling/importance_sampling_ratio/min": 0.7334407567977905, + "sampling/sampling_logp_difference/max": 0.42497920989990234, + "sampling/sampling_logp_difference/mean": 0.019550925120711327, + "step": 15 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/max_terminated_length": 269.0, + "completions/mean_length": 108.125, + "completions/mean_terminated_length": 108.125, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "entropy": 0.6516008954495192, + "epoch": 0.01582591493570722, + "grad_norm": 6.337355136871338, + "kl_approx": 0.19483184814453125, + "learning_rate": 2.9411764705882355e-06, + "loss": 0.1162, + "num_tokens": 399158.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.295330286026001, + "sampling/importance_sampling_ratio/mean": 1.000090479850769, + "sampling/importance_sampling_ratio/min": 0.7236149311065674, + "sampling/sampling_logp_difference/max": 0.32349586486816406, + "sampling/sampling_logp_difference/mean": 0.01921284943819046, + "step": 16 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.0, + "completions/max_terminated_length": 239.0, + "completions/mean_length": 130.125, + "completions/mean_terminated_length": 130.125, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 0.813218005001545, + "epoch": 0.016815034619188922, + "grad_norm": 6.062085151672363, + "kl_approx": 0.288330078125, + "learning_rate": 3.1372549019607846e-06, + "loss": 0.2161, + "num_tokens": 423210.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3300998210906982, + "sampling/importance_sampling_ratio/mean": 1.0006861686706543, + "sampling/importance_sampling_ratio/min": 0.7610476016998291, + "sampling/sampling_logp_difference/max": 0.28525400161743164, + "sampling/sampling_logp_difference/mean": 0.019886309280991554, + "step": 17 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/max_terminated_length": 268.0, + "completions/mean_length": 115.4375, + "completions/mean_terminated_length": 115.4375, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "entropy": 0.5778757254593074, + "epoch": 0.017804154302670624, + "grad_norm": 5.502050876617432, + "kl_approx": 0.20848846435546875, + "learning_rate": 3.3333333333333333e-06, + "loss": 0.1678, + "num_tokens": 448760.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.4366071224212646, + "sampling/importance_sampling_ratio/mean": 1.0004844665527344, + "sampling/importance_sampling_ratio/min": 0.7086447477340698, + "sampling/sampling_logp_difference/max": 0.36228418350219727, + "sampling/sampling_logp_difference/mean": 0.016118451952934265, + "step": 18 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 680.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 130.125, + "completions/mean_terminated_length": 130.125, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.543406939599663, + "epoch": 0.018793273986152326, + "grad_norm": 4.8573198318481445, + "kl_approx": 0.188812255859375, + "learning_rate": 3.529411764705883e-06, + "loss": 0.1383, + "num_tokens": 475740.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3149635791778564, + "sampling/importance_sampling_ratio/mean": 1.0015828609466553, + "sampling/importance_sampling_ratio/min": 0.630437433719635, + "sampling/sampling_logp_difference/max": 0.46134138107299805, + "sampling/sampling_logp_difference/mean": 0.015716973692178726, + "step": 19 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 121.0, + "completions/max_terminated_length": 121.0, + "completions/mean_length": 77.3125, + "completions/mean_terminated_length": 77.3125, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.4479383102734573, + "epoch": 0.019782393669634024, + "grad_norm": 4.919290542602539, + "kl_approx": 0.23265504837036133, + "learning_rate": 3.7254901960784316e-06, + "loss": 0.1596, + "num_tokens": 493678.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2743626832962036, + "sampling/importance_sampling_ratio/mean": 0.9999105930328369, + "sampling/importance_sampling_ratio/min": 0.6998186111450195, + "sampling/sampling_logp_difference/max": 0.3569340705871582, + "sampling/sampling_logp_difference/mean": 0.013152539730072021, + "step": 20 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 312.0, + "completions/max_terminated_length": 312.0, + "completions/mean_length": 124.0, + "completions/mean_terminated_length": 124.0, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "entropy": 0.5678619706304744, + "epoch": 0.020771513353115726, + "grad_norm": 5.2283101081848145, + "kl_approx": 0.23252105712890625, + "learning_rate": 3.92156862745098e-06, + "loss": 0.1748, + "num_tokens": 521166.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2499433755874634, + "sampling/importance_sampling_ratio/mean": 0.9997319579124451, + "sampling/importance_sampling_ratio/min": 0.7065470814704895, + "sampling/sampling_logp_difference/max": 0.3473653793334961, + "sampling/sampling_logp_difference/mean": 0.016745995730161667, + "step": 21 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 273.0, + "completions/max_terminated_length": 273.0, + "completions/mean_length": 90.28125, + "completions/mean_terminated_length": 90.28125, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.6121624982915819, + "epoch": 0.021760633036597428, + "grad_norm": 5.407651901245117, + "kl_approx": 0.23561859130859375, + "learning_rate": 4.11764705882353e-06, + "loss": 0.1638, + "num_tokens": 539255.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2807447910308838, + "sampling/importance_sampling_ratio/mean": 0.9981391429901123, + "sampling/importance_sampling_ratio/min": 0.787506639957428, + "sampling/sampling_logp_difference/max": 0.24744176864624023, + "sampling/sampling_logp_difference/mean": 0.01780509389936924, + "step": 22 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 350.0, + "completions/max_terminated_length": 350.0, + "completions/mean_length": 132.03125, + "completions/mean_terminated_length": 132.03125, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "entropy": 0.5304539701901376, + "epoch": 0.02274975272007913, + "grad_norm": 4.16707181930542, + "kl_approx": 0.15975189208984375, + "learning_rate": 4.313725490196079e-06, + "loss": 0.1284, + "num_tokens": 562352.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.242124080657959, + "sampling/importance_sampling_ratio/mean": 1.0004630088806152, + "sampling/importance_sampling_ratio/min": 0.7878240346908569, + "sampling/sampling_logp_difference/max": 0.2384805679321289, + "sampling/sampling_logp_difference/mean": 0.017122572287917137, + "step": 23 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.0, + "completions/max_terminated_length": 246.0, + "completions/mean_length": 113.65625, + "completions/mean_terminated_length": 113.65625, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "entropy": 0.4428328915964812, + "epoch": 0.02373887240356083, + "grad_norm": 5.808507919311523, + "kl_approx": 0.23372268676757812, + "learning_rate": 4.509803921568628e-06, + "loss": 0.1636, + "num_tokens": 586381.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2541260719299316, + "sampling/importance_sampling_ratio/mean": 1.0004154443740845, + "sampling/importance_sampling_ratio/min": 0.7925978899002075, + "sampling/sampling_logp_difference/max": 0.23243927955627441, + "sampling/sampling_logp_difference/mean": 0.013458705507218838, + "step": 24 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 157.53125, + "completions/mean_terminated_length": 157.53125, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.4033222822472453, + "epoch": 0.024727992087042534, + "grad_norm": 3.5725226402282715, + "kl_approx": 0.13341522216796875, + "learning_rate": 4.705882352941177e-06, + "loss": 0.0865, + "num_tokens": 609390.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.272744059562683, + "sampling/importance_sampling_ratio/mean": 0.9999245405197144, + "sampling/importance_sampling_ratio/min": 0.7086235284805298, + "sampling/sampling_logp_difference/max": 0.34443092346191406, + "sampling/sampling_logp_difference/mean": 0.012822979129850864, + "step": 25 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/max_terminated_length": 287.0, + "completions/mean_length": 148.5625, + "completions/mean_terminated_length": 148.5625, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "entropy": 0.4659550020005554, + "epoch": 0.025717111770524232, + "grad_norm": 3.973996639251709, + "kl_approx": 0.19349288940429688, + "learning_rate": 4.901960784313726e-06, + "loss": 0.1312, + "num_tokens": 634200.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2952141761779785, + "sampling/importance_sampling_ratio/mean": 1.0001169443130493, + "sampling/importance_sampling_ratio/min": 0.6906945705413818, + "sampling/sampling_logp_difference/max": 0.3700575828552246, + "sampling/sampling_logp_difference/mean": 0.014177861623466015, + "step": 26 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 361.0, + "completions/max_terminated_length": 361.0, + "completions/mean_length": 120.90625, + "completions/mean_terminated_length": 120.90625, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "entropy": 0.5012554116547108, + "epoch": 0.026706231454005934, + "grad_norm": 5.57405948638916, + "kl_approx": 0.2424774169921875, + "learning_rate": 5.098039215686274e-06, + "loss": 0.1595, + "num_tokens": 660749.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2622395753860474, + "sampling/importance_sampling_ratio/mean": 1.0014152526855469, + "sampling/importance_sampling_ratio/min": 0.7354806661605835, + "sampling/sampling_logp_difference/max": 0.307231068611145, + "sampling/sampling_logp_difference/mean": 0.01518701296299696, + "step": 27 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 433.0, + "completions/max_terminated_length": 433.0, + "completions/mean_length": 175.65625, + "completions/mean_terminated_length": 175.65625, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.6228614673018456, + "epoch": 0.027695351137487636, + "grad_norm": 4.616665363311768, + "kl_approx": 0.192138671875, + "learning_rate": 5.294117647058824e-06, + "loss": 0.1287, + "num_tokens": 687338.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2605739831924438, + "sampling/importance_sampling_ratio/mean": 1.0005100965499878, + "sampling/importance_sampling_ratio/min": 0.7780243158340454, + "sampling/sampling_logp_difference/max": 0.25099754333496094, + "sampling/sampling_logp_difference/mean": 0.01689489185810089, + "step": 28 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 100.28125, + "completions/mean_terminated_length": 100.28125, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.47152079176157713, + "epoch": 0.028684470820969338, + "grad_norm": 4.3633036613464355, + "kl_approx": 0.15057373046875, + "learning_rate": 5.4901960784313735e-06, + "loss": 0.0866, + "num_tokens": 713555.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2824264764785767, + "sampling/importance_sampling_ratio/mean": 1.0002281665802002, + "sampling/importance_sampling_ratio/min": 0.7465155720710754, + "sampling/sampling_logp_difference/max": 0.29233884811401367, + "sampling/sampling_logp_difference/mean": 0.014382078312337399, + "step": 29 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 798.0, + "completions/max_terminated_length": 798.0, + "completions/mean_length": 165.6875, + "completions/mean_terminated_length": 165.6875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.5833946452476084, + "epoch": 0.02967359050445104, + "grad_norm": 4.380878925323486, + "kl_approx": 0.2080535888671875, + "learning_rate": 5.686274509803922e-06, + "loss": 0.1481, + "num_tokens": 736977.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2449756860733032, + "sampling/importance_sampling_ratio/mean": 0.9996135830879211, + "sampling/importance_sampling_ratio/min": 0.7551159262657166, + "sampling/sampling_logp_difference/max": 0.2808840274810791, + "sampling/sampling_logp_difference/mean": 0.014716500416398048, + "step": 30 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 368.0, + "completions/max_terminated_length": 368.0, + "completions/mean_length": 118.34375, + "completions/mean_terminated_length": 118.34375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.609582512639463, + "epoch": 0.03066271018793274, + "grad_norm": 9.437041282653809, + "kl_approx": 0.2823333740234375, + "learning_rate": 5.882352941176471e-06, + "loss": 0.2094, + "num_tokens": 760068.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.5377250909805298, + "sampling/importance_sampling_ratio/mean": 0.9994555115699768, + "sampling/importance_sampling_ratio/min": 0.7325965762138367, + "sampling/sampling_logp_difference/max": 0.43030405044555664, + "sampling/sampling_logp_difference/mean": 0.017115885391831398, + "step": 31 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 211.0, + "completions/max_terminated_length": 211.0, + "completions/mean_length": 106.3125, + "completions/mean_terminated_length": 106.3125, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "entropy": 0.44338538870215416, + "epoch": 0.03165182987141444, + "grad_norm": 4.166123867034912, + "kl_approx": 0.1922454833984375, + "learning_rate": 6.07843137254902e-06, + "loss": 0.1223, + "num_tokens": 783790.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2705289125442505, + "sampling/importance_sampling_ratio/mean": 1.0000391006469727, + "sampling/importance_sampling_ratio/min": 0.7931848764419556, + "sampling/sampling_logp_difference/max": 0.23943328857421875, + "sampling/sampling_logp_difference/mean": 0.012508069165050983, + "step": 32 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/max_terminated_length": 378.0, + "completions/mean_length": 146.03125, + "completions/mean_terminated_length": 146.03125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.6924732802435756, + "epoch": 0.032640949554896145, + "grad_norm": 4.595773696899414, + "kl_approx": 0.298614501953125, + "learning_rate": 6.274509803921569e-06, + "loss": 0.1686, + "num_tokens": 810639.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2869395017623901, + "sampling/importance_sampling_ratio/mean": 0.999365508556366, + "sampling/importance_sampling_ratio/min": 0.736172616481781, + "sampling/sampling_logp_difference/max": 0.3062906265258789, + "sampling/sampling_logp_difference/mean": 0.020145833492279053, + "step": 33 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 136.71875, + "completions/mean_terminated_length": 136.71875, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.7962839929386973, + "epoch": 0.033630069238377844, + "grad_norm": 4.532458782196045, + "kl_approx": 0.2537841796875, + "learning_rate": 6.470588235294119e-06, + "loss": 0.1685, + "num_tokens": 833222.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2344743013381958, + "sampling/importance_sampling_ratio/mean": 1.0008091926574707, + "sampling/importance_sampling_ratio/min": 0.696750819683075, + "sampling/sampling_logp_difference/max": 0.3613274097442627, + "sampling/sampling_logp_difference/mean": 0.019129553809762, + "step": 34 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 432.0, + "completions/max_terminated_length": 432.0, + "completions/mean_length": 101.90625, + "completions/mean_terminated_length": 101.90625, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.6338094496168196, + "epoch": 0.03461918892185954, + "grad_norm": 3.7068588733673096, + "kl_approx": 0.21271514892578125, + "learning_rate": 6.666666666666667e-06, + "loss": 0.1188, + "num_tokens": 852627.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2575933933258057, + "sampling/importance_sampling_ratio/mean": 0.9996321797370911, + "sampling/importance_sampling_ratio/min": 0.6894897222518921, + "sampling/sampling_logp_difference/max": 0.37180352210998535, + "sampling/sampling_logp_difference/mean": 0.01760854385793209, + "step": 35 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/max_terminated_length": 225.0, + "completions/mean_length": 121.75, + "completions/mean_terminated_length": 121.75, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.49393809074535966, + "epoch": 0.03560830860534125, + "grad_norm": 3.0840585231781006, + "kl_approx": 0.1484966278076172, + "learning_rate": 6.862745098039216e-06, + "loss": 0.1154, + "num_tokens": 882491.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.215436339378357, + "sampling/importance_sampling_ratio/mean": 0.9996856451034546, + "sampling/importance_sampling_ratio/min": 0.7761471271514893, + "sampling/sampling_logp_difference/max": 0.25341320037841797, + "sampling/sampling_logp_difference/mean": 0.012671963311731815, + "step": 36 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 680.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 135.5, + "completions/mean_terminated_length": 135.5, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.5945717701688409, + "epoch": 0.036597428288822946, + "grad_norm": 5.685115337371826, + "kl_approx": 0.2591571807861328, + "learning_rate": 7.058823529411766e-06, + "loss": 0.1462, + "num_tokens": 905635.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2564952373504639, + "sampling/importance_sampling_ratio/mean": 1.0002124309539795, + "sampling/importance_sampling_ratio/min": 0.7755049467086792, + "sampling/sampling_logp_difference/max": 0.2542409896850586, + "sampling/sampling_logp_difference/mean": 0.015301528386771679, + "step": 37 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 138.4375, + "completions/mean_terminated_length": 138.4375, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "entropy": 0.6307068914175034, + "epoch": 0.03758654797230465, + "grad_norm": 4.414489269256592, + "kl_approx": 0.276824951171875, + "learning_rate": 7.2549019607843145e-06, + "loss": 0.1615, + "num_tokens": 931681.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3150644302368164, + "sampling/importance_sampling_ratio/mean": 0.9999549984931946, + "sampling/importance_sampling_ratio/min": 0.7587952017784119, + "sampling/sampling_logp_difference/max": 0.27602338790893555, + "sampling/sampling_logp_difference/mean": 0.01803310215473175, + "step": 38 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 406.0, + "completions/max_terminated_length": 406.0, + "completions/mean_length": 131.375, + "completions/mean_terminated_length": 131.375, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.5818085940554738, + "epoch": 0.03857566765578635, + "grad_norm": 3.9511404037475586, + "kl_approx": 0.2630462646484375, + "learning_rate": 7.450980392156863e-06, + "loss": 0.1447, + "num_tokens": 954197.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.257724404335022, + "sampling/importance_sampling_ratio/mean": 0.999701738357544, + "sampling/importance_sampling_ratio/min": 0.7506154179573059, + "sampling/sampling_logp_difference/max": 0.2868618965148926, + "sampling/sampling_logp_difference/mean": 0.014471353031694889, + "step": 39 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 500.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 124.28125, + "completions/mean_terminated_length": 124.28125, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.49119923263788223, + "epoch": 0.03956478733926805, + "grad_norm": 4.087814807891846, + "kl_approx": 0.203094482421875, + "learning_rate": 7.647058823529411e-06, + "loss": 0.136, + "num_tokens": 974838.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2847933769226074, + "sampling/importance_sampling_ratio/mean": 0.9999269843101501, + "sampling/importance_sampling_ratio/min": 0.7777205109596252, + "sampling/sampling_logp_difference/max": 0.2513880729675293, + "sampling/sampling_logp_difference/mean": 0.01348426379263401, + "step": 40 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 385.0, + "completions/max_terminated_length": 385.0, + "completions/mean_length": 137.9375, + "completions/mean_terminated_length": 137.9375, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.5795987108722329, + "epoch": 0.040553907022749754, + "grad_norm": 3.838060140609741, + "kl_approx": 0.2061767578125, + "learning_rate": 7.84313725490196e-06, + "loss": 0.1217, + "num_tokens": 999788.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.373495101928711, + "sampling/importance_sampling_ratio/mean": 1.0006526708602905, + "sampling/importance_sampling_ratio/min": 0.8049514889717102, + "sampling/sampling_logp_difference/max": 0.31735873222351074, + "sampling/sampling_logp_difference/mean": 0.015573837794363499, + "step": 41 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.0, + "completions/max_terminated_length": 355.0, + "completions/mean_length": 130.59375, + "completions/mean_terminated_length": 130.59375, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.4268115006852895, + "epoch": 0.04154302670623145, + "grad_norm": 3.38944411277771, + "kl_approx": 0.185943603515625, + "learning_rate": 8.03921568627451e-06, + "loss": 0.1044, + "num_tokens": 1026591.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.4606575965881348, + "sampling/importance_sampling_ratio/mean": 1.000106692314148, + "sampling/importance_sampling_ratio/min": 0.8157088756561279, + "sampling/sampling_logp_difference/max": 0.37888669967651367, + "sampling/sampling_logp_difference/mean": 0.011561281979084015, + "step": 42 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/max_terminated_length": 282.0, + "completions/mean_length": 109.84375, + "completions/mean_terminated_length": 109.84375, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.4379591876640916, + "epoch": 0.04253214638971316, + "grad_norm": 3.84601092338562, + "kl_approx": 0.203826904296875, + "learning_rate": 8.23529411764706e-06, + "loss": 0.0937, + "num_tokens": 1051242.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.281457781791687, + "sampling/importance_sampling_ratio/mean": 1.000130534172058, + "sampling/importance_sampling_ratio/min": 0.766370952129364, + "sampling/sampling_logp_difference/max": 0.26608896255493164, + "sampling/sampling_logp_difference/mean": 0.013051528483629227, + "step": 43 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.0, + "completions/max_terminated_length": 369.0, + "completions/mean_length": 119.6875, + "completions/mean_terminated_length": 119.6875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.4783370946533978, + "epoch": 0.043521266073194856, + "grad_norm": 4.2641401290893555, + "kl_approx": 0.2268524169921875, + "learning_rate": 8.43137254901961e-06, + "loss": 0.1197, + "num_tokens": 1076320.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2634073495864868, + "sampling/importance_sampling_ratio/mean": 0.9995564818382263, + "sampling/importance_sampling_ratio/min": 0.7962554693222046, + "sampling/sampling_logp_difference/max": 0.2338123321533203, + "sampling/sampling_logp_difference/mean": 0.013220756314694881, + "step": 44 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.0, + "completions/max_terminated_length": 328.0, + "completions/mean_length": 121.0625, + "completions/mean_terminated_length": 121.0625, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "entropy": 0.500814110506326, + "epoch": 0.04451038575667656, + "grad_norm": 5.393642902374268, + "kl_approx": 0.21185302734375, + "learning_rate": 8.627450980392157e-06, + "loss": 0.1338, + "num_tokens": 1098210.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3016771078109741, + "sampling/importance_sampling_ratio/mean": 1.0000889301300049, + "sampling/importance_sampling_ratio/min": 0.792271614074707, + "sampling/sampling_logp_difference/max": 0.2636535167694092, + "sampling/sampling_logp_difference/mean": 0.014255186542868614, + "step": 45 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 307.0, + "completions/max_terminated_length": 307.0, + "completions/mean_length": 111.6875, + "completions/mean_terminated_length": 111.6875, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.4824839881621301, + "epoch": 0.04549950544015826, + "grad_norm": 5.159158706665039, + "kl_approx": 0.26015472412109375, + "learning_rate": 8.823529411764707e-06, + "loss": 0.1726, + "num_tokens": 1119824.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2540059089660645, + "sampling/importance_sampling_ratio/mean": 1.0001651048660278, + "sampling/importance_sampling_ratio/min": 0.8153637647628784, + "sampling/sampling_logp_difference/max": 0.22634315490722656, + "sampling/sampling_logp_difference/mean": 0.012914080172777176, + "step": 46 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 166.34375, + "completions/mean_terminated_length": 166.34375, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.6073106471449137, + "epoch": 0.04648862512363996, + "grad_norm": 5.110304832458496, + "kl_approx": 0.268310546875, + "learning_rate": 9.019607843137256e-06, + "loss": 0.1865, + "num_tokens": 1138043.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.4095134735107422, + "sampling/importance_sampling_ratio/mean": 0.9999644160270691, + "sampling/importance_sampling_ratio/min": 0.7814053893089294, + "sampling/sampling_logp_difference/max": 0.3432445526123047, + "sampling/sampling_logp_difference/mean": 0.017521638423204422, + "step": 47 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 348.0, + "completions/max_terminated_length": 348.0, + "completions/mean_length": 166.875, + "completions/mean_terminated_length": 166.875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.6038749944418669, + "epoch": 0.04747774480712166, + "grad_norm": 3.6243886947631836, + "kl_approx": 0.2293853759765625, + "learning_rate": 9.215686274509804e-06, + "loss": 0.1409, + "num_tokens": 1163719.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.330727219581604, + "sampling/importance_sampling_ratio/mean": 0.9999634623527527, + "sampling/importance_sampling_ratio/min": 0.794928252696991, + "sampling/sampling_logp_difference/max": 0.28572559356689453, + "sampling/sampling_logp_difference/mean": 0.015606286935508251, + "step": 48 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.0, + "completions/max_terminated_length": 337.0, + "completions/mean_length": 100.40625, + "completions/mean_terminated_length": 100.40625, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "entropy": 0.500477098627016, + "epoch": 0.04846686449060336, + "grad_norm": 4.582136154174805, + "kl_approx": 0.26647186279296875, + "learning_rate": 9.411764705882354e-06, + "loss": 0.1496, + "num_tokens": 1182140.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.292516827583313, + "sampling/importance_sampling_ratio/mean": 1.000788688659668, + "sampling/importance_sampling_ratio/min": 0.8057621717453003, + "sampling/sampling_logp_difference/max": 0.2565913200378418, + "sampling/sampling_logp_difference/mean": 0.014075304381549358, + "step": 49 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 347.0, + "completions/max_terminated_length": 347.0, + "completions/mean_length": 144.5625, + "completions/mean_terminated_length": 144.5625, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.5469464962370694, + "epoch": 0.04945598417408507, + "grad_norm": 4.1229681968688965, + "kl_approx": 0.22538185119628906, + "learning_rate": 9.607843137254903e-06, + "loss": 0.1937, + "num_tokens": 1207590.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.207270622253418, + "sampling/importance_sampling_ratio/mean": 1.0007202625274658, + "sampling/importance_sampling_ratio/min": 0.8117063045501709, + "sampling/sampling_logp_difference/max": 0.2086167335510254, + "sampling/sampling_logp_difference/mean": 0.015022498555481434, + "step": 50 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 366.0, + "completions/max_terminated_length": 366.0, + "completions/mean_length": 132.53125, + "completions/mean_terminated_length": 132.53125, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.5224494733847678, + "epoch": 0.050445103857566766, + "grad_norm": 3.403608798980713, + "kl_approx": 0.18885040283203125, + "learning_rate": 9.803921568627451e-06, + "loss": 0.1213, + "num_tokens": 1228623.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2892875671386719, + "sampling/importance_sampling_ratio/mean": 0.99994957447052, + "sampling/importance_sampling_ratio/min": 0.7977840304374695, + "sampling/sampling_logp_difference/max": 0.2540898323059082, + "sampling/sampling_logp_difference/mean": 0.012245790101587772, + "step": 51 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 424.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 108.25, + "completions/mean_terminated_length": 108.25, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.6296374616213143, + "epoch": 0.051434223541048464, + "grad_norm": 4.969080924987793, + "kl_approx": 0.2586669921875, + "learning_rate": 1e-05, + "loss": 0.1311, + "num_tokens": 1252263.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.241348147392273, + "sampling/importance_sampling_ratio/mean": 0.9995492100715637, + "sampling/importance_sampling_ratio/min": 0.7735205292701721, + "sampling/sampling_logp_difference/max": 0.256803035736084, + "sampling/sampling_logp_difference/mean": 0.015211866237223148, + "step": 52 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 541.0, + "completions/max_terminated_length": 541.0, + "completions/mean_length": 161.34375, + "completions/mean_terminated_length": 161.34375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.7328842608258128, + "epoch": 0.05242334322453017, + "grad_norm": 5.320689678192139, + "kl_approx": 0.260223388671875, + "learning_rate": 1.0196078431372549e-05, + "loss": 0.1589, + "num_tokens": 1275330.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2876975536346436, + "sampling/importance_sampling_ratio/mean": 1.000205636024475, + "sampling/importance_sampling_ratio/min": 0.781684160232544, + "sampling/sampling_logp_difference/max": 0.2528557777404785, + "sampling/sampling_logp_difference/mean": 0.017746655270457268, + "step": 53 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 890.0, + "completions/max_terminated_length": 890.0, + "completions/mean_length": 180.8125, + "completions/mean_terminated_length": 180.8125, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.6511295037344098, + "epoch": 0.05341246290801187, + "grad_norm": 5.195742607116699, + "kl_approx": 0.25853729248046875, + "learning_rate": 1.03921568627451e-05, + "loss": 0.1672, + "num_tokens": 1300380.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2799780368804932, + "sampling/importance_sampling_ratio/mean": 0.9992867708206177, + "sampling/importance_sampling_ratio/min": 0.6605044007301331, + "sampling/sampling_logp_difference/max": 0.4147515892982483, + "sampling/sampling_logp_difference/mean": 0.01718878746032715, + "step": 54 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 118.90625, + "completions/mean_terminated_length": 118.90625, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.5597416623495519, + "epoch": 0.05440158259149357, + "grad_norm": 5.125489711761475, + "kl_approx": 0.2502403259277344, + "learning_rate": 1.0588235294117648e-05, + "loss": 0.157, + "num_tokens": 1320841.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2517250776290894, + "sampling/importance_sampling_ratio/mean": 1.000245213508606, + "sampling/importance_sampling_ratio/min": 0.7895849943161011, + "sampling/sampling_logp_difference/max": 0.23624777793884277, + "sampling/sampling_logp_difference/mean": 0.013802413828670979, + "step": 55 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/max_terminated_length": 166.0, + "completions/mean_length": 92.3125, + "completions/mean_terminated_length": 92.3125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.4637425309047103, + "epoch": 0.05539070227497527, + "grad_norm": 3.5693695545196533, + "kl_approx": 0.17520523071289062, + "learning_rate": 1.0784313725490196e-05, + "loss": 0.095, + "num_tokens": 1342763.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2677499055862427, + "sampling/importance_sampling_ratio/mean": 0.9992490410804749, + "sampling/importance_sampling_ratio/min": 0.7959326505661011, + "sampling/sampling_logp_difference/max": 0.23724365234375, + "sampling/sampling_logp_difference/mean": 0.011262464337050915, + "step": 56 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 140.3125, + "completions/mean_terminated_length": 140.3125, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.6376035045832396, + "epoch": 0.05637982195845697, + "grad_norm": 4.30594539642334, + "kl_approx": 0.2780914306640625, + "learning_rate": 1.0980392156862747e-05, + "loss": 0.1652, + "num_tokens": 1365981.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2821263074874878, + "sampling/importance_sampling_ratio/mean": 1.0000948905944824, + "sampling/importance_sampling_ratio/min": 0.7887351512908936, + "sampling/sampling_logp_difference/max": 0.2485198974609375, + "sampling/sampling_logp_difference/mean": 0.015333757735788822, + "step": 57 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/max_terminated_length": 137.0, + "completions/mean_length": 70.96875, + "completions/mean_terminated_length": 70.96875, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.5017928471788764, + "epoch": 0.057368941641938676, + "grad_norm": 4.030434608459473, + "kl_approx": 0.15564727783203125, + "learning_rate": 1.1176470588235295e-05, + "loss": 0.1109, + "num_tokens": 1388644.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1583945751190186, + "sampling/importance_sampling_ratio/mean": 1.0000076293945312, + "sampling/importance_sampling_ratio/min": 0.8063974976539612, + "sampling/sampling_logp_difference/max": 0.2151784896850586, + "sampling/sampling_logp_difference/mean": 0.01196204498410225, + "step": 58 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 366.0, + "completions/max_terminated_length": 366.0, + "completions/mean_length": 99.65625, + "completions/mean_terminated_length": 99.65625, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.5088606770150363, + "epoch": 0.058358061325420374, + "grad_norm": 5.449374675750732, + "kl_approx": 0.20304489135742188, + "learning_rate": 1.1372549019607844e-05, + "loss": 0.1051, + "num_tokens": 1409049.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2883384227752686, + "sampling/importance_sampling_ratio/mean": 1.0003690719604492, + "sampling/importance_sampling_ratio/min": 0.8127831816673279, + "sampling/sampling_logp_difference/max": 0.2533533573150635, + "sampling/sampling_logp_difference/mean": 0.015136120840907097, + "step": 59 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 173.0, + "completions/max_terminated_length": 173.0, + "completions/mean_length": 87.125, + "completions/mean_terminated_length": 87.125, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.2818223061040044, + "epoch": 0.05934718100890208, + "grad_norm": 3.715052366256714, + "kl_approx": 0.16213226318359375, + "learning_rate": 1.1568627450980394e-05, + "loss": 0.0902, + "num_tokens": 1434461.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2972204685211182, + "sampling/importance_sampling_ratio/mean": 0.9993278384208679, + "sampling/importance_sampling_ratio/min": 0.7910410165786743, + "sampling/sampling_logp_difference/max": 0.2602238655090332, + "sampling/sampling_logp_difference/mean": 0.0068718609400093555, + "step": 60 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 305.0, + "completions/max_terminated_length": 305.0, + "completions/mean_length": 110.21875, + "completions/mean_terminated_length": 110.21875, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.5369161823764443, + "epoch": 0.06033630069238378, + "grad_norm": 3.9810492992401123, + "kl_approx": 0.19330596923828125, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.1446, + "num_tokens": 1456820.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1823234558105469, + "sampling/importance_sampling_ratio/mean": 1.0000263452529907, + "sampling/importance_sampling_ratio/min": 0.7547599077224731, + "sampling/sampling_logp_difference/max": 0.281355619430542, + "sampling/sampling_logp_difference/mean": 0.012838956899940968, + "step": 61 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 143.1875, + "completions/mean_terminated_length": 143.1875, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.3660207106731832, + "epoch": 0.06132542037586548, + "grad_norm": 3.04774808883667, + "kl_approx": 0.18878936767578125, + "learning_rate": 1.1960784313725491e-05, + "loss": 0.1077, + "num_tokens": 1486658.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2588038444519043, + "sampling/importance_sampling_ratio/mean": 1.0000052452087402, + "sampling/importance_sampling_ratio/min": 0.8176159262657166, + "sampling/sampling_logp_difference/max": 0.2301619052886963, + "sampling/sampling_logp_difference/mean": 0.010408961214125156, + "step": 62 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 148.90625, + "completions/mean_terminated_length": 148.90625, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "entropy": 0.7495590569451451, + "epoch": 0.06231454005934718, + "grad_norm": 4.467825412750244, + "kl_approx": 0.285797119140625, + "learning_rate": 1.215686274509804e-05, + "loss": 0.2146, + "num_tokens": 1508367.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2406947612762451, + "sampling/importance_sampling_ratio/mean": 0.9999914169311523, + "sampling/importance_sampling_ratio/min": 0.793444037437439, + "sampling/sampling_logp_difference/max": 0.23137235641479492, + "sampling/sampling_logp_difference/mean": 0.017643310129642487, + "step": 63 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 139.9375, + "completions/mean_terminated_length": 139.9375, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5831932504661381, + "epoch": 0.06330365974282888, + "grad_norm": 7.5350823402404785, + "kl_approx": 0.2658271789550781, + "learning_rate": 1.235294117647059e-05, + "loss": 0.1766, + "num_tokens": 1529909.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2842048406600952, + "sampling/importance_sampling_ratio/mean": 1.000518798828125, + "sampling/importance_sampling_ratio/min": 0.7359094023704529, + "sampling/sampling_logp_difference/max": 0.30664825439453125, + "sampling/sampling_logp_difference/mean": 0.015831125900149345, + "step": 64 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/max_terminated_length": 292.0, + "completions/mean_length": 84.71875, + "completions/mean_terminated_length": 84.71875, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.5008618012070656, + "epoch": 0.06429277942631058, + "grad_norm": 4.710756301879883, + "kl_approx": 0.27294158935546875, + "learning_rate": 1.2549019607843138e-05, + "loss": 0.1516, + "num_tokens": 1551276.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3586218357086182, + "sampling/importance_sampling_ratio/mean": 1.0008198022842407, + "sampling/importance_sampling_ratio/min": 0.8010686039924622, + "sampling/sampling_logp_difference/max": 0.3064708709716797, + "sampling/sampling_logp_difference/mean": 0.013259034603834152, + "step": 65 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.0, + "completions/max_terminated_length": 328.0, + "completions/mean_length": 156.5625, + "completions/mean_terminated_length": 156.5625, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "entropy": 0.6333168176934123, + "epoch": 0.06528189910979229, + "grad_norm": 4.57286262512207, + "kl_approx": 0.258544921875, + "learning_rate": 1.2745098039215686e-05, + "loss": 0.2034, + "num_tokens": 1572078.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.330077052116394, + "sampling/importance_sampling_ratio/mean": 1.0002566576004028, + "sampling/importance_sampling_ratio/min": 0.7931466698646545, + "sampling/sampling_logp_difference/max": 0.28523683547973633, + "sampling/sampling_logp_difference/mean": 0.014598055742681026, + "step": 66 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 182.40625, + "completions/mean_terminated_length": 182.40625, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.7425875635817647, + "epoch": 0.06627101879327399, + "grad_norm": 4.569087982177734, + "kl_approx": 0.23430633544921875, + "learning_rate": 1.2941176470588238e-05, + "loss": 0.1771, + "num_tokens": 1596371.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3222166299819946, + "sampling/importance_sampling_ratio/mean": 1.0001628398895264, + "sampling/importance_sampling_ratio/min": 0.7878240346908569, + "sampling/sampling_logp_difference/max": 0.27930963039398193, + "sampling/sampling_logp_difference/mean": 0.01689918339252472, + "step": 67 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 246.65625, + "completions/mean_terminated_length": 246.65625, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.7379114059731364, + "epoch": 0.06726013847675569, + "grad_norm": 4.0915327072143555, + "kl_approx": 0.24199676513671875, + "learning_rate": 1.3137254901960785e-05, + "loss": 0.1662, + "num_tokens": 1622288.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.4416778087615967, + "sampling/importance_sampling_ratio/mean": 0.9996952414512634, + "sampling/importance_sampling_ratio/min": 0.7524896860122681, + "sampling/sampling_logp_difference/max": 0.36580753326416016, + "sampling/sampling_logp_difference/mean": 0.01640690304338932, + "step": 68 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/max_terminated_length": 178.0, + "completions/mean_length": 110.59375, + "completions/mean_terminated_length": 110.59375, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "entropy": 0.5114238555543125, + "epoch": 0.06824925816023739, + "grad_norm": 3.7289226055145264, + "kl_approx": 0.24538516998291016, + "learning_rate": 1.3333333333333333e-05, + "loss": 0.1315, + "num_tokens": 1646963.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2762572765350342, + "sampling/importance_sampling_ratio/mean": 0.9999822378158569, + "sampling/importance_sampling_ratio/min": 0.8291629552841187, + "sampling/sampling_logp_difference/max": 0.24393177032470703, + "sampling/sampling_logp_difference/mean": 0.01177594531327486, + "step": 69 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 476.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 145.125, + "completions/mean_terminated_length": 145.125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.7065270124003291, + "epoch": 0.06923837784371908, + "grad_norm": 3.5972189903259277, + "kl_approx": 0.2792510986328125, + "learning_rate": 1.3529411764705885e-05, + "loss": 0.1745, + "num_tokens": 1666839.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.212645173072815, + "sampling/importance_sampling_ratio/mean": 1.0000544786453247, + "sampling/importance_sampling_ratio/min": 0.4468255937099457, + "sampling/sampling_logp_difference/max": 0.8055869340896606, + "sampling/sampling_logp_difference/mean": 0.018307412043213844, + "step": 70 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 138.53125, + "completions/mean_terminated_length": 138.53125, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7211345219984651, + "epoch": 0.0702274975272008, + "grad_norm": 5.390006065368652, + "kl_approx": 0.393585205078125, + "learning_rate": 1.3725490196078432e-05, + "loss": 0.2178, + "num_tokens": 1686328.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.210097312927246, + "sampling/importance_sampling_ratio/mean": 0.9998346567153931, + "sampling/importance_sampling_ratio/min": 0.7745994329452515, + "sampling/sampling_logp_difference/max": 0.25540924072265625, + "sampling/sampling_logp_difference/mean": 0.017397606745362282, + "step": 71 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 193.09375, + "completions/mean_terminated_length": 166.29031372070312, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7085119737312198, + "epoch": 0.0712166172106825, + "grad_norm": 4.230503559112549, + "kl_approx": 0.2684783935546875, + "learning_rate": 1.392156862745098e-05, + "loss": 0.1732, + "num_tokens": 1716051.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2566486597061157, + "sampling/importance_sampling_ratio/mean": 1.0002861022949219, + "sampling/importance_sampling_ratio/min": 0.7851418256759644, + "sampling/sampling_logp_difference/max": 0.24189090728759766, + "sampling/sampling_logp_difference/mean": 0.016826679930090904, + "step": 72 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/max_terminated_length": 206.0, + "completions/mean_length": 82.5625, + "completions/mean_terminated_length": 82.5625, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "entropy": 0.38565474888309836, + "epoch": 0.0722057368941642, + "grad_norm": 3.6139774322509766, + "kl_approx": 0.16648483276367188, + "learning_rate": 1.4117647058823532e-05, + "loss": 0.1046, + "num_tokens": 1741701.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.159116268157959, + "sampling/importance_sampling_ratio/mean": 0.9989307522773743, + "sampling/importance_sampling_ratio/min": 0.8333114981651306, + "sampling/sampling_logp_difference/max": 0.18234777450561523, + "sampling/sampling_logp_difference/mean": 0.009898793883621693, + "step": 73 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 142.40625, + "completions/mean_terminated_length": 142.40625, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.6197466151788831, + "epoch": 0.07319485657764589, + "grad_norm": 4.112690448760986, + "kl_approx": 0.2643890380859375, + "learning_rate": 1.431372549019608e-05, + "loss": 0.1906, + "num_tokens": 1760690.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1888421773910522, + "sampling/importance_sampling_ratio/mean": 1.0007389783859253, + "sampling/importance_sampling_ratio/min": 0.7927209138870239, + "sampling/sampling_logp_difference/max": 0.2322840690612793, + "sampling/sampling_logp_difference/mean": 0.0161321759223938, + "step": 74 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 883.0, + "completions/max_terminated_length": 883.0, + "completions/mean_length": 162.8125, + "completions/mean_terminated_length": 162.8125, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.6469260104931891, + "epoch": 0.07418397626112759, + "grad_norm": 4.854889869689941, + "kl_approx": 0.210357666015625, + "learning_rate": 1.4509803921568629e-05, + "loss": 0.1751, + "num_tokens": 1787468.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2213832139968872, + "sampling/importance_sampling_ratio/mean": 0.9998693466186523, + "sampling/importance_sampling_ratio/min": 0.7829110622406006, + "sampling/sampling_logp_difference/max": 0.2447361946105957, + "sampling/sampling_logp_difference/mean": 0.01834258995950222, + "step": 75 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 349.0, + "completions/max_terminated_length": 349.0, + "completions/mean_length": 123.21875, + "completions/mean_terminated_length": 123.21875, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.5755073186010122, + "epoch": 0.0751730959446093, + "grad_norm": 3.652179479598999, + "kl_approx": 0.20203399658203125, + "learning_rate": 1.4705882352941179e-05, + "loss": 0.1461, + "num_tokens": 1808715.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2707154750823975, + "sampling/importance_sampling_ratio/mean": 0.9996893405914307, + "sampling/importance_sampling_ratio/min": 0.7929303646087646, + "sampling/sampling_logp_difference/max": 0.2395801544189453, + "sampling/sampling_logp_difference/mean": 0.013100972399115562, + "step": 76 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/max_terminated_length": 136.0, + "completions/mean_length": 67.1875, + "completions/mean_terminated_length": 67.1875, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.5051397397182882, + "epoch": 0.076162215628091, + "grad_norm": 5.326716423034668, + "kl_approx": 0.20829010009765625, + "learning_rate": 1.4901960784313726e-05, + "loss": 0.1792, + "num_tokens": 1828777.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.136636734008789, + "sampling/importance_sampling_ratio/mean": 0.9984632134437561, + "sampling/importance_sampling_ratio/min": 0.8360645174980164, + "sampling/sampling_logp_difference/max": 0.17904949188232422, + "sampling/sampling_logp_difference/mean": 0.011927456595003605, + "step": 77 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.0, + "completions/max_terminated_length": 340.0, + "completions/mean_length": 133.40625, + "completions/mean_terminated_length": 133.40625, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.5220940811559558, + "epoch": 0.0771513353115727, + "grad_norm": 3.3579773902893066, + "kl_approx": 0.2205352783203125, + "learning_rate": 1.5098039215686276e-05, + "loss": 0.1369, + "num_tokens": 1853606.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1806988716125488, + "sampling/importance_sampling_ratio/mean": 0.9996694326400757, + "sampling/importance_sampling_ratio/min": 0.7869690656661987, + "sampling/sampling_logp_difference/max": 0.23956632614135742, + "sampling/sampling_logp_difference/mean": 0.012044726870954037, + "step": 78 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.0, + "completions/max_terminated_length": 262.0, + "completions/mean_length": 93.1875, + "completions/mean_terminated_length": 93.1875, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.4660562495701015, + "epoch": 0.0781404549950544, + "grad_norm": 4.1132659912109375, + "kl_approx": 0.181182861328125, + "learning_rate": 1.5294117647058822e-05, + "loss": 0.1463, + "num_tokens": 1876068.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2498778104782104, + "sampling/importance_sampling_ratio/mean": 1.0000693798065186, + "sampling/importance_sampling_ratio/min": 0.839215099811554, + "sampling/sampling_logp_difference/max": 0.22304582595825195, + "sampling/sampling_logp_difference/mean": 0.010969881899654865, + "step": 79 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/max_terminated_length": 233.0, + "completions/mean_length": 98.8125, + "completions/mean_terminated_length": 98.8125, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "entropy": 0.5295102949021384, + "epoch": 0.0791295746785361, + "grad_norm": 3.925215721130371, + "kl_approx": 0.29920434951782227, + "learning_rate": 1.5490196078431373e-05, + "loss": 0.1502, + "num_tokens": 1897222.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2634944915771484, + "sampling/importance_sampling_ratio/mean": 0.9994741678237915, + "sampling/importance_sampling_ratio/min": 0.8290769457817078, + "sampling/sampling_logp_difference/max": 0.23388123512268066, + "sampling/sampling_logp_difference/mean": 0.012161962687969208, + "step": 80 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 887.0, + "completions/max_terminated_length": 887.0, + "completions/mean_length": 145.96875, + "completions/mean_terminated_length": 145.96875, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.658632637001574, + "epoch": 0.08011869436201781, + "grad_norm": 4.738213539123535, + "kl_approx": 0.29586029052734375, + "learning_rate": 1.568627450980392e-05, + "loss": 0.1785, + "num_tokens": 1920757.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2729096412658691, + "sampling/importance_sampling_ratio/mean": 1.0000708103179932, + "sampling/importance_sampling_ratio/min": 0.772980809211731, + "sampling/sampling_logp_difference/max": 0.2575010061264038, + "sampling/sampling_logp_difference/mean": 0.014527578838169575, + "step": 81 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 143.5, + "completions/mean_terminated_length": 143.5, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.7595312488265336, + "epoch": 0.08110781404549951, + "grad_norm": 4.447983264923096, + "kl_approx": 0.279693603515625, + "learning_rate": 1.5882352941176473e-05, + "loss": 0.2166, + "num_tokens": 1945133.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1928465366363525, + "sampling/importance_sampling_ratio/mean": 1.0001873970031738, + "sampling/importance_sampling_ratio/min": 0.7964349389076233, + "sampling/sampling_logp_difference/max": 0.22760987281799316, + "sampling/sampling_logp_difference/mean": 0.017735807225108147, + "step": 82 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 146.65625, + "completions/mean_terminated_length": 146.65625, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.6563051482662559, + "epoch": 0.0820969337289812, + "grad_norm": 4.7257304191589355, + "kl_approx": 0.24554443359375, + "learning_rate": 1.607843137254902e-05, + "loss": 0.1841, + "num_tokens": 1960418.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2431120872497559, + "sampling/importance_sampling_ratio/mean": 1.0000678300857544, + "sampling/importance_sampling_ratio/min": 0.7770373821258545, + "sampling/sampling_logp_difference/max": 0.25226688385009766, + "sampling/sampling_logp_difference/mean": 0.01416561659425497, + "step": 83 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 356.0, + "completions/max_terminated_length": 356.0, + "completions/mean_length": 106.0625, + "completions/mean_terminated_length": 106.0625, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.4924747720360756, + "epoch": 0.0830860534124629, + "grad_norm": 3.2110092639923096, + "kl_approx": 0.16954803466796875, + "learning_rate": 1.627450980392157e-05, + "loss": 0.1172, + "num_tokens": 1982412.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2176752090454102, + "sampling/importance_sampling_ratio/mean": 1.000216007232666, + "sampling/importance_sampling_ratio/min": 0.800976037979126, + "sampling/sampling_logp_difference/max": 0.2219243049621582, + "sampling/sampling_logp_difference/mean": 0.012952733784914017, + "step": 84 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 159.875, + "completions/mean_terminated_length": 132.0, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "entropy": 0.6197850131429732, + "epoch": 0.0840751730959446, + "grad_norm": 4.267870903015137, + "kl_approx": 0.3108978271484375, + "learning_rate": 1.647058823529412e-05, + "loss": 0.1784, + "num_tokens": 2010024.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2554748058319092, + "sampling/importance_sampling_ratio/mean": 0.9985430240631104, + "sampling/importance_sampling_ratio/min": 0.811707079410553, + "sampling/sampling_logp_difference/max": 0.22751379013061523, + "sampling/sampling_logp_difference/mean": 0.01452487614005804, + "step": 85 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 813.0, + "completions/max_terminated_length": 813.0, + "completions/mean_length": 141.90625, + "completions/mean_terminated_length": 141.90625, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.5942427241243422, + "epoch": 0.08506429277942631, + "grad_norm": 4.358031272888184, + "kl_approx": 0.347015380859375, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.1949, + "num_tokens": 2036893.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2746130228042603, + "sampling/importance_sampling_ratio/mean": 0.9995061159133911, + "sampling/importance_sampling_ratio/min": 0.8187244534492493, + "sampling/sampling_logp_difference/max": 0.24264264106750488, + "sampling/sampling_logp_difference/mean": 0.014404760673642159, + "step": 86 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 483.0, + "completions/max_terminated_length": 483.0, + "completions/mean_length": 116.28125, + "completions/mean_terminated_length": 116.28125, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.5420629633590579, + "epoch": 0.08605341246290801, + "grad_norm": 4.053826808929443, + "kl_approx": 0.23535537719726562, + "learning_rate": 1.686274509803922e-05, + "loss": 0.16, + "num_tokens": 2056062.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3019064664840698, + "sampling/importance_sampling_ratio/mean": 1.0003461837768555, + "sampling/importance_sampling_ratio/min": 0.7194052934646606, + "sampling/sampling_logp_difference/max": 0.3293304443359375, + "sampling/sampling_logp_difference/mean": 0.014076776802539825, + "step": 87 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.0, + "completions/max_terminated_length": 328.0, + "completions/mean_length": 106.3125, + "completions/mean_terminated_length": 106.3125, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.4662083578296006, + "epoch": 0.08704253214638971, + "grad_norm": 3.5453341007232666, + "kl_approx": 0.23992919921875, + "learning_rate": 1.7058823529411767e-05, + "loss": 0.1495, + "num_tokens": 2084784.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2618094682693481, + "sampling/importance_sampling_ratio/mean": 0.9999485611915588, + "sampling/importance_sampling_ratio/min": 0.7966673970222473, + "sampling/sampling_logp_difference/max": 0.23254680633544922, + "sampling/sampling_logp_difference/mean": 0.011845567263662815, + "step": 88 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 421.0, + "completions/max_terminated_length": 421.0, + "completions/mean_length": 147.4375, + "completions/mean_terminated_length": 147.4375, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.6805998277850449, + "epoch": 0.08803165182987141, + "grad_norm": 3.4955856800079346, + "kl_approx": 0.25186920166015625, + "learning_rate": 1.7254901960784314e-05, + "loss": 0.1666, + "num_tokens": 2110766.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.6132633686065674, + "sampling/importance_sampling_ratio/mean": 0.9999154210090637, + "sampling/importance_sampling_ratio/min": 0.703249454498291, + "sampling/sampling_logp_difference/max": 0.4782590866088867, + "sampling/sampling_logp_difference/mean": 0.014171433635056019, + "step": 89 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/max_terminated_length": 232.0, + "completions/mean_length": 92.96875, + "completions/mean_terminated_length": 92.96875, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.6654437128454447, + "epoch": 0.08902077151335312, + "grad_norm": 3.8866493701934814, + "kl_approx": 0.21889495849609375, + "learning_rate": 1.7450980392156866e-05, + "loss": 0.1781, + "num_tokens": 2133437.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2338017225265503, + "sampling/importance_sampling_ratio/mean": 1.0004531145095825, + "sampling/importance_sampling_ratio/min": 0.5772315263748169, + "sampling/sampling_logp_difference/max": 0.5495117902755737, + "sampling/sampling_logp_difference/mean": 0.015423045493662357, + "step": 90 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 422.0, + "completions/max_terminated_length": 422.0, + "completions/mean_length": 106.09375, + "completions/mean_terminated_length": 106.09375, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "entropy": 0.5722916247323155, + "epoch": 0.09000989119683482, + "grad_norm": 3.762639045715332, + "kl_approx": 0.30530548095703125, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.1607, + "num_tokens": 2154136.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1478874683380127, + "sampling/importance_sampling_ratio/mean": 0.9992611408233643, + "sampling/importance_sampling_ratio/min": 0.7661829590797424, + "sampling/sampling_logp_difference/max": 0.26633429527282715, + "sampling/sampling_logp_difference/mean": 0.01384639646857977, + "step": 91 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/max_terminated_length": 201.0, + "completions/mean_length": 87.21875, + "completions/mean_terminated_length": 87.21875, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.5211337637156248, + "epoch": 0.09099901088031652, + "grad_norm": 3.4914159774780273, + "kl_approx": 0.21887969970703125, + "learning_rate": 1.7843137254901965e-05, + "loss": 0.1241, + "num_tokens": 2180935.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2110965251922607, + "sampling/importance_sampling_ratio/mean": 1.0001884698867798, + "sampling/importance_sampling_ratio/min": 0.8818774819374084, + "sampling/sampling_logp_difference/max": 0.19152617454528809, + "sampling/sampling_logp_difference/mean": 0.011368767358362675, + "step": 92 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 93.5, + "completions/mean_terminated_length": 93.5, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.545645251404494, + "epoch": 0.09198813056379822, + "grad_norm": 4.095757484436035, + "kl_approx": 0.27759552001953125, + "learning_rate": 1.8039215686274513e-05, + "loss": 0.2176, + "num_tokens": 2196759.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2412327527999878, + "sampling/importance_sampling_ratio/mean": 0.9990619421005249, + "sampling/importance_sampling_ratio/min": 0.8541035652160645, + "sampling/sampling_logp_difference/max": 0.21610498428344727, + "sampling/sampling_logp_difference/mean": 0.012378948740661144, + "step": 93 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 982.0, + "completions/max_terminated_length": 982.0, + "completions/mean_length": 163.28125, + "completions/mean_terminated_length": 163.28125, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.7363119008950889, + "epoch": 0.09297725024727992, + "grad_norm": 3.907405138015747, + "kl_approx": 0.373992919921875, + "learning_rate": 1.823529411764706e-05, + "loss": 0.1871, + "num_tokens": 2222552.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1711499691009521, + "sampling/importance_sampling_ratio/mean": 1.0000168085098267, + "sampling/importance_sampling_ratio/min": 0.7566782236099243, + "sampling/sampling_logp_difference/max": 0.27881717681884766, + "sampling/sampling_logp_difference/mean": 0.01468683872371912, + "step": 94 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 971.0, + "completions/max_terminated_length": 971.0, + "completions/mean_length": 173.4375, + "completions/mean_terminated_length": 173.4375, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.5695787584409118, + "epoch": 0.09396636993076163, + "grad_norm": 4.660771369934082, + "kl_approx": 0.2996368408203125, + "learning_rate": 1.843137254901961e-05, + "loss": 0.2144, + "num_tokens": 2246502.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2650411128997803, + "sampling/importance_sampling_ratio/mean": 1.000042200088501, + "sampling/importance_sampling_ratio/min": 0.7962116599082947, + "sampling/sampling_logp_difference/max": 0.23510456085205078, + "sampling/sampling_logp_difference/mean": 0.01371805276721716, + "step": 95 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 981.0, + "completions/mean_length": 268.5, + "completions/mean_terminated_length": 244.1290283203125, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "entropy": 0.701228266581893, + "epoch": 0.09495548961424333, + "grad_norm": 3.1961963176727295, + "kl_approx": 0.26605987548828125, + "learning_rate": 1.862745098039216e-05, + "loss": 0.1678, + "num_tokens": 2278286.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2763711214065552, + "sampling/importance_sampling_ratio/mean": 1.0001819133758545, + "sampling/importance_sampling_ratio/min": 0.7788647413253784, + "sampling/sampling_logp_difference/max": 0.24991798400878906, + "sampling/sampling_logp_difference/mean": 0.016298644244670868, + "step": 96 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 789.0, + "completions/max_terminated_length": 789.0, + "completions/mean_length": 179.6875, + "completions/mean_terminated_length": 179.6875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.5723489276133478, + "epoch": 0.09594460929772503, + "grad_norm": 4.202455997467041, + "kl_approx": 0.280120849609375, + "learning_rate": 1.8823529411764708e-05, + "loss": 0.1963, + "num_tokens": 2303932.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2651300430297852, + "sampling/importance_sampling_ratio/mean": 0.999332845211029, + "sampling/importance_sampling_ratio/min": 0.7756439447402954, + "sampling/sampling_logp_difference/max": 0.2540616989135742, + "sampling/sampling_logp_difference/mean": 0.014458324760198593, + "step": 97 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/max_terminated_length": 218.0, + "completions/mean_length": 93.21875, + "completions/mean_terminated_length": 93.21875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.408864579629153, + "epoch": 0.09693372898120672, + "grad_norm": 3.407325029373169, + "kl_approx": 0.19140625, + "learning_rate": 1.9019607843137255e-05, + "loss": 0.126, + "num_tokens": 2327299.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1449666023254395, + "sampling/importance_sampling_ratio/mean": 1.0001323223114014, + "sampling/importance_sampling_ratio/min": 0.767653226852417, + "sampling/sampling_logp_difference/max": 0.2644171714782715, + "sampling/sampling_logp_difference/mean": 0.010124210268259048, + "step": 98 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 362.0, + "completions/max_terminated_length": 362.0, + "completions/mean_length": 106.9375, + "completions/mean_terminated_length": 106.9375, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.6095218281261623, + "epoch": 0.09792284866468842, + "grad_norm": 4.767866611480713, + "kl_approx": 0.3236122131347656, + "learning_rate": 1.9215686274509807e-05, + "loss": 0.2244, + "num_tokens": 2348033.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.164278507232666, + "sampling/importance_sampling_ratio/mean": 1.0005046129226685, + "sampling/importance_sampling_ratio/min": 0.7904103994369507, + "sampling/sampling_logp_difference/max": 0.23520302772521973, + "sampling/sampling_logp_difference/mean": 0.01352311298251152, + "step": 99 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 390.0, + "completions/max_terminated_length": 390.0, + "completions/mean_length": 119.0625, + "completions/mean_terminated_length": 119.0625, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.740677310153842, + "epoch": 0.09891196834817013, + "grad_norm": 4.633357048034668, + "kl_approx": 0.379608154296875, + "learning_rate": 1.9411764705882355e-05, + "loss": 0.2563, + "num_tokens": 2373531.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2093089818954468, + "sampling/importance_sampling_ratio/mean": 1.0009273290634155, + "sampling/importance_sampling_ratio/min": 0.8385035395622253, + "sampling/sampling_logp_difference/max": 0.1900491714477539, + "sampling/sampling_logp_difference/mean": 0.015810729935765266, + "step": 100 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 164.21875, + "completions/mean_terminated_length": 164.21875, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "entropy": 0.7902802284806967, + "epoch": 0.09990108803165183, + "grad_norm": 5.2603840827941895, + "kl_approx": 0.45441436767578125, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.2906, + "num_tokens": 2401018.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.258989930152893, + "sampling/importance_sampling_ratio/mean": 1.0002994537353516, + "sampling/importance_sampling_ratio/min": 0.7966687083244324, + "sampling/sampling_logp_difference/max": 0.23030972480773926, + "sampling/sampling_logp_difference/mean": 0.01634138822555542, + "step": 101 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 132.15625, + "completions/mean_terminated_length": 132.15625, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7276120446622372, + "epoch": 0.10089020771513353, + "grad_norm": 3.8905274868011475, + "kl_approx": 0.3837432861328125, + "learning_rate": 1.9803921568627454e-05, + "loss": 0.2333, + "num_tokens": 2425047.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2377561330795288, + "sampling/importance_sampling_ratio/mean": 0.9996985793113708, + "sampling/importance_sampling_ratio/min": 0.8417240977287292, + "sampling/sampling_logp_difference/max": 0.21330022811889648, + "sampling/sampling_logp_difference/mean": 0.015429042279720306, + "step": 102 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/max_terminated_length": 241.0, + "completions/mean_length": 114.0625, + "completions/mean_terminated_length": 114.0625, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7341065937653184, + "epoch": 0.10187932739861523, + "grad_norm": 4.386577606201172, + "kl_approx": 0.380645751953125, + "learning_rate": 2e-05, + "loss": 0.2009, + "num_tokens": 2449361.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3453978300094604, + "sampling/importance_sampling_ratio/mean": 1.0000827312469482, + "sampling/importance_sampling_ratio/min": 0.7842176556587219, + "sampling/sampling_logp_difference/max": 0.2966897487640381, + "sampling/sampling_logp_difference/mean": 0.014964519999921322, + "step": 103 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.0, + "completions/max_terminated_length": 328.0, + "completions/mean_length": 123.90625, + "completions/mean_terminated_length": 123.90625, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.7485779877752066, + "epoch": 0.10286844708209693, + "grad_norm": 3.574596881866455, + "kl_approx": 0.3981781005859375, + "learning_rate": 1.9999940277008807e-05, + "loss": 0.2063, + "num_tokens": 2474422.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.264679193496704, + "sampling/importance_sampling_ratio/mean": 0.9999602437019348, + "sampling/importance_sampling_ratio/min": 0.8279533982276917, + "sampling/sampling_logp_difference/max": 0.2348184585571289, + "sampling/sampling_logp_difference/mean": 0.01481439359486103, + "step": 104 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 836.0, + "completions/max_terminated_length": 836.0, + "completions/mean_length": 180.125, + "completions/mean_terminated_length": 180.125, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.8464941820129752, + "epoch": 0.10385756676557864, + "grad_norm": 5.77216100692749, + "kl_approx": 0.47412109375, + "learning_rate": 1.99997611087486e-05, + "loss": 0.2896, + "num_tokens": 2502322.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3012601137161255, + "sampling/importance_sampling_ratio/mean": 0.9996833205223083, + "sampling/importance_sampling_ratio/min": 0.8167773485183716, + "sampling/sampling_logp_difference/max": 0.26333320140838623, + "sampling/sampling_logp_difference/mean": 0.01482043694704771, + "step": 105 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 934.0, + "completions/mean_length": 319.875, + "completions/mean_terminated_length": 297.1612854003906, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.7510637398809195, + "epoch": 0.10484668644906034, + "grad_norm": 3.1025259494781494, + "kl_approx": 0.3179473876953125, + "learning_rate": 1.9999462497359468e-05, + "loss": 0.2132, + "num_tokens": 2538238.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2718979120254517, + "sampling/importance_sampling_ratio/mean": 0.9991446137428284, + "sampling/importance_sampling_ratio/min": 0.8031912446022034, + "sampling/sampling_logp_difference/max": 0.2405102252960205, + "sampling/sampling_logp_difference/mean": 0.014363830909132957, + "step": 106 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 507.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 173.875, + "completions/mean_terminated_length": 173.875, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.7316669309511781, + "epoch": 0.10583580613254204, + "grad_norm": 3.9798974990844727, + "kl_approx": 0.3847503662109375, + "learning_rate": 1.9999044446408203e-05, + "loss": 0.2302, + "num_tokens": 2562274.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2616320848464966, + "sampling/importance_sampling_ratio/mean": 1.0001888275146484, + "sampling/importance_sampling_ratio/min": 0.7377116680145264, + "sampling/sampling_logp_difference/max": 0.3042023181915283, + "sampling/sampling_logp_difference/mean": 0.014552840031683445, + "step": 107 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 894.0, + "completions/max_terminated_length": 894.0, + "completions/mean_length": 197.125, + "completions/mean_terminated_length": 197.125, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.8237380003556609, + "epoch": 0.10682492581602374, + "grad_norm": 3.964432954788208, + "kl_approx": 0.30419921875, + "learning_rate": 1.9998506960888258e-05, + "loss": 0.1923, + "num_tokens": 2593718.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2332961559295654, + "sampling/importance_sampling_ratio/mean": 1.0005275011062622, + "sampling/importance_sampling_ratio/min": 0.773954451084137, + "sampling/sampling_logp_difference/max": 0.2562422752380371, + "sampling/sampling_logp_difference/mean": 0.015181615017354488, + "step": 108 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 405.0, + "completions/max_terminated_length": 405.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.758779913187027, + "epoch": 0.10781404549950543, + "grad_norm": 4.285181522369385, + "kl_approx": 0.3531494140625, + "learning_rate": 1.999785004721968e-05, + "loss": 0.2097, + "num_tokens": 2618434.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.219534158706665, + "sampling/importance_sampling_ratio/mean": 0.9998952746391296, + "sampling/importance_sampling_ratio/min": 0.7177202105522156, + "sampling/sampling_logp_difference/max": 0.33167552947998047, + "sampling/sampling_logp_difference/mean": 0.015747712925076485, + "step": 109 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/max_terminated_length": 291.0, + "completions/mean_length": 115.3125, + "completions/mean_terminated_length": 115.3125, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "entropy": 0.5678953961469233, + "epoch": 0.10880316518298715, + "grad_norm": 2.912767171859741, + "kl_approx": 0.24812889099121094, + "learning_rate": 1.999707371324904e-05, + "loss": 0.1639, + "num_tokens": 2641724.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2738890647888184, + "sampling/importance_sampling_ratio/mean": 1.0002368688583374, + "sampling/importance_sampling_ratio/min": 0.80885910987854, + "sampling/sampling_logp_difference/max": 0.24207448959350586, + "sampling/sampling_logp_difference/mean": 0.013785050250589848, + "step": 110 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 210.0, + "completions/max_terminated_length": 210.0, + "completions/mean_length": 72.71875, + "completions/mean_terminated_length": 72.71875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.42386614764109254, + "epoch": 0.10979228486646884, + "grad_norm": 2.873330593109131, + "kl_approx": 0.21701812744140625, + "learning_rate": 1.9996177968249336e-05, + "loss": 0.125, + "num_tokens": 2670795.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2316755056381226, + "sampling/importance_sampling_ratio/mean": 0.9997743964195251, + "sampling/importance_sampling_ratio/min": 0.8671895861625671, + "sampling/sampling_logp_difference/max": 0.2083754539489746, + "sampling/sampling_logp_difference/mean": 0.009304158389568329, + "step": 111 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 78.65625, + "completions/mean_terminated_length": 78.65625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.5251203184016049, + "epoch": 0.11078140454995054, + "grad_norm": 4.083740234375, + "kl_approx": 0.3099822998046875, + "learning_rate": 1.999516282291988e-05, + "loss": 0.2088, + "num_tokens": 2688560.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1888399124145508, + "sampling/importance_sampling_ratio/mean": 0.9998668432235718, + "sampling/importance_sampling_ratio/min": 0.7867171168327332, + "sampling/sampling_logp_difference/max": 0.23988652229309082, + "sampling/sampling_logp_difference/mean": 0.009795577265322208, + "step": 112 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/max_terminated_length": 230.0, + "completions/mean_length": 98.3125, + "completions/mean_terminated_length": 98.3125, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.681112757883966, + "epoch": 0.11177052423343224, + "grad_norm": 5.098729610443115, + "kl_approx": 0.3492431640625, + "learning_rate": 1.999402828938618e-05, + "loss": 0.2723, + "num_tokens": 2707826.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.164537787437439, + "sampling/importance_sampling_ratio/mean": 0.9997772574424744, + "sampling/importance_sampling_ratio/min": 0.7510643601417542, + "sampling/sampling_logp_difference/max": 0.28626394271850586, + "sampling/sampling_logp_difference/mean": 0.014632935635745525, + "step": 113 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/max_terminated_length": 187.0, + "completions/mean_length": 92.375, + "completions/mean_terminated_length": 92.375, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.6517065521329641, + "epoch": 0.11275964391691394, + "grad_norm": 4.192362308502197, + "kl_approx": 0.3641204833984375, + "learning_rate": 1.999277438119978e-05, + "loss": 0.2464, + "num_tokens": 2729990.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1989916563034058, + "sampling/importance_sampling_ratio/mean": 1.0002601146697998, + "sampling/importance_sampling_ratio/min": 0.7930451035499573, + "sampling/sampling_logp_difference/max": 0.23187518119812012, + "sampling/sampling_logp_difference/mean": 0.01365387998521328, + "step": 114 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 386.0, + "completions/max_terminated_length": 386.0, + "completions/mean_length": 148.5, + "completions/mean_terminated_length": 148.5, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6783338310196996, + "epoch": 0.11374876360039565, + "grad_norm": 5.984393119812012, + "kl_approx": 0.4548492431640625, + "learning_rate": 1.9991401113338103e-05, + "loss": 0.3117, + "num_tokens": 2753750.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.215997576713562, + "sampling/importance_sampling_ratio/mean": 1.0003629922866821, + "sampling/importance_sampling_ratio/min": 0.795659601688385, + "sampling/sampling_logp_difference/max": 0.22858381271362305, + "sampling/sampling_logp_difference/mean": 0.013075390830636024, + "step": 115 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/max_terminated_length": 165.0, + "completions/mean_length": 70.71875, + "completions/mean_terminated_length": 70.71875, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "entropy": 0.41725221974775195, + "epoch": 0.11473788328387735, + "grad_norm": 4.085015773773193, + "kl_approx": 0.25689697265625, + "learning_rate": 1.9989908502204295e-05, + "loss": 0.1716, + "num_tokens": 2774245.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2599588632583618, + "sampling/importance_sampling_ratio/mean": 1.0001918077468872, + "sampling/importance_sampling_ratio/min": 0.8632153272628784, + "sampling/sampling_logp_difference/max": 0.2310791015625, + "sampling/sampling_logp_difference/mean": 0.008520031347870827, + "step": 116 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/max_terminated_length": 436.0, + "completions/mean_length": 130.34375, + "completions/mean_terminated_length": 130.34375, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.7754448829218745, + "epoch": 0.11572700296735905, + "grad_norm": 4.090041637420654, + "kl_approx": 0.4168243408203125, + "learning_rate": 1.9988296565626988e-05, + "loss": 0.266, + "num_tokens": 2795144.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2268414497375488, + "sampling/importance_sampling_ratio/mean": 0.9999029636383057, + "sampling/importance_sampling_ratio/min": 0.8270318508148193, + "sampling/sampling_logp_difference/max": 0.20444297790527344, + "sampling/sampling_logp_difference/mean": 0.01447058841586113, + "step": 117 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 137.53125, + "completions/mean_terminated_length": 137.53125, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.695324975065887, + "epoch": 0.11671612265084075, + "grad_norm": 2.9672799110412598, + "kl_approx": 0.27996826171875, + "learning_rate": 1.9986565322860117e-05, + "loss": 0.1866, + "num_tokens": 2823585.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2193204164505005, + "sampling/importance_sampling_ratio/mean": 0.9997122883796692, + "sampling/importance_sampling_ratio/min": 0.8274162411689758, + "sampling/sampling_logp_difference/max": 0.19829368591308594, + "sampling/sampling_logp_difference/mean": 0.01369547750800848, + "step": 118 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 564.0, + "completions/max_terminated_length": 564.0, + "completions/mean_length": 208.6875, + "completions/mean_terminated_length": 208.6875, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 1.0364877041429281, + "epoch": 0.11770524233432245, + "grad_norm": 3.7812628746032715, + "kl_approx": 0.41851806640625, + "learning_rate": 1.9984714794582682e-05, + "loss": 0.2543, + "num_tokens": 2846143.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2559202909469604, + "sampling/importance_sampling_ratio/mean": 0.9999114274978638, + "sampling/importance_sampling_ratio/min": 0.8150038719177246, + "sampling/sampling_logp_difference/max": 0.22786855697631836, + "sampling/sampling_logp_difference/mean": 0.017147187143564224, + "step": 119 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 437.0, + "completions/max_terminated_length": 437.0, + "completions/mean_length": 142.59375, + "completions/mean_terminated_length": 142.59375, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.9492260534316301, + "epoch": 0.11869436201780416, + "grad_norm": 3.706278085708618, + "kl_approx": 0.42816162109375, + "learning_rate": 1.99827450028985e-05, + "loss": 0.2319, + "num_tokens": 2869554.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.38479483127594, + "sampling/importance_sampling_ratio/mean": 1.0000927448272705, + "sampling/importance_sampling_ratio/min": 0.8255041241645813, + "sampling/sampling_logp_difference/max": 0.32555198669433594, + "sampling/sampling_logp_difference/mean": 0.017543373629450798, + "step": 120 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 348.0, + "completions/max_terminated_length": 348.0, + "completions/mean_length": 146.875, + "completions/mean_terminated_length": 146.875, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "entropy": 0.8445757003501058, + "epoch": 0.11968348170128586, + "grad_norm": 3.2209582328796387, + "kl_approx": 0.3781890869140625, + "learning_rate": 1.9980655971335944e-05, + "loss": 0.2553, + "num_tokens": 2890942.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.573632836341858, + "sampling/importance_sampling_ratio/mean": 0.9996671080589294, + "sampling/importance_sampling_ratio/min": 0.8194323778152466, + "sampling/sampling_logp_difference/max": 0.4533867835998535, + "sampling/sampling_logp_difference/mean": 0.01623055338859558, + "step": 121 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 994.0, + "completions/max_terminated_length": 994.0, + "completions/mean_length": 201.25, + "completions/mean_terminated_length": 201.25, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.8141225362196565, + "epoch": 0.12067260138476756, + "grad_norm": 3.3192756175994873, + "kl_approx": 0.331939697265625, + "learning_rate": 1.9978447724847655e-05, + "loss": 0.2015, + "num_tokens": 2913166.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2308330535888672, + "sampling/importance_sampling_ratio/mean": 1.0001190900802612, + "sampling/importance_sampling_ratio/min": 0.795708179473877, + "sampling/sampling_logp_difference/max": 0.22852277755737305, + "sampling/sampling_logp_difference/mean": 0.017103655263781548, + "step": 122 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 534.0, + "completions/max_terminated_length": 534.0, + "completions/mean_length": 219.375, + "completions/mean_terminated_length": 219.375, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 1.0134714087471366, + "epoch": 0.12166172106824925, + "grad_norm": 3.7369258403778076, + "kl_approx": 0.353790283203125, + "learning_rate": 1.9976120289810247e-05, + "loss": 0.2333, + "num_tokens": 2941234.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2188323736190796, + "sampling/importance_sampling_ratio/mean": 0.9995399117469788, + "sampling/importance_sampling_ratio/min": 0.7955179214477539, + "sampling/sampling_logp_difference/max": 0.22876191139221191, + "sampling/sampling_logp_difference/mean": 0.018029918894171715, + "step": 123 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 1012.0, + "completions/max_terminated_length": 1012.0, + "completions/mean_length": 175.15625, + "completions/mean_terminated_length": 175.15625, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.8084270162507892, + "epoch": 0.12265084075173097, + "grad_norm": 3.8425707817077637, + "kl_approx": 0.3931732177734375, + "learning_rate": 1.9973673694024002e-05, + "loss": 0.2143, + "num_tokens": 2964335.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.4137145280838013, + "sampling/importance_sampling_ratio/mean": 1.0000357627868652, + "sampling/importance_sampling_ratio/min": 0.7922927141189575, + "sampling/sampling_logp_difference/max": 0.3462207317352295, + "sampling/sampling_logp_difference/mean": 0.013710507191717625, + "step": 124 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 211.0, + "completions/max_terminated_length": 211.0, + "completions/mean_length": 92.5, + "completions/mean_terminated_length": 92.5, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.788776084780693, + "epoch": 0.12363996043521266, + "grad_norm": 4.461124420166016, + "kl_approx": 0.465850830078125, + "learning_rate": 1.9971107966712518e-05, + "loss": 0.2901, + "num_tokens": 2979183.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1524842977523804, + "sampling/importance_sampling_ratio/mean": 0.9991501569747925, + "sampling/importance_sampling_ratio/min": 0.8436844348907471, + "sampling/sampling_logp_difference/max": 0.16997671127319336, + "sampling/sampling_logp_difference/mean": 0.014075849205255508, + "step": 125 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 198.8125, + "completions/mean_terminated_length": 198.8125, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.7385151144117117, + "epoch": 0.12462908011869436, + "grad_norm": 3.1229376792907715, + "kl_approx": 0.2642364501953125, + "learning_rate": 1.9968423138522382e-05, + "loss": 0.2085, + "num_tokens": 3006497.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1916447877883911, + "sampling/importance_sampling_ratio/mean": 1.000024676322937, + "sampling/importance_sampling_ratio/min": 0.8050271272659302, + "sampling/sampling_logp_difference/max": 0.21687936782836914, + "sampling/sampling_logp_difference/mean": 0.013660969212651253, + "step": 126 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.0, + "completions/max_terminated_length": 318.0, + "completions/mean_length": 138.59375, + "completions/mean_terminated_length": 138.59375, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "entropy": 0.7264044368639588, + "epoch": 0.12561819980217606, + "grad_norm": 3.5425596237182617, + "kl_approx": 0.345550537109375, + "learning_rate": 1.996561924152278e-05, + "loss": 0.2292, + "num_tokens": 3027196.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1457940340042114, + "sampling/importance_sampling_ratio/mean": 0.9994900226593018, + "sampling/importance_sampling_ratio/min": 0.8245117664337158, + "sampling/sampling_logp_difference/max": 0.1929638385772705, + "sampling/sampling_logp_difference/mean": 0.013796227984130383, + "step": 127 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 100.65625, + "completions/mean_terminated_length": 100.65625, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.65774618787691, + "epoch": 0.12660731948565776, + "grad_norm": 3.4651596546173096, + "kl_approx": 0.26280975341796875, + "learning_rate": 1.9962696309205146e-05, + "loss": 0.2042, + "num_tokens": 3049025.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1447635889053345, + "sampling/importance_sampling_ratio/mean": 0.9992141723632812, + "sampling/importance_sampling_ratio/min": 0.8015051484107971, + "sampling/sampling_logp_difference/max": 0.22126388549804688, + "sampling/sampling_logp_difference/mean": 0.013278336264193058, + "step": 128 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 348.0, + "completions/max_terminated_length": 348.0, + "completions/mean_length": 90.125, + "completions/mean_terminated_length": 90.125, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.673021528404206, + "epoch": 0.12759643916913946, + "grad_norm": 3.7388739585876465, + "kl_approx": 0.348297119140625, + "learning_rate": 1.995965437648273e-05, + "loss": 0.2312, + "num_tokens": 3071413.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1890754699707031, + "sampling/importance_sampling_ratio/mean": 1.0004607439041138, + "sampling/importance_sampling_ratio/min": 0.7971530556678772, + "sampling/sampling_logp_difference/max": 0.22670865058898926, + "sampling/sampling_logp_difference/mean": 0.014202319085597992, + "step": 129 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 289.0, + "completions/max_terminated_length": 289.0, + "completions/mean_length": 83.25, + "completions/mean_terminated_length": 83.25, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.6157866641879082, + "epoch": 0.12858555885262116, + "grad_norm": 3.515709400177002, + "kl_approx": 0.301788330078125, + "learning_rate": 1.995649347969019e-05, + "loss": 0.1971, + "num_tokens": 3096157.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2816556692123413, + "sampling/importance_sampling_ratio/mean": 0.9996862411499023, + "sampling/importance_sampling_ratio/min": 0.8433333039283752, + "sampling/sampling_logp_difference/max": 0.2481527328491211, + "sampling/sampling_logp_difference/mean": 0.012496390379965305, + "step": 130 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 534.0, + "completions/max_terminated_length": 534.0, + "completions/mean_length": 125.9375, + "completions/mean_terminated_length": 125.9375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.7223977474495769, + "epoch": 0.12957467853610286, + "grad_norm": 3.493124008178711, + "kl_approx": 0.334381103515625, + "learning_rate": 1.995321365658317e-05, + "loss": 0.2321, + "num_tokens": 3122715.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.279219627380371, + "sampling/importance_sampling_ratio/mean": 1.0001983642578125, + "sampling/importance_sampling_ratio/min": 0.779522716999054, + "sampling/sampling_logp_difference/max": 0.24907350540161133, + "sampling/sampling_logp_difference/mean": 0.013877233490347862, + "step": 131 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 121.4375, + "completions/mean_terminated_length": 121.4375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.5878563146106899, + "epoch": 0.13056379821958458, + "grad_norm": 3.056612014770508, + "kl_approx": 0.257598876953125, + "learning_rate": 1.994981494633784e-05, + "loss": 0.1674, + "num_tokens": 3151913.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1949712038040161, + "sampling/importance_sampling_ratio/mean": 0.9996578693389893, + "sampling/importance_sampling_ratio/min": 0.8144047856330872, + "sampling/sampling_logp_difference/max": 0.20529770851135254, + "sampling/sampling_logp_difference/mean": 0.012141771614551544, + "step": 132 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 162.59375, + "completions/mean_terminated_length": 162.59375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.8467169459909201, + "epoch": 0.13155291790306628, + "grad_norm": 3.707963705062866, + "kl_approx": 0.359100341796875, + "learning_rate": 1.9946297389550433e-05, + "loss": 0.2646, + "num_tokens": 3175780.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.267918348312378, + "sampling/importance_sampling_ratio/mean": 0.9997239708900452, + "sampling/importance_sampling_ratio/min": 0.8259754776954651, + "sampling/sampling_logp_difference/max": 0.23737645149230957, + "sampling/sampling_logp_difference/mean": 0.01623581536114216, + "step": 133 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 814.0, + "completions/max_terminated_length": 814.0, + "completions/mean_length": 146.0625, + "completions/mean_terminated_length": 146.0625, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.7495677229017019, + "epoch": 0.13254203758654798, + "grad_norm": 3.92936635017395, + "kl_approx": 0.4515533447265625, + "learning_rate": 1.9942661028236746e-05, + "loss": 0.3051, + "num_tokens": 3202678.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1853837966918945, + "sampling/importance_sampling_ratio/mean": 0.9995786547660828, + "sampling/importance_sampling_ratio/min": 0.8259376883506775, + "sampling/sampling_logp_difference/max": 0.19123601913452148, + "sampling/sampling_logp_difference/mean": 0.012967344373464584, + "step": 134 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 877.0, + "completions/max_terminated_length": 877.0, + "completions/mean_length": 305.78125, + "completions/mean_terminated_length": 305.78125, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "entropy": 1.0335219977423549, + "epoch": 0.13353115727002968, + "grad_norm": 2.639463424682617, + "kl_approx": 0.344879150390625, + "learning_rate": 1.9938905905831657e-05, + "loss": 0.2526, + "num_tokens": 3231591.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2736576795578003, + "sampling/importance_sampling_ratio/mean": 1.0002832412719727, + "sampling/importance_sampling_ratio/min": 0.772063136100769, + "sampling/sampling_logp_difference/max": 0.25868892669677734, + "sampling/sampling_logp_difference/mean": 0.017690157517790794, + "step": 135 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 827.0, + "completions/max_terminated_length": 827.0, + "completions/mean_length": 275.21875, + "completions/mean_terminated_length": 275.21875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 1.0136343240737915, + "epoch": 0.13452027695351138, + "grad_norm": 2.8746609687805176, + "kl_approx": 0.364501953125, + "learning_rate": 1.993503206718859e-05, + "loss": 0.2383, + "num_tokens": 3262350.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2130011320114136, + "sampling/importance_sampling_ratio/mean": 1.0002808570861816, + "sampling/importance_sampling_ratio/min": 0.7758130431175232, + "sampling/sampling_logp_difference/max": 0.2538437843322754, + "sampling/sampling_logp_difference/mean": 0.016873760148882866, + "step": 136 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 703.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 271.90625, + "completions/mean_terminated_length": 271.90625, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.7439038986340165, + "epoch": 0.13550939663699307, + "grad_norm": 2.3194704055786133, + "kl_approx": 0.24530029296875, + "learning_rate": 1.9931039558578997e-05, + "loss": 0.1633, + "num_tokens": 3290419.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.8843318223953247, + "sampling/importance_sampling_ratio/mean": 1.0003113746643066, + "sampling/importance_sampling_ratio/min": 0.8047864437103271, + "sampling/sampling_logp_difference/max": 0.6335732936859131, + "sampling/sampling_logp_difference/mean": 0.013634921051561832, + "step": 137 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 140.78125, + "completions/mean_terminated_length": 140.78125, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "entropy": 0.665743570891209, + "epoch": 0.13649851632047477, + "grad_norm": 2.8517050743103027, + "kl_approx": 0.28572845458984375, + "learning_rate": 1.9926928427691788e-05, + "loss": 0.1952, + "num_tokens": 3309324.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2762024402618408, + "sampling/importance_sampling_ratio/mean": 0.9995846748352051, + "sampling/importance_sampling_ratio/min": 0.8159916996955872, + "sampling/sampling_logp_difference/max": 0.24388885498046875, + "sampling/sampling_logp_difference/mean": 0.014222693629562855, + "step": 138 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 741.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 150.4375, + "completions/mean_terminated_length": 150.4375, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.6878425776958466, + "epoch": 0.13748763600395647, + "grad_norm": 3.4530131816864014, + "kl_approx": 0.340484619140625, + "learning_rate": 1.992269872363277e-05, + "loss": 0.2157, + "num_tokens": 3338258.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2046599388122559, + "sampling/importance_sampling_ratio/mean": 1.0001176595687866, + "sampling/importance_sampling_ratio/min": 0.7968791723251343, + "sampling/sampling_logp_difference/max": 0.2270522117614746, + "sampling/sampling_logp_difference/mean": 0.012848662212491035, + "step": 139 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 887.0, + "completions/max_terminated_length": 887.0, + "completions/mean_length": 192.15625, + "completions/mean_terminated_length": 192.15625, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 1.00134711060673, + "epoch": 0.13847675568743817, + "grad_norm": 3.532534122467041, + "kl_approx": 0.370758056640625, + "learning_rate": 1.991835049692405e-05, + "loss": 0.2251, + "num_tokens": 3361239.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.297330617904663, + "sampling/importance_sampling_ratio/mean": 1.0006046295166016, + "sampling/importance_sampling_ratio/min": 0.8044227361679077, + "sampling/sampling_logp_difference/max": 0.26030874252319336, + "sampling/sampling_logp_difference/mean": 0.018435800448060036, + "step": 140 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 772.0, + "completions/max_terminated_length": 772.0, + "completions/mean_length": 214.8125, + "completions/mean_terminated_length": 214.8125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.7120185764506459, + "epoch": 0.1394658753709199, + "grad_norm": 2.9272654056549072, + "kl_approx": 0.3009490966796875, + "learning_rate": 1.991388379950346e-05, + "loss": 0.1939, + "num_tokens": 3390257.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2365518808364868, + "sampling/importance_sampling_ratio/mean": 1.0006357431411743, + "sampling/importance_sampling_ratio/min": 0.8355432152748108, + "sampling/sampling_logp_difference/max": 0.2123267650604248, + "sampling/sampling_logp_difference/mean": 0.01206715777516365, + "step": 141 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 410.0, + "completions/max_terminated_length": 410.0, + "completions/mean_length": 133.875, + "completions/mean_terminated_length": 133.875, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.6778213949874043, + "epoch": 0.1404549950544016, + "grad_norm": 3.4875831604003906, + "kl_approx": 0.3140869140625, + "learning_rate": 1.9909298684723905e-05, + "loss": 0.1903, + "num_tokens": 3413541.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1968603134155273, + "sampling/importance_sampling_ratio/mean": 0.9993901252746582, + "sampling/importance_sampling_ratio/min": 0.845062255859375, + "sampling/sampling_logp_difference/max": 0.1797018051147461, + "sampling/sampling_logp_difference/mean": 0.01377237867563963, + "step": 142 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 468.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 157.875, + "completions/mean_terminated_length": 157.875, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.7205053064972162, + "epoch": 0.1414441147378833, + "grad_norm": 3.157585620880127, + "kl_approx": 0.33448028564453125, + "learning_rate": 1.9904595207352736e-05, + "loss": 0.1991, + "num_tokens": 3433289.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2319239377975464, + "sampling/importance_sampling_ratio/mean": 1.000170111656189, + "sampling/importance_sampling_ratio/min": 0.7970343828201294, + "sampling/sampling_logp_difference/max": 0.22685742378234863, + "sampling/sampling_logp_difference/mean": 0.014586882665753365, + "step": 143 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 518.0, + "completions/max_terminated_length": 518.0, + "completions/mean_length": 192.4375, + "completions/mean_terminated_length": 192.4375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.7360031455755234, + "epoch": 0.142433234421365, + "grad_norm": 3.2515690326690674, + "kl_approx": 0.3509521484375, + "learning_rate": 1.9899773423571102e-05, + "loss": 0.2009, + "num_tokens": 3465903.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2092190980911255, + "sampling/importance_sampling_ratio/mean": 1.000055193901062, + "sampling/importance_sampling_ratio/min": 0.8104243874549866, + "sampling/sampling_logp_difference/max": 0.21019721031188965, + "sampling/sampling_logp_difference/mean": 0.014205585233867168, + "step": 144 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 288.0, + "completions/max_terminated_length": 288.0, + "completions/mean_length": 86.96875, + "completions/mean_terminated_length": 86.96875, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "entropy": 0.47909684432670474, + "epoch": 0.1434223541048467, + "grad_norm": 3.299182415008545, + "kl_approx": 0.2391510009765625, + "learning_rate": 1.9894833390973266e-05, + "loss": 0.1817, + "num_tokens": 3482566.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2213146686553955, + "sampling/importance_sampling_ratio/mean": 0.9997127056121826, + "sampling/importance_sampling_ratio/min": 0.8729819059371948, + "sampling/sampling_logp_difference/max": 0.19992780685424805, + "sampling/sampling_logp_difference/mean": 0.011355416849255562, + "step": 145 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 821.0, + "completions/max_terminated_length": 821.0, + "completions/mean_length": 104.15625, + "completions/mean_terminated_length": 104.15625, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "entropy": 0.6040189173072577, + "epoch": 0.1444114737883284, + "grad_norm": 2.6128745079040527, + "kl_approx": 0.29138946533203125, + "learning_rate": 1.9889775168565942e-05, + "loss": 0.1523, + "num_tokens": 3502923.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2069822549819946, + "sampling/importance_sampling_ratio/mean": 1.000307321548462, + "sampling/importance_sampling_ratio/min": 0.8034505248069763, + "sampling/sampling_logp_difference/max": 0.21883970499038696, + "sampling/sampling_logp_difference/mean": 0.01206815242767334, + "step": 146 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 517.0, + "completions/max_terminated_length": 517.0, + "completions/mean_length": 198.1875, + "completions/mean_terminated_length": 198.1875, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "entropy": 0.7451638225466013, + "epoch": 0.14540059347181009, + "grad_norm": 3.3588709831237793, + "kl_approx": 0.2706146240234375, + "learning_rate": 1.9884598816767563e-05, + "loss": 0.2074, + "num_tokens": 3531977.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2334710359573364, + "sampling/importance_sampling_ratio/mean": 1.000329852104187, + "sampling/importance_sampling_ratio/min": 0.7879751920700073, + "sampling/sampling_logp_difference/max": 0.23828864097595215, + "sampling/sampling_logp_difference/mean": 0.014222600497305393, + "step": 147 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 428.0, + "completions/max_terminated_length": 428.0, + "completions/mean_length": 158.875, + "completions/mean_terminated_length": 158.875, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.6897634696215391, + "epoch": 0.14638971315529178, + "grad_norm": 3.2062039375305176, + "kl_approx": 0.3004302978515625, + "learning_rate": 1.987930439740757e-05, + "loss": 0.2006, + "num_tokens": 3560021.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2588196992874146, + "sampling/importance_sampling_ratio/mean": 0.9999896287918091, + "sampling/importance_sampling_ratio/min": 0.8409462571144104, + "sampling/sampling_logp_difference/max": 0.23017454147338867, + "sampling/sampling_logp_difference/mean": 0.01389367040246725, + "step": 148 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/max_terminated_length": 221.0, + "completions/mean_length": 86.625, + "completions/mean_terminated_length": 86.625, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.52367312181741, + "epoch": 0.14737883283877348, + "grad_norm": 2.776329517364502, + "kl_approx": 0.2728729248046875, + "learning_rate": 1.9873891973725673e-05, + "loss": 0.1638, + "num_tokens": 3585089.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1584150791168213, + "sampling/importance_sampling_ratio/mean": 1.0003002882003784, + "sampling/importance_sampling_ratio/min": 0.8473848700523376, + "sampling/sampling_logp_difference/max": 0.16560029983520508, + "sampling/sampling_logp_difference/mean": 0.010013856925070286, + "step": 149 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/max_terminated_length": 298.0, + "completions/mean_length": 106.3125, + "completions/mean_terminated_length": 106.3125, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.5564784072339535, + "epoch": 0.14836795252225518, + "grad_norm": 2.86161208152771, + "kl_approx": 0.2273406982421875, + "learning_rate": 1.98683616103711e-05, + "loss": 0.1612, + "num_tokens": 3606235.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2571991682052612, + "sampling/importance_sampling_ratio/mean": 0.9994977712631226, + "sampling/importance_sampling_ratio/min": 0.7923993468284607, + "sampling/sampling_logp_difference/max": 0.23268985748291016, + "sampling/sampling_logp_difference/mean": 0.012471191585063934, + "step": 150 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 458.0, + "completions/max_terminated_length": 458.0, + "completions/mean_length": 176.5, + "completions/mean_terminated_length": 176.5, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "entropy": 0.8884472846984863, + "epoch": 0.1493570722057369, + "grad_norm": 3.5079596042633057, + "kl_approx": 0.347076416015625, + "learning_rate": 1.986271337340182e-05, + "loss": 0.2555, + "num_tokens": 3630483.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2813501358032227, + "sampling/importance_sampling_ratio/mean": 1.0000793933868408, + "sampling/importance_sampling_ratio/min": 0.7779307961463928, + "sampling/sampling_logp_difference/max": 0.2511177062988281, + "sampling/sampling_logp_difference/mean": 0.015513719990849495, + "step": 151 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 1021.0, + "completions/mean_length": 221.875, + "completions/mean_terminated_length": 196.0, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.8409735849127173, + "epoch": 0.1503461918892186, + "grad_norm": 2.913163423538208, + "kl_approx": 0.304290771484375, + "learning_rate": 1.9856947330283752e-05, + "loss": 0.2101, + "num_tokens": 3657103.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2671051025390625, + "sampling/importance_sampling_ratio/mean": 0.9994755983352661, + "sampling/importance_sampling_ratio/min": 0.7095960974693298, + "sampling/sampling_logp_difference/max": 0.34305936098098755, + "sampling/sampling_logp_difference/mean": 0.014956234022974968, + "step": 152 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 629.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 198.1875, + "completions/mean_terminated_length": 198.1875, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.7537503028288484, + "epoch": 0.1513353115727003, + "grad_norm": 3.0429210662841797, + "kl_approx": 0.3447265625, + "learning_rate": 1.985106354988997e-05, + "loss": 0.259, + "num_tokens": 3684909.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1934473514556885, + "sampling/importance_sampling_ratio/mean": 0.9998313784599304, + "sampling/importance_sampling_ratio/min": 0.8096355199813843, + "sampling/sampling_logp_difference/max": 0.21117115020751953, + "sampling/sampling_logp_difference/mean": 0.014254819601774216, + "step": 153 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 597.0, + "completions/max_terminated_length": 597.0, + "completions/mean_length": 124.78125, + "completions/mean_terminated_length": 124.78125, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.6247568116523325, + "epoch": 0.152324431256182, + "grad_norm": 2.672576427459717, + "kl_approx": 0.300811767578125, + "learning_rate": 1.984506210249986e-05, + "loss": 0.1681, + "num_tokens": 3708878.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1651464700698853, + "sampling/importance_sampling_ratio/mean": 0.9997557401657104, + "sampling/importance_sampling_ratio/min": 0.7907420992851257, + "sampling/sampling_logp_difference/max": 0.23478341102600098, + "sampling/sampling_logp_difference/mean": 0.012037264183163643, + "step": 154 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 979.0, + "completions/max_terminated_length": 979.0, + "completions/mean_length": 174.0625, + "completions/mean_terminated_length": 174.0625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.5881611919030547, + "epoch": 0.1533135509396637, + "grad_norm": 3.105463743209839, + "kl_approx": 0.23784637451171875, + "learning_rate": 1.9838943059798305e-05, + "loss": 0.1565, + "num_tokens": 3733720.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.195961356163025, + "sampling/importance_sampling_ratio/mean": 0.9997238516807556, + "sampling/importance_sampling_ratio/min": 0.8026215434074402, + "sampling/sampling_logp_difference/max": 0.21987199783325195, + "sampling/sampling_logp_difference/mean": 0.011494816280901432, + "step": 155 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 293.0, + "completions/max_terminated_length": 293.0, + "completions/mean_length": 114.78125, + "completions/mean_terminated_length": 114.78125, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.7020180281251669, + "epoch": 0.1543026706231454, + "grad_norm": 3.6018433570861816, + "kl_approx": 0.30696868896484375, + "learning_rate": 1.9832706494874812e-05, + "loss": 0.1799, + "num_tokens": 3754737.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2384921312332153, + "sampling/importance_sampling_ratio/mean": 1.0007057189941406, + "sampling/importance_sampling_ratio/min": 0.8484919667243958, + "sampling/sampling_logp_difference/max": 0.21389460563659668, + "sampling/sampling_logp_difference/mean": 0.012855030596256256, + "step": 156 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 145.59375, + "completions/mean_terminated_length": 145.59375, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.7499087369069457, + "epoch": 0.1552917903066271, + "grad_norm": 3.3212716579437256, + "kl_approx": 0.3294525146484375, + "learning_rate": 1.982635248222264e-05, + "loss": 0.2113, + "num_tokens": 3779452.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2396161556243896, + "sampling/importance_sampling_ratio/mean": 1.0000402927398682, + "sampling/importance_sampling_ratio/min": 0.7865886688232422, + "sampling/sampling_logp_difference/max": 0.2400498390197754, + "sampling/sampling_logp_difference/mean": 0.016655726358294487, + "step": 157 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 379.0, + "completions/max_terminated_length": 379.0, + "completions/mean_length": 132.9375, + "completions/mean_terminated_length": 132.9375, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.6739329663105309, + "epoch": 0.1562809099901088, + "grad_norm": 2.291365146636963, + "kl_approx": 0.23013877868652344, + "learning_rate": 1.9819881097737917e-05, + "loss": 0.1676, + "num_tokens": 3803258.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2654168605804443, + "sampling/importance_sampling_ratio/mean": 1.0008260011672974, + "sampling/importance_sampling_ratio/min": 0.854902446269989, + "sampling/sampling_logp_difference/max": 0.23540163040161133, + "sampling/sampling_logp_difference/mean": 0.012866603210568428, + "step": 158 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 89.53125, + "completions/mean_terminated_length": 89.53125, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.6572990431450307, + "epoch": 0.1572700296735905, + "grad_norm": 3.89337158203125, + "kl_approx": 0.3024749755859375, + "learning_rate": 1.9813292418718734e-05, + "loss": 0.2223, + "num_tokens": 3828891.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2767776250839233, + "sampling/importance_sampling_ratio/mean": 1.0000778436660767, + "sampling/importance_sampling_ratio/min": 0.869756281375885, + "sampling/sampling_logp_difference/max": 0.2443394660949707, + "sampling/sampling_logp_difference/mean": 0.012500524520874023, + "step": 159 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 131.875, + "completions/mean_terminated_length": 131.875, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6191426855511963, + "epoch": 0.1582591493570722, + "grad_norm": 4.415280818939209, + "kl_approx": 0.421478271484375, + "learning_rate": 1.9806586523864212e-05, + "loss": 0.3052, + "num_tokens": 3855055.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2650364637374878, + "sampling/importance_sampling_ratio/mean": 1.0005650520324707, + "sampling/importance_sampling_ratio/min": 0.7848238945007324, + "sampling/sampling_logp_difference/max": 0.2422959804534912, + "sampling/sampling_logp_difference/mean": 0.012121576815843582, + "step": 160 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 137.875, + "completions/mean_terminated_length": 109.29032135009766, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.7634169803932309, + "epoch": 0.15924826904055392, + "grad_norm": 3.610884189605713, + "kl_approx": 0.359588623046875, + "learning_rate": 1.9799763493273572e-05, + "loss": 0.2192, + "num_tokens": 3877203.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2720191478729248, + "sampling/importance_sampling_ratio/mean": 1.000068187713623, + "sampling/importance_sampling_ratio/min": 0.8429659008979797, + "sampling/sampling_logp_difference/max": 0.24060559272766113, + "sampling/sampling_logp_difference/mean": 0.012453525327146053, + "step": 161 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 853.0, + "completions/max_terminated_length": 853.0, + "completions/mean_length": 202.71875, + "completions/mean_terminated_length": 202.71875, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.8651245888322592, + "epoch": 0.16023738872403562, + "grad_norm": 2.9442763328552246, + "kl_approx": 0.31336212158203125, + "learning_rate": 1.9792823408445173e-05, + "loss": 0.2007, + "num_tokens": 3902882.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.258875846862793, + "sampling/importance_sampling_ratio/mean": 1.0000407695770264, + "sampling/importance_sampling_ratio/min": 0.7594712376594543, + "sampling/sampling_logp_difference/max": 0.2751328945159912, + "sampling/sampling_logp_difference/mean": 0.016275566071271896, + "step": 162 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 226.53125, + "completions/mean_terminated_length": 226.53125, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 0.7738486537709832, + "epoch": 0.16122650840751732, + "grad_norm": 2.517744302749634, + "kl_approx": 0.2826194763183594, + "learning_rate": 1.978576635227554e-05, + "loss": 0.242, + "num_tokens": 3930331.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2863456010818481, + "sampling/importance_sampling_ratio/mean": 0.9998080134391785, + "sampling/importance_sampling_ratio/min": 0.7640848755836487, + "sampling/sampling_logp_difference/max": 0.26907646656036377, + "sampling/sampling_logp_difference/mean": 0.015387672930955887, + "step": 163 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 193.40625, + "completions/mean_terminated_length": 193.40625, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 0.8344437694177032, + "epoch": 0.16221562809099901, + "grad_norm": 2.780697822570801, + "kl_approx": 0.28790283203125, + "learning_rate": 1.9778592409058376e-05, + "loss": 0.1961, + "num_tokens": 3958688.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1876567602157593, + "sampling/importance_sampling_ratio/mean": 0.9996064305305481, + "sampling/importance_sampling_ratio/min": 0.7988571524620056, + "sampling/sampling_logp_difference/max": 0.22457313537597656, + "sampling/sampling_logp_difference/mean": 0.014476037584245205, + "step": 164 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 137.3125, + "completions/mean_terminated_length": 137.3125, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.8471739897504449, + "epoch": 0.1632047477744807, + "grad_norm": 4.318841934204102, + "kl_approx": 0.38570404052734375, + "learning_rate": 1.9771301664483548e-05, + "loss": 0.2909, + "num_tokens": 3978818.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3852136135101318, + "sampling/importance_sampling_ratio/mean": 0.9999037384986877, + "sampling/importance_sampling_ratio/min": 0.8238513469696045, + "sampling/sampling_logp_difference/max": 0.3258543014526367, + "sampling/sampling_logp_difference/mean": 0.016974125057458878, + "step": 165 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 419.0, + "completions/max_terminated_length": 419.0, + "completions/mean_length": 128.46875, + "completions/mean_terminated_length": 128.46875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.8208950664848089, + "epoch": 0.1641938674579624, + "grad_norm": 2.9148755073547363, + "kl_approx": 0.3923187255859375, + "learning_rate": 1.976389420563607e-05, + "loss": 0.2515, + "num_tokens": 4006073.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3772600889205933, + "sampling/importance_sampling_ratio/mean": 1.0003610849380493, + "sampling/importance_sampling_ratio/min": 0.736176609992981, + "sampling/sampling_logp_difference/max": 0.3200960159301758, + "sampling/sampling_logp_difference/mean": 0.014962531626224518, + "step": 166 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 609.0, + "completions/max_terminated_length": 609.0, + "completions/mean_length": 115.375, + "completions/mean_terminated_length": 115.375, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7249600132927299, + "epoch": 0.1651829871414441, + "grad_norm": 3.2919387817382812, + "kl_approx": 0.3755035400390625, + "learning_rate": 1.975637012099507e-05, + "loss": 0.2123, + "num_tokens": 4029997.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.183044672012329, + "sampling/importance_sampling_ratio/mean": 1.0003975629806519, + "sampling/importance_sampling_ratio/min": 0.7889966368675232, + "sampling/sampling_logp_difference/max": 0.23699331283569336, + "sampling/sampling_logp_difference/mean": 0.012874506413936615, + "step": 167 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 95.46875, + "completions/mean_terminated_length": 95.46875, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "entropy": 0.7351692164083943, + "epoch": 0.1661721068249258, + "grad_norm": 3.296281099319458, + "kl_approx": 0.3929901123046875, + "learning_rate": 1.97487295004327e-05, + "loss": 0.2123, + "num_tokens": 4051188.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1956899166107178, + "sampling/importance_sampling_ratio/mean": 1.0005613565444946, + "sampling/importance_sampling_ratio/min": 0.7848471999168396, + "sampling/sampling_logp_difference/max": 0.24226617813110352, + "sampling/sampling_logp_difference/mean": 0.013580622151494026, + "step": 168 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 710.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 184.53125, + "completions/mean_terminated_length": 184.53125, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.7059888476505876, + "epoch": 0.1671612265084075, + "grad_norm": 4.150257110595703, + "kl_approx": 0.361114501953125, + "learning_rate": 1.9740972435213114e-05, + "loss": 0.241, + "num_tokens": 4075181.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.4256788492202759, + "sampling/importance_sampling_ratio/mean": 0.9996298551559448, + "sampling/importance_sampling_ratio/min": 0.8149135708808899, + "sampling/sampling_logp_difference/max": 0.3546481132507324, + "sampling/sampling_logp_difference/mean": 0.013830306939780712, + "step": 169 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 626.0, + "completions/max_terminated_length": 626.0, + "completions/mean_length": 155.625, + "completions/mean_terminated_length": 155.625, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.6838713185861707, + "epoch": 0.1681503461918892, + "grad_norm": 4.234602928161621, + "kl_approx": 0.3809967041015625, + "learning_rate": 1.9733099017991342e-05, + "loss": 0.2757, + "num_tokens": 4095057.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.303252100944519, + "sampling/importance_sampling_ratio/mean": 0.9999076724052429, + "sampling/importance_sampling_ratio/min": 0.8086825013160706, + "sampling/sampling_logp_difference/max": 0.2648627758026123, + "sampling/sampling_logp_difference/mean": 0.013494862243533134, + "step": 170 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 855.0, + "completions/max_terminated_length": 855.0, + "completions/mean_length": 243.15625, + "completions/mean_terminated_length": 243.15625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.6644512871280313, + "epoch": 0.16913946587537093, + "grad_norm": 2.594848871231079, + "kl_approx": 0.21328353881835938, + "learning_rate": 1.972510934281218e-05, + "loss": 0.1656, + "num_tokens": 4122542.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2951419353485107, + "sampling/importance_sampling_ratio/mean": 1.0000733137130737, + "sampling/importance_sampling_ratio/min": 0.7652145028114319, + "sampling/sampling_logp_difference/max": 0.26759910583496094, + "sampling/sampling_logp_difference/mean": 0.010878982953727245, + "step": 171 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 252.84375, + "completions/mean_terminated_length": 227.9677276611328, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 1.0398688837885857, + "epoch": 0.17012858555885263, + "grad_norm": 4.199376106262207, + "kl_approx": 0.5884552001953125, + "learning_rate": 1.9717003505109097e-05, + "loss": 0.3295, + "num_tokens": 4148985.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.272071123123169, + "sampling/importance_sampling_ratio/mean": 0.9997367262840271, + "sampling/importance_sampling_ratio/min": 0.796878457069397, + "sampling/sampling_logp_difference/max": 0.2406463623046875, + "sampling/sampling_logp_difference/mean": 0.016616344451904297, + "step": 172 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 715.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 232.9375, + "completions/mean_terminated_length": 232.9375, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.7784532429650426, + "epoch": 0.17111770524233433, + "grad_norm": 2.239717721939087, + "kl_approx": 0.2509613037109375, + "learning_rate": 1.9708781601703066e-05, + "loss": 0.1767, + "num_tokens": 4176015.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2318041324615479, + "sampling/importance_sampling_ratio/mean": 0.9998577237129211, + "sampling/importance_sampling_ratio/min": 0.8284434676170349, + "sampling/sampling_logp_difference/max": 0.2084798812866211, + "sampling/sampling_logp_difference/mean": 0.013383631594479084, + "step": 173 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 481.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 155.78125, + "completions/mean_terminated_length": 155.78125, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.792903590016067, + "epoch": 0.17210682492581603, + "grad_norm": 3.1732075214385986, + "kl_approx": 0.3128662109375, + "learning_rate": 1.9700443730801412e-05, + "loss": 0.2143, + "num_tokens": 4198488.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1857928037643433, + "sampling/importance_sampling_ratio/mean": 0.9998219013214111, + "sampling/importance_sampling_ratio/min": 0.8167960047721863, + "sampling/sampling_logp_difference/max": 0.20236587524414062, + "sampling/sampling_logp_difference/mean": 0.014955122023820877, + "step": 174 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 888.0, + "completions/max_terminated_length": 888.0, + "completions/mean_length": 200.375, + "completions/mean_terminated_length": 200.375, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "entropy": 0.668834628071636, + "epoch": 0.17309594460929772, + "grad_norm": 2.112032413482666, + "kl_approx": 0.2960700988769531, + "learning_rate": 1.9691989991996663e-05, + "loss": 0.1635, + "num_tokens": 4226844.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.355528712272644, + "sampling/importance_sampling_ratio/mean": 1.0006966590881348, + "sampling/importance_sampling_ratio/min": 0.8050819635391235, + "sampling/sampling_logp_difference/max": 0.30419158935546875, + "sampling/sampling_logp_difference/mean": 0.013798365369439125, + "step": 175 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.0, + "completions/max_terminated_length": 217.0, + "completions/mean_length": 77.0625, + "completions/mean_terminated_length": 77.0625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.43388065369799733, + "epoch": 0.17408506429277942, + "grad_norm": 1.8014647960662842, + "kl_approx": 0.17783355712890625, + "learning_rate": 1.9683420486265328e-05, + "loss": 0.0985, + "num_tokens": 4245606.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1526026725769043, + "sampling/importance_sampling_ratio/mean": 1.0000427961349487, + "sampling/importance_sampling_ratio/min": 0.8161384463310242, + "sampling/sampling_logp_difference/max": 0.2031712532043457, + "sampling/sampling_logp_difference/mean": 0.008749328553676605, + "step": 176 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/max_terminated_length": 169.0, + "completions/mean_length": 66.09375, + "completions/mean_terminated_length": 66.09375, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "entropy": 0.5946537521667778, + "epoch": 0.17507418397626112, + "grad_norm": 4.021422386169434, + "kl_approx": 0.40715789794921875, + "learning_rate": 1.967473531596671e-05, + "loss": 0.2421, + "num_tokens": 4265769.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1555269956588745, + "sampling/importance_sampling_ratio/mean": 0.9993851780891418, + "sampling/importance_sampling_ratio/min": 0.8805910348892212, + "sampling/sampling_logp_difference/max": 0.14455652236938477, + "sampling/sampling_logp_difference/mean": 0.010368101298809052, + "step": 177 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 142.0, + "completions/max_terminated_length": 142.0, + "completions/mean_length": 68.90625, + "completions/mean_terminated_length": 68.90625, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.5327342487871647, + "epoch": 0.17606330365974282, + "grad_norm": 6.317509651184082, + "kl_approx": 0.4254322052001953, + "learning_rate": 1.966593458484168e-05, + "loss": 0.2733, + "num_tokens": 4286398.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2253488302230835, + "sampling/importance_sampling_ratio/mean": 1.0002055168151855, + "sampling/importance_sampling_ratio/min": 0.879186749458313, + "sampling/sampling_logp_difference/max": 0.20322561264038086, + "sampling/sampling_logp_difference/mean": 0.009690026752650738, + "step": 178 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 142.0, + "completions/max_terminated_length": 142.0, + "completions/mean_length": 81.46875, + "completions/mean_terminated_length": 81.46875, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.5916108139790595, + "epoch": 0.17705242334322452, + "grad_norm": 3.7381250858306885, + "kl_approx": 0.3475494384765625, + "learning_rate": 1.9657018398011435e-05, + "loss": 0.2458, + "num_tokens": 4309397.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1364266872406006, + "sampling/importance_sampling_ratio/mean": 1.0005912780761719, + "sampling/importance_sampling_ratio/min": 0.8377895951271057, + "sampling/sampling_logp_difference/max": 0.1769883632659912, + "sampling/sampling_logp_difference/mean": 0.009945884346961975, + "step": 179 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/max_terminated_length": 160.0, + "completions/mean_length": 90.0, + "completions/mean_terminated_length": 90.0, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.5743699269369245, + "epoch": 0.17804154302670624, + "grad_norm": 4.380485534667969, + "kl_approx": 0.485565185546875, + "learning_rate": 1.9647986861976246e-05, + "loss": 0.3441, + "num_tokens": 4332237.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.150006651878357, + "sampling/importance_sampling_ratio/mean": 0.9991024732589722, + "sampling/importance_sampling_ratio/min": 0.8479262590408325, + "sampling/sampling_logp_difference/max": 0.164961576461792, + "sampling/sampling_logp_difference/mean": 0.011140280403196812, + "step": 180 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 475.0, + "completions/max_terminated_length": 475.0, + "completions/mean_length": 134.71875, + "completions/mean_terminated_length": 134.71875, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "entropy": 0.6700577416922897, + "epoch": 0.17903066271018794, + "grad_norm": 3.176434278488159, + "kl_approx": 0.35842132568359375, + "learning_rate": 1.9638840084614182e-05, + "loss": 0.2196, + "num_tokens": 4362252.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2185258865356445, + "sampling/importance_sampling_ratio/mean": 0.9991312623023987, + "sampling/importance_sampling_ratio/min": 0.821160614490509, + "sampling/sampling_logp_difference/max": 0.19764184951782227, + "sampling/sampling_logp_difference/mean": 0.013197019696235657, + "step": 181 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 997.0, + "completions/max_terminated_length": 997.0, + "completions/mean_length": 253.8125, + "completions/mean_terminated_length": 253.8125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "entropy": 1.0195479076355696, + "epoch": 0.18001978239366964, + "grad_norm": 2.866442918777466, + "kl_approx": 0.397796630859375, + "learning_rate": 1.9629578175179823e-05, + "loss": 0.2639, + "num_tokens": 4389190.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2505145072937012, + "sampling/importance_sampling_ratio/mean": 1.0000097751617432, + "sampling/importance_sampling_ratio/min": 0.7541719079017639, + "sampling/sampling_logp_difference/max": 0.282135009765625, + "sampling/sampling_logp_difference/mean": 0.015476331114768982, + "step": 182 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/max_terminated_length": 295.0, + "completions/mean_length": 106.40625, + "completions/mean_terminated_length": 106.40625, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.7956465752795339, + "epoch": 0.18100890207715134, + "grad_norm": 2.7353873252868652, + "kl_approx": 0.35871124267578125, + "learning_rate": 1.9620201244302952e-05, + "loss": 0.2052, + "num_tokens": 4409491.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2233108282089233, + "sampling/importance_sampling_ratio/mean": 1.0005443096160889, + "sampling/importance_sampling_ratio/min": 0.8413002490997314, + "sampling/sampling_logp_difference/max": 0.20156097412109375, + "sampling/sampling_logp_difference/mean": 0.014676181599497795, + "step": 183 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 747.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 196.90625, + "completions/mean_terminated_length": 196.90625, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.8832181142643094, + "epoch": 0.18199802176063304, + "grad_norm": 2.3406519889831543, + "kl_approx": 0.320037841796875, + "learning_rate": 1.9610709403987248e-05, + "loss": 0.1994, + "num_tokens": 4433656.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1645704507827759, + "sampling/importance_sampling_ratio/mean": 1.0000042915344238, + "sampling/importance_sampling_ratio/min": 0.7931860089302063, + "sampling/sampling_logp_difference/max": 0.23169755935668945, + "sampling/sampling_logp_difference/mean": 0.014447847381234169, + "step": 184 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.0, + "completions/max_terminated_length": 369.0, + "completions/mean_length": 141.1875, + "completions/mean_terminated_length": 141.1875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.8309785891324282, + "epoch": 0.18298714144411474, + "grad_norm": 3.316915988922119, + "kl_approx": 0.369415283203125, + "learning_rate": 1.9601102767608924e-05, + "loss": 0.2373, + "num_tokens": 4459342.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1812007427215576, + "sampling/importance_sampling_ratio/mean": 1.000186562538147, + "sampling/importance_sampling_ratio/min": 0.7918558120727539, + "sampling/sampling_logp_difference/max": 0.23337602615356445, + "sampling/sampling_logp_difference/mean": 0.013928555883467197, + "step": 185 + }, + { + "completions/clipped_ratio": 0.09375, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 208.65625, + "completions/mean_terminated_length": 124.31034088134766, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "entropy": 0.8102191786165349, + "epoch": 0.18397626112759644, + "grad_norm": 3.627206563949585, + "kl_approx": 0.3878021240234375, + "learning_rate": 1.95913814499154e-05, + "loss": 0.2484, + "num_tokens": 4484379.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2079881429672241, + "sampling/importance_sampling_ratio/mean": 0.9997969269752502, + "sampling/importance_sampling_ratio/min": 0.7932904362678528, + "sampling/sampling_logp_difference/max": 0.2315659523010254, + "sampling/sampling_logp_difference/mean": 0.009880214929580688, + "step": 186 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 115.0625, + "completions/mean_terminated_length": 115.0625, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.8650742191821337, + "epoch": 0.18496538081107813, + "grad_norm": 3.8704757690429688, + "kl_approx": 0.44366455078125, + "learning_rate": 1.95815455670239e-05, + "loss": 0.3256, + "num_tokens": 4503637.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1381938457489014, + "sampling/importance_sampling_ratio/mean": 1.000834584236145, + "sampling/importance_sampling_ratio/min": 0.8320397138595581, + "sampling/sampling_logp_difference/max": 0.18387508392333984, + "sampling/sampling_logp_difference/mean": 0.014146720990538597, + "step": 187 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 147.03125, + "completions/mean_terminated_length": 147.03125, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.7689765151590109, + "epoch": 0.18595450049455983, + "grad_norm": 3.563326120376587, + "kl_approx": 0.3994140625, + "learning_rate": 1.9571595236420103e-05, + "loss": 0.2779, + "num_tokens": 4524022.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1522051095962524, + "sampling/importance_sampling_ratio/mean": 1.0002191066741943, + "sampling/importance_sampling_ratio/min": 0.8480837941169739, + "sampling/sampling_logp_difference/max": 0.16477584838867188, + "sampling/sampling_logp_difference/mean": 0.013784163631498814, + "step": 188 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 535.0, + "completions/max_terminated_length": 535.0, + "completions/mean_length": 177.0625, + "completions/mean_terminated_length": 177.0625, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 1.0105805043131113, + "epoch": 0.18694362017804153, + "grad_norm": 3.023618698120117, + "kl_approx": 0.341705322265625, + "learning_rate": 1.9561530576956703e-05, + "loss": 0.2472, + "num_tokens": 4548872.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2522778511047363, + "sampling/importance_sampling_ratio/mean": 1.000182867050171, + "sampling/importance_sampling_ratio/min": 0.8739991188049316, + "sampling/sampling_logp_difference/max": 0.22496414184570312, + "sampling/sampling_logp_difference/mean": 0.01618615910410881, + "step": 189 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 776.0, + "completions/max_terminated_length": 776.0, + "completions/mean_length": 251.125, + "completions/mean_terminated_length": 251.125, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.9955198168754578, + "epoch": 0.18793273986152326, + "grad_norm": 3.141435146331787, + "kl_approx": 0.37359619140625, + "learning_rate": 1.955135170885202e-05, + "loss": 0.2296, + "num_tokens": 4571380.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.196488380432129, + "sampling/importance_sampling_ratio/mean": 1.001082181930542, + "sampling/importance_sampling_ratio/min": 0.7715004682540894, + "sampling/sampling_logp_difference/max": 0.2594180107116699, + "sampling/sampling_logp_difference/mean": 0.015606801956892014, + "step": 190 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 375.0, + "completions/max_terminated_length": 375.0, + "completions/mean_length": 119.0625, + "completions/mean_terminated_length": 119.0625, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.7149736005812883, + "epoch": 0.18892185954500496, + "grad_norm": 2.953836679458618, + "kl_approx": 0.2916107177734375, + "learning_rate": 1.9541058753688538e-05, + "loss": 0.1574, + "num_tokens": 4598206.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1896014213562012, + "sampling/importance_sampling_ratio/mean": 1.0003082752227783, + "sampling/importance_sampling_ratio/min": 0.8298519849777222, + "sampling/sampling_logp_difference/max": 0.1865079402923584, + "sampling/sampling_logp_difference/mean": 0.013138935901224613, + "step": 191 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 432.0, + "completions/max_terminated_length": 432.0, + "completions/mean_length": 129.75, + "completions/mean_terminated_length": 129.75, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.8066576132550836, + "epoch": 0.18991097922848665, + "grad_norm": 2.5305604934692383, + "kl_approx": 0.371826171875, + "learning_rate": 1.9530651834411477e-05, + "loss": 0.1943, + "num_tokens": 4624694.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1441923379898071, + "sampling/importance_sampling_ratio/mean": 0.9994572997093201, + "sampling/importance_sampling_ratio/min": 0.8220016360282898, + "sampling/sampling_logp_difference/max": 0.19601285457611084, + "sampling/sampling_logp_difference/mean": 0.014570243656635284, + "step": 192 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 477.0, + "completions/max_terminated_length": 477.0, + "completions/mean_length": 159.21875, + "completions/mean_terminated_length": 159.21875, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.9491471033543348, + "epoch": 0.19090009891196835, + "grad_norm": 2.8296799659729004, + "kl_approx": 0.3559112548828125, + "learning_rate": 1.95201310753273e-05, + "loss": 0.2463, + "num_tokens": 4654013.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2658840417861938, + "sampling/importance_sampling_ratio/mean": 1.0003011226654053, + "sampling/importance_sampling_ratio/min": 0.8553217053413391, + "sampling/sampling_logp_difference/max": 0.23577070236206055, + "sampling/sampling_logp_difference/mean": 0.016150332987308502, + "step": 193 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 389.0, + "completions/max_terminated_length": 389.0, + "completions/mean_length": 102.28125, + "completions/mean_terminated_length": 102.28125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.8161912616342306, + "epoch": 0.19188921859545005, + "grad_norm": 3.4798426628112793, + "kl_approx": 0.3473243713378906, + "learning_rate": 1.9509496602102253e-05, + "loss": 0.214, + "num_tokens": 4680094.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2022815942764282, + "sampling/importance_sampling_ratio/mean": 0.9997754693031311, + "sampling/importance_sampling_ratio/min": 0.8727205395698547, + "sampling/sampling_logp_difference/max": 0.1842210292816162, + "sampling/sampling_logp_difference/mean": 0.01524446438997984, + "step": 194 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/max_terminated_length": 177.0, + "completions/mean_length": 93.5625, + "completions/mean_terminated_length": 93.5625, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.6369808427989483, + "epoch": 0.19287833827893175, + "grad_norm": 3.127537965774536, + "kl_approx": 0.2580108642578125, + "learning_rate": 1.9498748541760845e-05, + "loss": 0.2354, + "num_tokens": 4702680.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2261836528778076, + "sampling/importance_sampling_ratio/mean": 1.0004708766937256, + "sampling/importance_sampling_ratio/min": 0.8368150591850281, + "sampling/sampling_logp_difference/max": 0.20390665531158447, + "sampling/sampling_logp_difference/mean": 0.011521467007696629, + "step": 195 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/max_terminated_length": 159.0, + "completions/mean_length": 92.21875, + "completions/mean_terminated_length": 92.21875, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.6702957535162568, + "epoch": 0.19386745796241345, + "grad_norm": 2.4470937252044678, + "kl_approx": 0.28216552734375, + "learning_rate": 1.9487887022684336e-05, + "loss": 0.1685, + "num_tokens": 4721303.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2678173780441284, + "sampling/importance_sampling_ratio/mean": 0.9994837641716003, + "sampling/importance_sampling_ratio/min": 0.8375304937362671, + "sampling/sampling_logp_difference/max": 0.23729681968688965, + "sampling/sampling_logp_difference/mean": 0.011245638132095337, + "step": 196 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 173.0, + "completions/max_terminated_length": 173.0, + "completions/mean_length": 86.125, + "completions/mean_terminated_length": 86.125, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.5091623235493898, + "epoch": 0.19485657764589515, + "grad_norm": 2.0916755199432373, + "kl_approx": 0.2183971405029297, + "learning_rate": 1.947691217460921e-05, + "loss": 0.1276, + "num_tokens": 4742603.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1455672979354858, + "sampling/importance_sampling_ratio/mean": 0.9997662901878357, + "sampling/importance_sampling_ratio/min": 0.8272221684455872, + "sampling/sampling_logp_difference/max": 0.1896820068359375, + "sampling/sampling_logp_difference/mean": 0.01040777750313282, + "step": 197 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 687.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 191.78125, + "completions/mean_terminated_length": 191.78125, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.8770013572648168, + "epoch": 0.19584569732937684, + "grad_norm": 3.199310064315796, + "kl_approx": 0.3926849365234375, + "learning_rate": 1.946582412862562e-05, + "loss": 0.2737, + "num_tokens": 4765476.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3287382125854492, + "sampling/importance_sampling_ratio/mean": 0.9993873238563538, + "sampling/importance_sampling_ratio/min": 0.7851613163948059, + "sampling/sampling_logp_difference/max": 0.28422975540161133, + "sampling/sampling_logp_difference/mean": 0.0156437736004591, + "step": 198 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 171.4375, + "completions/mean_terminated_length": 171.4375, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.7275398671627045, + "epoch": 0.19683481701285854, + "grad_norm": 2.7614235877990723, + "kl_approx": 0.272003173828125, + "learning_rate": 1.9454623017175814e-05, + "loss": 0.2234, + "num_tokens": 4792330.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1939396858215332, + "sampling/importance_sampling_ratio/mean": 0.9999340176582336, + "sampling/importance_sampling_ratio/min": 0.7815227508544922, + "sampling/sampling_logp_difference/max": 0.24651098251342773, + "sampling/sampling_logp_difference/mean": 0.013386149890720844, + "step": 199 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/max_terminated_length": 259.0, + "completions/mean_length": 77.8125, + "completions/mean_terminated_length": 77.8125, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "entropy": 0.5949868741445243, + "epoch": 0.19782393669634027, + "grad_norm": 3.8966453075408936, + "kl_approx": 0.30572509765625, + "learning_rate": 1.9443308974052574e-05, + "loss": 0.2292, + "num_tokens": 4809532.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1457360982894897, + "sampling/importance_sampling_ratio/mean": 0.9992508292198181, + "sampling/importance_sampling_ratio/min": 0.7327111959457397, + "sampling/sampling_logp_difference/max": 0.3110036849975586, + "sampling/sampling_logp_difference/mean": 0.011897114105522633, + "step": 200 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 413.0, + "completions/max_terminated_length": 413.0, + "completions/mean_length": 92.4375, + "completions/mean_terminated_length": 92.4375, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.5635983040556312, + "epoch": 0.19881305637982197, + "grad_norm": 5.359744548797607, + "kl_approx": 0.34445953369140625, + "learning_rate": 1.9431882134397596e-05, + "loss": 0.2546, + "num_tokens": 4827146.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1715437173843384, + "sampling/importance_sampling_ratio/mean": 1.0001041889190674, + "sampling/importance_sampling_ratio/min": 0.8740832209587097, + "sampling/sampling_logp_difference/max": 0.15832233428955078, + "sampling/sampling_logp_difference/mean": 0.011723761446774006, + "step": 201 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 654.0, + "completions/max_terminated_length": 654.0, + "completions/mean_length": 155.875, + "completions/mean_terminated_length": 155.875, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "entropy": 0.7137188259512186, + "epoch": 0.19980217606330367, + "grad_norm": 3.809016466140747, + "kl_approx": 0.458099365234375, + "learning_rate": 1.9420342634699893e-05, + "loss": 0.2864, + "num_tokens": 4854150.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2270963191986084, + "sampling/importance_sampling_ratio/mean": 0.9997861981391907, + "sampling/importance_sampling_ratio/min": 0.8192124366760254, + "sampling/sampling_logp_difference/max": 0.2046506404876709, + "sampling/sampling_logp_difference/mean": 0.014872337691485882, + "step": 202 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 664.0, + "completions/mean_length": 239.53125, + "completions/mean_terminated_length": 214.22579956054688, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.7787833148613572, + "epoch": 0.20079129574678536, + "grad_norm": 3.1124439239501953, + "kl_approx": 0.321380615234375, + "learning_rate": 1.9408690612794146e-05, + "loss": 0.2715, + "num_tokens": 4889903.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3740170001983643, + "sampling/importance_sampling_ratio/mean": 1.0001214742660522, + "sampling/importance_sampling_ratio/min": 0.7406344413757324, + "sampling/sampling_logp_difference/max": 0.31773853302001953, + "sampling/sampling_logp_difference/mean": 0.013337602838873863, + "step": 203 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 721.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 210.875, + "completions/mean_terminated_length": 210.875, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.7800484076142311, + "epoch": 0.20178041543026706, + "grad_norm": 3.4030117988586426, + "kl_approx": 0.412445068359375, + "learning_rate": 1.9396926207859085e-05, + "loss": 0.2629, + "num_tokens": 4919331.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2188256978988647, + "sampling/importance_sampling_ratio/mean": 1.0005685091018677, + "sampling/importance_sampling_ratio/min": 0.8302646279335022, + "sampling/sampling_logp_difference/max": 0.19788789749145508, + "sampling/sampling_logp_difference/mean": 0.014459380879998207, + "step": 204 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 896.0, + "completions/max_terminated_length": 896.0, + "completions/mean_length": 251.1875, + "completions/mean_terminated_length": 251.1875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.892274959012866, + "epoch": 0.20276953511374876, + "grad_norm": 2.918339252471924, + "kl_approx": 0.353759765625, + "learning_rate": 1.9385049560415794e-05, + "loss": 0.2188, + "num_tokens": 4949625.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2151859998703003, + "sampling/importance_sampling_ratio/mean": 1.0001851320266724, + "sampling/importance_sampling_ratio/min": 0.7375075221061707, + "sampling/sampling_logp_difference/max": 0.3044790029525757, + "sampling/sampling_logp_difference/mean": 0.014378399588167667, + "step": 205 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 140.375, + "completions/mean_terminated_length": 140.375, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7373186573386192, + "epoch": 0.20375865479723046, + "grad_norm": 2.4946844577789307, + "kl_approx": 0.3034515380859375, + "learning_rate": 1.9373060812326053e-05, + "loss": 0.1599, + "num_tokens": 4973933.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2447043657302856, + "sampling/importance_sampling_ratio/mean": 1.0001064538955688, + "sampling/importance_sampling_ratio/min": 0.8452403545379639, + "sampling/sampling_logp_difference/max": 0.21889805793762207, + "sampling/sampling_logp_difference/mean": 0.013263982720673084, + "step": 206 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 474.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 185.59375, + "completions/mean_terminated_length": 185.59375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 1.1881454810500145, + "epoch": 0.20474777448071216, + "grad_norm": 3.8895764350891113, + "kl_approx": 0.4849853515625, + "learning_rate": 1.9360960106790645e-05, + "loss": 0.3703, + "num_tokens": 4994240.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2132391929626465, + "sampling/importance_sampling_ratio/mean": 1.0001754760742188, + "sampling/importance_sampling_ratio/min": 0.7871976494789124, + "sampling/sampling_logp_difference/max": 0.23927593231201172, + "sampling/sampling_logp_difference/mean": 0.018709952011704445, + "step": 207 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 160.375, + "completions/mean_terminated_length": 160.375, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.7769688349217176, + "epoch": 0.20573689416419386, + "grad_norm": 2.359301805496216, + "kl_approx": 0.27413177490234375, + "learning_rate": 1.9348747588347637e-05, + "loss": 0.1788, + "num_tokens": 5015228.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2356021404266357, + "sampling/importance_sampling_ratio/mean": 1.0001152753829956, + "sampling/importance_sampling_ratio/min": 0.8298988938331604, + "sampling/sampling_logp_difference/max": 0.21155834197998047, + "sampling/sampling_logp_difference/mean": 0.01686965487897396, + "step": 208 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 462.0, + "completions/max_terminated_length": 462.0, + "completions/mean_length": 113.75, + "completions/mean_terminated_length": 113.75, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7697482267394662, + "epoch": 0.20672601384767555, + "grad_norm": 2.464195489883423, + "kl_approx": 0.2992095947265625, + "learning_rate": 1.9336423402870655e-05, + "loss": 0.1803, + "num_tokens": 5037452.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1645135879516602, + "sampling/importance_sampling_ratio/mean": 1.000540018081665, + "sampling/importance_sampling_ratio/min": 0.8467459082603455, + "sampling/sampling_logp_difference/max": 0.16635465621948242, + "sampling/sampling_logp_difference/mean": 0.012847894802689552, + "step": 209 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/max_terminated_length": 190.0, + "completions/mean_length": 77.75, + "completions/mean_terminated_length": 77.75, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.6320470701903105, + "epoch": 0.20771513353115728, + "grad_norm": 3.374441146850586, + "kl_approx": 0.3346385955810547, + "learning_rate": 1.932398769756714e-05, + "loss": 0.2166, + "num_tokens": 5058676.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1435011625289917, + "sampling/importance_sampling_ratio/mean": 0.9997200965881348, + "sampling/importance_sampling_ratio/min": 0.8282894492149353, + "sampling/sampling_logp_difference/max": 0.18839263916015625, + "sampling/sampling_logp_difference/mean": 0.010547553189098835, + "step": 210 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 123.5, + "completions/mean_terminated_length": 123.5, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7704211338423193, + "epoch": 0.20870425321463898, + "grad_norm": 2.5807645320892334, + "kl_approx": 0.29001617431640625, + "learning_rate": 1.9311440620976597e-05, + "loss": 0.1891, + "num_tokens": 5085948.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1862140893936157, + "sampling/importance_sampling_ratio/mean": 0.9999226927757263, + "sampling/importance_sampling_ratio/min": 0.8195359110832214, + "sampling/sampling_logp_difference/max": 0.19901704788208008, + "sampling/sampling_logp_difference/mean": 0.014977291226387024, + "step": 211 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 848.0, + "completions/max_terminated_length": 848.0, + "completions/mean_length": 257.5625, + "completions/mean_terminated_length": 257.5625, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.9993455754593015, + "epoch": 0.20969337289812068, + "grad_norm": 3.080603837966919, + "kl_approx": 0.40655517578125, + "learning_rate": 1.9298782322968817e-05, + "loss": 0.2901, + "num_tokens": 5116694.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.245516300201416, + "sampling/importance_sampling_ratio/mean": 0.9999556541442871, + "sampling/importance_sampling_ratio/min": 0.7899016737937927, + "sampling/sampling_logp_difference/max": 0.23584675788879395, + "sampling/sampling_logp_difference/mean": 0.015802502632141113, + "step": 212 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 188.25, + "completions/mean_terminated_length": 188.25, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "entropy": 0.7308742143213749, + "epoch": 0.21068249258160238, + "grad_norm": 2.7129364013671875, + "kl_approx": 0.3378143310546875, + "learning_rate": 1.9286012954742078e-05, + "loss": 0.2051, + "num_tokens": 5143694.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2486017942428589, + "sampling/importance_sampling_ratio/mean": 1.000115990638733, + "sampling/importance_sampling_ratio/min": 0.7331138253211975, + "sampling/sampling_logp_difference/max": 0.3104543685913086, + "sampling/sampling_logp_difference/mean": 0.012803388759493828, + "step": 213 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 423.0, + "completions/max_terminated_length": 423.0, + "completions/mean_length": 130.15625, + "completions/mean_terminated_length": 130.15625, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7911368329077959, + "epoch": 0.21167161226508407, + "grad_norm": 2.9650793075561523, + "kl_approx": 0.36651611328125, + "learning_rate": 1.9273132668821363e-05, + "loss": 0.2241, + "num_tokens": 5166619.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1962522268295288, + "sampling/importance_sampling_ratio/mean": 0.9995303750038147, + "sampling/importance_sampling_ratio/min": 0.8319612145423889, + "sampling/sampling_logp_difference/max": 0.18396949768066406, + "sampling/sampling_logp_difference/mean": 0.013538091443479061, + "step": 214 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 103.0625, + "completions/mean_terminated_length": 103.0625, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.6631857696920633, + "epoch": 0.21266073194856577, + "grad_norm": 4.841701507568359, + "kl_approx": 0.40521240234375, + "learning_rate": 1.9260141619056507e-05, + "loss": 0.2922, + "num_tokens": 5187493.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.196144938468933, + "sampling/importance_sampling_ratio/mean": 1.0002715587615967, + "sampling/importance_sampling_ratio/min": 0.822303056716919, + "sampling/sampling_logp_difference/max": 0.1956462860107422, + "sampling/sampling_logp_difference/mean": 0.01218469813466072, + "step": 215 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 263.0, + "completions/max_terminated_length": 263.0, + "completions/mean_length": 90.25, + "completions/mean_terminated_length": 90.25, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.5698326015844941, + "epoch": 0.21364985163204747, + "grad_norm": 3.1044483184814453, + "kl_approx": 0.26737213134765625, + "learning_rate": 1.924703996062038e-05, + "loss": 0.1826, + "num_tokens": 5209101.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2458637952804565, + "sampling/importance_sampling_ratio/mean": 0.9994869828224182, + "sampling/importance_sampling_ratio/min": 0.8788732290267944, + "sampling/sampling_logp_difference/max": 0.21982908248901367, + "sampling/sampling_logp_difference/mean": 0.0105671975761652, + "step": 216 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 457.0, + "completions/max_terminated_length": 457.0, + "completions/mean_length": 155.0, + "completions/mean_terminated_length": 155.0, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.8520530294626951, + "epoch": 0.21463897131552917, + "grad_norm": 2.76305890083313, + "kl_approx": 0.377349853515625, + "learning_rate": 1.9233827850007028e-05, + "loss": 0.2426, + "num_tokens": 5234229.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1866172552108765, + "sampling/importance_sampling_ratio/mean": 0.9993882775306702, + "sampling/importance_sampling_ratio/min": 0.8335623741149902, + "sampling/sampling_logp_difference/max": 0.1820467710494995, + "sampling/sampling_logp_difference/mean": 0.014644963666796684, + "step": 217 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 919.0, + "completions/max_terminated_length": 919.0, + "completions/mean_length": 235.8125, + "completions/mean_terminated_length": 235.8125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.8813197785057127, + "epoch": 0.21562809099901087, + "grad_norm": 2.8763091564178467, + "kl_approx": 0.3265724182128906, + "learning_rate": 1.9220505445029803e-05, + "loss": 0.2413, + "num_tokens": 5261023.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2905473709106445, + "sampling/importance_sampling_ratio/mean": 1.0000096559524536, + "sampling/importance_sampling_ratio/min": 0.6972465515136719, + "sampling/sampling_logp_difference/max": 0.36061620712280273, + "sampling/sampling_logp_difference/mean": 0.014830242842435837, + "step": 218 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 500.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 144.15625, + "completions/mean_terminated_length": 144.15625, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.7217067535966635, + "epoch": 0.2166172106824926, + "grad_norm": 3.456519365310669, + "kl_approx": 0.322509765625, + "learning_rate": 1.9207072904819484e-05, + "loss": 0.2156, + "num_tokens": 5289956.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3540971279144287, + "sampling/importance_sampling_ratio/mean": 1.0006392002105713, + "sampling/importance_sampling_ratio/min": 0.8137744069099426, + "sampling/sampling_logp_difference/max": 0.3031349182128906, + "sampling/sampling_logp_difference/mean": 0.011853271164000034, + "step": 219 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 471.0, + "completions/max_terminated_length": 471.0, + "completions/mean_length": 187.625, + "completions/mean_terminated_length": 187.625, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.7304103774949908, + "epoch": 0.2176063303659743, + "grad_norm": 2.696822166442871, + "kl_approx": 0.313201904296875, + "learning_rate": 1.9193530389822364e-05, + "loss": 0.2273, + "num_tokens": 5315472.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1479991674423218, + "sampling/importance_sampling_ratio/mean": 0.9998998045921326, + "sampling/importance_sampling_ratio/min": 0.8383550047874451, + "sampling/sampling_logp_difference/max": 0.1763136386871338, + "sampling/sampling_logp_difference/mean": 0.01179458200931549, + "step": 220 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 164.0, + "completions/max_terminated_length": 164.0, + "completions/mean_length": 75.46875, + "completions/mean_terminated_length": 75.46875, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.5871479194611311, + "epoch": 0.218595450049456, + "grad_norm": 2.9542453289031982, + "kl_approx": 0.2622222900390625, + "learning_rate": 1.9179878061798347e-05, + "loss": 0.1563, + "num_tokens": 5336255.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1370563507080078, + "sampling/importance_sampling_ratio/mean": 0.9996902346611023, + "sampling/importance_sampling_ratio/min": 0.7966554164886475, + "sampling/sampling_logp_difference/max": 0.22733306884765625, + "sampling/sampling_logp_difference/mean": 0.010305657051503658, + "step": 221 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 146.65625, + "completions/mean_terminated_length": 146.65625, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.9522844757884741, + "epoch": 0.2195845697329377, + "grad_norm": 2.627192735671997, + "kl_approx": 0.40423583984375, + "learning_rate": 1.9166116083819002e-05, + "loss": 0.2683, + "num_tokens": 5359988.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1700776815414429, + "sampling/importance_sampling_ratio/mean": 1.0001568794250488, + "sampling/importance_sampling_ratio/min": 0.8344256281852722, + "sampling/sampling_logp_difference/max": 0.18101167678833008, + "sampling/sampling_logp_difference/mean": 0.016865020617842674, + "step": 222 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 377.0, + "completions/max_terminated_length": 377.0, + "completions/mean_length": 143.65625, + "completions/mean_terminated_length": 143.65625, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.7796209808439016, + "epoch": 0.2205736894164194, + "grad_norm": 2.666339874267578, + "kl_approx": 0.34142303466796875, + "learning_rate": 1.915224462026563e-05, + "loss": 0.2159, + "num_tokens": 5383921.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.182591199874878, + "sampling/importance_sampling_ratio/mean": 1.0003803968429565, + "sampling/importance_sampling_ratio/min": 0.8004521727561951, + "sampling/sampling_logp_difference/max": 0.2225785255432129, + "sampling/sampling_logp_difference/mean": 0.013402228243649006, + "step": 223 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 179.8125, + "completions/mean_terminated_length": 179.8125, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.8673951933160424, + "epoch": 0.2215628090999011, + "grad_norm": 2.7892537117004395, + "kl_approx": 0.36529541015625, + "learning_rate": 1.913826383682729e-05, + "loss": 0.2501, + "num_tokens": 5408003.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.18012535572052, + "sampling/importance_sampling_ratio/mean": 1.0003184080123901, + "sampling/importance_sampling_ratio/min": 0.8448547720909119, + "sampling/sampling_logp_difference/max": 0.16859054565429688, + "sampling/sampling_logp_difference/mean": 0.013767481781542301, + "step": 224 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 220.0, + "completions/mean_length": 118.21875, + "completions/mean_terminated_length": 89.0, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.5930796023458242, + "epoch": 0.22255192878338279, + "grad_norm": 2.3983278274536133, + "kl_approx": 0.2525482177734375, + "learning_rate": 1.912417390049882e-05, + "loss": 0.1595, + "num_tokens": 5427994.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2002683877944946, + "sampling/importance_sampling_ratio/mean": 0.9999665021896362, + "sampling/importance_sampling_ratio/min": 0.7997391223907471, + "sampling/sampling_logp_difference/max": 0.22346973419189453, + "sampling/sampling_logp_difference/mean": 0.011154585517942905, + "step": 225 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 89.875, + "completions/mean_terminated_length": 89.875, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.7915005348622799, + "epoch": 0.22354104846686448, + "grad_norm": 4.020339012145996, + "kl_approx": 0.456451416015625, + "learning_rate": 1.9109974979578852e-05, + "loss": 0.2336, + "num_tokens": 5448518.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2413122653961182, + "sampling/importance_sampling_ratio/mean": 1.0008007287979126, + "sampling/importance_sampling_ratio/min": 0.8584089875221252, + "sampling/sampling_logp_difference/max": 0.21616911888122559, + "sampling/sampling_logp_difference/mean": 0.013304656371474266, + "step": 226 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 99.3125, + "completions/mean_terminated_length": 99.3125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.5809233691543341, + "epoch": 0.22453016815034618, + "grad_norm": 2.259439706802368, + "kl_approx": 0.219329833984375, + "learning_rate": 1.909566724366779e-05, + "loss": 0.1398, + "num_tokens": 5466576.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1431745290756226, + "sampling/importance_sampling_ratio/mean": 0.999667763710022, + "sampling/importance_sampling_ratio/min": 0.8343027234077454, + "sampling/sampling_logp_difference/max": 0.18115901947021484, + "sampling/sampling_logp_difference/mean": 0.01166341919451952, + "step": 227 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 769.0, + "completions/mean_length": 201.40625, + "completions/mean_terminated_length": 174.87095642089844, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.5947200027294457, + "epoch": 0.22551928783382788, + "grad_norm": 3.1953752040863037, + "kl_approx": 0.25662994384765625, + "learning_rate": 1.9081250863665794e-05, + "loss": 0.1925, + "num_tokens": 5491141.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.288362741470337, + "sampling/importance_sampling_ratio/mean": 1.0006479024887085, + "sampling/importance_sampling_ratio/min": 0.8294237852096558, + "sampling/sampling_logp_difference/max": 0.2533721923828125, + "sampling/sampling_logp_difference/mean": 0.012894628569483757, + "step": 228 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 200.0, + "completions/mean_terminated_length": 200.0, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.7009008713066578, + "epoch": 0.2265084075173096, + "grad_norm": 2.094667911529541, + "kl_approx": 0.23578262329101562, + "learning_rate": 1.9066726011770725e-05, + "loss": 0.1923, + "num_tokens": 5517933.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.21649968624115, + "sampling/importance_sampling_ratio/mean": 0.999966561794281, + "sampling/importance_sampling_ratio/min": 0.8073489665985107, + "sampling/sampling_logp_difference/max": 0.21399927139282227, + "sampling/sampling_logp_difference/mean": 0.01398975308984518, + "step": 229 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 205.875, + "completions/mean_terminated_length": 205.875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.6458039940334857, + "epoch": 0.2274975272007913, + "grad_norm": 1.9702178239822388, + "kl_approx": 0.2554359436035156, + "learning_rate": 1.905209286147611e-05, + "loss": 0.1765, + "num_tokens": 5544073.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3105998039245605, + "sampling/importance_sampling_ratio/mean": 0.999586820602417, + "sampling/importance_sampling_ratio/min": 0.7754350304603577, + "sampling/sampling_logp_difference/max": 0.27048492431640625, + "sampling/sampling_logp_difference/mean": 0.01276719756424427, + "step": 230 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 874.0, + "completions/max_terminated_length": 874.0, + "completions/mean_length": 284.59375, + "completions/mean_terminated_length": 284.59375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.6856138175353408, + "epoch": 0.228486646884273, + "grad_norm": 2.3308005332946777, + "kl_approx": 0.2988739013671875, + "learning_rate": 1.903735158756905e-05, + "loss": 0.2188, + "num_tokens": 5570516.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2148922681808472, + "sampling/importance_sampling_ratio/mean": 1.0006022453308105, + "sampling/importance_sampling_ratio/min": 0.8102623820304871, + "sampling/sampling_logp_difference/max": 0.21039724349975586, + "sampling/sampling_logp_difference/mean": 0.012063105590641499, + "step": 231 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 860.0, + "completions/max_terminated_length": 860.0, + "completions/mean_length": 229.90625, + "completions/mean_terminated_length": 229.90625, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.8164218720048666, + "epoch": 0.2294757665677547, + "grad_norm": 2.1481070518493652, + "kl_approx": 0.331634521484375, + "learning_rate": 1.9022502366128136e-05, + "loss": 0.2188, + "num_tokens": 5597193.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2343577146530151, + "sampling/importance_sampling_ratio/mean": 0.9999073147773743, + "sampling/importance_sampling_ratio/min": 0.7946333885192871, + "sampling/sampling_logp_difference/max": 0.2298743724822998, + "sampling/sampling_logp_difference/mean": 0.012741408310830593, + "step": 232 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/max_terminated_length": 252.0, + "completions/mean_length": 98.25, + "completions/mean_terminated_length": 98.25, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.528086791280657, + "epoch": 0.2304648862512364, + "grad_norm": 3.4126057624816895, + "kl_approx": 0.2632617950439453, + "learning_rate": 1.9007545374521354e-05, + "loss": 0.1583, + "num_tokens": 5622449.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1884492635726929, + "sampling/importance_sampling_ratio/mean": 1.0001106262207031, + "sampling/importance_sampling_ratio/min": 0.8104504942893982, + "sampling/sampling_logp_difference/max": 0.21016502380371094, + "sampling/sampling_logp_difference/mean": 0.009452199563384056, + "step": 233 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/max_terminated_length": 181.0, + "completions/mean_length": 69.6875, + "completions/mean_terminated_length": 69.6875, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.5038614354562014, + "epoch": 0.2314540059347181, + "grad_norm": 3.4858715534210205, + "kl_approx": 0.243865966796875, + "learning_rate": 1.8992480791403957e-05, + "loss": 0.2044, + "num_tokens": 5643543.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1353437900543213, + "sampling/importance_sampling_ratio/mean": 0.9996883869171143, + "sampling/importance_sampling_ratio/min": 0.8652843236923218, + "sampling/sampling_logp_difference/max": 0.1446971893310547, + "sampling/sampling_logp_difference/mean": 0.009055567905306816, + "step": 234 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.0, + "completions/max_terminated_length": 327.0, + "completions/mean_length": 111.75, + "completions/mean_terminated_length": 111.75, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.6546653714030981, + "epoch": 0.2324431256181998, + "grad_norm": 3.8027219772338867, + "kl_approx": 0.2834320068359375, + "learning_rate": 1.897730879671634e-05, + "loss": 0.1741, + "num_tokens": 5668935.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.184888482093811, + "sampling/importance_sampling_ratio/mean": 0.9995028376579285, + "sampling/importance_sampling_ratio/min": 0.8584259152412415, + "sampling/sampling_logp_difference/max": 0.1696486473083496, + "sampling/sampling_logp_difference/mean": 0.011030866764485836, + "step": 235 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 434.0, + "completions/max_terminated_length": 434.0, + "completions/mean_length": 101.5, + "completions/mean_terminated_length": 101.5, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.7878697253763676, + "epoch": 0.2334322453016815, + "grad_norm": 2.6981778144836426, + "kl_approx": 0.2628021240234375, + "learning_rate": 1.8962029571681887e-05, + "loss": 0.1891, + "num_tokens": 5689815.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2150640487670898, + "sampling/importance_sampling_ratio/mean": 1.000068187713623, + "sampling/importance_sampling_ratio/min": 0.46216318011283875, + "sampling/sampling_logp_difference/max": 0.7718372344970703, + "sampling/sampling_logp_difference/mean": 0.01477569155395031, + "step": 236 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 113.6875, + "completions/mean_terminated_length": 113.6875, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.6975641055032611, + "epoch": 0.2344213649851632, + "grad_norm": 2.8937313556671143, + "kl_approx": 0.3151702880859375, + "learning_rate": 1.8946643298804794e-05, + "loss": 0.2132, + "num_tokens": 5708613.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1440235376358032, + "sampling/importance_sampling_ratio/mean": 1.000037431716919, + "sampling/importance_sampling_ratio/min": 0.8411276936531067, + "sampling/sampling_logp_difference/max": 0.17301177978515625, + "sampling/sampling_logp_difference/mean": 0.011513817124068737, + "step": 237 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/max_terminated_length": 167.0, + "completions/mean_length": 91.59375, + "completions/mean_terminated_length": 91.59375, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.708747148513794, + "epoch": 0.2354104846686449, + "grad_norm": 2.9254322052001953, + "kl_approx": 0.27355194091796875, + "learning_rate": 1.8931150161867917e-05, + "loss": 0.1669, + "num_tokens": 5730800.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1886290311813354, + "sampling/importance_sampling_ratio/mean": 1.000718116760254, + "sampling/importance_sampling_ratio/min": 0.8469982743263245, + "sampling/sampling_logp_difference/max": 0.17280054092407227, + "sampling/sampling_logp_difference/mean": 0.011884743347764015, + "step": 238 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/max_terminated_length": 182.0, + "completions/mean_length": 79.8125, + "completions/mean_terminated_length": 79.8125, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.6106561003252864, + "epoch": 0.23639960435212662, + "grad_norm": 2.5883443355560303, + "kl_approx": 0.31252288818359375, + "learning_rate": 1.891555034593055e-05, + "loss": 0.1829, + "num_tokens": 5748074.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.178993821144104, + "sampling/importance_sampling_ratio/mean": 1.0005333423614502, + "sampling/importance_sampling_ratio/min": 0.7861150503158569, + "sampling/sampling_logp_difference/max": 0.24065208435058594, + "sampling/sampling_logp_difference/mean": 0.011805953457951546, + "step": 239 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/max_terminated_length": 330.0, + "completions/mean_length": 137.15625, + "completions/mean_terminated_length": 137.15625, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.8554209163412452, + "epoch": 0.23738872403560832, + "grad_norm": 2.745689868927002, + "kl_approx": 0.3905029296875, + "learning_rate": 1.8899844037326227e-05, + "loss": 0.2393, + "num_tokens": 5775855.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1931133270263672, + "sampling/importance_sampling_ratio/mean": 0.9999681711196899, + "sampling/importance_sampling_ratio/min": 0.8318423628807068, + "sampling/sampling_logp_difference/max": 0.1841123104095459, + "sampling/sampling_logp_difference/mean": 0.013768631964921951, + "step": 240 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 547.0, + "completions/max_terminated_length": 547.0, + "completions/mean_length": 142.0625, + "completions/mean_terminated_length": 142.0625, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.5729648259002715, + "epoch": 0.23837784371909002, + "grad_norm": 2.084716558456421, + "kl_approx": 0.17239665985107422, + "learning_rate": 1.8884031423660492e-05, + "loss": 0.1379, + "num_tokens": 5795225.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2653734683990479, + "sampling/importance_sampling_ratio/mean": 0.9999136328697205, + "sampling/importance_sampling_ratio/min": 0.8735803365707397, + "sampling/sampling_logp_difference/max": 0.2353672981262207, + "sampling/sampling_logp_difference/mean": 0.010552264750003815, + "step": 241 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 533.0, + "completions/max_terminated_length": 533.0, + "completions/mean_length": 136.84375, + "completions/mean_terminated_length": 136.84375, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.736097046174109, + "epoch": 0.23936696340257171, + "grad_norm": 2.923643112182617, + "kl_approx": 0.2623291015625, + "learning_rate": 1.8868112693808664e-05, + "loss": 0.2281, + "num_tokens": 5818588.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.253665804862976, + "sampling/importance_sampling_ratio/mean": 0.9993937611579895, + "sampling/importance_sampling_ratio/min": 0.8383716344833374, + "sampling/sampling_logp_difference/max": 0.22607183456420898, + "sampling/sampling_logp_difference/mean": 0.012744957581162453, + "step": 242 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 772.0, + "completions/max_terminated_length": 772.0, + "completions/mean_length": 249.15625, + "completions/mean_terminated_length": 249.15625, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7830150690861046, + "epoch": 0.2403560830860534, + "grad_norm": 2.332535982131958, + "kl_approx": 0.26373291015625, + "learning_rate": 1.8852088037913577e-05, + "loss": 0.1922, + "num_tokens": 5846961.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1901663541793823, + "sampling/importance_sampling_ratio/mean": 1.0005383491516113, + "sampling/importance_sampling_ratio/min": 0.8108292818069458, + "sampling/sampling_logp_difference/max": 0.20969772338867188, + "sampling/sampling_logp_difference/mean": 0.01449556089937687, + "step": 243 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 445.0, + "completions/max_terminated_length": 445.0, + "completions/mean_length": 127.5, + "completions/mean_terminated_length": 127.5, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.7224850584752858, + "epoch": 0.2413452027695351, + "grad_norm": 2.4544849395751953, + "kl_approx": 0.2571220397949219, + "learning_rate": 1.8835957647383304e-05, + "loss": 0.175, + "num_tokens": 5872689.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2584421634674072, + "sampling/importance_sampling_ratio/mean": 1.0000208616256714, + "sampling/importance_sampling_ratio/min": 0.8083779215812683, + "sampling/sampling_logp_difference/max": 0.2298746109008789, + "sampling/sampling_logp_difference/mean": 0.015055437572300434, + "step": 244 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/max_terminated_length": 299.0, + "completions/mean_length": 113.5, + "completions/mean_terminated_length": 113.5, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "entropy": 0.6309742857702076, + "epoch": 0.2423343224530168, + "grad_norm": 3.1405208110809326, + "kl_approx": 0.30637454986572266, + "learning_rate": 1.8819721714888878e-05, + "loss": 0.2624, + "num_tokens": 5897081.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1957024335861206, + "sampling/importance_sampling_ratio/mean": 1.000077724456787, + "sampling/importance_sampling_ratio/min": 0.8335638046264648, + "sampling/sampling_logp_difference/max": 0.18204498291015625, + "sampling/sampling_logp_difference/mean": 0.011632377281785011, + "step": 245 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 479.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 109.25, + "completions/mean_terminated_length": 109.25, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.5707564014010131, + "epoch": 0.2433234421364985, + "grad_norm": 2.7850465774536133, + "kl_approx": 0.2675743103027344, + "learning_rate": 1.8803380434362e-05, + "loss": 0.1802, + "num_tokens": 5914281.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1616880893707275, + "sampling/importance_sampling_ratio/mean": 1.0002697706222534, + "sampling/importance_sampling_ratio/min": 0.8415634036064148, + "sampling/sampling_logp_difference/max": 0.17249393463134766, + "sampling/sampling_logp_difference/mean": 0.010010476224124432, + "step": 246 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 386.0, + "completions/max_terminated_length": 386.0, + "completions/mean_length": 118.90625, + "completions/mean_terminated_length": 118.90625, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "entropy": 0.6378051619976759, + "epoch": 0.2443125618199802, + "grad_norm": 2.910785675048828, + "kl_approx": 0.2985076904296875, + "learning_rate": 1.878693400099269e-05, + "loss": 0.1761, + "num_tokens": 5936782.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1438119411468506, + "sampling/importance_sampling_ratio/mean": 1.0003470182418823, + "sampling/importance_sampling_ratio/min": 0.8807020783424377, + "sampling/sampling_logp_difference/max": 0.13436651229858398, + "sampling/sampling_logp_difference/mean": 0.01189448032528162, + "step": 247 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 202.84375, + "completions/mean_terminated_length": 202.84375, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.8097506552003324, + "epoch": 0.24530168150346193, + "grad_norm": 3.200960874557495, + "kl_approx": 0.3280487060546875, + "learning_rate": 1.877038261122699e-05, + "loss": 0.2236, + "num_tokens": 5962041.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2076621055603027, + "sampling/importance_sampling_ratio/mean": 0.999841570854187, + "sampling/importance_sampling_ratio/min": 0.8294474482536316, + "sampling/sampling_logp_difference/max": 0.18868637084960938, + "sampling/sampling_logp_difference/mean": 0.015314118005335331, + "step": 248 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/max_terminated_length": 251.0, + "completions/mean_length": 77.75, + "completions/mean_terminated_length": 77.75, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.5310401674360037, + "epoch": 0.24629080118694363, + "grad_norm": 3.114018201828003, + "kl_approx": 0.25604248046875, + "learning_rate": 1.87537264627646e-05, + "loss": 0.1655, + "num_tokens": 5984201.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.135076880455017, + "sampling/importance_sampling_ratio/mean": 1.0014406442642212, + "sampling/importance_sampling_ratio/min": 0.8749927282333374, + "sampling/sampling_logp_difference/max": 0.13353967666625977, + "sampling/sampling_logp_difference/mean": 0.00955023430287838, + "step": 249 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 165.59375, + "completions/mean_terminated_length": 165.59375, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.9683704702183604, + "epoch": 0.24727992087042533, + "grad_norm": 2.9160985946655273, + "kl_approx": 0.34423828125, + "learning_rate": 1.8736965754556527e-05, + "loss": 0.2257, + "num_tokens": 6008380.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1938070058822632, + "sampling/importance_sampling_ratio/mean": 0.9996144771575928, + "sampling/importance_sampling_ratio/min": 0.8227328658103943, + "sampling/sampling_logp_difference/max": 0.19512367248535156, + "sampling/sampling_logp_difference/mean": 0.015261182561516762, + "step": 250 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 696.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 206.53125, + "completions/mean_terminated_length": 206.53125, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.734200544655323, + "epoch": 0.24826904055390703, + "grad_norm": 2.265373945236206, + "kl_approx": 0.2452392578125, + "learning_rate": 1.8720100686802693e-05, + "loss": 0.173, + "num_tokens": 6043661.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2176592350006104, + "sampling/importance_sampling_ratio/mean": 0.9999580383300781, + "sampling/importance_sampling_ratio/min": 0.7978817820549011, + "sampling/sampling_logp_difference/max": 0.22579479217529297, + "sampling/sampling_logp_difference/mean": 0.011789221316576004, + "step": 251 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 488.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 116.71875, + "completions/mean_terminated_length": 116.71875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.8170098252594471, + "epoch": 0.24925816023738873, + "grad_norm": 3.788538694381714, + "kl_approx": 0.35205078125, + "learning_rate": 1.8703131460949555e-05, + "loss": 0.2262, + "num_tokens": 6069884.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2335857152938843, + "sampling/importance_sampling_ratio/mean": 1.0003721714019775, + "sampling/importance_sampling_ratio/min": 0.8483299612998962, + "sampling/sampling_logp_difference/max": 0.20992517471313477, + "sampling/sampling_logp_difference/mean": 0.013158032670617104, + "step": 252 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 361.0, + "completions/max_terminated_length": 361.0, + "completions/mean_length": 124.15625, + "completions/mean_terminated_length": 124.15625, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.6976150772534311, + "epoch": 0.2502472799208704, + "grad_norm": 2.345968723297119, + "kl_approx": 0.29544830322265625, + "learning_rate": 1.86860582796877e-05, + "loss": 0.1778, + "num_tokens": 6094665.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1928038597106934, + "sampling/importance_sampling_ratio/mean": 1.0006217956542969, + "sampling/importance_sampling_ratio/min": 0.8621042370796204, + "sampling/sampling_logp_difference/max": 0.17630672454833984, + "sampling/sampling_logp_difference/mean": 0.012578437104821205, + "step": 253 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 987.0, + "completions/max_terminated_length": 987.0, + "completions/mean_length": 207.8125, + "completions/mean_terminated_length": 207.8125, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.6181977167725563, + "epoch": 0.2512363996043521, + "grad_norm": 2.2176454067230225, + "kl_approx": 0.26081085205078125, + "learning_rate": 1.866888134694942e-05, + "loss": 0.1735, + "num_tokens": 6122067.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1774410009384155, + "sampling/importance_sampling_ratio/mean": 0.999228298664093, + "sampling/importance_sampling_ratio/min": 0.8295028209686279, + "sampling/sampling_logp_difference/max": 0.18692874908447266, + "sampling/sampling_logp_difference/mean": 0.011136235669255257, + "step": 254 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 802.0, + "completions/max_terminated_length": 802.0, + "completions/mean_length": 166.5625, + "completions/mean_terminated_length": 166.5625, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "entropy": 0.7053878409788013, + "epoch": 0.2522255192878338, + "grad_norm": 2.4710538387298584, + "kl_approx": 0.3014373779296875, + "learning_rate": 1.865160086790627e-05, + "loss": 0.1963, + "num_tokens": 6143221.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2361165285110474, + "sampling/importance_sampling_ratio/mean": 1.0000426769256592, + "sampling/importance_sampling_ratio/min": 0.850463330745697, + "sampling/sampling_logp_difference/max": 0.2119746208190918, + "sampling/sampling_logp_difference/mean": 0.014212892390787601, + "step": 255 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 1018.0, + "completions/max_terminated_length": 1018.0, + "completions/mean_length": 219.5, + "completions/mean_terminated_length": 219.5, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7651667045429349, + "epoch": 0.2532146389713155, + "grad_norm": 2.52970027923584, + "kl_approx": 0.285430908203125, + "learning_rate": 1.8634217048966638e-05, + "loss": 0.208, + "num_tokens": 6168901.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1996407508850098, + "sampling/importance_sampling_ratio/mean": 1.0000035762786865, + "sampling/importance_sampling_ratio/min": 0.7754191160202026, + "sampling/sampling_logp_difference/max": 0.2543516159057617, + "sampling/sampling_logp_difference/mean": 0.011865036562085152, + "step": 256 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 749.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 248.4375, + "completions/mean_terminated_length": 248.4375, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "entropy": 0.8611204288899899, + "epoch": 0.2542037586547972, + "grad_norm": 3.3415417671203613, + "kl_approx": 0.3981170654296875, + "learning_rate": 1.861673009777325e-05, + "loss": 0.2339, + "num_tokens": 6198211.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.27825129032135, + "sampling/importance_sampling_ratio/mean": 0.9999289512634277, + "sampling/importance_sampling_ratio/min": 0.8141903281211853, + "sampling/sampling_logp_difference/max": 0.24549293518066406, + "sampling/sampling_logp_difference/mean": 0.013347550295293331, + "step": 257 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 1003.0, + "completions/max_terminated_length": 1003.0, + "completions/mean_length": 315.1875, + "completions/mean_terminated_length": 315.1875, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.9514345768839121, + "epoch": 0.2551928783382789, + "grad_norm": 3.32955265045166, + "kl_approx": 0.31549072265625, + "learning_rate": 1.8599140223200716e-05, + "loss": 0.2785, + "num_tokens": 6228697.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2381187677383423, + "sampling/importance_sampling_ratio/mean": 0.9999781250953674, + "sampling/importance_sampling_ratio/min": 0.789922833442688, + "sampling/sampling_logp_difference/max": 0.23582005500793457, + "sampling/sampling_logp_difference/mean": 0.014978324994444847, + "step": 258 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 118.5625, + "completions/mean_terminated_length": 118.5625, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.6374960239045322, + "epoch": 0.2561819980217606, + "grad_norm": 2.5630977153778076, + "kl_approx": 0.21505355834960938, + "learning_rate": 1.858144763535302e-05, + "loss": 0.1644, + "num_tokens": 6254027.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1675320863723755, + "sampling/importance_sampling_ratio/mean": 1.000195860862732, + "sampling/importance_sampling_ratio/min": 0.8264777660369873, + "sampling/sampling_logp_difference/max": 0.190582275390625, + "sampling/sampling_logp_difference/mean": 0.012435130774974823, + "step": 259 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 746.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 232.03125, + "completions/mean_terminated_length": 232.03125, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.8175475019961596, + "epoch": 0.2571711177052423, + "grad_norm": 3.6637983322143555, + "kl_approx": 0.39014434814453125, + "learning_rate": 1.8563652545561014e-05, + "loss": 0.2757, + "num_tokens": 6278756.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.194381594657898, + "sampling/importance_sampling_ratio/mean": 1.0003455877304077, + "sampling/importance_sampling_ratio/min": 0.8545555472373962, + "sampling/sampling_logp_difference/max": 0.1776285171508789, + "sampling/sampling_logp_difference/mean": 0.01306745782494545, + "step": 260 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 875.0, + "completions/max_terminated_length": 875.0, + "completions/mean_length": 229.84375, + "completions/mean_terminated_length": 229.84375, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.6362983407452703, + "epoch": 0.258160237388724, + "grad_norm": 2.0252318382263184, + "kl_approx": 0.20578742027282715, + "learning_rate": 1.8545755166379898e-05, + "loss": 0.1786, + "num_tokens": 6307399.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.255062460899353, + "sampling/importance_sampling_ratio/mean": 1.000187873840332, + "sampling/importance_sampling_ratio/min": 0.7375921010971069, + "sampling/sampling_logp_difference/max": 0.30436432361602783, + "sampling/sampling_logp_difference/mean": 0.01274816133081913, + "step": 261 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 954.0, + "completions/max_terminated_length": 954.0, + "completions/mean_length": 267.46875, + "completions/mean_terminated_length": 267.46875, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.8469747696071863, + "epoch": 0.2591493570722057, + "grad_norm": 2.2046382427215576, + "kl_approx": 0.3128204345703125, + "learning_rate": 1.852775571158668e-05, + "loss": 0.1908, + "num_tokens": 6335310.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2168724536895752, + "sampling/importance_sampling_ratio/mean": 0.999741792678833, + "sampling/importance_sampling_ratio/min": 0.8333194255828857, + "sampling/sampling_logp_difference/max": 0.19628405570983887, + "sampling/sampling_logp_difference/mean": 0.013291655108332634, + "step": 262 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 523.0, + "completions/max_terminated_length": 523.0, + "completions/mean_length": 170.375, + "completions/mean_terminated_length": 170.375, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.837700417265296, + "epoch": 0.26013847675568746, + "grad_norm": 2.7334036827087402, + "kl_approx": 0.349151611328125, + "learning_rate": 1.850965439617761e-05, + "loss": 0.2329, + "num_tokens": 6357842.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1958190202713013, + "sampling/importance_sampling_ratio/mean": 1.0001704692840576, + "sampling/importance_sampling_ratio/min": 0.8276202082633972, + "sampling/sampling_logp_difference/max": 0.18920087814331055, + "sampling/sampling_logp_difference/mean": 0.01420552097260952, + "step": 263 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 151.75, + "completions/mean_terminated_length": 151.75, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.9040473736822605, + "epoch": 0.26112759643916916, + "grad_norm": 3.7835257053375244, + "kl_approx": 0.4090423583984375, + "learning_rate": 1.8491451436365628e-05, + "loss": 0.2823, + "num_tokens": 6380514.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.171353816986084, + "sampling/importance_sampling_ratio/mean": 0.9996326565742493, + "sampling/importance_sampling_ratio/min": 0.8048141002655029, + "sampling/sampling_logp_difference/max": 0.21714401245117188, + "sampling/sampling_logp_difference/mean": 0.014794974587857723, + "step": 264 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/max_terminated_length": 285.0, + "completions/mean_length": 101.6875, + "completions/mean_terminated_length": 101.6875, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.7273328183218837, + "epoch": 0.26211671612265086, + "grad_norm": 2.64215350151062, + "kl_approx": 0.283447265625, + "learning_rate": 1.8473147049577777e-05, + "loss": 0.1988, + "num_tokens": 6399712.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1332677602767944, + "sampling/importance_sampling_ratio/mean": 0.9994909167289734, + "sampling/importance_sampling_ratio/min": 0.8055465817451477, + "sampling/sampling_logp_difference/max": 0.2162342071533203, + "sampling/sampling_logp_difference/mean": 0.01294513139873743, + "step": 265 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 695.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 179.46875, + "completions/mean_terminated_length": 179.46875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.7899589100852609, + "epoch": 0.26310583580613256, + "grad_norm": 2.4730005264282227, + "kl_approx": 0.2939167022705078, + "learning_rate": 1.8454741454452604e-05, + "loss": 0.2065, + "num_tokens": 6428239.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1971691846847534, + "sampling/importance_sampling_ratio/mean": 1.0003697872161865, + "sampling/importance_sampling_ratio/min": 0.8207126259803772, + "sampling/sampling_logp_difference/max": 0.19758224487304688, + "sampling/sampling_logp_difference/mean": 0.014770924113690853, + "step": 266 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 117.78125, + "completions/mean_terminated_length": 117.78125, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.7121186791919172, + "epoch": 0.26409495548961426, + "grad_norm": 2.5290145874023438, + "kl_approx": 0.2938804626464844, + "learning_rate": 1.843623487083755e-05, + "loss": 0.2051, + "num_tokens": 6445832.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2087434530258179, + "sampling/importance_sampling_ratio/mean": 1.0003143548965454, + "sampling/importance_sampling_ratio/min": 0.8200746178627014, + "sampling/sampling_logp_difference/max": 0.19835996627807617, + "sampling/sampling_logp_difference/mean": 0.013039608485996723, + "step": 267 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 335.0, + "completions/max_terminated_length": 335.0, + "completions/mean_length": 131.71875, + "completions/mean_terminated_length": 131.71875, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.7606870573945343, + "epoch": 0.26508407517309596, + "grad_norm": 2.677860736846924, + "kl_approx": 0.3374805450439453, + "learning_rate": 1.8417627519786317e-05, + "loss": 0.2237, + "num_tokens": 6469295.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2280875444412231, + "sampling/importance_sampling_ratio/mean": 0.9997721314430237, + "sampling/importance_sampling_ratio/min": 0.8464064002037048, + "sampling/sampling_logp_difference/max": 0.2054581642150879, + "sampling/sampling_logp_difference/mean": 0.013750326819717884, + "step": 268 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/max_terminated_length": 206.0, + "completions/mean_length": 89.25, + "completions/mean_terminated_length": 89.25, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.6565246256068349, + "epoch": 0.26607319485657766, + "grad_norm": 3.0166139602661133, + "kl_approx": 0.35611724853515625, + "learning_rate": 1.839891962355624e-05, + "loss": 0.2312, + "num_tokens": 6489311.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1516737937927246, + "sampling/importance_sampling_ratio/mean": 1.0002615451812744, + "sampling/importance_sampling_ratio/min": 0.8777641654014587, + "sampling/sampling_logp_difference/max": 0.14121627807617188, + "sampling/sampling_logp_difference/mean": 0.012450135312974453, + "step": 269 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 153.65625, + "completions/mean_terminated_length": 153.65625, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.5829455158673227, + "epoch": 0.26706231454005935, + "grad_norm": 1.89301335811615, + "kl_approx": 0.17298126220703125, + "learning_rate": 1.838011140560562e-05, + "loss": 0.136, + "num_tokens": 6510988.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2037020921707153, + "sampling/importance_sampling_ratio/mean": 1.0001550912857056, + "sampling/importance_sampling_ratio/min": 0.7927284836769104, + "sampling/sampling_logp_difference/max": 0.23227453231811523, + "sampling/sampling_logp_difference/mean": 0.012998693622648716, + "step": 270 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 545.0, + "completions/max_terminated_length": 545.0, + "completions/mean_length": 163.6875, + "completions/mean_terminated_length": 163.6875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.7352368859574199, + "epoch": 0.26805143422354105, + "grad_norm": 2.6686573028564453, + "kl_approx": 0.23332977294921875, + "learning_rate": 1.836120309059107e-05, + "loss": 0.1802, + "num_tokens": 6533562.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2149933576583862, + "sampling/importance_sampling_ratio/mean": 1.0000017881393433, + "sampling/importance_sampling_ratio/min": 0.8244803547859192, + "sampling/sampling_logp_difference/max": 0.19473862648010254, + "sampling/sampling_logp_difference/mean": 0.013422228395938873, + "step": 271 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 816.0, + "completions/max_terminated_length": 816.0, + "completions/mean_length": 172.53125, + "completions/mean_terminated_length": 172.53125, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.7640396486967802, + "epoch": 0.26904055390702275, + "grad_norm": 2.455402374267578, + "kl_approx": 0.291748046875, + "learning_rate": 1.8342194904364815e-05, + "loss": 0.1728, + "num_tokens": 6556835.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.15168035030365, + "sampling/importance_sampling_ratio/mean": 0.9995319843292236, + "sampling/importance_sampling_ratio/min": 0.7965874075889587, + "sampling/sampling_logp_difference/max": 0.2274184226989746, + "sampling/sampling_logp_difference/mean": 0.014608344994485378, + "step": 272 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 949.0, + "completions/max_terminated_length": 949.0, + "completions/mean_length": 203.09375, + "completions/mean_terminated_length": 203.09375, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.9471575552597642, + "epoch": 0.27002967359050445, + "grad_norm": 2.6995186805725098, + "kl_approx": 0.363067626953125, + "learning_rate": 1.8323087073971996e-05, + "loss": 0.2283, + "num_tokens": 6582102.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2342381477355957, + "sampling/importance_sampling_ratio/mean": 1.0002514123916626, + "sampling/importance_sampling_ratio/min": 0.8057738542556763, + "sampling/sampling_logp_difference/max": 0.21595215797424316, + "sampling/sampling_logp_difference/mean": 0.014374662190675735, + "step": 273 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 152.34375, + "completions/mean_terminated_length": 152.34375, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.7210090886801481, + "epoch": 0.27101879327398615, + "grad_norm": 2.835953950881958, + "kl_approx": 0.293792724609375, + "learning_rate": 1.8303879827647977e-05, + "loss": 0.2646, + "num_tokens": 6612553.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.529760718345642, + "sampling/importance_sampling_ratio/mean": 0.9997280836105347, + "sampling/importance_sampling_ratio/min": 0.8696050643920898, + "sampling/sampling_logp_difference/max": 0.4251112937927246, + "sampling/sampling_logp_difference/mean": 0.011369016952812672, + "step": 274 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 694.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 128.53125, + "completions/mean_terminated_length": 128.53125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.8139790697023273, + "epoch": 0.27200791295746785, + "grad_norm": 2.8997466564178467, + "kl_approx": 0.33301544189453125, + "learning_rate": 1.8284573394815596e-05, + "loss": 0.2046, + "num_tokens": 6635178.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1900774240493774, + "sampling/importance_sampling_ratio/mean": 0.9996291399002075, + "sampling/importance_sampling_ratio/min": 0.8119118809700012, + "sampling/sampling_logp_difference/max": 0.20836353302001953, + "sampling/sampling_logp_difference/mean": 0.014103001914918423, + "step": 275 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/max_terminated_length": 146.0, + "completions/mean_length": 78.46875, + "completions/mean_terminated_length": 78.46875, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "entropy": 0.6132251652888954, + "epoch": 0.27299703264094954, + "grad_norm": 3.064506769180298, + "kl_approx": 0.28894805908203125, + "learning_rate": 1.826516800608244e-05, + "loss": 0.1735, + "num_tokens": 6655881.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2053059339523315, + "sampling/importance_sampling_ratio/mean": 1.0005022287368774, + "sampling/importance_sampling_ratio/min": 0.8730798363685608, + "sampling/sampling_logp_difference/max": 0.18673348426818848, + "sampling/sampling_logp_difference/mean": 0.010536580346524715, + "step": 276 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 379.0, + "completions/max_terminated_length": 379.0, + "completions/mean_length": 110.34375, + "completions/mean_terminated_length": 110.34375, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "entropy": 0.6002367818728089, + "epoch": 0.27398615232443124, + "grad_norm": 2.6671674251556396, + "kl_approx": 0.21013832092285156, + "learning_rate": 1.8245663893238075e-05, + "loss": 0.1583, + "num_tokens": 6678148.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1797600984573364, + "sampling/importance_sampling_ratio/mean": 1.0004637241363525, + "sampling/importance_sampling_ratio/min": 0.828397274017334, + "sampling/sampling_logp_difference/max": 0.1882624626159668, + "sampling/sampling_logp_difference/mean": 0.012014534324407578, + "step": 277 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 464.0, + "completions/max_terminated_length": 464.0, + "completions/mean_length": 140.0625, + "completions/mean_terminated_length": 140.0625, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.7178203221410513, + "epoch": 0.27497527200791294, + "grad_norm": 2.621427059173584, + "kl_approx": 0.30187225341796875, + "learning_rate": 1.8226061289251297e-05, + "loss": 0.1901, + "num_tokens": 6701358.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1766642332077026, + "sampling/importance_sampling_ratio/mean": 0.9992976188659668, + "sampling/importance_sampling_ratio/min": 0.8742595911026001, + "sampling/sampling_logp_difference/max": 0.16268348693847656, + "sampling/sampling_logp_difference/mean": 0.01123831793665886, + "step": 278 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/max_terminated_length": 278.0, + "completions/mean_length": 124.71875, + "completions/mean_terminated_length": 124.71875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7544533615000546, + "epoch": 0.27596439169139464, + "grad_norm": 2.844728946685791, + "kl_approx": 0.29319000244140625, + "learning_rate": 1.8206360428267332e-05, + "loss": 0.2182, + "num_tokens": 6723125.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1419024467468262, + "sampling/importance_sampling_ratio/mean": 0.9990918636322021, + "sampling/importance_sampling_ratio/min": 0.8635880947113037, + "sampling/sampling_logp_difference/max": 0.14665937423706055, + "sampling/sampling_logp_difference/mean": 0.012254110537469387, + "step": 279 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 609.0, + "completions/max_terminated_length": 609.0, + "completions/mean_length": 246.40625, + "completions/mean_terminated_length": 246.40625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.9971765512600541, + "epoch": 0.27695351137487634, + "grad_norm": 2.899927854537964, + "kl_approx": 0.36212158203125, + "learning_rate": 1.8186561545605055e-05, + "loss": 0.2412, + "num_tokens": 6750634.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1603909730911255, + "sampling/importance_sampling_ratio/mean": 0.999725878238678, + "sampling/importance_sampling_ratio/min": 0.8646631240844727, + "sampling/sampling_logp_difference/max": 0.1487569808959961, + "sampling/sampling_logp_difference/mean": 0.01443057507276535, + "step": 280 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 173.6875, + "completions/mean_terminated_length": 173.6875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.8997823763638735, + "epoch": 0.27794263105835804, + "grad_norm": 2.642136812210083, + "kl_approx": 0.3118133544921875, + "learning_rate": 1.816666487775416e-05, + "loss": 0.2171, + "num_tokens": 6775848.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2122026681900024, + "sampling/importance_sampling_ratio/mean": 0.999157190322876, + "sampling/importance_sampling_ratio/min": 0.7813576459884644, + "sampling/sampling_logp_difference/max": 0.24672222137451172, + "sampling/sampling_logp_difference/mean": 0.015035804361104965, + "step": 281 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 408.0, + "completions/max_terminated_length": 408.0, + "completions/mean_length": 137.96875, + "completions/mean_terminated_length": 137.96875, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.6917269062250853, + "epoch": 0.2789317507418398, + "grad_norm": 1.9701180458068848, + "kl_approx": 0.2175750732421875, + "learning_rate": 1.8146670662372353e-05, + "loss": 0.1549, + "num_tokens": 6795791.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1528571844100952, + "sampling/importance_sampling_ratio/mean": 1.0004239082336426, + "sampling/importance_sampling_ratio/min": 0.8303714990615845, + "sampling/sampling_logp_difference/max": 0.1858820915222168, + "sampling/sampling_logp_difference/mean": 0.013751442544162273, + "step": 282 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 205.8125, + "completions/mean_terminated_length": 205.8125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "entropy": 0.9559988332912326, + "epoch": 0.2799208704253215, + "grad_norm": 2.5761358737945557, + "kl_approx": 0.343414306640625, + "learning_rate": 1.8126579138282502e-05, + "loss": 0.2287, + "num_tokens": 6828121.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2181378602981567, + "sampling/importance_sampling_ratio/mean": 0.9998427033424377, + "sampling/importance_sampling_ratio/min": 0.8122193217277527, + "sampling/sampling_logp_difference/max": 0.20798492431640625, + "sampling/sampling_logp_difference/mean": 0.016082478687167168, + "step": 283 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 461.0, + "completions/max_terminated_length": 461.0, + "completions/mean_length": 170.59375, + "completions/mean_terminated_length": 170.59375, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.7337540173903108, + "epoch": 0.2809099901088032, + "grad_norm": 2.3644371032714844, + "kl_approx": 0.244873046875, + "learning_rate": 1.8106390545469797e-05, + "loss": 0.203, + "num_tokens": 6855796.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2447283267974854, + "sampling/importance_sampling_ratio/mean": 1.0003597736358643, + "sampling/importance_sampling_ratio/min": 0.8401584029197693, + "sampling/sampling_logp_difference/max": 0.2189173698425293, + "sampling/sampling_logp_difference/mean": 0.012584760785102844, + "step": 284 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 638.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 185.0, + "completions/mean_terminated_length": 185.0, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.769881590269506, + "epoch": 0.2818991097922849, + "grad_norm": 2.6485514640808105, + "kl_approx": 0.316741943359375, + "learning_rate": 1.8086105125078858e-05, + "loss": 0.2133, + "num_tokens": 6883068.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2408983707427979, + "sampling/importance_sampling_ratio/mean": 1.00046706199646, + "sampling/importance_sampling_ratio/min": 0.7734773755073547, + "sampling/sampling_logp_difference/max": 0.25685882568359375, + "sampling/sampling_logp_difference/mean": 0.013372906483709812, + "step": 285 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 857.0, + "completions/max_terminated_length": 857.0, + "completions/mean_length": 188.40625, + "completions/mean_terminated_length": 188.40625, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.6370646376162767, + "epoch": 0.2828882294757666, + "grad_norm": 2.3806402683258057, + "kl_approx": 0.2541618347167969, + "learning_rate": 1.8065723119410885e-05, + "loss": 0.1784, + "num_tokens": 6915793.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2556909322738647, + "sampling/importance_sampling_ratio/mean": 1.0005565881729126, + "sampling/importance_sampling_ratio/min": 0.7984816431999207, + "sampling/sampling_logp_difference/max": 0.22768592834472656, + "sampling/sampling_logp_difference/mean": 0.011392634361982346, + "step": 286 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.0, + "completions/max_terminated_length": 297.0, + "completions/mean_length": 115.46875, + "completions/mean_terminated_length": 115.46875, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "entropy": 0.6798480600118637, + "epoch": 0.2838773491592483, + "grad_norm": 3.0986886024475098, + "kl_approx": 0.2765960693359375, + "learning_rate": 1.804524477192075e-05, + "loss": 0.2277, + "num_tokens": 6933992.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1889050006866455, + "sampling/importance_sampling_ratio/mean": 1.0004689693450928, + "sampling/importance_sampling_ratio/min": 0.8637964725494385, + "sampling/sampling_logp_difference/max": 0.1730327606201172, + "sampling/sampling_logp_difference/mean": 0.01372337993234396, + "step": 287 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/max_terminated_length": 304.0, + "completions/mean_length": 135.84375, + "completions/mean_terminated_length": 135.84375, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.7850636513903737, + "epoch": 0.28486646884273, + "grad_norm": 3.6997897624969482, + "kl_approx": 0.384735107421875, + "learning_rate": 1.8024670327214084e-05, + "loss": 0.2873, + "num_tokens": 6955331.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1435002088546753, + "sampling/importance_sampling_ratio/mean": 0.9999605417251587, + "sampling/importance_sampling_ratio/min": 0.8320849537849426, + "sampling/sampling_logp_difference/max": 0.1838207244873047, + "sampling/sampling_logp_difference/mean": 0.013304715976119041, + "step": 288 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/max_terminated_length": 301.0, + "completions/mean_length": 128.125, + "completions/mean_terminated_length": 128.125, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.7551005519926548, + "epoch": 0.2858555885262117, + "grad_norm": 2.53135085105896, + "kl_approx": 0.28559112548828125, + "learning_rate": 1.8004000031044363e-05, + "loss": 0.1794, + "num_tokens": 6983119.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1393771171569824, + "sampling/importance_sampling_ratio/mean": 0.999236524105072, + "sampling/importance_sampling_ratio/min": 0.8413154482841492, + "sampling/sampling_logp_difference/max": 0.1727886199951172, + "sampling/sampling_logp_difference/mean": 0.011013248935341835, + "step": 289 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 144.0625, + "completions/mean_terminated_length": 144.0625, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.7586485026404262, + "epoch": 0.2868447082096934, + "grad_norm": 2.2351346015930176, + "kl_approx": 0.29018402099609375, + "learning_rate": 1.798323413030997e-05, + "loss": 0.1841, + "num_tokens": 7005489.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1876126527786255, + "sampling/importance_sampling_ratio/mean": 1.0005043745040894, + "sampling/importance_sampling_ratio/min": 0.8623186349868774, + "sampling/sampling_logp_difference/max": 0.17194509506225586, + "sampling/sampling_logp_difference/mean": 0.014061874710023403, + "step": 290 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 163.84375, + "completions/mean_terminated_length": 163.84375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.8267401978373528, + "epoch": 0.2878338278931751, + "grad_norm": 2.4321110248565674, + "kl_approx": 0.3395042419433594, + "learning_rate": 1.796237287305125e-05, + "loss": 0.2321, + "num_tokens": 7029076.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1893455982208252, + "sampling/importance_sampling_ratio/mean": 1.0011012554168701, + "sampling/importance_sampling_ratio/min": 0.8425796627998352, + "sampling/sampling_logp_difference/max": 0.17340326309204102, + "sampling/sampling_logp_difference/mean": 0.01425518561154604, + "step": 291 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 452.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 138.0625, + "completions/mean_terminated_length": 138.0625, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.79865199374035, + "epoch": 0.2888229475766568, + "grad_norm": 2.358048439025879, + "kl_approx": 0.28905487060546875, + "learning_rate": 1.7941416508447537e-05, + "loss": 0.2169, + "num_tokens": 7053126.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2217631340026855, + "sampling/importance_sampling_ratio/mean": 1.0000317096710205, + "sampling/importance_sampling_ratio/min": 0.8115402460098267, + "sampling/sampling_logp_difference/max": 0.20882129669189453, + "sampling/sampling_logp_difference/mean": 0.013350498862564564, + "step": 292 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 821.0, + "completions/max_terminated_length": 821.0, + "completions/mean_length": 249.90625, + "completions/mean_terminated_length": 249.90625, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.9886434320360422, + "epoch": 0.2898120672601385, + "grad_norm": 3.1051204204559326, + "kl_approx": 0.32391357421875, + "learning_rate": 1.792036528681418e-05, + "loss": 0.247, + "num_tokens": 7077731.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2592612504959106, + "sampling/importance_sampling_ratio/mean": 0.999854326248169, + "sampling/importance_sampling_ratio/min": 0.775351881980896, + "sampling/sampling_logp_difference/max": 0.2544384002685547, + "sampling/sampling_logp_difference/mean": 0.014785322360694408, + "step": 293 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 135.3125, + "completions/mean_terminated_length": 135.3125, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.9014694644138217, + "epoch": 0.29080118694362017, + "grad_norm": 3.2792179584503174, + "kl_approx": 0.36553955078125, + "learning_rate": 1.789921945959958e-05, + "loss": 0.2509, + "num_tokens": 7097925.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.180361270904541, + "sampling/importance_sampling_ratio/mean": 1.0005325078964233, + "sampling/importance_sampling_ratio/min": 0.8205909729003906, + "sampling/sampling_logp_difference/max": 0.19773054122924805, + "sampling/sampling_logp_difference/mean": 0.014601066708564758, + "step": 294 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 789.0, + "completions/max_terminated_length": 789.0, + "completions/mean_length": 262.75, + "completions/mean_terminated_length": 262.75, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.7073164647445083, + "epoch": 0.29179030662710187, + "grad_norm": 1.7222764492034912, + "kl_approx": 0.22618865966796875, + "learning_rate": 1.7877979279382135e-05, + "loss": 0.1689, + "num_tokens": 7128677.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1963460445404053, + "sampling/importance_sampling_ratio/mean": 0.9996522068977356, + "sampling/importance_sampling_ratio/min": 0.7813910245895386, + "sampling/sampling_logp_difference/max": 0.24667954444885254, + "sampling/sampling_logp_difference/mean": 0.012343580834567547, + "step": 295 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 581.0, + "completions/max_terminated_length": 581.0, + "completions/mean_length": 153.71875, + "completions/mean_terminated_length": 153.71875, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.7062114709988236, + "epoch": 0.29277942631058357, + "grad_norm": 2.4552528858184814, + "kl_approx": 0.26074981689453125, + "learning_rate": 1.7856644999867264e-05, + "loss": 0.1951, + "num_tokens": 7152228.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.308112382888794, + "sampling/importance_sampling_ratio/mean": 1.0005593299865723, + "sampling/importance_sampling_ratio/min": 0.8344243764877319, + "sampling/sampling_logp_difference/max": 0.268585205078125, + "sampling/sampling_logp_difference/mean": 0.012547975406050682, + "step": 296 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/max_terminated_length": 178.0, + "completions/mean_length": 87.34375, + "completions/mean_terminated_length": 87.34375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.6183590926229954, + "epoch": 0.29376854599406527, + "grad_norm": 1.9936422109603882, + "kl_approx": 0.21903157234191895, + "learning_rate": 1.783521687588437e-05, + "loss": 0.1514, + "num_tokens": 7170575.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1669435501098633, + "sampling/importance_sampling_ratio/mean": 1.0008279085159302, + "sampling/importance_sampling_ratio/min": 0.8506006002426147, + "sampling/sampling_logp_difference/max": 0.16181254386901855, + "sampling/sampling_logp_difference/mean": 0.011522624641656876, + "step": 297 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 126.78125, + "completions/mean_terminated_length": 126.78125, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.6006322121247649, + "epoch": 0.29475766567754697, + "grad_norm": 2.3153836727142334, + "kl_approx": 0.24176025390625, + "learning_rate": 1.781369516338378e-05, + "loss": 0.1633, + "num_tokens": 7193752.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1500394344329834, + "sampling/importance_sampling_ratio/mean": 1.0003947019577026, + "sampling/importance_sampling_ratio/min": 0.7957252264022827, + "sampling/sampling_logp_difference/max": 0.2285013198852539, + "sampling/sampling_logp_difference/mean": 0.010313395410776138, + "step": 298 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 604.0, + "completions/max_terminated_length": 604.0, + "completions/mean_length": 165.5, + "completions/mean_terminated_length": 165.5, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7353044738993049, + "epoch": 0.29574678536102866, + "grad_norm": 2.7496542930603027, + "kl_approx": 0.3392333984375, + "learning_rate": 1.779208011943371e-05, + "loss": 0.2359, + "num_tokens": 7213720.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1633710861206055, + "sampling/importance_sampling_ratio/mean": 0.9998388290405273, + "sampling/importance_sampling_ratio/min": 0.8292375206947327, + "sampling/sampling_logp_difference/max": 0.18724870681762695, + "sampling/sampling_logp_difference/mean": 0.013257890939712524, + "step": 299 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/max_terminated_length": 275.0, + "completions/mean_length": 118.625, + "completions/mean_terminated_length": 118.625, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.7141266879625618, + "epoch": 0.29673590504451036, + "grad_norm": 2.838669538497925, + "kl_approx": 0.309112548828125, + "learning_rate": 1.777037200221717e-05, + "loss": 0.2379, + "num_tokens": 7238452.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2296544313430786, + "sampling/importance_sampling_ratio/mean": 1.0002495050430298, + "sampling/importance_sampling_ratio/min": 0.8321012258529663, + "sampling/sampling_logp_difference/max": 0.20673322677612305, + "sampling/sampling_logp_difference/mean": 0.011981537565588951, + "step": 300 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 113.6875, + "completions/mean_terminated_length": 113.6875, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.7095110211521387, + "epoch": 0.29772502472799206, + "grad_norm": 2.6985971927642822, + "kl_approx": 0.291595458984375, + "learning_rate": 1.77485710710289e-05, + "loss": 0.1773, + "num_tokens": 7262042.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1687511205673218, + "sampling/importance_sampling_ratio/mean": 1.0000685453414917, + "sampling/importance_sampling_ratio/min": 0.8543300032615662, + "sampling/sampling_logp_difference/max": 0.15743780136108398, + "sampling/sampling_logp_difference/mean": 0.013693314045667648, + "step": 301 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/max_terminated_length": 292.0, + "completions/mean_length": 113.3125, + "completions/mean_terminated_length": 113.3125, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.7567874817177653, + "epoch": 0.2987141444114738, + "grad_norm": 2.950988531112671, + "kl_approx": 0.38045501708984375, + "learning_rate": 1.7726677586272263e-05, + "loss": 0.257, + "num_tokens": 7286220.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2035075426101685, + "sampling/importance_sampling_ratio/mean": 0.999890923500061, + "sampling/importance_sampling_ratio/min": 0.812684953212738, + "sampling/sampling_logp_difference/max": 0.2074117660522461, + "sampling/sampling_logp_difference/mean": 0.012761159799993038, + "step": 302 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 672.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 185.5, + "completions/mean_terminated_length": 185.5, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.5831036274321377, + "epoch": 0.2997032640949555, + "grad_norm": 2.4150466918945312, + "kl_approx": 0.24143218994140625, + "learning_rate": 1.7704691809456142e-05, + "loss": 0.1821, + "num_tokens": 7311620.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.5507961511611938, + "sampling/importance_sampling_ratio/mean": 1.0000437498092651, + "sampling/importance_sampling_ratio/min": 0.8100433349609375, + "sampling/sampling_logp_difference/max": 0.4387683868408203, + "sampling/sampling_logp_difference/mean": 0.010963356122374535, + "step": 303 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 776.0, + "completions/max_terminated_length": 776.0, + "completions/mean_length": 172.875, + "completions/mean_terminated_length": 172.875, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.7722360463812947, + "epoch": 0.3006923837784372, + "grad_norm": 2.2434005737304688, + "kl_approx": 0.29010009765625, + "learning_rate": 1.7682614003191807e-05, + "loss": 0.2088, + "num_tokens": 7332672.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1927436590194702, + "sampling/importance_sampling_ratio/mean": 1.000109076499939, + "sampling/importance_sampling_ratio/min": 0.7187306880950928, + "sampling/sampling_logp_difference/max": 0.33026862144470215, + "sampling/sampling_logp_difference/mean": 0.01385170966386795, + "step": 304 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 749.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 266.3125, + "completions/mean_terminated_length": 266.3125, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.8784914426505566, + "epoch": 0.3016815034619189, + "grad_norm": 2.5032622814178467, + "kl_approx": 0.33319091796875, + "learning_rate": 1.766044443118978e-05, + "loss": 0.2532, + "num_tokens": 7362874.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2380260229110718, + "sampling/importance_sampling_ratio/mean": 1.0003330707550049, + "sampling/importance_sampling_ratio/min": 0.8043450713157654, + "sampling/sampling_logp_difference/max": 0.2177269458770752, + "sampling/sampling_logp_difference/mean": 0.014404799789190292, + "step": 305 + }, + { + "completions/clipped_ratio": 0.125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 848.0, + "completions/mean_length": 331.75, + "completions/mean_terminated_length": 232.85714721679688, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "entropy": 0.9333799220621586, + "epoch": 0.3026706231454006, + "grad_norm": 1.9041674137115479, + "kl_approx": 0.265899658203125, + "learning_rate": 1.76381833582567e-05, + "loss": 0.1712, + "num_tokens": 7400154.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2089046239852905, + "sampling/importance_sampling_ratio/mean": 1.000046730041504, + "sampling/importance_sampling_ratio/min": 0.7312637567520142, + "sampling/sampling_logp_difference/max": 0.31298112869262695, + "sampling/sampling_logp_difference/mean": 0.014311689883470535, + "step": 306 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 352.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 124.5, + "completions/mean_terminated_length": 124.5, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 0.8359832325950265, + "epoch": 0.3036597428288823, + "grad_norm": 2.4830639362335205, + "kl_approx": 0.333831787109375, + "learning_rate": 1.761583105029213e-05, + "loss": 0.2109, + "num_tokens": 7423450.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.185927391052246, + "sampling/importance_sampling_ratio/mean": 1.000803828239441, + "sampling/importance_sampling_ratio/min": 0.8344119191169739, + "sampling/sampling_logp_difference/max": 0.18102812767028809, + "sampling/sampling_logp_difference/mean": 0.013677140697836876, + "step": 307 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 745.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 150.15625, + "completions/mean_terminated_length": 150.15625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.9037802591919899, + "epoch": 0.304648862512364, + "grad_norm": 3.295927047729492, + "kl_approx": 0.34511566162109375, + "learning_rate": 1.7593387774285412e-05, + "loss": 0.2455, + "num_tokens": 7445503.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.29276704788208, + "sampling/importance_sampling_ratio/mean": 1.000915765762329, + "sampling/importance_sampling_ratio/min": 0.8182945847511292, + "sampling/sampling_logp_difference/max": 0.25678491592407227, + "sampling/sampling_logp_difference/mean": 0.01416583452373743, + "step": 308 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 204.28125, + "completions/mean_terminated_length": 204.28125, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.9801548514515162, + "epoch": 0.3056379821958457, + "grad_norm": 2.721508502960205, + "kl_approx": 0.3337554931640625, + "learning_rate": 1.7570853798312462e-05, + "loss": 0.2328, + "num_tokens": 7472768.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2582334280014038, + "sampling/importance_sampling_ratio/mean": 0.9999727010726929, + "sampling/importance_sampling_ratio/min": 0.7821531891822815, + "sampling/sampling_logp_difference/max": 0.24570465087890625, + "sampling/sampling_logp_difference/mean": 0.015173878520727158, + "step": 309 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 160.75, + "completions/mean_terminated_length": 160.75, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.811941085383296, + "epoch": 0.3066271018793274, + "grad_norm": 3.168898582458496, + "kl_approx": 0.3369140625, + "learning_rate": 1.7548229391532572e-05, + "loss": 0.1955, + "num_tokens": 7496200.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.179616928100586, + "sampling/importance_sampling_ratio/mean": 1.0002747774124146, + "sampling/importance_sampling_ratio/min": 0.8288637399673462, + "sampling/sampling_logp_difference/max": 0.187699556350708, + "sampling/sampling_logp_difference/mean": 0.011781209148466587, + "step": 310 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 558.0, + "completions/max_terminated_length": 558.0, + "completions/mean_length": 142.5, + "completions/mean_terminated_length": 142.5, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.7952607600018382, + "epoch": 0.3076162215628091, + "grad_norm": 2.547868251800537, + "kl_approx": 0.3126983642578125, + "learning_rate": 1.7525514824185187e-05, + "loss": 0.2062, + "num_tokens": 7521456.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1848186254501343, + "sampling/importance_sampling_ratio/mean": 0.9998703598976135, + "sampling/importance_sampling_ratio/min": 0.8693347573280334, + "sampling/sampling_logp_difference/max": 0.16958975791931152, + "sampling/sampling_logp_difference/mean": 0.01422309409826994, + "step": 311 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 188.75, + "completions/mean_terminated_length": 188.75, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "entropy": 0.9201382072642446, + "epoch": 0.3086053412462908, + "grad_norm": 2.5949835777282715, + "kl_approx": 0.305145263671875, + "learning_rate": 1.750271036758669e-05, + "loss": 0.2291, + "num_tokens": 7544608.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.243618369102478, + "sampling/importance_sampling_ratio/mean": 1.000194787979126, + "sampling/importance_sampling_ratio/min": 0.8223516941070557, + "sampling/sampling_logp_difference/max": 0.21802520751953125, + "sampling/sampling_logp_difference/mean": 0.014224348589777946, + "step": 312 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 294.0, + "completions/max_terminated_length": 294.0, + "completions/mean_length": 66.78125, + "completions/mean_terminated_length": 66.78125, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.4123487868346274, + "epoch": 0.3095944609297725, + "grad_norm": 2.082361936569214, + "kl_approx": 0.15377044677734375, + "learning_rate": 1.747981629412715e-05, + "loss": 0.1249, + "num_tokens": 7559497.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1596769094467163, + "sampling/importance_sampling_ratio/mean": 0.998990535736084, + "sampling/importance_sampling_ratio/min": 0.8791318535804749, + "sampling/sampling_logp_difference/max": 0.14814138412475586, + "sampling/sampling_logp_difference/mean": 0.00934837106615305, + "step": 313 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 851.0, + "completions/max_terminated_length": 851.0, + "completions/mean_length": 226.5, + "completions/mean_terminated_length": 226.5, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "entropy": 0.6805483684875071, + "epoch": 0.3105835806132542, + "grad_norm": 2.3783607482910156, + "kl_approx": 0.27045440673828125, + "learning_rate": 1.7456832877267083e-05, + "loss": 0.1623, + "num_tokens": 7588977.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2857998609542847, + "sampling/importance_sampling_ratio/mean": 0.9996693134307861, + "sampling/importance_sampling_ratio/min": 0.8160499334335327, + "sampling/sampling_logp_difference/max": 0.25138092041015625, + "sampling/sampling_logp_difference/mean": 0.013767233118414879, + "step": 314 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 213.875, + "completions/mean_terminated_length": 213.875, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 1.061541692353785, + "epoch": 0.3115727002967359, + "grad_norm": 2.9245636463165283, + "kl_approx": 0.339569091796875, + "learning_rate": 1.7433760391534166e-05, + "loss": 0.2708, + "num_tokens": 7610517.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2627816200256348, + "sampling/importance_sampling_ratio/mean": 1.0001723766326904, + "sampling/importance_sampling_ratio/min": 0.8057682514190674, + "sampling/sampling_logp_difference/max": 0.23331689834594727, + "sampling/sampling_logp_difference/mean": 0.01707172766327858, + "step": 315 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 368.0, + "completions/max_terminated_length": 368.0, + "completions/mean_length": 176.4375, + "completions/mean_terminated_length": 176.4375, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 0.7591512352228165, + "epoch": 0.3125618199802176, + "grad_norm": 2.69907283782959, + "kl_approx": 0.30389404296875, + "learning_rate": 1.741059911251997e-05, + "loss": 0.2101, + "num_tokens": 7638563.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1901832818984985, + "sampling/importance_sampling_ratio/mean": 0.9997280836105347, + "sampling/importance_sampling_ratio/min": 0.8279536366462708, + "sampling/sampling_logp_difference/max": 0.188798189163208, + "sampling/sampling_logp_difference/mean": 0.012978103011846542, + "step": 316 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/max_terminated_length": 250.0, + "completions/mean_length": 114.625, + "completions/mean_terminated_length": 114.625, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.7344440892338753, + "epoch": 0.3135509396636993, + "grad_norm": 2.5733983516693115, + "kl_approx": 0.2907562255859375, + "learning_rate": 1.7387349316876668e-05, + "loss": 0.1854, + "num_tokens": 7663679.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1945958137512207, + "sampling/importance_sampling_ratio/mean": 0.9997657537460327, + "sampling/importance_sampling_ratio/min": 0.8389726281166077, + "sampling/sampling_logp_difference/max": 0.17780792713165283, + "sampling/sampling_logp_difference/mean": 0.012271163985133171, + "step": 317 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 975.0, + "completions/max_terminated_length": 975.0, + "completions/mean_length": 188.625, + "completions/mean_terminated_length": 188.625, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6078087952919304, + "epoch": 0.314540059347181, + "grad_norm": 4.636024475097656, + "kl_approx": 0.27092742919921875, + "learning_rate": 1.7364011282313732e-05, + "loss": 0.2253, + "num_tokens": 7685579.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.413045048713684, + "sampling/importance_sampling_ratio/mean": 1.0004764795303345, + "sampling/importance_sampling_ratio/min": 0.6871237754821777, + "sampling/sampling_logp_difference/max": 0.3752408027648926, + "sampling/sampling_logp_difference/mean": 0.01063003484159708, + "step": 318 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 353.0, + "completions/max_terminated_length": 353.0, + "completions/mean_length": 117.6875, + "completions/mean_terminated_length": 117.6875, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.538060920778662, + "epoch": 0.3155291790306627, + "grad_norm": 2.4417264461517334, + "kl_approx": 0.22021102905273438, + "learning_rate": 1.7340585287594605e-05, + "loss": 0.1708, + "num_tokens": 7710905.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1498738527297974, + "sampling/importance_sampling_ratio/mean": 1.0002936124801636, + "sampling/importance_sampling_ratio/min": 0.8082682490348816, + "sampling/sampling_logp_difference/max": 0.2128612995147705, + "sampling/sampling_logp_difference/mean": 0.010301164351403713, + "step": 319 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 703.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 197.25, + "completions/mean_terminated_length": 197.25, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.7277012141421437, + "epoch": 0.3165182987141444, + "grad_norm": 1.8807249069213867, + "kl_approx": 0.2105560302734375, + "learning_rate": 1.731707161253338e-05, + "loss": 0.1725, + "num_tokens": 7737369.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2102413177490234, + "sampling/importance_sampling_ratio/mean": 1.0001676082611084, + "sampling/importance_sampling_ratio/min": 0.7940690517425537, + "sampling/sampling_logp_difference/max": 0.23058485984802246, + "sampling/sampling_logp_difference/mean": 0.01430986262857914, + "step": 320 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 441.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 125.3125, + "completions/mean_terminated_length": 125.3125, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.734582195058465, + "epoch": 0.31750741839762614, + "grad_norm": 2.896090507507324, + "kl_approx": 0.3123779296875, + "learning_rate": 1.7293470537991463e-05, + "loss": 0.2412, + "num_tokens": 7762707.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1922818422317505, + "sampling/importance_sampling_ratio/mean": 1.0006396770477295, + "sampling/importance_sampling_ratio/min": 0.7879047989845276, + "sampling/sampling_logp_difference/max": 0.23837804794311523, + "sampling/sampling_logp_difference/mean": 0.013731887564063072, + "step": 321 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 144.59375, + "completions/mean_terminated_length": 144.59375, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.8370183417573571, + "epoch": 0.31849653808110784, + "grad_norm": 2.4357006549835205, + "kl_approx": 0.278076171875, + "learning_rate": 1.7269782345874204e-05, + "loss": 0.1784, + "num_tokens": 7787902.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1571754217147827, + "sampling/importance_sampling_ratio/mean": 0.9991551637649536, + "sampling/importance_sampling_ratio/min": 0.8420906662940979, + "sampling/sampling_logp_difference/max": 0.17186760902404785, + "sampling/sampling_logp_difference/mean": 0.014920658431947231, + "step": 322 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 480.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 117.0, + "completions/mean_terminated_length": 117.0, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.7496029892936349, + "epoch": 0.31948565776458954, + "grad_norm": 3.4023690223693848, + "kl_approx": 0.296173095703125, + "learning_rate": 1.7246007319127547e-05, + "loss": 0.1955, + "num_tokens": 7811558.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1666719913482666, + "sampling/importance_sampling_ratio/mean": 1.000491976737976, + "sampling/importance_sampling_ratio/min": 0.8648551106452942, + "sampling/sampling_logp_difference/max": 0.15415525436401367, + "sampling/sampling_logp_difference/mean": 0.013773888349533081, + "step": 323 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 159.28125, + "completions/mean_terminated_length": 159.28125, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "entropy": 0.6421801568940282, + "epoch": 0.32047477744807124, + "grad_norm": 2.133829116821289, + "kl_approx": 0.2075653076171875, + "learning_rate": 1.7222145741734625e-05, + "loss": 0.1409, + "num_tokens": 7832951.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1435959339141846, + "sampling/importance_sampling_ratio/mean": 1.0000251531600952, + "sampling/importance_sampling_ratio/min": 0.8439325094223022, + "sampling/sampling_logp_difference/max": 0.16968274116516113, + "sampling/sampling_logp_difference/mean": 0.010992590337991714, + "step": 324 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/max_terminated_length": 258.0, + "completions/mean_length": 111.71875, + "completions/mean_terminated_length": 111.71875, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "entropy": 0.5186840426176786, + "epoch": 0.32146389713155293, + "grad_norm": 2.301877021789551, + "kl_approx": 0.17468643188476562, + "learning_rate": 1.7198197898712402e-05, + "loss": 0.1228, + "num_tokens": 7854902.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.152989149093628, + "sampling/importance_sampling_ratio/mean": 1.0004150867462158, + "sampling/importance_sampling_ratio/min": 0.8422931432723999, + "sampling/sampling_logp_difference/max": 0.17162716388702393, + "sampling/sampling_logp_difference/mean": 0.00966770201921463, + "step": 325 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 485.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 132.34375, + "completions/mean_terminated_length": 132.34375, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "entropy": 0.7781155416741967, + "epoch": 0.32245301681503463, + "grad_norm": 2.694973945617676, + "kl_approx": 0.320892333984375, + "learning_rate": 1.717416407610824e-05, + "loss": 0.2261, + "num_tokens": 7876761.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2620285749435425, + "sampling/importance_sampling_ratio/mean": 1.0005673170089722, + "sampling/importance_sampling_ratio/min": 0.7546735405921936, + "sampling/sampling_logp_difference/max": 0.28147006034851074, + "sampling/sampling_logp_difference/mean": 0.013519820757210255, + "step": 326 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/max_terminated_length": 267.0, + "completions/mean_length": 98.0625, + "completions/mean_terminated_length": 98.0625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.639696694444865, + "epoch": 0.32344213649851633, + "grad_norm": 2.9873499870300293, + "kl_approx": 0.3207893371582031, + "learning_rate": 1.7150044560996488e-05, + "loss": 0.2431, + "num_tokens": 7899611.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2218170166015625, + "sampling/importance_sampling_ratio/mean": 0.9997280836105347, + "sampling/importance_sampling_ratio/min": 0.8298965096473694, + "sampling/sampling_logp_difference/max": 0.20033907890319824, + "sampling/sampling_logp_difference/mean": 0.011493654921650887, + "step": 327 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 432.0, + "completions/max_terminated_length": 432.0, + "completions/mean_length": 150.65625, + "completions/mean_terminated_length": 150.65625, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.9651678632944822, + "epoch": 0.32443125618199803, + "grad_norm": 3.117753028869629, + "kl_approx": 0.324951171875, + "learning_rate": 1.7125839641475074e-05, + "loss": 0.2387, + "num_tokens": 7924992.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2002174854278564, + "sampling/importance_sampling_ratio/mean": 1.0000826120376587, + "sampling/importance_sampling_ratio/min": 0.8420368432998657, + "sampling/sampling_logp_difference/max": 0.18250274658203125, + "sampling/sampling_logp_difference/mean": 0.015069739893078804, + "step": 328 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 583.0, + "completions/max_terminated_length": 583.0, + "completions/mean_length": 211.78125, + "completions/mean_terminated_length": 211.78125, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.8704281989485025, + "epoch": 0.3254203758654797, + "grad_norm": 2.6333398818969727, + "kl_approx": 0.411834716796875, + "learning_rate": 1.7101549606662025e-05, + "loss": 0.2627, + "num_tokens": 7957913.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.202318549156189, + "sampling/importance_sampling_ratio/mean": 1.0005874633789062, + "sampling/importance_sampling_ratio/min": 0.8206818699836731, + "sampling/sampling_logp_difference/max": 0.19761979579925537, + "sampling/sampling_logp_difference/mean": 0.014082243666052818, + "step": 329 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/max_terminated_length": 295.0, + "completions/mean_length": 121.90625, + "completions/mean_terminated_length": 121.90625, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "entropy": 0.7842995952814817, + "epoch": 0.3264094955489614, + "grad_norm": 2.698920249938965, + "kl_approx": 0.32391357421875, + "learning_rate": 1.7077174746692054e-05, + "loss": 0.1895, + "num_tokens": 7981846.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1493991613388062, + "sampling/importance_sampling_ratio/mean": 1.0004703998565674, + "sampling/importance_sampling_ratio/min": 0.7930014729499817, + "sampling/sampling_logp_difference/max": 0.23193025588989258, + "sampling/sampling_logp_difference/mean": 0.013386135920882225, + "step": 330 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 118.40625, + "completions/mean_terminated_length": 118.40625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.6225263751111925, + "epoch": 0.3273986152324431, + "grad_norm": 2.9460132122039795, + "kl_approx": 0.2357025146484375, + "learning_rate": 1.7052715352713076e-05, + "loss": 0.1615, + "num_tokens": 8006019.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2382031679153442, + "sampling/importance_sampling_ratio/mean": 0.9998303055763245, + "sampling/importance_sampling_ratio/min": 0.7931509613990784, + "sampling/sampling_logp_difference/max": 0.2317417860031128, + "sampling/sampling_logp_difference/mean": 0.013472470454871655, + "step": 331 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 458.0, + "completions/max_terminated_length": 458.0, + "completions/mean_length": 138.1875, + "completions/mean_terminated_length": 138.1875, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.6855741366744041, + "epoch": 0.3283877349159248, + "grad_norm": 3.0839734077453613, + "kl_approx": 0.281585693359375, + "learning_rate": 1.7028171716882714e-05, + "loss": 0.2259, + "num_tokens": 8026849.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2909655570983887, + "sampling/importance_sampling_ratio/mean": 1.0004098415374756, + "sampling/importance_sampling_ratio/min": 0.7945570945739746, + "sampling/sampling_logp_difference/max": 0.2553904056549072, + "sampling/sampling_logp_difference/mean": 0.012955369427800179, + "step": 332 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 96.8125, + "completions/mean_terminated_length": 96.8125, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "entropy": 0.5918765431270003, + "epoch": 0.3293768545994065, + "grad_norm": 2.312124013900757, + "kl_approx": 0.20192718505859375, + "learning_rate": 1.7003544132364847e-05, + "loss": 0.1528, + "num_tokens": 8050707.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2106544971466064, + "sampling/importance_sampling_ratio/mean": 0.9998504519462585, + "sampling/importance_sampling_ratio/min": 0.8512711524963379, + "sampling/sampling_logp_difference/max": 0.1911611557006836, + "sampling/sampling_logp_difference/mean": 0.011819862760603428, + "step": 333 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 735.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 185.15625, + "completions/mean_terminated_length": 185.15625, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.8883398249745369, + "epoch": 0.3303659742828882, + "grad_norm": 2.7629902362823486, + "kl_approx": 0.32049560546875, + "learning_rate": 1.6978832893326074e-05, + "loss": 0.2088, + "num_tokens": 8074592.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.186069369316101, + "sampling/importance_sampling_ratio/mean": 0.9999062418937683, + "sampling/importance_sampling_ratio/min": 0.6885431408882141, + "sampling/sampling_logp_difference/max": 0.37317728996276855, + "sampling/sampling_logp_difference/mean": 0.01694898121058941, + "step": 334 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 560.0, + "completions/max_terminated_length": 560.0, + "completions/mean_length": 216.21875, + "completions/mean_terminated_length": 216.21875, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.9462208957411349, + "epoch": 0.3313550939663699, + "grad_norm": 2.465144634246826, + "kl_approx": 0.3285369873046875, + "learning_rate": 1.6954038294932215e-05, + "loss": 0.2229, + "num_tokens": 8106047.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1920123100280762, + "sampling/importance_sampling_ratio/mean": 0.999750018119812, + "sampling/importance_sampling_ratio/min": 0.7935164570808411, + "sampling/sampling_logp_difference/max": 0.23128104209899902, + "sampling/sampling_logp_difference/mean": 0.015693508088588715, + "step": 335 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 993.0, + "completions/mean_length": 271.53125, + "completions/mean_terminated_length": 247.258056640625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.735797509085387, + "epoch": 0.3323442136498516, + "grad_norm": 2.240936517715454, + "kl_approx": 0.20676040649414062, + "learning_rate": 1.692916063334479e-05, + "loss": 0.1582, + "num_tokens": 8138712.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.178946614265442, + "sampling/importance_sampling_ratio/mean": 1.0002467632293701, + "sampling/importance_sampling_ratio/min": 0.8015815615653992, + "sampling/sampling_logp_difference/max": 0.22116851806640625, + "sampling/sampling_logp_difference/mean": 0.013347666710615158, + "step": 336 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 791.0, + "completions/max_terminated_length": 791.0, + "completions/mean_length": 213.0625, + "completions/mean_terminated_length": 213.0625, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "entropy": 0.9242281476035714, + "epoch": 0.3333333333333333, + "grad_norm": 2.6171796321868896, + "kl_approx": 0.34039306640625, + "learning_rate": 1.690420020571747e-05, + "loss": 0.2334, + "num_tokens": 8163234.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1770601272583008, + "sampling/importance_sampling_ratio/mean": 0.9999088644981384, + "sampling/importance_sampling_ratio/min": 0.8223693370819092, + "sampling/sampling_logp_difference/max": 0.19556570053100586, + "sampling/sampling_logp_difference/mean": 0.015630025416612625, + "step": 337 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 254.65625, + "completions/mean_terminated_length": 254.65625, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "entropy": 0.7351608574390411, + "epoch": 0.334322453016815, + "grad_norm": 2.166008949279785, + "kl_approx": 0.27698516845703125, + "learning_rate": 1.6879157310192537e-05, + "loss": 0.2154, + "num_tokens": 8195903.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2003525495529175, + "sampling/importance_sampling_ratio/mean": 1.0004019737243652, + "sampling/importance_sampling_ratio/min": 0.8317449688911438, + "sampling/sampling_logp_difference/max": 0.18422937393188477, + "sampling/sampling_logp_difference/mean": 0.012019136920571327, + "step": 338 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/max_terminated_length": 300.0, + "completions/mean_length": 138.0, + "completions/mean_terminated_length": 138.0, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.681645249016583, + "epoch": 0.3353115727002967, + "grad_norm": 2.7927026748657227, + "kl_approx": 0.2854576110839844, + "learning_rate": 1.685403224589731e-05, + "loss": 0.1845, + "num_tokens": 8218079.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1737463474273682, + "sampling/importance_sampling_ratio/mean": 0.9995397329330444, + "sampling/importance_sampling_ratio/min": 0.8553330898284912, + "sampling/sampling_logp_difference/max": 0.1602005958557129, + "sampling/sampling_logp_difference/mean": 0.012534228153526783, + "step": 339 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 145.375, + "completions/mean_terminated_length": 145.375, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "entropy": 0.6974824108183384, + "epoch": 0.3363006923837784, + "grad_norm": 7.834415435791016, + "kl_approx": 0.2658843994140625, + "learning_rate": 1.6828825312940594e-05, + "loss": 0.1807, + "num_tokens": 8245331.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1552565097808838, + "sampling/importance_sampling_ratio/mean": 1.0003085136413574, + "sampling/importance_sampling_ratio/min": 0.8286535143852234, + "sampling/sampling_logp_difference/max": 0.18795323371887207, + "sampling/sampling_logp_difference/mean": 0.011320590041577816, + "step": 340 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/max_terminated_length": 254.0, + "completions/mean_length": 96.78125, + "completions/mean_terminated_length": 96.78125, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.6893311282619834, + "epoch": 0.33728981206726016, + "grad_norm": 2.5862979888916016, + "kl_approx": 0.3193511962890625, + "learning_rate": 1.6803536812409077e-05, + "loss": 0.1912, + "num_tokens": 8267524.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1379252672195435, + "sampling/importance_sampling_ratio/mean": 0.9997590184211731, + "sampling/importance_sampling_ratio/min": 0.8460214138031006, + "sampling/sampling_logp_difference/max": 0.16721057891845703, + "sampling/sampling_logp_difference/mean": 0.011431217193603516, + "step": 341 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 142.125, + "completions/mean_terminated_length": 142.125, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.642220621695742, + "epoch": 0.33827893175074186, + "grad_norm": 2.7381465435028076, + "kl_approx": 0.23992156982421875, + "learning_rate": 1.6778167046363735e-05, + "loss": 0.1889, + "num_tokens": 8294360.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.267491340637207, + "sampling/importance_sampling_ratio/mean": 1.0000215768814087, + "sampling/importance_sampling_ratio/min": 0.8366511464118958, + "sampling/sampling_logp_difference/max": 0.23703956604003906, + "sampling/sampling_logp_difference/mean": 0.012712901458144188, + "step": 342 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 354.0, + "completions/max_terminated_length": 354.0, + "completions/mean_length": 121.59375, + "completions/mean_terminated_length": 121.59375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.8493235902860761, + "epoch": 0.33926805143422356, + "grad_norm": 2.7640316486358643, + "kl_approx": 0.32525634765625, + "learning_rate": 1.675271631783623e-05, + "loss": 0.2215, + "num_tokens": 8319907.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.203780174255371, + "sampling/importance_sampling_ratio/mean": 1.0004667043685913, + "sampling/importance_sampling_ratio/min": 0.781249463558197, + "sampling/sampling_logp_difference/max": 0.24686074256896973, + "sampling/sampling_logp_difference/mean": 0.013187232427299023, + "step": 343 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/max_terminated_length": 126.0, + "completions/mean_length": 69.0625, + "completions/mean_terminated_length": 69.0625, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.449458361370489, + "epoch": 0.34025717111770526, + "grad_norm": 2.7708165645599365, + "kl_approx": 0.21734619140625, + "learning_rate": 1.672718493082529e-05, + "loss": 0.1339, + "num_tokens": 8338093.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1871144771575928, + "sampling/importance_sampling_ratio/mean": 0.9998024702072144, + "sampling/importance_sampling_ratio/min": 0.7909983396530151, + "sampling/sampling_logp_difference/max": 0.23445940017700195, + "sampling/sampling_logp_difference/mean": 0.008594024926424026, + "step": 344 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 120.0, + "completions/max_terminated_length": 120.0, + "completions/mean_length": 72.4375, + "completions/mean_terminated_length": 72.4375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.5927004376426339, + "epoch": 0.34124629080118696, + "grad_norm": 3.043832302093506, + "kl_approx": 0.28568267822265625, + "learning_rate": 1.6701573190293076e-05, + "loss": 0.1649, + "num_tokens": 8356003.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1451889276504517, + "sampling/importance_sampling_ratio/mean": 0.9995126724243164, + "sampling/importance_sampling_ratio/min": 0.3746478259563446, + "sampling/sampling_logp_difference/max": 0.9817688465118408, + "sampling/sampling_logp_difference/mean": 0.011849264614284039, + "step": 345 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 366.0, + "completions/max_terminated_length": 366.0, + "completions/mean_length": 115.71875, + "completions/mean_terminated_length": 115.71875, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.5215674787759781, + "epoch": 0.34223541048466866, + "grad_norm": 5.8786845207214355, + "kl_approx": 0.2333230972290039, + "learning_rate": 1.667588140216154e-05, + "loss": 0.1512, + "num_tokens": 8385234.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2515674829483032, + "sampling/importance_sampling_ratio/mean": 0.9995107054710388, + "sampling/importance_sampling_ratio/min": 0.4951493740081787, + "sampling/sampling_logp_difference/max": 0.7028958201408386, + "sampling/sampling_logp_difference/mean": 0.010219091549515724, + "step": 346 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 184.15625, + "completions/mean_terminated_length": 184.15625, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.9323503030464053, + "epoch": 0.34322453016815035, + "grad_norm": 2.77953839302063, + "kl_approx": 0.354583740234375, + "learning_rate": 1.6650109873308763e-05, + "loss": 0.2593, + "num_tokens": 8412191.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3314311504364014, + "sampling/importance_sampling_ratio/mean": 1.0003751516342163, + "sampling/importance_sampling_ratio/min": 0.8249359130859375, + "sampling/sampling_logp_difference/max": 0.2862544059753418, + "sampling/sampling_logp_difference/mean": 0.014484494924545288, + "step": 347 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 784.0, + "completions/max_terminated_length": 784.0, + "completions/mean_length": 160.3125, + "completions/mean_terminated_length": 160.3125, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "entropy": 0.8016514405608177, + "epoch": 0.34421364985163205, + "grad_norm": 3.0054924488067627, + "kl_approx": 0.4635162353515625, + "learning_rate": 1.6624258911565312e-05, + "loss": 0.251, + "num_tokens": 8437329.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2469606399536133, + "sampling/importance_sampling_ratio/mean": 0.9995077848434448, + "sampling/importance_sampling_ratio/min": 0.7698357701301575, + "sampling/sampling_logp_difference/max": 0.2615780830383301, + "sampling/sampling_logp_difference/mean": 0.014198815450072289, + "step": 348 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 435.0, + "completions/max_terminated_length": 435.0, + "completions/mean_length": 137.6875, + "completions/mean_terminated_length": 137.6875, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.6603802088648081, + "epoch": 0.34520276953511375, + "grad_norm": 2.484124183654785, + "kl_approx": 0.2418975830078125, + "learning_rate": 1.6598328825710536e-05, + "loss": 0.1786, + "num_tokens": 8457511.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.143612265586853, + "sampling/importance_sampling_ratio/mean": 0.9998595118522644, + "sampling/importance_sampling_ratio/min": 0.8481531739234924, + "sampling/sampling_logp_difference/max": 0.16469407081604004, + "sampling/sampling_logp_difference/mean": 0.012545258738100529, + "step": 349 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 373.15625, + "completions/mean_terminated_length": 352.1612854003906, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "entropy": 1.0246247667819262, + "epoch": 0.34619188921859545, + "grad_norm": 2.636146068572998, + "kl_approx": 0.34759521484375, + "learning_rate": 1.6572319925468892e-05, + "loss": 0.2268, + "num_tokens": 8490700.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2696646451950073, + "sampling/importance_sampling_ratio/mean": 1.0000004768371582, + "sampling/importance_sampling_ratio/min": 0.8414121866226196, + "sampling/sampling_logp_difference/max": 0.2387528419494629, + "sampling/sampling_logp_difference/mean": 0.01419394463300705, + "step": 350 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 711.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 202.3125, + "completions/mean_terminated_length": 202.3125, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.8651395379565656, + "epoch": 0.34718100890207715, + "grad_norm": 2.050278663635254, + "kl_approx": 0.28633880615234375, + "learning_rate": 1.654623252150624e-05, + "loss": 0.1871, + "num_tokens": 8514446.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2784003019332886, + "sampling/importance_sampling_ratio/mean": 1.0002435445785522, + "sampling/importance_sampling_ratio/min": 0.8417696952819824, + "sampling/sampling_logp_difference/max": 0.24560952186584473, + "sampling/sampling_logp_difference/mean": 0.013517262414097786, + "step": 351 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 515.0, + "completions/max_terminated_length": 515.0, + "completions/mean_length": 121.625, + "completions/mean_terminated_length": 121.625, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.6628641174174845, + "epoch": 0.34817012858555885, + "grad_norm": 2.132230758666992, + "kl_approx": 0.2456207275390625, + "learning_rate": 1.6520066925426146e-05, + "loss": 0.1358, + "num_tokens": 8539658.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1653947830200195, + "sampling/importance_sampling_ratio/mean": 0.9997344613075256, + "sampling/importance_sampling_ratio/min": 0.8472093343734741, + "sampling/sampling_logp_difference/max": 0.16580748558044434, + "sampling/sampling_logp_difference/mean": 0.012784886173903942, + "step": 352 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 729.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 203.75, + "completions/mean_terminated_length": 203.75, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.8103084731847048, + "epoch": 0.34915924826904055, + "grad_norm": 1.8776968717575073, + "kl_approx": 0.2314300537109375, + "learning_rate": 1.6493823449766137e-05, + "loss": 0.1697, + "num_tokens": 8564722.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2023262977600098, + "sampling/importance_sampling_ratio/mean": 1.0000507831573486, + "sampling/importance_sampling_ratio/min": 0.8180822134017944, + "sampling/sampling_logp_difference/max": 0.20079243183135986, + "sampling/sampling_logp_difference/mean": 0.014451484195888042, + "step": 353 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 420.0, + "completions/max_terminated_length": 420.0, + "completions/mean_length": 160.84375, + "completions/mean_terminated_length": 160.84375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.9071384118869901, + "epoch": 0.35014836795252224, + "grad_norm": 2.7739357948303223, + "kl_approx": 0.319610595703125, + "learning_rate": 1.6467502407993995e-05, + "loss": 0.2192, + "num_tokens": 8587109.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.18180251121521, + "sampling/importance_sampling_ratio/mean": 1.0002802610397339, + "sampling/importance_sampling_ratio/min": 0.8440707325935364, + "sampling/sampling_logp_difference/max": 0.16951894760131836, + "sampling/sampling_logp_difference/mean": 0.01523625198751688, + "step": 354 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 591.0, + "completions/max_terminated_length": 591.0, + "completions/mean_length": 158.65625, + "completions/mean_terminated_length": 158.65625, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.683738776948303, + "epoch": 0.35113748763600394, + "grad_norm": 1.9370007514953613, + "kl_approx": 0.2371063232421875, + "learning_rate": 1.644110411450398e-05, + "loss": 0.1695, + "num_tokens": 8608258.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.201197624206543, + "sampling/importance_sampling_ratio/mean": 1.0001832246780396, + "sampling/importance_sampling_ratio/min": 0.8108100295066833, + "sampling/sampling_logp_difference/max": 0.20972156524658203, + "sampling/sampling_logp_difference/mean": 0.012913151644170284, + "step": 355 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 293.0, + "completions/max_terminated_length": 293.0, + "completions/mean_length": 115.9375, + "completions/mean_terminated_length": 115.9375, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "entropy": 0.5942155963275582, + "epoch": 0.35212660731948564, + "grad_norm": 5.682674884796143, + "kl_approx": 0.261138916015625, + "learning_rate": 1.6414628884613106e-05, + "loss": 0.1502, + "num_tokens": 8631088.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1730141639709473, + "sampling/importance_sampling_ratio/mean": 1.0002061128616333, + "sampling/importance_sampling_ratio/min": 0.694856584072113, + "sampling/sampling_logp_difference/max": 0.3640497922897339, + "sampling/sampling_logp_difference/mean": 0.012141243554651737, + "step": 356 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 753.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 151.03125, + "completions/mean_terminated_length": 151.03125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "entropy": 0.620893164537847, + "epoch": 0.35311572700296734, + "grad_norm": 2.415186882019043, + "kl_approx": 0.22493362426757812, + "learning_rate": 1.6388077034557355e-05, + "loss": 0.1905, + "num_tokens": 8662361.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2184079885482788, + "sampling/importance_sampling_ratio/mean": 0.9998778104782104, + "sampling/importance_sampling_ratio/min": 0.8003460764884949, + "sampling/sampling_logp_difference/max": 0.22271108627319336, + "sampling/sampling_logp_difference/mean": 0.011820074170827866, + "step": 357 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.0, + "completions/max_terminated_length": 223.0, + "completions/mean_length": 74.78125, + "completions/mean_terminated_length": 74.78125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.3338526857551187, + "epoch": 0.35410484668644904, + "grad_norm": 2.7763822078704834, + "kl_approx": 0.16136932373046875, + "learning_rate": 1.6361448881487913e-05, + "loss": 0.0984, + "num_tokens": 8684290.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1372718811035156, + "sampling/importance_sampling_ratio/mean": 0.9997091889381409, + "sampling/importance_sampling_ratio/min": 0.8464376926422119, + "sampling/sampling_logp_difference/max": 0.1667187213897705, + "sampling/sampling_logp_difference/mean": 0.007359681185334921, + "step": 358 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/max_terminated_length": 193.0, + "completions/mean_length": 91.34375, + "completions/mean_terminated_length": 91.34375, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.3891828968189657, + "epoch": 0.35509396636993074, + "grad_norm": 1.951055884361267, + "kl_approx": 0.20671463012695312, + "learning_rate": 1.6334744743467366e-05, + "loss": 0.1032, + "num_tokens": 8706949.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1983528137207031, + "sampling/importance_sampling_ratio/mean": 0.9994255304336548, + "sampling/importance_sampling_ratio/min": 0.8256502151489258, + "sampling/sampling_logp_difference/max": 0.19158411026000977, + "sampling/sampling_logp_difference/mean": 0.008102335967123508, + "step": 359 + }, + { + "completions/clipped_ratio": 0.0625, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 869.0, + "completions/mean_length": 280.21875, + "completions/mean_terminated_length": 230.6333465576172, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "entropy": 0.6012633121572435, + "epoch": 0.3560830860534125, + "grad_norm": 1.8356751203536987, + "kl_approx": 0.24257278442382812, + "learning_rate": 1.6307964939465914e-05, + "loss": 0.1798, + "num_tokens": 8737716.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1807500123977661, + "sampling/importance_sampling_ratio/mean": 0.9999646544456482, + "sampling/importance_sampling_ratio/min": 0.7804945707321167, + "sampling/sampling_logp_difference/max": 0.24782752990722656, + "sampling/sampling_logp_difference/mean": 0.010727438144385815, + "step": 360 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 188.6875, + "completions/mean_terminated_length": 188.6875, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.8121414370834827, + "epoch": 0.3570722057368942, + "grad_norm": 6.7955827713012695, + "kl_approx": 0.338165283203125, + "learning_rate": 1.628110978935756e-05, + "loss": 0.2335, + "num_tokens": 8764346.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3206875324249268, + "sampling/importance_sampling_ratio/mean": 1.0007163286209106, + "sampling/importance_sampling_ratio/min": 0.8089335560798645, + "sampling/sampling_logp_difference/max": 0.2781524658203125, + "sampling/sampling_logp_difference/mean": 0.012952751480042934, + "step": 361 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 190.6875, + "completions/mean_terminated_length": 190.6875, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "entropy": 0.781624068506062, + "epoch": 0.3580613254203759, + "grad_norm": 2.255150318145752, + "kl_approx": 0.277740478515625, + "learning_rate": 1.625417961391628e-05, + "loss": 0.2026, + "num_tokens": 8791160.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2455625534057617, + "sampling/importance_sampling_ratio/mean": 1.0005948543548584, + "sampling/importance_sampling_ratio/min": 0.792005717754364, + "sampling/sampling_logp_difference/max": 0.2331867218017578, + "sampling/sampling_logp_difference/mean": 0.01357511430978775, + "step": 362 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 161.375, + "completions/mean_terminated_length": 161.375, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "entropy": 0.7261916175484657, + "epoch": 0.3590504451038576, + "grad_norm": 2.2667558193206787, + "kl_approx": 0.300811767578125, + "learning_rate": 1.62271747348122e-05, + "loss": 0.2104, + "num_tokens": 8809700.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.3577779531478882, + "sampling/importance_sampling_ratio/mean": 1.000759243965149, + "sampling/importance_sampling_ratio/min": 0.8300924301147461, + "sampling/sampling_logp_difference/max": 0.305849552154541, + "sampling/sampling_logp_difference/mean": 0.011888876557350159, + "step": 363 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 718.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 173.15625, + "completions/mean_terminated_length": 173.15625, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.4828813658095896, + "epoch": 0.3600395647873393, + "grad_norm": 1.7517290115356445, + "kl_approx": 0.14997100830078125, + "learning_rate": 1.6200095474607753e-05, + "loss": 0.1149, + "num_tokens": 8835841.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.229462742805481, + "sampling/importance_sampling_ratio/mean": 0.9999154210090637, + "sampling/importance_sampling_ratio/min": 0.837244987487793, + "sampling/sampling_logp_difference/max": 0.20657730102539062, + "sampling/sampling_logp_difference/mean": 0.008864673785865307, + "step": 364 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 236.78125, + "completions/mean_terminated_length": 211.3870849609375, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.8053218652494252, + "epoch": 0.361028684470821, + "grad_norm": 2.0709829330444336, + "kl_approx": 0.23779964447021484, + "learning_rate": 1.6172942156753822e-05, + "loss": 0.1859, + "num_tokens": 8861034.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.187152624130249, + "sampling/importance_sampling_ratio/mean": 1.0002837181091309, + "sampling/importance_sampling_ratio/min": 0.831664502620697, + "sampling/sampling_logp_difference/max": 0.184326171875, + "sampling/sampling_logp_difference/mean": 0.014402704313397408, + "step": 365 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 530.0, + "completions/max_terminated_length": 530.0, + "completions/mean_length": 129.40625, + "completions/mean_terminated_length": 129.40625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "entropy": 0.5976112964563072, + "epoch": 0.3620178041543027, + "grad_norm": 2.368210792541504, + "kl_approx": 0.240325927734375, + "learning_rate": 1.614571510558588e-05, + "loss": 0.1495, + "num_tokens": 8882807.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2851083278656006, + "sampling/importance_sampling_ratio/mean": 0.9999275803565979, + "sampling/importance_sampling_ratio/min": 0.8389502763748169, + "sampling/sampling_logp_difference/max": 0.2508430480957031, + "sampling/sampling_logp_difference/mean": 0.011203138157725334, + "step": 366 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 877.0, + "completions/max_terminated_length": 877.0, + "completions/mean_length": 132.875, + "completions/mean_terminated_length": 132.875, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.6403806004673243, + "epoch": 0.3630069238377844, + "grad_norm": 2.9379289150238037, + "kl_approx": 0.25250244140625, + "learning_rate": 1.6118414646320115e-05, + "loss": 0.1924, + "num_tokens": 8904643.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.199820876121521, + "sampling/importance_sampling_ratio/mean": 0.999147891998291, + "sampling/importance_sampling_ratio/min": 0.7935819029808044, + "sampling/sampling_logp_difference/max": 0.23119854927062988, + "sampling/sampling_logp_difference/mean": 0.013297569938004017, + "step": 367 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/max_terminated_length": 365.0, + "completions/mean_length": 119.28125, + "completions/mean_terminated_length": 119.28125, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.7997883390635252, + "epoch": 0.3639960435212661, + "grad_norm": 2.318758249282837, + "kl_approx": 0.2836151123046875, + "learning_rate": 1.6091041105049542e-05, + "loss": 0.166, + "num_tokens": 8924444.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1700506210327148, + "sampling/importance_sampling_ratio/mean": 0.9996182322502136, + "sampling/importance_sampling_ratio/min": 0.8777026534080505, + "sampling/sampling_logp_difference/max": 0.15704703330993652, + "sampling/sampling_logp_difference/mean": 0.01342426985502243, + "step": 368 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.0, + "completions/max_terminated_length": 332.0, + "completions/mean_length": 102.9375, + "completions/mean_terminated_length": 102.9375, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "entropy": 0.5861324635334313, + "epoch": 0.3649851632047478, + "grad_norm": 2.1033666133880615, + "kl_approx": 0.24227523803710938, + "learning_rate": 1.6063594808740112e-05, + "loss": 0.1381, + "num_tokens": 8950930.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.144827961921692, + "sampling/importance_sampling_ratio/mean": 0.9999030232429504, + "sampling/importance_sampling_ratio/min": 0.839012861251831, + "sampling/sampling_logp_difference/max": 0.17552924156188965, + "sampling/sampling_logp_difference/mean": 0.011686254292726517, + "step": 369 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 645.0, + "completions/max_terminated_length": 645.0, + "completions/mean_length": 188.84375, + "completions/mean_terminated_length": 188.84375, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "entropy": 0.581179931294173, + "epoch": 0.3659742828882295, + "grad_norm": 2.0337040424346924, + "kl_approx": 0.24546051025390625, + "learning_rate": 1.6036076085226813e-05, + "loss": 0.1844, + "num_tokens": 8975365.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2810795307159424, + "sampling/importance_sampling_ratio/mean": 1.0002517700195312, + "sampling/importance_sampling_ratio/min": 0.814808189868927, + "sampling/sampling_logp_difference/max": 0.24770307540893555, + "sampling/sampling_logp_difference/mean": 0.008068220689892769, + "step": 370 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 184.53125, + "completions/mean_terminated_length": 184.53125, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.9399522272869945, + "epoch": 0.3669634025717112, + "grad_norm": 3.404837131500244, + "kl_approx": 0.3153228759765625, + "learning_rate": 1.6008485263209742e-05, + "loss": 0.2106, + "num_tokens": 8999518.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2140047550201416, + "sampling/importance_sampling_ratio/mean": 0.999363899230957, + "sampling/importance_sampling_ratio/min": 0.7802701592445374, + "sampling/sampling_logp_difference/max": 0.24811506271362305, + "sampling/sampling_logp_difference/mean": 0.014915591105818748, + "step": 371 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 664.0, + "completions/max_terminated_length": 664.0, + "completions/mean_length": 160.34375, + "completions/mean_terminated_length": 160.34375, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.6273523410782218, + "epoch": 0.36795252225519287, + "grad_norm": 1.9187628030776978, + "kl_approx": 0.21495819091796875, + "learning_rate": 1.598082267225018e-05, + "loss": 0.1576, + "num_tokens": 9019545.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2169606685638428, + "sampling/importance_sampling_ratio/mean": 1.0002495050430298, + "sampling/importance_sampling_ratio/min": 0.8490124344825745, + "sampling/sampling_logp_difference/max": 0.19635653495788574, + "sampling/sampling_logp_difference/mean": 0.012951545417308807, + "step": 372 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 745.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 129.0625, + "completions/mean_terminated_length": 129.0625, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.5108460397459567, + "epoch": 0.36894164193867457, + "grad_norm": 1.8111895322799683, + "kl_approx": 0.16696548461914062, + "learning_rate": 1.595308864276666e-05, + "loss": 0.1019, + "num_tokens": 9042443.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2525469064712524, + "sampling/importance_sampling_ratio/mean": 1.0001943111419678, + "sampling/importance_sampling_ratio/min": 0.8316817283630371, + "sampling/sampling_logp_difference/max": 0.22517895698547363, + "sampling/sampling_logp_difference/mean": 0.01172676682472229, + "step": 373 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 635.0, + "completions/max_terminated_length": 635.0, + "completions/mean_length": 128.3125, + "completions/mean_terminated_length": 128.3125, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.8318022666499019, + "epoch": 0.36993076162215627, + "grad_norm": 2.2463698387145996, + "kl_approx": 0.22014617919921875, + "learning_rate": 1.592528350603103e-05, + "loss": 0.174, + "num_tokens": 9062837.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1957305669784546, + "sampling/importance_sampling_ratio/mean": 0.99953693151474, + "sampling/importance_sampling_ratio/min": 0.8314329385757446, + "sampling/sampling_logp_difference/max": 0.18460464477539062, + "sampling/sampling_logp_difference/mean": 0.01514038722962141, + "step": 374 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 861.0, + "completions/max_terminated_length": 861.0, + "completions/mean_length": 285.09375, + "completions/mean_terminated_length": 285.09375, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 1.0577433137223125, + "epoch": 0.37091988130563797, + "grad_norm": 2.101933479309082, + "kl_approx": 0.275665283203125, + "learning_rate": 1.5897407594164468e-05, + "loss": 0.2014, + "num_tokens": 9088552.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.208816647529602, + "sampling/importance_sampling_ratio/mean": 1.0002485513687134, + "sampling/importance_sampling_ratio/min": 0.8247970938682556, + "sampling/sampling_logp_difference/max": 0.19261789321899414, + "sampling/sampling_logp_difference/mean": 0.0155795868486166, + "step": 375 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 129.84375, + "completions/mean_terminated_length": 129.84375, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "entropy": 0.6944182328879833, + "epoch": 0.37190900098911966, + "grad_norm": 2.2824857234954834, + "kl_approx": 0.25750732421875, + "learning_rate": 1.586946124013354e-05, + "loss": 0.1888, + "num_tokens": 9110787.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.295641303062439, + "sampling/importance_sampling_ratio/mean": 1.0001964569091797, + "sampling/importance_sampling_ratio/min": 0.8404685854911804, + "sampling/sampling_logp_difference/max": 0.2590057849884033, + "sampling/sampling_logp_difference/mean": 0.013446149416267872, + "step": 376 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 138.0625, + "completions/mean_terminated_length": 138.0625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.7542224321514368, + "epoch": 0.37289812067260136, + "grad_norm": 2.739356756210327, + "kl_approx": 0.3043975830078125, + "learning_rate": 1.5841444777746232e-05, + "loss": 0.2464, + "num_tokens": 9134509.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1727577447891235, + "sampling/importance_sampling_ratio/mean": 0.9995575547218323, + "sampling/importance_sampling_ratio/min": 0.8480107188224792, + "sampling/sampling_logp_difference/max": 0.1648620367050171, + "sampling/sampling_logp_difference/mean": 0.012854214757680893, + "step": 377 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 164.4375, + "completions/mean_terminated_length": 164.4375, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "entropy": 0.8976317401975393, + "epoch": 0.37388724035608306, + "grad_norm": 2.6244962215423584, + "kl_approx": 0.2871246337890625, + "learning_rate": 1.5813358541647915e-05, + "loss": 0.2266, + "num_tokens": 9156251.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2180379629135132, + "sampling/importance_sampling_ratio/mean": 1.0011508464813232, + "sampling/importance_sampling_ratio/min": 0.8355380296707153, + "sampling/sampling_logp_difference/max": 0.19724130630493164, + "sampling/sampling_logp_difference/mean": 0.014126446098089218, + "step": 378 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 815.0, + "completions/max_terminated_length": 815.0, + "completions/mean_length": 135.875, + "completions/mean_terminated_length": 135.875, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.7467193491756916, + "epoch": 0.37487636003956476, + "grad_norm": 3.120851516723633, + "kl_approx": 0.3340606689453125, + "learning_rate": 1.578520286731741e-05, + "loss": 0.239, + "num_tokens": 9179791.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.189101219177246, + "sampling/importance_sampling_ratio/mean": 0.9994665384292603, + "sampling/importance_sampling_ratio/min": 0.8026962280273438, + "sampling/sampling_logp_difference/max": 0.21977901458740234, + "sampling/sampling_logp_difference/mean": 0.01388273574411869, + "step": 379 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 852.0, + "completions/mean_length": 298.09375, + "completions/mean_terminated_length": 274.6773986816406, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "entropy": 0.8252547108568251, + "epoch": 0.3758654797230465, + "grad_norm": 2.0605688095092773, + "kl_approx": 0.2386627197265625, + "learning_rate": 1.575697809106292e-05, + "loss": 0.2061, + "num_tokens": 9209314.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1771339178085327, + "sampling/importance_sampling_ratio/mean": 0.9999204277992249, + "sampling/importance_sampling_ratio/min": 0.7959311008453369, + "sampling/sampling_logp_difference/max": 0.2282426357269287, + "sampling/sampling_logp_difference/mean": 0.013232769444584846, + "step": 380 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 473.0, + "completions/max_terminated_length": 473.0, + "completions/mean_length": 181.6875, + "completions/mean_terminated_length": 181.6875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.8465917864814401, + "epoch": 0.3768545994065282, + "grad_norm": 2.4323086738586426, + "kl_approx": 0.27507781982421875, + "learning_rate": 1.5728684550018066e-05, + "loss": 0.1975, + "num_tokens": 9234760.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2688701152801514, + "sampling/importance_sampling_ratio/mean": 0.9999415874481201, + "sampling/importance_sampling_ratio/min": 0.8298973441123962, + "sampling/sampling_logp_difference/max": 0.2381267547607422, + "sampling/sampling_logp_difference/mean": 0.014800166711211205, + "step": 381 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 421.0, + "completions/max_terminated_length": 421.0, + "completions/mean_length": 133.5, + "completions/mean_terminated_length": 133.5, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.7986752595752478, + "epoch": 0.3778437190900099, + "grad_norm": 3.27736496925354, + "kl_approx": 0.384521484375, + "learning_rate": 1.570032258213783e-05, + "loss": 0.2563, + "num_tokens": 9258696.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1461498737335205, + "sampling/importance_sampling_ratio/mean": 0.9991601705551147, + "sampling/importance_sampling_ratio/min": 0.8503790497779846, + "sampling/sampling_logp_difference/max": 0.16207313537597656, + "sampling/sampling_logp_difference/mean": 0.013560895808041096, + "step": 382 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/max_terminated_length": 163.0, + "completions/mean_length": 82.96875, + "completions/mean_terminated_length": 82.96875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.6561488835141063, + "epoch": 0.3788328387734916, + "grad_norm": 5.923593521118164, + "kl_approx": 0.296844482421875, + "learning_rate": 1.5671892526194515e-05, + "loss": 0.202, + "num_tokens": 9280183.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1768373250961304, + "sampling/importance_sampling_ratio/mean": 1.0004161596298218, + "sampling/importance_sampling_ratio/min": 0.8592154383659363, + "sampling/sampling_logp_difference/max": 0.16283059120178223, + "sampling/sampling_logp_difference/mean": 0.012121755629777908, + "step": 383 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 762.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 120.125, + "completions/mean_terminated_length": 120.125, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.5308083277195692, + "epoch": 0.3798219584569733, + "grad_norm": 3.1105942726135254, + "kl_approx": 0.2258453369140625, + "learning_rate": 1.564339472177373e-05, + "loss": 0.1476, + "num_tokens": 9300907.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1908764839172363, + "sampling/importance_sampling_ratio/mean": 0.9999884963035583, + "sampling/importance_sampling_ratio/min": 0.8166634440422058, + "sampling/sampling_logp_difference/max": 0.2025282382965088, + "sampling/sampling_logp_difference/mean": 0.010241501033306122, + "step": 384 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 528.0, + "completions/max_terminated_length": 528.0, + "completions/mean_length": 136.15625, + "completions/mean_terminated_length": 136.15625, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.6417626738548279, + "epoch": 0.380811078140455, + "grad_norm": 2.15088152885437, + "kl_approx": 0.19831085205078125, + "learning_rate": 1.561482950927029e-05, + "loss": 0.1404, + "num_tokens": 9326216.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1713711023330688, + "sampling/importance_sampling_ratio/mean": 1.0000646114349365, + "sampling/importance_sampling_ratio/min": 0.8562986254692078, + "sampling/sampling_logp_difference/max": 0.15817499160766602, + "sampling/sampling_logp_difference/mean": 0.011397141963243484, + "step": 385 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 362.0, + "completions/max_terminated_length": 362.0, + "completions/mean_length": 115.0, + "completions/mean_terminated_length": 115.0, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.80773267429322, + "epoch": 0.3818001978239367, + "grad_norm": 2.1561625003814697, + "kl_approx": 0.268707275390625, + "learning_rate": 1.5586197229884185e-05, + "loss": 0.1743, + "num_tokens": 9347240.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1532036066055298, + "sampling/importance_sampling_ratio/mean": 1.000807523727417, + "sampling/importance_sampling_ratio/min": 0.8788111805915833, + "sampling/sampling_logp_difference/max": 0.14254379272460938, + "sampling/sampling_logp_difference/mean": 0.01283793430775404, + "step": 386 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 156.625, + "completions/mean_terminated_length": 156.625, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.7017519646324217, + "epoch": 0.3827893175074184, + "grad_norm": 2.0136001110076904, + "kl_approx": 0.23560333251953125, + "learning_rate": 1.5557498225616488e-05, + "loss": 0.1532, + "num_tokens": 9374556.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1784998178482056, + "sampling/importance_sampling_ratio/mean": 0.999943196773529, + "sampling/importance_sampling_ratio/min": 0.8654402494430542, + "sampling/sampling_logp_difference/max": 0.16424226760864258, + "sampling/sampling_logp_difference/mean": 0.01358980406075716, + "step": 387 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/max_terminated_length": 345.0, + "completions/mean_length": 140.6875, + "completions/mean_terminated_length": 140.6875, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "entropy": 0.7591469082981348, + "epoch": 0.3837784371909001, + "grad_norm": 2.4345834255218506, + "kl_approx": 0.3016510009765625, + "learning_rate": 1.5528732839265272e-05, + "loss": 0.2035, + "num_tokens": 9397754.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1635252237319946, + "sampling/importance_sampling_ratio/mean": 0.9998807311058044, + "sampling/importance_sampling_ratio/min": 0.8415124416351318, + "sampling/sampling_logp_difference/max": 0.17255449295043945, + "sampling/sampling_logp_difference/mean": 0.013161800801753998, + "step": 388 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 863.0, + "completions/max_terminated_length": 863.0, + "completions/mean_length": 219.5625, + "completions/mean_terminated_length": 219.5625, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.6951826056465507, + "epoch": 0.3847675568743818, + "grad_norm": 1.8261902332305908, + "kl_approx": 0.21053504943847656, + "learning_rate": 1.549990141442153e-05, + "loss": 0.1308, + "num_tokens": 9423876.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1618504524230957, + "sampling/importance_sampling_ratio/mean": 0.9996642470359802, + "sampling/importance_sampling_ratio/min": 0.739203691482544, + "sampling/sampling_logp_difference/max": 0.3021817207336426, + "sampling/sampling_logp_difference/mean": 0.011931492015719414, + "step": 389 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 204.1875, + "completions/mean_terminated_length": 204.1875, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.7856100546196103, + "epoch": 0.3857566765578635, + "grad_norm": 2.470937728881836, + "kl_approx": 0.25396728515625, + "learning_rate": 1.5471004295465034e-05, + "loss": 0.1992, + "num_tokens": 9447730.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1737536191940308, + "sampling/importance_sampling_ratio/mean": 0.9996646046638489, + "sampling/importance_sampling_ratio/min": 0.8353384733200073, + "sampling/sampling_logp_difference/max": 0.1799182891845703, + "sampling/sampling_logp_difference/mean": 0.013104341924190521, + "step": 390 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 598.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 162.59375, + "completions/mean_terminated_length": 162.59375, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.7058191308751702, + "epoch": 0.3867457962413452, + "grad_norm": 2.589174747467041, + "kl_approx": 0.25220489501953125, + "learning_rate": 1.5442041827560274e-05, + "loss": 0.1762, + "num_tokens": 9471261.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2235606908798218, + "sampling/importance_sampling_ratio/mean": 0.9999344944953918, + "sampling/importance_sampling_ratio/min": 0.8511485457420349, + "sampling/sampling_logp_difference/max": 0.20176517963409424, + "sampling/sampling_logp_difference/mean": 0.01232069730758667, + "step": 391 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 749.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 163.09375, + "completions/mean_terminated_length": 163.09375, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "entropy": 0.7082721563056111, + "epoch": 0.3877349159248269, + "grad_norm": 3.146322250366211, + "kl_approx": 0.4079132080078125, + "learning_rate": 1.5413014356652287e-05, + "loss": 0.2618, + "num_tokens": 9500744.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2651423215866089, + "sampling/importance_sampling_ratio/mean": 0.9999623894691467, + "sampling/importance_sampling_ratio/min": 0.7710256576538086, + "sampling/sampling_logp_difference/max": 0.26003360748291016, + "sampling/sampling_logp_difference/mean": 0.012805013917386532, + "step": 392 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 213.46875, + "completions/mean_terminated_length": 213.46875, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.6826157849282026, + "epoch": 0.3887240356083086, + "grad_norm": 1.7774665355682373, + "kl_approx": 0.20732879638671875, + "learning_rate": 1.538392222946255e-05, + "loss": 0.1532, + "num_tokens": 9525527.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1476131677627563, + "sampling/importance_sampling_ratio/mean": 0.9997501373291016, + "sampling/importance_sampling_ratio/min": 0.833834171295166, + "sampling/sampling_logp_difference/max": 0.18172073364257812, + "sampling/sampling_logp_difference/mean": 0.010913715697824955, + "step": 393 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 209.9375, + "completions/mean_terminated_length": 183.6774139404297, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "entropy": 0.7142486907541752, + "epoch": 0.3897131552917903, + "grad_norm": 3.1309561729431152, + "kl_approx": 0.3763885498046875, + "learning_rate": 1.5354765793484834e-05, + "loss": 0.2582, + "num_tokens": 9551133.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2187414169311523, + "sampling/importance_sampling_ratio/mean": 0.9996961951255798, + "sampling/importance_sampling_ratio/min": 0.7896600961685181, + "sampling/sampling_logp_difference/max": 0.23615264892578125, + "sampling/sampling_logp_difference/mean": 0.011643128469586372, + "step": 394 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 862.0, + "completions/max_terminated_length": 862.0, + "completions/mean_length": 123.4375, + "completions/mean_terminated_length": 123.4375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.6525642424821854, + "epoch": 0.390702274975272, + "grad_norm": 2.606126308441162, + "kl_approx": 0.25640869140625, + "learning_rate": 1.5325545396981053e-05, + "loss": 0.1638, + "num_tokens": 9571747.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2033898830413818, + "sampling/importance_sampling_ratio/mean": 0.9994295239448547, + "sampling/importance_sampling_ratio/min": 0.7732419371604919, + "sampling/sampling_logp_difference/max": 0.25716328620910645, + "sampling/sampling_logp_difference/mean": 0.011888217180967331, + "step": 395 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 559.0, + "completions/max_terminated_length": 559.0, + "completions/mean_length": 170.0625, + "completions/mean_terminated_length": 170.0625, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "entropy": 0.6335577010177076, + "epoch": 0.3916913946587537, + "grad_norm": 1.9306553602218628, + "kl_approx": 0.189788818359375, + "learning_rate": 1.5296261388977107e-05, + "loss": 0.1605, + "num_tokens": 9591941.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.192777395248413, + "sampling/importance_sampling_ratio/mean": 1.0005309581756592, + "sampling/importance_sampling_ratio/min": 0.8286999464035034, + "sampling/sampling_logp_difference/max": 0.1878972053527832, + "sampling/sampling_logp_difference/mean": 0.01298566348850727, + "step": 396 + }, + { + "completions/clipped_ratio": 0.03125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 1008.0, + "completions/mean_length": 253.90625, + "completions/mean_terminated_length": 229.06451416015625, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.7864647079259157, + "epoch": 0.3926805143422354, + "grad_norm": 2.409792423248291, + "kl_approx": 0.23056793212890625, + "learning_rate": 1.52669141192587e-05, + "loss": 0.1709, + "num_tokens": 9623842.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.174325704574585, + "sampling/importance_sampling_ratio/mean": 0.9997132420539856, + "sampling/importance_sampling_ratio/min": 0.8205747008323669, + "sampling/sampling_logp_difference/max": 0.19775032997131348, + "sampling/sampling_logp_difference/mean": 0.01377257239073515, + "step": 397 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 781.0, + "completions/max_terminated_length": 781.0, + "completions/mean_length": 156.0, + "completions/mean_terminated_length": 156.0, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.5018252991139889, + "epoch": 0.3936696340257171, + "grad_norm": 1.9581407308578491, + "kl_approx": 0.15090179443359375, + "learning_rate": 1.5237503938367186e-05, + "loss": 0.1215, + "num_tokens": 9647722.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1466683149337769, + "sampling/importance_sampling_ratio/mean": 1.0001895427703857, + "sampling/importance_sampling_ratio/min": 0.8531631827354431, + "sampling/sampling_logp_difference/max": 0.15880447626113892, + "sampling/sampling_logp_difference/mean": 0.009894904680550098, + "step": 398 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 310.0, + "completions/max_terminated_length": 310.0, + "completions/mean_length": 133.375, + "completions/mean_terminated_length": 133.375, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "entropy": 0.6371352691203356, + "epoch": 0.39465875370919884, + "grad_norm": 1.943426489830017, + "kl_approx": 0.19311904907226562, + "learning_rate": 1.5208031197595357e-05, + "loss": 0.1583, + "num_tokens": 9672902.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.1892338991165161, + "sampling/importance_sampling_ratio/mean": 0.9997968077659607, + "sampling/importance_sampling_ratio/min": 0.823902428150177, + "sampling/sampling_logp_difference/max": 0.19370317459106445, + "sampling/sampling_logp_difference/mean": 0.010704712010920048, + "step": 399 + }, + { + "completions/clipped_ratio": 0.0, + "completions/max_length": 882.0, + "completions/max_terminated_length": 882.0, + "completions/mean_length": 232.1875, + "completions/mean_terminated_length": 232.1875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.8602967103943229, + "epoch": 0.39564787339268054, + "grad_norm": 2.228437662124634, + "kl_approx": 0.2644805908203125, + "learning_rate": 1.5178496248983254e-05, + "loss": 0.2105, + "num_tokens": 9701420.0, + "rewards": 0.0, + "sampling/importance_sampling_ratio/max": 1.2045495510101318, + "sampling/importance_sampling_ratio/mean": 1.000301718711853, + "sampling/importance_sampling_ratio/min": 0.7740485072135925, + "sampling/sampling_logp_difference/max": 0.2561206817626953, + "sampling/sampling_logp_difference/mean": 0.01424815971404314, + "step": 400 + } + ], + "logging_steps": 1, + "max_steps": 1011, + "num_input_tokens_seen": 9701420, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}