diff --git "a/checkpoint-1930/trainer_state.json" "b/checkpoint-1930/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-1930/trainer_state.json" @@ -0,0 +1,25123 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 10, + "global_step": 1930, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completion_length": 426.8250160217285, + "epoch": 0.0010368066355624676, + "grad_norm": 14.147396087646484, + "kl": 0.0, + "learning_rate": 1.0344827586206896e-08, + "loss": 0.1143, + "reward": 0.30000000447034836, + "reward_std": 0.353316530585289, + "rewards/accuracy_reward": 0.22500000894069672, + "rewards/format_reward": 0.07500000111758709, + "step": 1 + }, + { + "completion_length": 345.9749946594238, + "epoch": 0.002073613271124935, + "grad_norm": 0.5298608541488647, + "kl": 0.0, + "learning_rate": 2.0689655172413793e-08, + "loss": 0.0471, + "reward": 0.3000000026077032, + "reward_std": 0.34968600049614906, + "rewards/accuracy_reward": 0.27500000409781933, + "rewards/format_reward": 0.02500000037252903, + "step": 2 + }, + { + "completion_length": 321.2000045776367, + "epoch": 0.003110419906687403, + "grad_norm": 3.317950487136841, + "kl": 0.00021088123321533203, + "learning_rate": 3.103448275862069e-08, + "loss": 0.0899, + "reward": 0.4750000014901161, + "reward_std": 0.42900275811553, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.02500000037252903, + "step": 3 + }, + { + "completion_length": 283.5250053405762, + "epoch": 0.00414722654224987, + "grad_norm": 0.7831366062164307, + "kl": 0.0001735687255859375, + "learning_rate": 4.1379310344827585e-08, + "loss": -0.0534, + "reward": 0.5250000096857548, + "reward_std": 0.4415663778781891, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.0, + "step": 4 + }, + { + "completion_length": 311.0750045776367, + "epoch": 0.005184033177812338, + "grad_norm": 0.6258169412612915, + "kl": 0.00017344951629638672, + "learning_rate": 5.172413793103448e-08, + "loss": -0.0563, + "reward": 0.24999999813735485, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.20000000670552254, + "rewards/format_reward": 0.05000000074505806, + "step": 5 + }, + { + "completion_length": 363.50000381469727, + "epoch": 0.006220839813374806, + "grad_norm": 3.521343946456909, + "kl": 0.0002512931823730469, + "learning_rate": 6.206896551724137e-08, + "loss": 0.1373, + "reward": 0.550000011920929, + "reward_std": 0.517391100525856, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.07500000111758709, + "step": 6 + }, + { + "completion_length": 387.9500045776367, + "epoch": 0.007257646448937273, + "grad_norm": 3.267216205596924, + "kl": 0.0003764629364013672, + "learning_rate": 7.241379310344827e-08, + "loss": 0.0081, + "reward": 0.2750000059604645, + "reward_std": 0.36053743213415146, + "rewards/accuracy_reward": 0.20000000670552254, + "rewards/format_reward": 0.07500000111758709, + "step": 7 + }, + { + "completion_length": 340.6500053405762, + "epoch": 0.00829445308449974, + "grad_norm": 2.6857998371124268, + "kl": 0.0002391338348388672, + "learning_rate": 8.275862068965517e-08, + "loss": 0.055, + "reward": 0.375, + "reward_std": 0.39822831004858017, + "rewards/accuracy_reward": 0.34999999962747097, + "rewards/format_reward": 0.02500000037252903, + "step": 8 + }, + { + "completion_length": 488.42501068115234, + "epoch": 0.00933125972006221, + "grad_norm": 0.6219092607498169, + "kl": 0.00023365020751953125, + "learning_rate": 9.310344827586207e-08, + "loss": -0.011, + "reward": 0.22500000521540642, + "reward_std": 0.3297629803419113, + "rewards/accuracy_reward": 0.22500000521540642, + "rewards/format_reward": 0.0, + "step": 9 + }, + { + "completion_length": 360.625, + "epoch": 0.010368066355624676, + "grad_norm": 0.5460860133171082, + "kl": 0.000164031982421875, + "learning_rate": 1.0344827586206897e-07, + "loss": 0.0558, + "reward": 0.25000000558793545, + "reward_std": 0.3856646828353405, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.02500000037252903, + "step": 10 + }, + { + "completion_length": 341.00000762939453, + "epoch": 0.011404872991187144, + "grad_norm": 1.1156655550003052, + "kl": 0.00022840499877929688, + "learning_rate": 1.1379310344827586e-07, + "loss": 0.0461, + "reward": 0.4000000059604645, + "reward_std": 0.497468076646328, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.0, + "step": 11 + }, + { + "completion_length": 354.0999984741211, + "epoch": 0.012441679626749611, + "grad_norm": 0.694392740726471, + "kl": 0.00033974647521972656, + "learning_rate": 1.2413793103448275e-07, + "loss": 0.0749, + "reward": 0.3250000011175871, + "reward_std": 0.3046357296407223, + "rewards/accuracy_reward": 0.3250000011175871, + "rewards/format_reward": 0.0, + "step": 12 + }, + { + "completion_length": 371.7750015258789, + "epoch": 0.01347848626231208, + "grad_norm": 1.2974847555160522, + "kl": 0.0002536773681640625, + "learning_rate": 1.3448275862068965e-07, + "loss": 0.0034, + "reward": 0.42500000447034836, + "reward_std": 0.5263643711805344, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.05000000074505806, + "step": 13 + }, + { + "completion_length": 377.55001068115234, + "epoch": 0.014515292897874546, + "grad_norm": 0.8971682786941528, + "kl": 0.0002269744873046875, + "learning_rate": 1.4482758620689654e-07, + "loss": 0.0266, + "reward": 0.4000000078231096, + "reward_std": 0.4415663816034794, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.0, + "step": 14 + }, + { + "completion_length": 382.6500053405762, + "epoch": 0.015552099533437015, + "grad_norm": 1.676780104637146, + "kl": 0.00034689903259277344, + "learning_rate": 1.5517241379310344e-07, + "loss": 0.0303, + "reward": 0.25000000558793545, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.02500000037252903, + "step": 15 + }, + { + "completion_length": 237.32500457763672, + "epoch": 0.01658890616899948, + "grad_norm": 0.6732295155525208, + "kl": 0.0001780986785888672, + "learning_rate": 1.6551724137931034e-07, + "loss": 0.0088, + "reward": 0.2000000048428774, + "reward_std": 0.31719936057925224, + "rewards/accuracy_reward": 0.1750000026077032, + "rewards/format_reward": 0.02500000037252903, + "step": 16 + }, + { + "completion_length": 246.77500343322754, + "epoch": 0.01762571280456195, + "grad_norm": 0.8578572273254395, + "kl": 0.00020766258239746094, + "learning_rate": 1.7586206896551724e-07, + "loss": -0.0679, + "reward": 0.2500000037252903, + "reward_std": 0.23617041110992432, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/format_reward": 0.0, + "step": 17 + }, + { + "completion_length": 287.7250061035156, + "epoch": 0.01866251944012442, + "grad_norm": 1.177450180053711, + "kl": 0.0002694129943847656, + "learning_rate": 1.8620689655172414e-07, + "loss": 0.1109, + "reward": 0.5750000178813934, + "reward_std": 0.5335852541029453, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.07500000111758709, + "step": 18 + }, + { + "completion_length": 276.7000045776367, + "epoch": 0.019699326075686883, + "grad_norm": 0.9230583906173706, + "kl": 0.0002455711364746094, + "learning_rate": 1.9655172413793106e-07, + "loss": -0.0118, + "reward": 0.6000000163912773, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.0, + "step": 19 + }, + { + "completion_length": 389.7750015258789, + "epoch": 0.020736132711249352, + "grad_norm": 0.8271303772926331, + "kl": 0.000316619873046875, + "learning_rate": 2.0689655172413793e-07, + "loss": -0.0191, + "reward": 0.450000012293458, + "reward_std": 0.25609347224235535, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.05000000074505806, + "step": 20 + }, + { + "completion_length": 313.29999923706055, + "epoch": 0.02177293934681182, + "grad_norm": 3.3089773654937744, + "kl": 0.0006225109100341797, + "learning_rate": 2.1724137931034486e-07, + "loss": -0.0103, + "reward": 0.4000000096857548, + "reward_std": 0.26129765808582306, + "rewards/accuracy_reward": 0.4000000096857548, + "rewards/format_reward": 0.0, + "step": 21 + }, + { + "completion_length": 342.8750114440918, + "epoch": 0.02280974598237429, + "grad_norm": 0.981069803237915, + "kl": 0.0006477832794189453, + "learning_rate": 2.2758620689655173e-07, + "loss": 0.0766, + "reward": 0.5250000059604645, + "reward_std": 0.44892579317092896, + "rewards/accuracy_reward": 0.44999999925494194, + "rewards/format_reward": 0.07500000111758709, + "step": 22 + }, + { + "completion_length": 429.30001068115234, + "epoch": 0.023846552617936754, + "grad_norm": 0.4381193518638611, + "kl": 0.0002999305725097656, + "learning_rate": 2.3793103448275865e-07, + "loss": -0.018, + "reward": 0.3750000037252903, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.0, + "step": 23 + }, + { + "completion_length": 341.90000915527344, + "epoch": 0.024883359253499222, + "grad_norm": 0.8023965954780579, + "kl": 0.0002903938293457031, + "learning_rate": 2.482758620689655e-07, + "loss": 0.0741, + "reward": 0.5750000048428774, + "reward_std": 0.36053744703531265, + "rewards/accuracy_reward": 0.5000000167638063, + "rewards/format_reward": 0.07500000111758709, + "step": 24 + }, + { + "completion_length": 414.6750183105469, + "epoch": 0.02592016588906169, + "grad_norm": 1.5727014541625977, + "kl": 0.00022554397583007812, + "learning_rate": 2.5862068965517245e-07, + "loss": -0.0566, + "reward": 0.42500000819563866, + "reward_std": 0.5225953236222267, + "rewards/accuracy_reward": 0.42500000819563866, + "rewards/format_reward": 0.0, + "step": 25 + }, + { + "completion_length": 285.1750030517578, + "epoch": 0.02695697252462416, + "grad_norm": 0.7682381272315979, + "kl": 0.0005116462707519531, + "learning_rate": 2.689655172413793e-07, + "loss": -0.0143, + "reward": 0.35000000335276127, + "reward_std": 0.45413001254200935, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.02500000037252903, + "step": 26 + }, + { + "completion_length": 340.00000762939453, + "epoch": 0.027993779160186624, + "grad_norm": 0.6389257907867432, + "kl": 0.00046062469482421875, + "learning_rate": 2.7931034482758624e-07, + "loss": -0.0795, + "reward": 0.20000000670552254, + "reward_std": 0.31719936430454254, + "rewards/accuracy_reward": 0.20000000670552254, + "rewards/format_reward": 0.0, + "step": 27 + }, + { + "completion_length": 273.35000228881836, + "epoch": 0.029030585795749093, + "grad_norm": 1.1553552150726318, + "kl": 0.00037384033203125, + "learning_rate": 2.896551724137931e-07, + "loss": -0.035, + "reward": 0.30000000447034836, + "reward_std": 0.3046357333660126, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.0, + "step": 28 + }, + { + "completion_length": 294.79999923706055, + "epoch": 0.03006739243131156, + "grad_norm": 2.068281650543213, + "kl": 0.0003008842468261719, + "learning_rate": 3.0000000000000004e-07, + "loss": 0.0542, + "reward": 0.8250000029802322, + "reward_std": 0.3371223881840706, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.05000000074505806, + "step": 29 + }, + { + "completion_length": 307.32500076293945, + "epoch": 0.03110419906687403, + "grad_norm": 4348.3486328125, + "kl": 6.1257734298706055, + "learning_rate": 3.103448275862069e-07, + "loss": 0.1656, + "reward": 0.4750000163912773, + "reward_std": 0.39822830632328987, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.0, + "step": 30 + }, + { + "completion_length": 387.575008392334, + "epoch": 0.0321410057024365, + "grad_norm": 1.4455534219741821, + "kl": 0.00043702125549316406, + "learning_rate": 3.2068965517241384e-07, + "loss": 0.0563, + "reward": 0.32500000298023224, + "reward_std": 0.3966545984148979, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.02500000037252903, + "step": 31 + }, + { + "completion_length": 281.0250015258789, + "epoch": 0.03317781233799896, + "grad_norm": 1.362587332725525, + "kl": 0.0006682872772216797, + "learning_rate": 3.310344827586207e-07, + "loss": -0.0157, + "reward": 0.5750000104308128, + "reward_std": 0.5587125271558762, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.02500000037252903, + "step": 32 + }, + { + "completion_length": 451.30001068115234, + "epoch": 0.03421461897356143, + "grad_norm": 0.9912571310997009, + "kl": 0.0003376007080078125, + "learning_rate": 3.4137931034482763e-07, + "loss": 0.0678, + "reward": 0.5000000074505806, + "reward_std": 0.45413000881671906, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.02500000037252903, + "step": 33 + }, + { + "completion_length": 398.0, + "epoch": 0.0352514256091239, + "grad_norm": 2.6609644889831543, + "kl": 0.0003285408020019531, + "learning_rate": 3.517241379310345e-07, + "loss": 0.0505, + "reward": 0.3500000089406967, + "reward_std": 0.37310105934739113, + "rewards/accuracy_reward": 0.32500001043081284, + "rewards/format_reward": 0.02500000037252903, + "step": 34 + }, + { + "completion_length": 316.0250015258789, + "epoch": 0.036288232244686365, + "grad_norm": 0.6625248193740845, + "kl": 0.000457763671875, + "learning_rate": 3.6206896551724143e-07, + "loss": 0.0455, + "reward": 0.5500000007450581, + "reward_std": 0.3856646716594696, + "rewards/accuracy_reward": 0.47499999590218067, + "rewards/format_reward": 0.07500000111758709, + "step": 35 + }, + { + "completion_length": 335.325008392334, + "epoch": 0.03732503888024884, + "grad_norm": 9.054551124572754, + "kl": 0.0006432533264160156, + "learning_rate": 3.7241379310344827e-07, + "loss": 0.0427, + "reward": 0.40000000037252903, + "reward_std": 0.32818928733468056, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.05000000074505806, + "step": 36 + }, + { + "completion_length": 380.825008392334, + "epoch": 0.0383618455158113, + "grad_norm": 0.7966611385345459, + "kl": 0.0003523826599121094, + "learning_rate": 3.827586206896552e-07, + "loss": -0.0201, + "reward": 0.3250000085681677, + "reward_std": 0.3856646828353405, + "rewards/accuracy_reward": 0.25000000931322575, + "rewards/format_reward": 0.07500000111758709, + "step": 37 + }, + { + "completion_length": 338.87500381469727, + "epoch": 0.03939865215137377, + "grad_norm": 0.723584771156311, + "kl": 0.00029540061950683594, + "learning_rate": 3.931034482758621e-07, + "loss": 0.0589, + "reward": 0.4000000059604645, + "reward_std": 0.553369790315628, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.07500000111758709, + "step": 38 + }, + { + "completion_length": 285.6000061035156, + "epoch": 0.04043545878693624, + "grad_norm": 1.385939121246338, + "kl": 0.0009665489196777344, + "learning_rate": 4.0344827586206897e-07, + "loss": 0.0266, + "reward": 0.3000000063329935, + "reward_std": 0.3731010630726814, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 0.0, + "step": 39 + }, + { + "completion_length": 342.0750045776367, + "epoch": 0.041472265422498704, + "grad_norm": 1.392383098602295, + "kl": 0.0010743141174316406, + "learning_rate": 4.1379310344827586e-07, + "loss": 0.037, + "reward": 0.42500001564621925, + "reward_std": 0.4469091147184372, + "rewards/accuracy_reward": 0.4000000096857548, + "rewards/format_reward": 0.02500000037252903, + "step": 40 + }, + { + "completion_length": 321.325008392334, + "epoch": 0.04250907205806117, + "grad_norm": 0.7412877678871155, + "kl": 0.0005884170532226562, + "learning_rate": 4.2413793103448276e-07, + "loss": 0.0443, + "reward": 0.6000000014901161, + "reward_std": 0.3856646791100502, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.10000000335276127, + "step": 41 + }, + { + "completion_length": 302.37500762939453, + "epoch": 0.04354587869362364, + "grad_norm": 0.8022830486297607, + "kl": 0.0013997554779052734, + "learning_rate": 4.344827586206897e-07, + "loss": 0.1065, + "reward": 0.20000000670552254, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.1750000063329935, + "rewards/format_reward": 0.02500000037252903, + "step": 42 + }, + { + "completion_length": 348.3499984741211, + "epoch": 0.044582685329186106, + "grad_norm": 0.5473265647888184, + "kl": 0.0008740425109863281, + "learning_rate": 4.4482758620689656e-07, + "loss": -0.0136, + "reward": 0.6000000089406967, + "reward_std": 0.3297629877924919, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.05000000074505806, + "step": 43 + }, + { + "completion_length": 475.4000015258789, + "epoch": 0.04561949196474858, + "grad_norm": 0.7117322683334351, + "kl": 0.0007257461547851562, + "learning_rate": 4.5517241379310346e-07, + "loss": 0.0024, + "reward": 0.25000000558793545, + "reward_std": 0.31719936057925224, + "rewards/accuracy_reward": 0.25000000558793545, + "rewards/format_reward": 0.0, + "step": 44 + }, + { + "completion_length": 364.4750099182129, + "epoch": 0.04665629860031104, + "grad_norm": 0.810202419757843, + "kl": 0.0007882118225097656, + "learning_rate": 4.6551724137931035e-07, + "loss": 0.0617, + "reward": 0.22500000335276127, + "reward_std": 0.3479738272726536, + "rewards/accuracy_reward": 0.2000000048428774, + "rewards/format_reward": 0.02500000037252903, + "step": 45 + }, + { + "completion_length": 276.65000915527344, + "epoch": 0.04769310523587351, + "grad_norm": 0.6206042766571045, + "kl": 0.0006685256958007812, + "learning_rate": 4.758620689655173e-07, + "loss": 0.0294, + "reward": 0.2750000022351742, + "reward_std": 0.41643913090229034, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/format_reward": 0.02500000037252903, + "step": 46 + }, + { + "completion_length": 358.450008392334, + "epoch": 0.04872991187143598, + "grad_norm": 3.410537004470825, + "kl": 0.002124786376953125, + "learning_rate": 4.862068965517241e-07, + "loss": 0.0181, + "reward": 0.3500000052154064, + "reward_std": 0.4415663778781891, + "rewards/accuracy_reward": 0.30000000819563866, + "rewards/format_reward": 0.05000000074505806, + "step": 47 + }, + { + "completion_length": 307.2500114440918, + "epoch": 0.049766718506998445, + "grad_norm": 2.8998336791992188, + "kl": 0.0014314651489257812, + "learning_rate": 4.96551724137931e-07, + "loss": 0.0167, + "reward": 0.6250000074505806, + "reward_std": 0.558712512254715, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.10000000149011612, + "step": 48 + }, + { + "completion_length": 303.54999923706055, + "epoch": 0.05080352514256091, + "grad_norm": 0.7844371199607849, + "kl": 0.0009660720825195312, + "learning_rate": 5.068965517241379e-07, + "loss": 0.0628, + "reward": 0.42500001192092896, + "reward_std": 0.47768356651067734, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.02500000037252903, + "step": 49 + }, + { + "completion_length": 461.3499984741211, + "epoch": 0.05184033177812338, + "grad_norm": 0.5534600615501404, + "kl": 0.002307891845703125, + "learning_rate": 5.172413793103449e-07, + "loss": -0.0613, + "reward": 0.3750000037252903, + "reward_std": 0.35704541951417923, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.05000000074505806, + "step": 50 + }, + { + "completion_length": 301.3500061035156, + "epoch": 0.05287713841368585, + "grad_norm": 0.9939333200454712, + "kl": 0.0011472702026367188, + "learning_rate": 5.275862068965517e-07, + "loss": -0.0638, + "reward": 0.5500000081956387, + "reward_std": 0.38409100100398064, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.05000000074505806, + "step": 51 + }, + { + "completion_length": 293.6750030517578, + "epoch": 0.05391394504924832, + "grad_norm": 0.6740232110023499, + "kl": 0.0012159347534179688, + "learning_rate": 5.379310344827586e-07, + "loss": 0.0925, + "reward": 0.5250000152736902, + "reward_std": 0.5210216604173183, + "rewards/accuracy_reward": 0.4500000048428774, + "rewards/format_reward": 0.07500000111758709, + "step": 52 + }, + { + "completion_length": 326.12500762939453, + "epoch": 0.054950751684810784, + "grad_norm": 0.8232463002204895, + "kl": 0.0014095306396484375, + "learning_rate": 5.482758620689655e-07, + "loss": 0.0658, + "reward": 0.5250000171363354, + "reward_std": 0.4415663816034794, + "rewards/accuracy_reward": 0.45000001415610313, + "rewards/format_reward": 0.07500000298023224, + "step": 53 + }, + { + "completion_length": 285.4000053405762, + "epoch": 0.05598755832037325, + "grad_norm": 1.9464935064315796, + "kl": 0.002483367919921875, + "learning_rate": 5.586206896551725e-07, + "loss": -0.1223, + "reward": 0.32500000670552254, + "reward_std": 0.4723408371210098, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/format_reward": 0.07500000111758709, + "step": 54 + }, + { + "completion_length": 331.77500915527344, + "epoch": 0.05702436495593572, + "grad_norm": 3.9341742992401123, + "kl": 0.0035686492919921875, + "learning_rate": 5.689655172413793e-07, + "loss": 0.0667, + "reward": 0.4750000163912773, + "reward_std": 0.36053743213415146, + "rewards/accuracy_reward": 0.42500000819563866, + "rewards/format_reward": 0.05000000074505806, + "step": 55 + }, + { + "completion_length": 326.8500099182129, + "epoch": 0.058061171591498185, + "grad_norm": 1.525079369544983, + "kl": 0.0031280517578125, + "learning_rate": 5.793103448275862e-07, + "loss": -0.0055, + "reward": 0.37500000186264515, + "reward_std": 0.37310105934739113, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.02500000037252903, + "step": 56 + }, + { + "completion_length": 267.37500762939453, + "epoch": 0.05909797822706065, + "grad_norm": 0.823444128036499, + "kl": 0.002056121826171875, + "learning_rate": 5.896551724137931e-07, + "loss": 0.0713, + "reward": 0.6500000059604645, + "reward_std": 0.46669362485408783, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.10000000149011612, + "step": 57 + }, + { + "completion_length": 222.75, + "epoch": 0.06013478486262312, + "grad_norm": 1.267547845840454, + "kl": 0.0028076171875, + "learning_rate": 6.000000000000001e-07, + "loss": 0.0757, + "reward": 0.600000012665987, + "reward_std": 0.4092182368040085, + "rewards/accuracy_reward": 0.5000000167638063, + "rewards/format_reward": 0.10000000149011612, + "step": 58 + }, + { + "completion_length": 252.62500762939453, + "epoch": 0.06117159149818559, + "grad_norm": 1.6207536458969116, + "kl": 0.0028018951416015625, + "learning_rate": 6.103448275862069e-07, + "loss": 0.0935, + "reward": 0.3750000074505806, + "reward_std": 0.4614894315600395, + "rewards/accuracy_reward": 0.3000000026077032, + "rewards/format_reward": 0.07500000111758709, + "step": 59 + }, + { + "completion_length": 295.5250053405762, + "epoch": 0.06220839813374806, + "grad_norm": 1.536206603050232, + "kl": 0.010120391845703125, + "learning_rate": 6.206896551724138e-07, + "loss": 0.0602, + "reward": 0.6750000156462193, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.05000000074505806, + "step": 60 + }, + { + "completion_length": 265.5250015258789, + "epoch": 0.06324520476931052, + "grad_norm": 14.682554244995117, + "kl": 0.020404815673828125, + "learning_rate": 6.310344827586207e-07, + "loss": -0.0303, + "reward": 0.6000000089406967, + "reward_std": 0.5896254852414131, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.12500000186264515, + "step": 61 + }, + { + "completion_length": 288.37500762939453, + "epoch": 0.064282011404873, + "grad_norm": 5.919084548950195, + "kl": 0.006771087646484375, + "learning_rate": 6.413793103448277e-07, + "loss": 0.0497, + "reward": 0.5500000044703484, + "reward_std": 0.4704626724123955, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.10000000149011612, + "step": 62 + }, + { + "completion_length": 301.6250114440918, + "epoch": 0.06531881804043546, + "grad_norm": 16.421817779541016, + "kl": 0.005756378173828125, + "learning_rate": 6.517241379310345e-07, + "loss": 0.1564, + "reward": 0.6250000149011612, + "reward_std": 0.5894869714975357, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.17500000447034836, + "step": 63 + }, + { + "completion_length": 284.77500915527344, + "epoch": 0.06635562467599793, + "grad_norm": 7.291428565979004, + "kl": 0.0056972503662109375, + "learning_rate": 6.620689655172414e-07, + "loss": 0.0605, + "reward": 0.5250000022351742, + "reward_std": 0.46148940920829773, + "rewards/accuracy_reward": 0.37499999813735485, + "rewards/format_reward": 0.15000000223517418, + "step": 64 + }, + { + "completion_length": 343.8000030517578, + "epoch": 0.06739243131156039, + "grad_norm": 6.103272914886475, + "kl": 0.004486083984375, + "learning_rate": 6.724137931034483e-07, + "loss": -0.0342, + "reward": 0.4750000089406967, + "reward_std": 0.5461488887667656, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.02500000037252903, + "step": 65 + }, + { + "completion_length": 276.20001220703125, + "epoch": 0.06842923794712286, + "grad_norm": 0.6578114032745361, + "kl": 0.004322052001953125, + "learning_rate": 6.827586206896553e-07, + "loss": 0.1695, + "reward": 0.3000000063329935, + "reward_std": 0.47234082594513893, + "rewards/accuracy_reward": 0.15000000409781933, + "rewards/format_reward": 0.15000000223517418, + "step": 66 + }, + { + "completion_length": 246.37500381469727, + "epoch": 0.06946604458268534, + "grad_norm": 7.143874645233154, + "kl": 0.004947662353515625, + "learning_rate": 6.931034482758621e-07, + "loss": 0.0603, + "reward": 0.45000001043081284, + "reward_std": 0.4902471899986267, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.10000000149011612, + "step": 67 + }, + { + "completion_length": 179.375, + "epoch": 0.0705028512182478, + "grad_norm": 0.9650272727012634, + "kl": 0.00789642333984375, + "learning_rate": 7.03448275862069e-07, + "loss": 0.1452, + "reward": 0.75, + "reward_std": 0.5985585674643517, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.27500000409781933, + "step": 68 + }, + { + "completion_length": 284.3250045776367, + "epoch": 0.07153965785381027, + "grad_norm": 1.4530069828033447, + "kl": 0.0058746337890625, + "learning_rate": 7.137931034482759e-07, + "loss": -0.0084, + "reward": 0.775000024586916, + "reward_std": 0.5697024464607239, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.2000000048428774, + "step": 69 + }, + { + "completion_length": 290.325008392334, + "epoch": 0.07257646448937273, + "grad_norm": 0.7363834381103516, + "kl": 0.01473236083984375, + "learning_rate": 7.241379310344829e-07, + "loss": -0.0061, + "reward": 0.4000000078231096, + "reward_std": 0.41643914952874184, + "rewards/accuracy_reward": 0.3250000011175871, + "rewards/format_reward": 0.07500000111758709, + "step": 70 + }, + { + "completion_length": 194.20000076293945, + "epoch": 0.0736132711249352, + "grad_norm": 1.855769395828247, + "kl": 0.01556396484375, + "learning_rate": 7.344827586206896e-07, + "loss": 0.0088, + "reward": 0.8250000216066837, + "reward_std": 0.5607292056083679, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.35000001080334187, + "step": 71 + }, + { + "completion_length": 244.00000381469727, + "epoch": 0.07465007776049767, + "grad_norm": 1.9801136255264282, + "kl": 0.0140533447265625, + "learning_rate": 7.448275862068965e-07, + "loss": 0.0587, + "reward": 0.9500000327825546, + "reward_std": 0.5624414160847664, + "rewards/accuracy_reward": 0.6500000208616257, + "rewards/format_reward": 0.30000000819563866, + "step": 72 + }, + { + "completion_length": 223.0250015258789, + "epoch": 0.07568688439606014, + "grad_norm": 0.857162594795227, + "kl": 0.01824951171875, + "learning_rate": 7.551724137931035e-07, + "loss": 0.0944, + "reward": 0.7750000059604645, + "reward_std": 0.5263643562793732, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.350000012665987, + "step": 73 + }, + { + "completion_length": 267.02500915527344, + "epoch": 0.0767236910316226, + "grad_norm": 1.467808723449707, + "kl": 0.0139923095703125, + "learning_rate": 7.655172413793104e-07, + "loss": 0.0959, + "reward": 0.8000000268220901, + "reward_std": 0.4850429780781269, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.4000000096857548, + "step": 74 + }, + { + "completion_length": 222.25000381469727, + "epoch": 0.07776049766718507, + "grad_norm": 1.6067920923233032, + "kl": 0.0166168212890625, + "learning_rate": 7.758620689655173e-07, + "loss": 0.1242, + "reward": 0.7500000037252903, + "reward_std": 0.6815058067440987, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.3500000014901161, + "step": 75 + }, + { + "completion_length": 199.7750015258789, + "epoch": 0.07879730430274753, + "grad_norm": 1.2611151933670044, + "kl": 0.03570556640625, + "learning_rate": 7.862068965517242e-07, + "loss": 0.0485, + "reward": 0.8250000178813934, + "reward_std": 0.582266092300415, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.4000000134110451, + "step": 76 + }, + { + "completion_length": 173.07500457763672, + "epoch": 0.07983411093831, + "grad_norm": 3.898858070373535, + "kl": 0.02545166015625, + "learning_rate": 7.96551724137931e-07, + "loss": 0.2017, + "reward": 1.0250000208616257, + "reward_std": 0.5948297046124935, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.5750000029802322, + "step": 77 + }, + { + "completion_length": 187.725004196167, + "epoch": 0.08087091757387248, + "grad_norm": 0.8916759490966797, + "kl": 0.040130615234375, + "learning_rate": 8.068965517241379e-07, + "loss": 0.0355, + "reward": 1.1250000149011612, + "reward_std": 0.6099044531583786, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.45000000670552254, + "step": 78 + }, + { + "completion_length": 177.35000610351562, + "epoch": 0.08190772420943494, + "grad_norm": 1.327777624130249, + "kl": 0.04638671875, + "learning_rate": 8.172413793103448e-07, + "loss": 0.0438, + "reward": 0.9500000178813934, + "reward_std": 0.7595372572541237, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.6000000089406967, + "step": 79 + }, + { + "completion_length": 183.1750030517578, + "epoch": 0.08294453084499741, + "grad_norm": 1.4906061887741089, + "kl": 0.054718017578125, + "learning_rate": 8.275862068965517e-07, + "loss": 0.1092, + "reward": 0.8500000089406967, + "reward_std": 0.6130405142903328, + "rewards/accuracy_reward": 0.15000000223517418, + "rewards/format_reward": 0.7000000029802322, + "step": 80 + }, + { + "completion_length": 199.35000228881836, + "epoch": 0.08398133748055987, + "grad_norm": 3.892275333404541, + "kl": 0.0404052734375, + "learning_rate": 8.379310344827587e-07, + "loss": 0.1296, + "reward": 1.175000011920929, + "reward_std": 0.6471008583903313, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.7250000089406967, + "step": 81 + }, + { + "completion_length": 141.80000114440918, + "epoch": 0.08501814411612234, + "grad_norm": 0.9681069254875183, + "kl": 0.091522216796875, + "learning_rate": 8.482758620689655e-07, + "loss": 0.0576, + "reward": 1.0500000417232513, + "reward_std": 0.5101702138781548, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.7249999940395355, + "step": 82 + }, + { + "completion_length": 141.00000190734863, + "epoch": 0.08605495075168482, + "grad_norm": 1.6170580387115479, + "kl": 0.0736083984375, + "learning_rate": 8.586206896551725e-07, + "loss": 0.091, + "reward": 1.1250000149011612, + "reward_std": 0.5112976878881454, + "rewards/accuracy_reward": 0.3500000052154064, + "rewards/format_reward": 0.7750000059604645, + "step": 83 + }, + { + "completion_length": 226.37499809265137, + "epoch": 0.08709175738724728, + "grad_norm": 2.232787847518921, + "kl": 0.05548095703125, + "learning_rate": 8.689655172413794e-07, + "loss": 0.1663, + "reward": 1.2000000327825546, + "reward_std": 0.5984200611710548, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 0.7500000149011612, + "step": 84 + }, + { + "completion_length": 137.9500026702881, + "epoch": 0.08812856402280975, + "grad_norm": 1.994882583618164, + "kl": 0.058135986328125, + "learning_rate": 8.793103448275862e-07, + "loss": 0.0491, + "reward": 1.4500000029802322, + "reward_std": 0.44892580062150955, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.800000011920929, + "step": 85 + }, + { + "completion_length": 121.20000076293945, + "epoch": 0.08916537065837221, + "grad_norm": 1.0910072326660156, + "kl": 0.056640625, + "learning_rate": 8.896551724137931e-07, + "loss": 0.054, + "reward": 1.175000011920929, + "reward_std": 0.40558772534132004, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.800000011920929, + "step": 86 + }, + { + "completion_length": 148.9000015258789, + "epoch": 0.09020217729393468, + "grad_norm": 2.2213571071624756, + "kl": 0.220458984375, + "learning_rate": 9e-07, + "loss": 0.0789, + "reward": 1.3250000327825546, + "reward_std": 0.5624815374612808, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.7750000059604645, + "step": 87 + }, + { + "completion_length": 157.7750015258789, + "epoch": 0.09123898392949716, + "grad_norm": 1.2787593603134155, + "kl": 0.067138671875, + "learning_rate": 9.103448275862069e-07, + "loss": 0.0193, + "reward": 1.2250000536441803, + "reward_std": 0.510664701461792, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.7500000149011612, + "step": 88 + }, + { + "completion_length": 141.32500076293945, + "epoch": 0.09227579056505962, + "grad_norm": 1.5759501457214355, + "kl": 0.109130859375, + "learning_rate": 9.206896551724139e-07, + "loss": 0.1089, + "reward": 1.4500000476837158, + "reward_std": 0.3804604858160019, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.8500000238418579, + "step": 89 + }, + { + "completion_length": 142.72500228881836, + "epoch": 0.09331259720062209, + "grad_norm": 0.9323010444641113, + "kl": 0.079803466796875, + "learning_rate": 9.310344827586207e-07, + "loss": 0.0734, + "reward": 1.3000000417232513, + "reward_std": 0.4657529257237911, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.8500000238418579, + "step": 90 + }, + { + "completion_length": 122.37500190734863, + "epoch": 0.09434940383618455, + "grad_norm": 1.3933780193328857, + "kl": 0.1185302734375, + "learning_rate": 9.413793103448277e-07, + "loss": 0.0271, + "reward": 1.2000000476837158, + "reward_std": 0.6617213189601898, + "rewards/accuracy_reward": 0.3500000052154064, + "rewards/format_reward": 0.8500000238418579, + "step": 91 + }, + { + "completion_length": 126.70000457763672, + "epoch": 0.09538621047174702, + "grad_norm": 7.927647113800049, + "kl": 0.4715576171875, + "learning_rate": 9.517241379310346e-07, + "loss": 0.0897, + "reward": 1.2000000178813934, + "reward_std": 0.6092714741826057, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 0.75, + "step": 92 + }, + { + "completion_length": 172.30000114440918, + "epoch": 0.09642301710730948, + "grad_norm": 1.3404741287231445, + "kl": 0.078857421875, + "learning_rate": 9.620689655172413e-07, + "loss": 0.0283, + "reward": 1.3750000298023224, + "reward_std": 0.46511995419859886, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 93 + }, + { + "completion_length": 113.35000038146973, + "epoch": 0.09745982374287196, + "grad_norm": 7.312382221221924, + "kl": 0.0780029296875, + "learning_rate": 9.724137931034483e-07, + "loss": 0.0145, + "reward": 1.4000000059604645, + "reward_std": 0.4092182442545891, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 94 + }, + { + "completion_length": 109.42500114440918, + "epoch": 0.09849663037843442, + "grad_norm": 1.404718041419983, + "kl": 0.08563232421875, + "learning_rate": 9.827586206896552e-07, + "loss": 0.072, + "reward": 1.4250000417232513, + "reward_std": 0.4740530401468277, + "rewards/accuracy_reward": 0.45000001788139343, + "rewards/format_reward": 0.9750000089406967, + "step": 95 + }, + { + "completion_length": 117.55000305175781, + "epoch": 0.09953343701399689, + "grad_norm": 1.4174925088882446, + "kl": 0.1512451171875, + "learning_rate": 9.93103448275862e-07, + "loss": -0.0339, + "reward": 1.4000000357627869, + "reward_std": 0.578635536134243, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.8250000178813934, + "step": 96 + }, + { + "completion_length": 105.30000114440918, + "epoch": 0.10057024364955935, + "grad_norm": 6.377532005310059, + "kl": 0.1229248046875, + "learning_rate": 1.003448275862069e-06, + "loss": -0.0262, + "reward": 1.3500000536441803, + "reward_std": 0.4958944097161293, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.8500000238418579, + "step": 97 + }, + { + "completion_length": 122.80000114440918, + "epoch": 0.10160705028512182, + "grad_norm": 3.010532855987549, + "kl": 0.09405517578125, + "learning_rate": 1.0137931034482759e-06, + "loss": 0.0275, + "reward": 1.3250000178813934, + "reward_std": 0.2795085050165653, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9250000268220901, + "step": 98 + }, + { + "completion_length": 117.97500038146973, + "epoch": 0.1026438569206843, + "grad_norm": 1.6175012588500977, + "kl": 0.09100341796875, + "learning_rate": 1.0241379310344828e-06, + "loss": 0.0431, + "reward": 1.2500000447034836, + "reward_std": 0.5535082966089249, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/format_reward": 0.875, + "step": 99 + }, + { + "completion_length": 123.85000419616699, + "epoch": 0.10368066355624676, + "grad_norm": 1.486450433731079, + "kl": 0.08172607421875, + "learning_rate": 1.0344827586206898e-06, + "loss": 0.0611, + "reward": 1.475000023841858, + "reward_std": 0.4525563009083271, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.8500000089406967, + "step": 100 + }, + { + "completion_length": 118.80000114440918, + "epoch": 0.10471747019180923, + "grad_norm": 1.8546826839447021, + "kl": 0.07427978515625, + "learning_rate": 1.0448275862068965e-06, + "loss": -0.0648, + "reward": 1.2500000298023224, + "reward_std": 0.45412999391555786, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.8500000089406967, + "step": 101 + }, + { + "completion_length": 107.62500190734863, + "epoch": 0.1057542768273717, + "grad_norm": 1.0969343185424805, + "kl": 0.1190185546875, + "learning_rate": 1.0551724137931035e-06, + "loss": 0.0655, + "reward": 1.4000000059604645, + "reward_std": 0.3804604709148407, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9500000178813934, + "step": 102 + }, + { + "completion_length": 125.30000114440918, + "epoch": 0.10679108346293416, + "grad_norm": 2.0286576747894287, + "kl": 0.1263427734375, + "learning_rate": 1.0655172413793104e-06, + "loss": 0.1098, + "reward": 1.350000023841858, + "reward_std": 0.44156638905406, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.9000000357627869, + "step": 103 + }, + { + "completion_length": 115.07500076293945, + "epoch": 0.10782789009849664, + "grad_norm": 2.2312369346618652, + "kl": 0.1448974609375, + "learning_rate": 1.0758620689655172e-06, + "loss": 0.0629, + "reward": 1.3250000178813934, + "reward_std": 0.47234081476926804, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.9500000178813934, + "step": 104 + }, + { + "completion_length": 119.22500228881836, + "epoch": 0.1088646967340591, + "grad_norm": 0.6519957184791565, + "kl": 0.0863037109375, + "learning_rate": 1.0862068965517241e-06, + "loss": -0.0487, + "reward": 1.3500000089406967, + "reward_std": 0.33353203162550926, + "rewards/accuracy_reward": 0.4500000011175871, + "rewards/format_reward": 0.9000000059604645, + "step": 105 + }, + { + "completion_length": 123.47500228881836, + "epoch": 0.10990150336962157, + "grad_norm": 1.1476188898086548, + "kl": 0.10223388671875, + "learning_rate": 1.096551724137931e-06, + "loss": 0.0576, + "reward": 1.2500000149011612, + "reward_std": 0.3533165417611599, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.9000000208616257, + "step": 106 + }, + { + "completion_length": 113.65000343322754, + "epoch": 0.11093831000518403, + "grad_norm": 1.9051084518432617, + "kl": 0.08203125, + "learning_rate": 1.106896551724138e-06, + "loss": 0.0674, + "reward": 1.4000000357627869, + "reward_std": 0.5806522481143475, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.800000011920929, + "step": 107 + }, + { + "completion_length": 123.80000305175781, + "epoch": 0.1119751166407465, + "grad_norm": 1.0134373903274536, + "kl": 0.1263427734375, + "learning_rate": 1.117241379310345e-06, + "loss": 0.1178, + "reward": 1.6500000357627869, + "reward_std": 0.47234082967042923, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.8750000298023224, + "step": 108 + }, + { + "completion_length": 189.0250072479248, + "epoch": 0.11301192327630896, + "grad_norm": 2.021437883377075, + "kl": 0.06475830078125, + "learning_rate": 1.1275862068965517e-06, + "loss": 0.046, + "reward": 1.2750000357627869, + "reward_std": 0.6544602662324905, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.8000000268220901, + "step": 109 + }, + { + "completion_length": 125.30000495910645, + "epoch": 0.11404872991187144, + "grad_norm": 1.5949187278747559, + "kl": 0.07537841796875, + "learning_rate": 1.1379310344827587e-06, + "loss": 0.1211, + "reward": 1.2750000059604645, + "reward_std": 0.5409446507692337, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.800000011920929, + "step": 110 + }, + { + "completion_length": 112.60000228881836, + "epoch": 0.1150855365474339, + "grad_norm": 0.9887787103652954, + "kl": 0.09454345703125, + "learning_rate": 1.1482758620689656e-06, + "loss": 0.0565, + "reward": 1.225000023841858, + "reward_std": 0.5337237864732742, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.8750000298023224, + "step": 111 + }, + { + "completion_length": 113.90000534057617, + "epoch": 0.11612234318299637, + "grad_norm": 1.2379708290100098, + "kl": 0.1116943359375, + "learning_rate": 1.1586206896551724e-06, + "loss": 0.0541, + "reward": 1.225000023841858, + "reward_std": 0.5572371743619442, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.7750000208616257, + "step": 112 + }, + { + "completion_length": 140.9500026702881, + "epoch": 0.11715914981855884, + "grad_norm": 2.4870309829711914, + "kl": 0.076171875, + "learning_rate": 1.1689655172413793e-06, + "loss": 0.0255, + "reward": 1.4750000536441803, + "reward_std": 0.4415663853287697, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.8250000178813934, + "step": 113 + }, + { + "completion_length": 143.625, + "epoch": 0.1181959564541213, + "grad_norm": 2.575742721557617, + "kl": 0.05718994140625, + "learning_rate": 1.1793103448275863e-06, + "loss": 0.071, + "reward": 1.225000023841858, + "reward_std": 0.562441386282444, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.800000011920929, + "step": 114 + }, + { + "completion_length": 150.72500228881836, + "epoch": 0.11923276308968378, + "grad_norm": 4.119329929351807, + "kl": 0.07586669921875, + "learning_rate": 1.1896551724137932e-06, + "loss": 0.0335, + "reward": 1.2000000178813934, + "reward_std": 0.5210216380655766, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.800000011920929, + "step": 115 + }, + { + "completion_length": 129.87500190734863, + "epoch": 0.12026956972524624, + "grad_norm": 0.9842166304588318, + "kl": 0.0792236328125, + "learning_rate": 1.2000000000000002e-06, + "loss": 0.0267, + "reward": 1.3750000298023224, + "reward_std": 0.6417581215500832, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.8250000178813934, + "step": 116 + }, + { + "completion_length": 125.87500381469727, + "epoch": 0.12130637636080871, + "grad_norm": 1.5760966539382935, + "kl": 0.0994873046875, + "learning_rate": 1.210344827586207e-06, + "loss": 0.0867, + "reward": 1.2000000029802322, + "reward_std": 0.6293330267071724, + "rewards/accuracy_reward": 0.4500000011175871, + "rewards/format_reward": 0.75, + "step": 117 + }, + { + "completion_length": 144.975004196167, + "epoch": 0.12234318299637117, + "grad_norm": 0.8227043747901917, + "kl": 0.0821533203125, + "learning_rate": 1.2206896551724138e-06, + "loss": 0.0002, + "reward": 1.3500000089406967, + "reward_std": 0.472973819822073, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.800000011920929, + "step": 118 + }, + { + "completion_length": 151.1500015258789, + "epoch": 0.12337998963193364, + "grad_norm": 3.4477927684783936, + "kl": 0.11090087890625, + "learning_rate": 1.2310344827586206e-06, + "loss": 0.057, + "reward": 1.5000000596046448, + "reward_std": 0.5665664002299309, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.7750000059604645, + "step": 119 + }, + { + "completion_length": 133.72500228881836, + "epoch": 0.12441679626749612, + "grad_norm": 1.916924238204956, + "kl": 0.14306640625, + "learning_rate": 1.2413793103448275e-06, + "loss": 0.1373, + "reward": 1.4749999940395355, + "reward_std": 0.41643914580345154, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.800000011920929, + "step": 120 + }, + { + "completion_length": 144.70000457763672, + "epoch": 0.12545360290305857, + "grad_norm": 1.4124659299850464, + "kl": 0.11669921875, + "learning_rate": 1.2517241379310345e-06, + "loss": 0.0701, + "reward": 1.600000023841858, + "reward_std": 0.5210216268897057, + "rewards/accuracy_reward": 0.7500000298023224, + "rewards/format_reward": 0.8500000238418579, + "step": 121 + }, + { + "completion_length": 143.50000381469727, + "epoch": 0.12649040953862103, + "grad_norm": 2.2720749378204346, + "kl": 0.09759521484375, + "learning_rate": 1.2620689655172414e-06, + "loss": 0.0201, + "reward": 1.449999988079071, + "reward_std": 0.5462873950600624, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.7750000059604645, + "step": 122 + }, + { + "completion_length": 125.87500190734863, + "epoch": 0.12752721617418353, + "grad_norm": 1.0222535133361816, + "kl": 0.07147216796875, + "learning_rate": 1.2724137931034484e-06, + "loss": 0.0006, + "reward": 1.4500000476837158, + "reward_std": 0.3856646865606308, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.9500000029802322, + "step": 123 + }, + { + "completion_length": 145.07500648498535, + "epoch": 0.128564022809746, + "grad_norm": 4.159966945648193, + "kl": 0.08453369140625, + "learning_rate": 1.2827586206896553e-06, + "loss": 0.0968, + "reward": 1.300000011920929, + "reward_std": 0.5659333989024162, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.7750000208616257, + "step": 124 + }, + { + "completion_length": 118.10000038146973, + "epoch": 0.12960082944530846, + "grad_norm": 1.1424363851547241, + "kl": 0.0953369140625, + "learning_rate": 1.293103448275862e-06, + "loss": -0.0023, + "reward": 1.3250000178813934, + "reward_std": 0.3245587684214115, + "rewards/accuracy_reward": 0.3999999985098839, + "rewards/format_reward": 0.925000011920929, + "step": 125 + }, + { + "completion_length": 108.20000076293945, + "epoch": 0.13063763608087092, + "grad_norm": 3.2891056537628174, + "kl": 0.100830078125, + "learning_rate": 1.303448275862069e-06, + "loss": -0.036, + "reward": 1.2750000208616257, + "reward_std": 0.37323955446481705, + "rewards/accuracy_reward": 0.40000000409781933, + "rewards/format_reward": 0.8750000149011612, + "step": 126 + }, + { + "completion_length": 128.57500457763672, + "epoch": 0.1316744427164334, + "grad_norm": 1.2664824724197388, + "kl": 0.07080078125, + "learning_rate": 1.3137931034482758e-06, + "loss": 0.0349, + "reward": 1.6750000417232513, + "reward_std": 0.36053745076060295, + "rewards/accuracy_reward": 0.75, + "rewards/format_reward": 0.925000011920929, + "step": 127 + }, + { + "completion_length": 145.00000190734863, + "epoch": 0.13271124935199585, + "grad_norm": 1.661323070526123, + "kl": 0.08135986328125, + "learning_rate": 1.3241379310344827e-06, + "loss": 0.0568, + "reward": 1.5000000298023224, + "reward_std": 0.4850430078804493, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.8750000298023224, + "step": 128 + }, + { + "completion_length": 129.42500114440918, + "epoch": 0.13374805598755832, + "grad_norm": 1.649943232536316, + "kl": 0.0888671875, + "learning_rate": 1.3344827586206897e-06, + "loss": -0.0105, + "reward": 1.3750000298023224, + "reward_std": 0.36053745076060295, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 129 + }, + { + "completion_length": 119.15000343322754, + "epoch": 0.13478486262312078, + "grad_norm": 6.876035690307617, + "kl": 0.19091796875, + "learning_rate": 1.3448275862068966e-06, + "loss": -0.0036, + "reward": 1.475000023841858, + "reward_std": 0.4849044345319271, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8250000178813934, + "step": 130 + }, + { + "completion_length": 108.72500038146973, + "epoch": 0.13582166925868325, + "grad_norm": 1.3705264329910278, + "kl": 0.1119384765625, + "learning_rate": 1.3551724137931036e-06, + "loss": 0.0473, + "reward": 1.1250000149011612, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.17500000074505806, + "rewards/format_reward": 0.9500000029802322, + "step": 131 + }, + { + "completion_length": 132.72500038146973, + "epoch": 0.1368584758942457, + "grad_norm": 9.587868690490723, + "kl": 0.10333251953125, + "learning_rate": 1.3655172413793105e-06, + "loss": 0.1013, + "reward": 1.3750000596046448, + "reward_std": 0.5335852727293968, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.8750000298023224, + "step": 132 + }, + { + "completion_length": 122.40000343322754, + "epoch": 0.13789528252980818, + "grad_norm": 5.267590045928955, + "kl": 0.120849609375, + "learning_rate": 1.3758620689655173e-06, + "loss": -0.0325, + "reward": 1.475000023841858, + "reward_std": 0.49746808409690857, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 133 + }, + { + "completion_length": 121.22500228881836, + "epoch": 0.13893208916537067, + "grad_norm": 2.1384124755859375, + "kl": 0.184814453125, + "learning_rate": 1.3862068965517242e-06, + "loss": 0.0501, + "reward": 1.2750000357627869, + "reward_std": 0.4291412718594074, + "rewards/accuracy_reward": 0.4250000137835741, + "rewards/format_reward": 0.8500000089406967, + "step": 134 + }, + { + "completion_length": 108.37500381469727, + "epoch": 0.13996889580093314, + "grad_norm": 4.733362674713135, + "kl": 0.0623779296875, + "learning_rate": 1.396551724137931e-06, + "loss": 0.0074, + "reward": 1.550000011920929, + "reward_std": 0.23617041110992432, + "rewards/accuracy_reward": 0.5749999973922968, + "rewards/format_reward": 0.9750000089406967, + "step": 135 + }, + { + "completion_length": 133.85000038146973, + "epoch": 0.1410057024364956, + "grad_norm": 1.9882097244262695, + "kl": 0.07232666015625, + "learning_rate": 1.406896551724138e-06, + "loss": 0.048, + "reward": 1.574999988079071, + "reward_std": 0.4290027506649494, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 136 + }, + { + "completion_length": 142.67500114440918, + "epoch": 0.14204250907205807, + "grad_norm": 2.703753709793091, + "kl": 0.1512451171875, + "learning_rate": 1.4172413793103449e-06, + "loss": 0.0586, + "reward": 1.375, + "reward_std": 0.4578990526497364, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.800000011920929, + "step": 137 + }, + { + "completion_length": 119.35000228881836, + "epoch": 0.14307931570762053, + "grad_norm": 2.8184866905212402, + "kl": 0.10125732421875, + "learning_rate": 1.4275862068965518e-06, + "loss": -0.0393, + "reward": 1.5000000298023224, + "reward_std": 0.4290027655661106, + "rewards/accuracy_reward": 0.5250000208616257, + "rewards/format_reward": 0.9750000089406967, + "step": 138 + }, + { + "completion_length": 111.85000228881836, + "epoch": 0.144116122343183, + "grad_norm": 4.2106242179870605, + "kl": 0.1044921875, + "learning_rate": 1.4379310344827588e-06, + "loss": 0.0386, + "reward": 1.0750000178813934, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/format_reward": 0.8250000178813934, + "step": 139 + }, + { + "completion_length": 117.80000305175781, + "epoch": 0.14515292897874546, + "grad_norm": 4.182285785675049, + "kl": 0.0821533203125, + "learning_rate": 1.4482758620689657e-06, + "loss": -0.0056, + "reward": 1.3500000536441803, + "reward_std": 0.5786355212330818, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.925000011920929, + "step": 140 + }, + { + "completion_length": 135.22500228881836, + "epoch": 0.14618973561430793, + "grad_norm": 1.8042055368423462, + "kl": 0.1070556640625, + "learning_rate": 1.4586206896551724e-06, + "loss": 0.0296, + "reward": 1.5, + "reward_std": 0.3856646940112114, + "rewards/accuracy_reward": 0.5500000063329935, + "rewards/format_reward": 0.9500000178813934, + "step": 141 + }, + { + "completion_length": 126.97500228881836, + "epoch": 0.1472265422498704, + "grad_norm": 4.371006011962891, + "kl": 0.09185791015625, + "learning_rate": 1.4689655172413792e-06, + "loss": 0.0361, + "reward": 1.75, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 142 + }, + { + "completion_length": 131.6750030517578, + "epoch": 0.14826334888543286, + "grad_norm": 17.638200759887695, + "kl": 0.11883544921875, + "learning_rate": 1.4793103448275861e-06, + "loss": 0.0239, + "reward": 1.375, + "reward_std": 0.24873404577374458, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9500000178813934, + "step": 143 + }, + { + "completion_length": 135.87500381469727, + "epoch": 0.14930015552099535, + "grad_norm": 10.907958984375, + "kl": 0.119384765625, + "learning_rate": 1.489655172413793e-06, + "loss": 0.0428, + "reward": 1.2250000089406967, + "reward_std": 0.3930240869522095, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.8500000089406967, + "step": 144 + }, + { + "completion_length": 147.6250057220459, + "epoch": 0.1503369621565578, + "grad_norm": 921.281005859375, + "kl": 3.2681884765625, + "learning_rate": 1.5e-06, + "loss": 0.2005, + "reward": 1.225000038743019, + "reward_std": 0.30463574081659317, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 145 + }, + { + "completion_length": 148.72500228881836, + "epoch": 0.15137376879212028, + "grad_norm": 14.504183769226074, + "kl": 0.0855712890625, + "learning_rate": 1.510344827586207e-06, + "loss": 0.0652, + "reward": 1.675000011920929, + "reward_std": 0.34448179230093956, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 146 + }, + { + "completion_length": 173.975004196167, + "epoch": 0.15241057542768274, + "grad_norm": 251.97561645507812, + "kl": 0.8280029296875, + "learning_rate": 1.520689655172414e-06, + "loss": 0.0497, + "reward": 1.5250000059604645, + "reward_std": 0.4343454875051975, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8750000149011612, + "step": 147 + }, + { + "completion_length": 180.57500457763672, + "epoch": 0.1534473820632452, + "grad_norm": 206.3589630126953, + "kl": 0.494384765625, + "learning_rate": 1.5310344827586209e-06, + "loss": 0.1159, + "reward": 1.2500000149011612, + "reward_std": 0.37310104444622993, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.800000011920929, + "step": 148 + }, + { + "completion_length": 170.9250030517578, + "epoch": 0.15448418869880767, + "grad_norm": 61797.4140625, + "kl": 1144.1375732421875, + "learning_rate": 1.5413793103448276e-06, + "loss": 27.6538, + "reward": 0.9000000208616257, + "reward_std": 0.5481655634939671, + "rewards/accuracy_reward": 0.1750000026077032, + "rewards/format_reward": 0.7250000089406967, + "step": 149 + }, + { + "completion_length": 165.55000495910645, + "epoch": 0.15552099533437014, + "grad_norm": 886645.125, + "kl": 669.49169921875, + "learning_rate": 1.5517241379310346e-06, + "loss": 31.4775, + "reward": 1.2750000208616257, + "reward_std": 0.49226387962698936, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.800000011920929, + "step": 150 + }, + { + "completion_length": 118.1500015258789, + "epoch": 0.1565578019699326, + "grad_norm": 120.34538269042969, + "kl": 1.54296875, + "learning_rate": 1.5620689655172415e-06, + "loss": 0.0637, + "reward": 1.425000011920929, + "reward_std": 0.26865706592798233, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.9500000178813934, + "step": 151 + }, + { + "completion_length": 126.35000419616699, + "epoch": 0.15759460860549507, + "grad_norm": 93.3667984008789, + "kl": 1.257568359375, + "learning_rate": 1.5724137931034485e-06, + "loss": -0.0012, + "reward": 1.5, + "reward_std": 0.4291412904858589, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 152 + }, + { + "completion_length": 185.10000610351562, + "epoch": 0.15863141524105753, + "grad_norm": 607.76611328125, + "kl": 1.4993896484375, + "learning_rate": 1.5827586206896554e-06, + "loss": 0.1878, + "reward": 1.5500000417232513, + "reward_std": 0.4902471825480461, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 153 + }, + { + "completion_length": 150.1500015258789, + "epoch": 0.15966822187662, + "grad_norm": 621.0995483398438, + "kl": 0.4017333984375, + "learning_rate": 1.593103448275862e-06, + "loss": 0.0861, + "reward": 1.5750000178813934, + "reward_std": 0.27228758856654167, + "rewards/accuracy_reward": 0.6750000007450581, + "rewards/format_reward": 0.9000000208616257, + "step": 154 + }, + { + "completion_length": 148.475004196167, + "epoch": 0.1607050285121825, + "grad_norm": 1060.59033203125, + "kl": 5.067626953125, + "learning_rate": 1.603448275862069e-06, + "loss": 0.3634, + "reward": 1.2999999970197678, + "reward_std": 0.48553744331002235, + "rewards/accuracy_reward": 0.4500000011175871, + "rewards/format_reward": 0.8500000238418579, + "step": 155 + }, + { + "completion_length": 134.1500015258789, + "epoch": 0.16174183514774496, + "grad_norm": 100.48238372802734, + "kl": 0.8262939453125, + "learning_rate": 1.6137931034482759e-06, + "loss": 0.0642, + "reward": 1.2750000059604645, + "reward_std": 0.248734038323164, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.8500000089406967, + "step": 156 + }, + { + "completion_length": 147.5750026702881, + "epoch": 0.16277864178330742, + "grad_norm": 62.48579788208008, + "kl": 0.651611328125, + "learning_rate": 1.6241379310344828e-06, + "loss": 0.151, + "reward": 1.25, + "reward_std": 0.41643912717700005, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.8500000238418579, + "step": 157 + }, + { + "completion_length": 220.97500610351562, + "epoch": 0.16381544841886989, + "grad_norm": 575.1019287109375, + "kl": 2.5919189453125, + "learning_rate": 1.6344827586206896e-06, + "loss": 0.2918, + "reward": 1.4000000357627869, + "reward_std": 0.36053742468357086, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.9000000059604645, + "step": 158 + }, + { + "completion_length": 144.225004196167, + "epoch": 0.16485225505443235, + "grad_norm": 7.095518112182617, + "kl": 0.1251220703125, + "learning_rate": 1.6448275862068965e-06, + "loss": 0.0249, + "reward": 1.5, + "reward_std": 0.39728759601712227, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9000000059604645, + "step": 159 + }, + { + "completion_length": 142.92500114440918, + "epoch": 0.16588906168999482, + "grad_norm": 33.05251693725586, + "kl": 0.8707275390625, + "learning_rate": 1.6551724137931035e-06, + "loss": 0.0677, + "reward": 1.2750000357627869, + "reward_std": 0.3371223881840706, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.925000011920929, + "step": 160 + }, + { + "completion_length": 154.0750026702881, + "epoch": 0.16692586832555728, + "grad_norm": 65645.4609375, + "kl": 342.103515625, + "learning_rate": 1.6655172413793104e-06, + "loss": 13.3002, + "reward": 1.5000000149011612, + "reward_std": 0.41643913090229034, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 161 + }, + { + "completion_length": 143.15000534057617, + "epoch": 0.16796267496111975, + "grad_norm": 230979.875, + "kl": 1004.058837890625, + "learning_rate": 1.6758620689655174e-06, + "loss": 41.8463, + "reward": 1.4500000476837158, + "reward_std": 0.3099784553050995, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9000000059604645, + "step": 162 + }, + { + "completion_length": 162.97500228881836, + "epoch": 0.1689994815966822, + "grad_norm": 104.41748809814453, + "kl": 0.559814453125, + "learning_rate": 1.6862068965517243e-06, + "loss": 0.0662, + "reward": 1.2750000208616257, + "reward_std": 0.45977721735835075, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.800000011920929, + "step": 163 + }, + { + "completion_length": 125.02500534057617, + "epoch": 0.17003628823224468, + "grad_norm": 422910.1875, + "kl": 7556.533203125, + "learning_rate": 1.696551724137931e-06, + "loss": 274.9914, + "reward": 1.4500000178813934, + "reward_std": 0.39302410185337067, + "rewards/accuracy_reward": 0.5500000081956387, + "rewards/format_reward": 0.9000000208616257, + "step": 164 + }, + { + "completion_length": 136.90000534057617, + "epoch": 0.17107309486780714, + "grad_norm": 39.6871452331543, + "kl": 0.236572265625, + "learning_rate": 1.706896551724138e-06, + "loss": 0.0168, + "reward": 1.324999988079071, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 165 + }, + { + "completion_length": 190.02500343322754, + "epoch": 0.17210990150336963, + "grad_norm": 2343.3095703125, + "kl": 3.414794921875, + "learning_rate": 1.717241379310345e-06, + "loss": 0.3358, + "reward": 1.4000000357627869, + "reward_std": 0.5968463532626629, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.8250000178813934, + "step": 166 + }, + { + "completion_length": 123.57500267028809, + "epoch": 0.1731467081389321, + "grad_norm": 1024.3455810546875, + "kl": 4.07666015625, + "learning_rate": 1.727586206896552e-06, + "loss": 0.2434, + "reward": 1.550000011920929, + "reward_std": 0.39665462449193, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.8500000238418579, + "step": 167 + }, + { + "completion_length": 127.15000343322754, + "epoch": 0.17418351477449456, + "grad_norm": 11.419363021850586, + "kl": 0.1678466796875, + "learning_rate": 1.7379310344827588e-06, + "loss": 0.0772, + "reward": 1.475000023841858, + "reward_std": 0.5659334138035774, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8500000238418579, + "step": 168 + }, + { + "completion_length": 158.8250026702881, + "epoch": 0.17522032141005703, + "grad_norm": 812.2886962890625, + "kl": 10.5908203125, + "learning_rate": 1.7482758620689658e-06, + "loss": 0.3363, + "reward": 1.7500000298023224, + "reward_std": 0.1928323283791542, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 169 + }, + { + "completion_length": 142.80000114440918, + "epoch": 0.1762571280456195, + "grad_norm": 114.91448974609375, + "kl": 0.28302001953125, + "learning_rate": 1.7586206896551723e-06, + "loss": 0.0374, + "reward": 1.5000000596046448, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 170 + }, + { + "completion_length": 175.60000038146973, + "epoch": 0.17729393468118196, + "grad_norm": 19.830774307250977, + "kl": 0.1810302734375, + "learning_rate": 1.7689655172413793e-06, + "loss": 0.0822, + "reward": 1.5250000059604645, + "reward_std": 0.324558787047863, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9750000089406967, + "step": 171 + }, + { + "completion_length": 159.2500057220459, + "epoch": 0.17833074131674442, + "grad_norm": 154360.703125, + "kl": 380.17138671875, + "learning_rate": 1.7793103448275862e-06, + "loss": 27.3536, + "reward": 1.4000000655651093, + "reward_std": 0.6183832511305809, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.8250000178813934, + "step": 172 + }, + { + "completion_length": 135.850004196167, + "epoch": 0.1793675479523069, + "grad_norm": 10.023521423339844, + "kl": 0.11395263671875, + "learning_rate": 1.789655172413793e-06, + "loss": 0.0159, + "reward": 1.4500000178813934, + "reward_std": 0.22894952446222305, + "rewards/accuracy_reward": 0.4749999977648258, + "rewards/format_reward": 0.9750000089406967, + "step": 173 + }, + { + "completion_length": 179.9250030517578, + "epoch": 0.18040435458786935, + "grad_norm": 25.44389533996582, + "kl": 0.44427490234375, + "learning_rate": 1.8e-06, + "loss": 0.0691, + "reward": 1.350000023841858, + "reward_std": 0.5138007514178753, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8000000268220901, + "step": 174 + }, + { + "completion_length": 157.3000030517578, + "epoch": 0.18144116122343182, + "grad_norm": 648.756103515625, + "kl": 1.504638671875, + "learning_rate": 1.8103448275862069e-06, + "loss": 0.1569, + "reward": 1.4500000476837158, + "reward_std": 0.5697024464607239, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.8750000298023224, + "step": 175 + }, + { + "completion_length": 140.4250030517578, + "epoch": 0.1824779678589943, + "grad_norm": 16.518726348876953, + "kl": 0.1947021484375, + "learning_rate": 1.8206896551724138e-06, + "loss": 0.0508, + "reward": 1.3499999940395355, + "reward_std": 0.30463574081659317, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 176 + }, + { + "completion_length": 168.4000015258789, + "epoch": 0.18351477449455678, + "grad_norm": 12.404158592224121, + "kl": 0.2431640625, + "learning_rate": 1.8310344827586208e-06, + "loss": -0.0125, + "reward": 1.4250000417232513, + "reward_std": 0.26129765436053276, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 1.0, + "step": 177 + }, + { + "completion_length": 164.92500114440918, + "epoch": 0.18455158113011924, + "grad_norm": 29.254444122314453, + "kl": 0.3502197265625, + "learning_rate": 1.8413793103448277e-06, + "loss": 0.1379, + "reward": 1.4000000357627869, + "reward_std": 0.5660719126462936, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.8750000149011612, + "step": 178 + }, + { + "completion_length": 149.7750015258789, + "epoch": 0.1855883877656817, + "grad_norm": 6.365128517150879, + "kl": 0.097900390625, + "learning_rate": 1.8517241379310345e-06, + "loss": 0.0811, + "reward": 1.2750000208616257, + "reward_std": 0.39302409440279007, + "rewards/accuracy_reward": 0.32500001043081284, + "rewards/format_reward": 0.9500000178813934, + "step": 179 + }, + { + "completion_length": 123.5, + "epoch": 0.18662519440124417, + "grad_norm": 19.429393768310547, + "kl": 0.1754150390625, + "learning_rate": 1.8620689655172414e-06, + "loss": 0.0367, + "reward": 1.4500000327825546, + "reward_std": 0.3804604858160019, + "rewards/accuracy_reward": 0.5500000063329935, + "rewards/format_reward": 0.9000000208616257, + "step": 180 + }, + { + "completion_length": 149.12500381469727, + "epoch": 0.18766200103680664, + "grad_norm": 14.26073932647705, + "kl": 0.134765625, + "learning_rate": 1.8724137931034484e-06, + "loss": 0.0374, + "reward": 1.5750000178813934, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 0.9750000089406967, + "step": 181 + }, + { + "completion_length": 144.2750072479248, + "epoch": 0.1886988076723691, + "grad_norm": 4.313653945922852, + "kl": 0.19671630859375, + "learning_rate": 1.8827586206896553e-06, + "loss": 0.0236, + "reward": 1.4000000208616257, + "reward_std": 0.28485122695565224, + "rewards/accuracy_reward": 0.4749999940395355, + "rewards/format_reward": 0.925000011920929, + "step": 182 + }, + { + "completion_length": 132.47500228881836, + "epoch": 0.18973561430793157, + "grad_norm": 4.134154796600342, + "kl": 0.1151123046875, + "learning_rate": 1.8931034482758623e-06, + "loss": 0.0525, + "reward": 1.4250000417232513, + "reward_std": 0.37310105189681053, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.8750000298023224, + "step": 183 + }, + { + "completion_length": 171.25000381469727, + "epoch": 0.19077242094349403, + "grad_norm": 7.449805736541748, + "kl": 0.124755859375, + "learning_rate": 1.9034482758620692e-06, + "loss": 0.0377, + "reward": 1.1250000447034836, + "reward_std": 0.4688488021492958, + "rewards/accuracy_reward": 0.2750000022351742, + "rewards/format_reward": 0.8500000238418579, + "step": 184 + }, + { + "completion_length": 134.2750015258789, + "epoch": 0.1918092275790565, + "grad_norm": 4.306768417358398, + "kl": 0.11016845703125, + "learning_rate": 1.913793103448276e-06, + "loss": 0.0353, + "reward": 1.4000000059604645, + "reward_std": 0.36053743958473206, + "rewards/accuracy_reward": 0.4499999973922968, + "rewards/format_reward": 0.9500000178813934, + "step": 185 + }, + { + "completion_length": 162.60000610351562, + "epoch": 0.19284603421461896, + "grad_norm": 4.144135475158691, + "kl": 0.0718994140625, + "learning_rate": 1.9241379310344827e-06, + "loss": 0.0482, + "reward": 1.4250000417232513, + "reward_std": 0.31719936802983284, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 186 + }, + { + "completion_length": 168.5, + "epoch": 0.19388284085018145, + "grad_norm": 2.6224422454833984, + "kl": 0.1402587890625, + "learning_rate": 1.9344827586206894e-06, + "loss": -0.0576, + "reward": 1.4500000476837158, + "reward_std": 0.5263643860816956, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.8750000149011612, + "step": 187 + }, + { + "completion_length": 155.87500190734863, + "epoch": 0.19491964748574392, + "grad_norm": 2.5991694927215576, + "kl": 0.1173095703125, + "learning_rate": 1.9448275862068966e-06, + "loss": 0.0525, + "reward": 1.425000011920929, + "reward_std": 0.4129471443593502, + "rewards/accuracy_reward": 0.5250000040978193, + "rewards/format_reward": 0.9000000357627869, + "step": 188 + }, + { + "completion_length": 157.60000228881836, + "epoch": 0.19595645412130638, + "grad_norm": 11.603816986083984, + "kl": 0.08740234375, + "learning_rate": 1.9551724137931033e-06, + "loss": 0.0086, + "reward": 1.5999999940395355, + "reward_std": 0.4525563307106495, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.8750000149011612, + "step": 189 + }, + { + "completion_length": 163.6750030517578, + "epoch": 0.19699326075686885, + "grad_norm": 2.4964864253997803, + "kl": 0.08258056640625, + "learning_rate": 1.9655172413793105e-06, + "loss": 0.007, + "reward": 1.5250000357627869, + "reward_std": 0.329762976616621, + "rewards/accuracy_reward": 0.5500000063329935, + "rewards/format_reward": 0.9750000089406967, + "step": 190 + }, + { + "completion_length": 163.37500381469727, + "epoch": 0.1980300673924313, + "grad_norm": 2.807250499725342, + "kl": 0.0880126953125, + "learning_rate": 1.9758620689655172e-06, + "loss": -0.0182, + "reward": 1.3000000417232513, + "reward_std": 0.4849044606089592, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.8499999940395355, + "step": 191 + }, + { + "completion_length": 113.47500228881836, + "epoch": 0.19906687402799378, + "grad_norm": 0.9696298241615295, + "kl": 0.1170654296875, + "learning_rate": 1.986206896551724e-06, + "loss": -0.0394, + "reward": 1.725000023841858, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 1.0, + "step": 192 + }, + { + "completion_length": 139.8250026702881, + "epoch": 0.20010368066355624, + "grad_norm": 1.4813657999038696, + "kl": 0.0897216796875, + "learning_rate": 1.996551724137931e-06, + "loss": 0.0159, + "reward": 1.3750000298023224, + "reward_std": 0.41643912717700005, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9250000268220901, + "step": 193 + }, + { + "completion_length": 186.35000228881836, + "epoch": 0.2011404872991187, + "grad_norm": 2.646305561065674, + "kl": 0.1492919921875, + "learning_rate": 2.006896551724138e-06, + "loss": 0.0906, + "reward": 1.5250000357627869, + "reward_std": 0.40387552604079247, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9000000357627869, + "step": 194 + }, + { + "completion_length": 178.600004196167, + "epoch": 0.20217729393468117, + "grad_norm": 3.9357028007507324, + "kl": 0.1038818359375, + "learning_rate": 2.017241379310345e-06, + "loss": 0.013, + "reward": 1.2750000059604645, + "reward_std": 0.3966546133160591, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.8750000149011612, + "step": 195 + }, + { + "completion_length": 175.0500030517578, + "epoch": 0.20321410057024364, + "grad_norm": 6.768120288848877, + "kl": 0.12518310546875, + "learning_rate": 2.0275862068965518e-06, + "loss": 0.0436, + "reward": 1.4749999940395355, + "reward_std": 0.472340852022171, + "rewards/accuracy_reward": 0.6250000223517418, + "rewards/format_reward": 0.8499999940395355, + "step": 196 + }, + { + "completion_length": 145.22500038146973, + "epoch": 0.2042509072058061, + "grad_norm": 2.82489275932312, + "kl": 0.0924072265625, + "learning_rate": 2.037931034482759e-06, + "loss": 0.0287, + "reward": 1.475000023841858, + "reward_std": 0.4290027730166912, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.8750000298023224, + "step": 197 + }, + { + "completion_length": 186.82500076293945, + "epoch": 0.2052877138413686, + "grad_norm": 6.81854772567749, + "kl": 0.0621337890625, + "learning_rate": 2.0482758620689657e-06, + "loss": -0.1108, + "reward": 1.225000038743019, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.32500001043081284, + "rewards/format_reward": 0.9000000059604645, + "step": 198 + }, + { + "completion_length": 160.30000114440918, + "epoch": 0.20632452047693106, + "grad_norm": 3.4426076412200928, + "kl": 0.06744384765625, + "learning_rate": 2.0586206896551724e-06, + "loss": 0.0542, + "reward": 1.1750000268220901, + "reward_std": 0.4614894203841686, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.8250000178813934, + "step": 199 + }, + { + "completion_length": 190.45000457763672, + "epoch": 0.20736132711249353, + "grad_norm": 2.7901272773742676, + "kl": 0.095458984375, + "learning_rate": 2.0689655172413796e-06, + "loss": 0.0244, + "reward": 1.5000000298023224, + "reward_std": 0.5173911228775978, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 200 + }, + { + "completion_length": 175.1500015258789, + "epoch": 0.208398133748056, + "grad_norm": 5.208185195922852, + "kl": 0.0733642578125, + "learning_rate": 2.0793103448275863e-06, + "loss": 0.0926, + "reward": 1.449999988079071, + "reward_std": 0.4525563232600689, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 201 + }, + { + "completion_length": 162.62500190734863, + "epoch": 0.20943494038361846, + "grad_norm": 1.4480936527252197, + "kl": 0.1016845703125, + "learning_rate": 2.089655172413793e-06, + "loss": 0.0509, + "reward": 1.4000000059604645, + "reward_std": 0.45977722853422165, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 202 + }, + { + "completion_length": 164.6750030517578, + "epoch": 0.21047174701918092, + "grad_norm": 4.689480304718018, + "kl": 0.0806884765625, + "learning_rate": 2.1e-06, + "loss": 0.0337, + "reward": 1.425000011920929, + "reward_std": 0.37310105934739113, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.8500000238418579, + "step": 203 + }, + { + "completion_length": 165.60000228881836, + "epoch": 0.2115085536547434, + "grad_norm": 5.075602054595947, + "kl": 0.078369140625, + "learning_rate": 2.110344827586207e-06, + "loss": 0.0524, + "reward": 1.3250000327825546, + "reward_std": 0.3407529257237911, + "rewards/accuracy_reward": 0.4500000011175871, + "rewards/format_reward": 0.875, + "step": 204 + }, + { + "completion_length": 155.70000457763672, + "epoch": 0.21254536029030585, + "grad_norm": 1.853857398033142, + "kl": 0.09381103515625, + "learning_rate": 2.1206896551724137e-06, + "loss": -0.0339, + "reward": 1.2250000089406967, + "reward_std": 0.5588510297238827, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.8250000029802322, + "step": 205 + }, + { + "completion_length": 183.0250015258789, + "epoch": 0.21358216692586832, + "grad_norm": 0.6376509666442871, + "kl": 0.068603515625, + "learning_rate": 2.131034482758621e-06, + "loss": 0.082, + "reward": 1.3750000149011612, + "reward_std": 0.36067599430680275, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.8000000044703484, + "step": 206 + }, + { + "completion_length": 186.22500228881836, + "epoch": 0.21461897356143078, + "grad_norm": 0.9813095927238464, + "kl": 0.0640869140625, + "learning_rate": 2.1413793103448276e-06, + "loss": 0.0382, + "reward": 1.3000000417232513, + "reward_std": 0.45413000881671906, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8750000149011612, + "step": 207 + }, + { + "completion_length": 151.05000114440918, + "epoch": 0.21565578019699327, + "grad_norm": 0.8648532032966614, + "kl": 0.0977783203125, + "learning_rate": 2.1517241379310343e-06, + "loss": 0.0119, + "reward": 1.4749999940395355, + "reward_std": 0.3804604560136795, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.925000011920929, + "step": 208 + }, + { + "completion_length": 152.5250015258789, + "epoch": 0.21669258683255574, + "grad_norm": 0.7379907965660095, + "kl": 0.09222412109375, + "learning_rate": 2.1620689655172415e-06, + "loss": 0.0774, + "reward": 1.300000011920929, + "reward_std": 0.44892578572034836, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9250000268220901, + "step": 209 + }, + { + "completion_length": 143.6500015258789, + "epoch": 0.2177293934681182, + "grad_norm": 0.6392645835876465, + "kl": 0.07421875, + "learning_rate": 2.1724137931034482e-06, + "loss": 0.0565, + "reward": 1.4249999821186066, + "reward_std": 0.23617041110992432, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 1.0, + "step": 210 + }, + { + "completion_length": 156.05000495910645, + "epoch": 0.21876620010368067, + "grad_norm": 0.8323525786399841, + "kl": 0.0712890625, + "learning_rate": 2.1827586206896554e-06, + "loss": -0.0088, + "reward": 1.5250000357627869, + "reward_std": 0.5156789347529411, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 211 + }, + { + "completion_length": 198.5500030517578, + "epoch": 0.21980300673924313, + "grad_norm": 0.6546644568443298, + "kl": 0.07940673828125, + "learning_rate": 2.193103448275862e-06, + "loss": 0.0794, + "reward": 1.5750000178813934, + "reward_std": 0.42178186774253845, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.925000011920929, + "step": 212 + }, + { + "completion_length": 146.5000057220459, + "epoch": 0.2208398133748056, + "grad_norm": 0.8499487042427063, + "kl": 0.11083984375, + "learning_rate": 2.203448275862069e-06, + "loss": 0.0293, + "reward": 1.2999999970197678, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9250000268220901, + "step": 213 + }, + { + "completion_length": 145.57500076293945, + "epoch": 0.22187662001036806, + "grad_norm": 0.687872588634491, + "kl": 0.07757568359375, + "learning_rate": 2.213793103448276e-06, + "loss": 0.0198, + "reward": 1.3000000417232513, + "reward_std": 0.3856646753847599, + "rewards/accuracy_reward": 0.4500000048428774, + "rewards/format_reward": 0.8500000089406967, + "step": 214 + }, + { + "completion_length": 133.89999961853027, + "epoch": 0.22291342664593053, + "grad_norm": 0.5972610116004944, + "kl": 0.09869384765625, + "learning_rate": 2.2241379310344828e-06, + "loss": -0.0411, + "reward": 1.6250000298023224, + "reward_std": 0.1928323432803154, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 215 + }, + { + "completion_length": 121.60000419616699, + "epoch": 0.223950233281493, + "grad_norm": 0.7216851711273193, + "kl": 0.14794921875, + "learning_rate": 2.23448275862069e-06, + "loss": -0.0187, + "reward": 1.5750000178813934, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 216 + }, + { + "completion_length": 178.8499984741211, + "epoch": 0.22498703991705546, + "grad_norm": 0.6720969676971436, + "kl": 0.113037109375, + "learning_rate": 2.2448275862068967e-06, + "loss": -0.0698, + "reward": 1.2500000447034836, + "reward_std": 0.40558772161602974, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.8750000149011612, + "step": 217 + }, + { + "completion_length": 189.72500228881836, + "epoch": 0.22602384655261792, + "grad_norm": 0.719994068145752, + "kl": 0.1602783203125, + "learning_rate": 2.2551724137931034e-06, + "loss": 0.0713, + "reward": 1.4250000417232513, + "reward_std": 0.45413000136613846, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.925000011920929, + "step": 218 + }, + { + "completion_length": 133.8250026702881, + "epoch": 0.22706065318818042, + "grad_norm": 0.4400331377983093, + "kl": 0.1085205078125, + "learning_rate": 2.26551724137931e-06, + "loss": 0.0333, + "reward": 1.5250000208616257, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 219 + }, + { + "completion_length": 154.80000686645508, + "epoch": 0.22809745982374288, + "grad_norm": 0.6450486779212952, + "kl": 0.131103515625, + "learning_rate": 2.2758620689655173e-06, + "loss": 0.0126, + "reward": 1.3250000327825546, + "reward_std": 0.31719936430454254, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 220 + }, + { + "completion_length": 145.72500610351562, + "epoch": 0.22913426645930535, + "grad_norm": 1.3167037963867188, + "kl": 0.135986328125, + "learning_rate": 2.286206896551724e-06, + "loss": 0.0465, + "reward": 1.8250000178813934, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.9750000089406967, + "rewards/format_reward": 0.8500000238418579, + "step": 221 + }, + { + "completion_length": 172.0500030517578, + "epoch": 0.2301710730948678, + "grad_norm": 0.7341930866241455, + "kl": 0.148193359375, + "learning_rate": 2.2965517241379312e-06, + "loss": 0.0681, + "reward": 1.0500000417232513, + "reward_std": 0.393024068325758, + "rewards/accuracy_reward": 0.22500000149011612, + "rewards/format_reward": 0.8250000178813934, + "step": 222 + }, + { + "completion_length": 146.42500495910645, + "epoch": 0.23120787973043028, + "grad_norm": 0.8762367367744446, + "kl": 0.123046875, + "learning_rate": 2.306896551724138e-06, + "loss": 0.1031, + "reward": 1.550000011920929, + "reward_std": 0.5210216194391251, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 223 + }, + { + "completion_length": 157.7750072479248, + "epoch": 0.23224468636599274, + "grad_norm": 0.8583253026008606, + "kl": 0.13037109375, + "learning_rate": 2.3172413793103447e-06, + "loss": 0.1291, + "reward": 1.6250000298023224, + "reward_std": 0.41643912717700005, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 0.875, + "step": 224 + }, + { + "completion_length": 146.5500030517578, + "epoch": 0.2332814930015552, + "grad_norm": 0.8889981508255005, + "kl": 0.113525390625, + "learning_rate": 2.327586206896552e-06, + "loss": -0.024, + "reward": 1.5999999940395355, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.6250000223517418, + "rewards/format_reward": 0.9750000089406967, + "step": 225 + }, + { + "completion_length": 137.5500030517578, + "epoch": 0.23431829963711767, + "grad_norm": 0.7300592064857483, + "kl": 0.1185302734375, + "learning_rate": 2.3379310344827586e-06, + "loss": 0.0438, + "reward": 1.3499999940395355, + "reward_std": 0.46511995419859886, + "rewards/accuracy_reward": 0.4000000096857548, + "rewards/format_reward": 0.9500000029802322, + "step": 226 + }, + { + "completion_length": 119.70000457763672, + "epoch": 0.23535510627268014, + "grad_norm": 0.727929413318634, + "kl": 0.137451171875, + "learning_rate": 2.3482758620689658e-06, + "loss": 0.0458, + "reward": 1.350000023841858, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.3999999985098839, + "rewards/format_reward": 0.9500000178813934, + "step": 227 + }, + { + "completion_length": 147.70000076293945, + "epoch": 0.2363919129082426, + "grad_norm": 0.9192894101142883, + "kl": 0.139404296875, + "learning_rate": 2.3586206896551725e-06, + "loss": 0.0975, + "reward": 1.350000023841858, + "reward_std": 0.39302409067749977, + "rewards/accuracy_reward": 0.4250000063329935, + "rewards/format_reward": 0.9250000268220901, + "step": 228 + }, + { + "completion_length": 156.4000015258789, + "epoch": 0.23742871954380507, + "grad_norm": 0.7950097918510437, + "kl": 0.1527099609375, + "learning_rate": 2.3689655172413793e-06, + "loss": 0.0755, + "reward": 1.600000023841858, + "reward_std": 0.3245587833225727, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 229 + }, + { + "completion_length": 109.77500343322754, + "epoch": 0.23846552617936756, + "grad_norm": 0.933161735534668, + "kl": 0.142333984375, + "learning_rate": 2.3793103448275864e-06, + "loss": 0.0125, + "reward": 1.449999988079071, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 230 + }, + { + "completion_length": 125.25, + "epoch": 0.23950233281493002, + "grad_norm": 0.715398371219635, + "kl": 0.1328125, + "learning_rate": 2.389655172413793e-06, + "loss": -0.0194, + "reward": 1.5500000417232513, + "reward_std": 0.3407529182732105, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 231 + }, + { + "completion_length": 106.65000343322754, + "epoch": 0.2405391394504925, + "grad_norm": 1.061570644378662, + "kl": 0.205810546875, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.0093, + "reward": 1.5250000059604645, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 1.0, + "step": 232 + }, + { + "completion_length": 107.42500305175781, + "epoch": 0.24157594608605495, + "grad_norm": 0.8809791207313538, + "kl": 0.1182861328125, + "learning_rate": 2.410344827586207e-06, + "loss": 0.0425, + "reward": 1.4250000417232513, + "reward_std": 0.40558769553899765, + "rewards/accuracy_reward": 0.44999999552965164, + "rewards/format_reward": 0.9750000089406967, + "step": 233 + }, + { + "completion_length": 118.6500015258789, + "epoch": 0.24261275272161742, + "grad_norm": 0.6223216652870178, + "kl": 0.1490478515625, + "learning_rate": 2.420689655172414e-06, + "loss": 0.1029, + "reward": 1.475000023841858, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.9750000089406967, + "step": 234 + }, + { + "completion_length": 116.7750015258789, + "epoch": 0.24364955935717988, + "grad_norm": 1.0359702110290527, + "kl": 0.21630859375, + "learning_rate": 2.4310344827586205e-06, + "loss": 0.1095, + "reward": 1.4500000178813934, + "reward_std": 0.5409446731209755, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9000000208616257, + "step": 235 + }, + { + "completion_length": 101.80000114440918, + "epoch": 0.24468636599274235, + "grad_norm": 0.6835160255432129, + "kl": 0.20654296875, + "learning_rate": 2.4413793103448277e-06, + "loss": 0.0014, + "reward": 1.4500000476837158, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 1.0, + "step": 236 + }, + { + "completion_length": 110.22500038146973, + "epoch": 0.24572317262830481, + "grad_norm": 0.50115966796875, + "kl": 0.1844482421875, + "learning_rate": 2.4517241379310344e-06, + "loss": 0.0505, + "reward": 1.7249999940395355, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 237 + }, + { + "completion_length": 113.75000190734863, + "epoch": 0.24675997926386728, + "grad_norm": 7.211286544799805, + "kl": 0.42578125, + "learning_rate": 2.462068965517241e-06, + "loss": 0.0999, + "reward": 1.550000011920929, + "reward_std": 0.3966546207666397, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8750000298023224, + "step": 238 + }, + { + "completion_length": 112.39999961853027, + "epoch": 0.24779678589942974, + "grad_norm": 0.547843873500824, + "kl": 0.156494140625, + "learning_rate": 2.4724137931034483e-06, + "loss": -0.0177, + "reward": 1.2750000059604645, + "reward_std": 0.1928323470056057, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 1.0, + "step": 239 + }, + { + "completion_length": 111.02500343322754, + "epoch": 0.24883359253499224, + "grad_norm": 0.553968608379364, + "kl": 0.1614990234375, + "learning_rate": 2.482758620689655e-06, + "loss": -0.0008, + "reward": 1.5, + "reward_std": 0.20019177719950676, + "rewards/accuracy_reward": 0.5500000100582838, + "rewards/format_reward": 0.9500000178813934, + "step": 240 + }, + { + "completion_length": 101.9000015258789, + "epoch": 0.2498703991705547, + "grad_norm": 0.6699236631393433, + "kl": 0.17431640625, + "learning_rate": 2.4931034482758622e-06, + "loss": 0.0162, + "reward": 1.3500000089406967, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.37500000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 241 + }, + { + "completion_length": 114.30000114440918, + "epoch": 0.25090720580611714, + "grad_norm": 0.8109186291694641, + "kl": 0.145751953125, + "learning_rate": 2.503448275862069e-06, + "loss": -0.009, + "reward": 1.475000023841858, + "reward_std": 0.3371223919093609, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 242 + }, + { + "completion_length": 150.85000228881836, + "epoch": 0.25194401244167963, + "grad_norm": 0.7055056691169739, + "kl": 0.14892578125, + "learning_rate": 2.513793103448276e-06, + "loss": 0.0376, + "reward": 1.2500000298023224, + "reward_std": 0.3930240757763386, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.925000011920929, + "step": 243 + }, + { + "completion_length": 116.4000015258789, + "epoch": 0.25298081907724207, + "grad_norm": 0.8164229393005371, + "kl": 0.1978759765625, + "learning_rate": 2.524137931034483e-06, + "loss": -0.0283, + "reward": 1.5500000417232513, + "reward_std": 0.36053745076060295, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 244 + }, + { + "completion_length": 149.54999923706055, + "epoch": 0.25401762571280456, + "grad_norm": 0.702789843082428, + "kl": 0.1356201171875, + "learning_rate": 2.5344827586206896e-06, + "loss": 0.0248, + "reward": 1.4500000476837158, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 245 + }, + { + "completion_length": 139.87500381469727, + "epoch": 0.25505443234836706, + "grad_norm": 0.8374885320663452, + "kl": 0.1865234375, + "learning_rate": 2.5448275862068968e-06, + "loss": 0.0117, + "reward": 1.3750000298023224, + "reward_std": 0.31719936057925224, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 246 + }, + { + "completion_length": 154.17500686645508, + "epoch": 0.2560912389839295, + "grad_norm": 0.5410239696502686, + "kl": 0.1728515625, + "learning_rate": 2.5551724137931035e-06, + "loss": 0.0571, + "reward": 1.5750000178813934, + "reward_std": 0.26143617182970047, + "rewards/accuracy_reward": 0.6249999944120646, + "rewards/format_reward": 0.9500000178813934, + "step": 247 + }, + { + "completion_length": 132.79999923706055, + "epoch": 0.257128045619492, + "grad_norm": 0.733527660369873, + "kl": 0.1473388671875, + "learning_rate": 2.5655172413793107e-06, + "loss": 0.0043, + "reward": 1.5999999940395355, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 248 + }, + { + "completion_length": 250.8000030517578, + "epoch": 0.2581648522550544, + "grad_norm": 0.6939677596092224, + "kl": 0.1351318359375, + "learning_rate": 2.5758620689655174e-06, + "loss": 0.0938, + "reward": 1.5750000476837158, + "reward_std": 0.3856646791100502, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.8750000149011612, + "step": 249 + }, + { + "completion_length": 151.17500495910645, + "epoch": 0.2592016588906169, + "grad_norm": 0.3936190605163574, + "kl": 0.166015625, + "learning_rate": 2.586206896551724e-06, + "loss": 0.0012, + "reward": 1.5999999940395355, + "reward_std": 0.16770508512854576, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9750000089406967, + "step": 250 + }, + { + "completion_length": 235.57499885559082, + "epoch": 0.26023846552617935, + "grad_norm": 1.5798455476760864, + "kl": 0.26806640625, + "learning_rate": 2.596551724137931e-06, + "loss": 0.2016, + "reward": 1.4000000357627869, + "reward_std": 0.6146142035722733, + "rewards/accuracy_reward": 0.5749999955296516, + "rewards/format_reward": 0.8250000029802322, + "step": 251 + }, + { + "completion_length": 187.70000457763672, + "epoch": 0.26127527216174184, + "grad_norm": 0.5298973917961121, + "kl": 0.16845703125, + "learning_rate": 2.606896551724138e-06, + "loss": 0.0335, + "reward": 1.3250000029802322, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 252 + }, + { + "completion_length": 160.1750030517578, + "epoch": 0.2623120787973043, + "grad_norm": 0.6976402401924133, + "kl": 0.157958984375, + "learning_rate": 2.617241379310345e-06, + "loss": 0.0418, + "reward": 1.5750000178813934, + "reward_std": 0.5587124899029732, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 253 + }, + { + "completion_length": 284.1750144958496, + "epoch": 0.2633488854328668, + "grad_norm": 0.6427121758460999, + "kl": 0.1856689453125, + "learning_rate": 2.6275862068965515e-06, + "loss": 0.1826, + "reward": 1.350000023841858, + "reward_std": 0.4850429743528366, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.8750000298023224, + "step": 254 + }, + { + "completion_length": 304.4750061035156, + "epoch": 0.2643856920684292, + "grad_norm": 3.0576741695404053, + "kl": 0.264404296875, + "learning_rate": 2.6379310344827587e-06, + "loss": 0.1303, + "reward": 1.225000038743019, + "reward_std": 0.6094100177288055, + "rewards/accuracy_reward": 0.4249999988824129, + "rewards/format_reward": 0.800000011920929, + "step": 255 + }, + { + "completion_length": 373.6000099182129, + "epoch": 0.2654224987039917, + "grad_norm": 0.5253204107284546, + "kl": 0.1571044921875, + "learning_rate": 2.6482758620689654e-06, + "loss": 0.1922, + "reward": 0.9750000089406967, + "reward_std": 0.47768357396125793, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 0.6750000044703484, + "step": 256 + }, + { + "completion_length": 355.32501220703125, + "epoch": 0.2664593053395542, + "grad_norm": 0.9178584218025208, + "kl": 0.2265625, + "learning_rate": 2.6586206896551726e-06, + "loss": 0.3166, + "reward": 1.0000000298023224, + "reward_std": 0.7397527396678925, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.6250000149011612, + "step": 257 + }, + { + "completion_length": 237.27500534057617, + "epoch": 0.26749611197511663, + "grad_norm": 0.6554264426231384, + "kl": 0.17626953125, + "learning_rate": 2.6689655172413793e-06, + "loss": 0.1707, + "reward": 1.0750000327825546, + "reward_std": 0.6130405068397522, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.7000000178813934, + "step": 258 + }, + { + "completion_length": 530.1000061035156, + "epoch": 0.2685329186106791, + "grad_norm": 0.6588838696479797, + "kl": 0.1446533203125, + "learning_rate": 2.6793103448275865e-06, + "loss": 0.1994, + "reward": 1.0000000149011612, + "reward_std": 0.6033165566623211, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.5750000104308128, + "step": 259 + }, + { + "completion_length": 372.25000762939453, + "epoch": 0.26956972524624156, + "grad_norm": 0.8617604970932007, + "kl": 0.279052734375, + "learning_rate": 2.6896551724137932e-06, + "loss": 0.2631, + "reward": 1.1500000059604645, + "reward_std": 0.7571920454502106, + "rewards/accuracy_reward": 0.5250000022351742, + "rewards/format_reward": 0.625, + "step": 260 + }, + { + "completion_length": 370.0750045776367, + "epoch": 0.27060653188180406, + "grad_norm": 0.8590409755706787, + "kl": 0.257080078125, + "learning_rate": 2.7e-06, + "loss": 0.1892, + "reward": 1.2000000476837158, + "reward_std": 0.7344100549817085, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.800000011920929, + "step": 261 + }, + { + "completion_length": 197.8250026702881, + "epoch": 0.2716433385173665, + "grad_norm": 0.921504020690918, + "kl": 0.262939453125, + "learning_rate": 2.710344827586207e-06, + "loss": 0.2638, + "reward": 1.4500000178813934, + "reward_std": 0.4723408371210098, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.8500000089406967, + "step": 262 + }, + { + "completion_length": 230.94999885559082, + "epoch": 0.272680145152929, + "grad_norm": 0.7630376219749451, + "kl": 0.238525390625, + "learning_rate": 2.720689655172414e-06, + "loss": 0.1966, + "reward": 1.300000011920929, + "reward_std": 0.6491576656699181, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.7750000059604645, + "step": 263 + }, + { + "completion_length": 208.65000343322754, + "epoch": 0.2737169517884914, + "grad_norm": 0.9253503680229187, + "kl": 0.29345703125, + "learning_rate": 2.731034482758621e-06, + "loss": 0.2176, + "reward": 1.1250000447034836, + "reward_std": 0.5414391383528709, + "rewards/accuracy_reward": 0.3250000011175871, + "rewards/format_reward": 0.7999999970197678, + "step": 264 + }, + { + "completion_length": 141.70000457763672, + "epoch": 0.2747537584240539, + "grad_norm": 0.9654400944709778, + "kl": 0.34521484375, + "learning_rate": 2.7413793103448278e-06, + "loss": 0.1135, + "reward": 1.1250000298023224, + "reward_std": 0.4170721210539341, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.8500000238418579, + "step": 265 + }, + { + "completion_length": 121.32500267028809, + "epoch": 0.27579056505961635, + "grad_norm": 1.2874776124954224, + "kl": 0.447509765625, + "learning_rate": 2.7517241379310345e-06, + "loss": 0.1512, + "reward": 1.3750000298023224, + "reward_std": 0.41643913462758064, + "rewards/accuracy_reward": 0.4500000048428774, + "rewards/format_reward": 0.925000011920929, + "step": 266 + }, + { + "completion_length": 124.97500324249268, + "epoch": 0.27682737169517885, + "grad_norm": 0.7988603711128235, + "kl": 0.448486328125, + "learning_rate": 2.7620689655172413e-06, + "loss": 0.1464, + "reward": 1.5249999910593033, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9500000029802322, + "step": 267 + }, + { + "completion_length": 43.82500076293945, + "epoch": 0.27786417833074134, + "grad_norm": 1.037258505821228, + "kl": 0.646484375, + "learning_rate": 2.7724137931034484e-06, + "loss": 0.0372, + "reward": 1.4500000178813934, + "reward_std": 0.2487340308725834, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 1.0, + "step": 268 + }, + { + "completion_length": 47.30000019073486, + "epoch": 0.2789009849663038, + "grad_norm": 0.7092971205711365, + "kl": 0.41455078125, + "learning_rate": 2.782758620689655e-06, + "loss": 0.0284, + "reward": 1.6500000059604645, + "reward_std": 0.24151314795017242, + "rewards/accuracy_reward": 0.6750000007450581, + "rewards/format_reward": 0.9750000089406967, + "step": 269 + }, + { + "completion_length": 94.62500286102295, + "epoch": 0.27993779160186627, + "grad_norm": 0.4653095304965973, + "kl": 0.7783203125, + "learning_rate": 2.793103448275862e-06, + "loss": 0.0589, + "reward": 1.425000011920929, + "reward_std": 0.15685367584228516, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9750000089406967, + "step": 270 + }, + { + "completion_length": 151.97500228881836, + "epoch": 0.2809745982374287, + "grad_norm": 0.9383624792098999, + "kl": 0.50537109375, + "learning_rate": 2.803448275862069e-06, + "loss": 0.0766, + "reward": 1.5500000417232513, + "reward_std": 0.32818931341171265, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 271 + }, + { + "completion_length": 193.1000051498413, + "epoch": 0.2820114048729912, + "grad_norm": 0.7510073781013489, + "kl": 0.46533203125, + "learning_rate": 2.813793103448276e-06, + "loss": 0.1247, + "reward": 1.4500000178813934, + "reward_std": 0.41657765209674835, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 272 + }, + { + "completion_length": 219.00000762939453, + "epoch": 0.28304821150855364, + "grad_norm": 0.8484097719192505, + "kl": 0.505859375, + "learning_rate": 2.824137931034483e-06, + "loss": 0.2461, + "reward": 1.475000023841858, + "reward_std": 0.4783165566623211, + "rewards/accuracy_reward": 0.5750000141561031, + "rewards/format_reward": 0.9000000059604645, + "step": 273 + }, + { + "completion_length": 54.650001525878906, + "epoch": 0.28408501814411613, + "grad_norm": 0.9514493346214294, + "kl": 0.64794921875, + "learning_rate": 2.8344827586206897e-06, + "loss": 0.041, + "reward": 1.300000011920929, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 274 + }, + { + "completion_length": 203.30000114440918, + "epoch": 0.28512182477967857, + "grad_norm": 0.680706262588501, + "kl": 0.5048828125, + "learning_rate": 2.8448275862068964e-06, + "loss": 0.1976, + "reward": 1.350000023841858, + "reward_std": 0.3611704185605049, + "rewards/accuracy_reward": 0.44999999925494194, + "rewards/format_reward": 0.9000000059604645, + "step": 275 + }, + { + "completion_length": 150.50000381469727, + "epoch": 0.28615863141524106, + "grad_norm": 0.5486094355583191, + "kl": 0.56982421875, + "learning_rate": 2.8551724137931036e-06, + "loss": 0.0903, + "reward": 1.4000000059604645, + "reward_std": 0.24936703965067863, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9500000029802322, + "step": 276 + }, + { + "completion_length": 193.65000534057617, + "epoch": 0.2871954380508035, + "grad_norm": 0.9081488847732544, + "kl": 0.53564453125, + "learning_rate": 2.8655172413793104e-06, + "loss": 0.0827, + "reward": 1.3000000417232513, + "reward_std": 0.3407529219985008, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.925000011920929, + "step": 277 + }, + { + "completion_length": 118.72500324249268, + "epoch": 0.288232244686366, + "grad_norm": 0.8595836758613586, + "kl": 0.46435546875, + "learning_rate": 2.8758620689655175e-06, + "loss": 0.096, + "reward": 1.300000011920929, + "reward_std": 0.3858031928539276, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9250000268220901, + "step": 278 + }, + { + "completion_length": 197.52500247955322, + "epoch": 0.2892690513219285, + "grad_norm": 0.6958838105201721, + "kl": 0.708984375, + "learning_rate": 2.8862068965517243e-06, + "loss": 0.1442, + "reward": 1.2000000029802322, + "reward_std": 0.26129765436053276, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.9000000059604645, + "step": 279 + }, + { + "completion_length": 150.2000026702881, + "epoch": 0.2903058579574909, + "grad_norm": 0.8232119679450989, + "kl": 0.640625, + "learning_rate": 2.8965517241379314e-06, + "loss": 0.136, + "reward": 1.350000023841858, + "reward_std": 0.335410188883543, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 280 + }, + { + "completion_length": 104.10000324249268, + "epoch": 0.2913426645930534, + "grad_norm": 0.7683313488960266, + "kl": 0.57568359375, + "learning_rate": 2.906896551724138e-06, + "loss": 0.084, + "reward": 1.2500000447034836, + "reward_std": 0.3046357296407223, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 281 + }, + { + "completion_length": 179.05000019073486, + "epoch": 0.29237947122861585, + "grad_norm": 0.7294011116027832, + "kl": 0.525390625, + "learning_rate": 2.917241379310345e-06, + "loss": 0.2052, + "reward": 1.4000000357627869, + "reward_std": 0.4237985461950302, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.925000011920929, + "step": 282 + }, + { + "completion_length": 99.7750015258789, + "epoch": 0.29341627786417834, + "grad_norm": 0.783625602722168, + "kl": 0.57958984375, + "learning_rate": 2.9275862068965516e-06, + "loss": 0.0685, + "reward": 1.425000011920929, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.4500000048428774, + "rewards/format_reward": 0.9750000089406967, + "step": 283 + }, + { + "completion_length": 150.60000228881836, + "epoch": 0.2944530844997408, + "grad_norm": 0.9992136359214783, + "kl": 0.68603515625, + "learning_rate": 2.9379310344827584e-06, + "loss": 0.1626, + "reward": 1.4500000029802322, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9500000029802322, + "step": 284 + }, + { + "completion_length": 100.15000057220459, + "epoch": 0.2954898911353033, + "grad_norm": 0.5232768654823303, + "kl": 0.70703125, + "learning_rate": 2.9482758620689655e-06, + "loss": 0.0727, + "reward": 1.2500000149011612, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 285 + }, + { + "completion_length": 146.7750015258789, + "epoch": 0.2965266977708657, + "grad_norm": 1.0567251443862915, + "kl": 0.83203125, + "learning_rate": 2.9586206896551723e-06, + "loss": 0.1205, + "reward": 1.225000023841858, + "reward_std": 0.37323957309126854, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 0.925000011920929, + "step": 286 + }, + { + "completion_length": 96.50000286102295, + "epoch": 0.2975635044064282, + "grad_norm": 0.7808656096458435, + "kl": 0.728515625, + "learning_rate": 2.9689655172413794e-06, + "loss": 0.0931, + "reward": 1.4000000357627869, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.4250000100582838, + "rewards/format_reward": 0.9750000089406967, + "step": 287 + }, + { + "completion_length": 196.0250015258789, + "epoch": 0.2986003110419907, + "grad_norm": 0.7538319230079651, + "kl": 0.6689453125, + "learning_rate": 2.979310344827586e-06, + "loss": 0.1413, + "reward": 1.4500000327825546, + "reward_std": 0.26129764690995216, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.925000011920929, + "step": 288 + }, + { + "completion_length": 144.70000076293945, + "epoch": 0.29963711767755313, + "grad_norm": 1.2029356956481934, + "kl": 1.0400390625, + "learning_rate": 2.9896551724137933e-06, + "loss": 0.1362, + "reward": 1.175000011920929, + "reward_std": 0.31199513748288155, + "rewards/accuracy_reward": 0.22500000707805157, + "rewards/format_reward": 0.9500000178813934, + "step": 289 + }, + { + "completion_length": 354.9250087738037, + "epoch": 0.3006739243131156, + "grad_norm": 0.8506849408149719, + "kl": 0.8056640625, + "learning_rate": 3e-06, + "loss": 0.3898, + "reward": 1.25, + "reward_std": 0.5210216455161572, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.800000011920929, + "step": 290 + }, + { + "completion_length": 223.7750062942505, + "epoch": 0.30171073094867806, + "grad_norm": 0.479114294052124, + "kl": 0.705078125, + "learning_rate": 2.999998906682319e-06, + "loss": 0.135, + "reward": 1.2750000357627869, + "reward_std": 0.22531899064779282, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.8750000149011612, + "step": 291 + }, + { + "completion_length": 140.07500171661377, + "epoch": 0.30274753758424056, + "grad_norm": 0.6931892037391663, + "kl": 0.734375, + "learning_rate": 2.9999956267308692e-06, + "loss": 0.1173, + "reward": 1.5000000149011612, + "reward_std": 0.36053743958473206, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.9500000178813934, + "step": 292 + }, + { + "completion_length": 172.12500286102295, + "epoch": 0.303784344219803, + "grad_norm": 0.528679609298706, + "kl": 0.580078125, + "learning_rate": 2.9999901601504326e-06, + "loss": 0.1734, + "reward": 1.2749999910593033, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.925000011920929, + "step": 293 + }, + { + "completion_length": 313.8000030517578, + "epoch": 0.3048211508553655, + "grad_norm": 0.8700326681137085, + "kl": 0.62646484375, + "learning_rate": 2.9999825069489775e-06, + "loss": 0.2371, + "reward": 1.0500000268220901, + "reward_std": 0.47768355160951614, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.8250000178813934, + "step": 294 + }, + { + "completion_length": 239.125, + "epoch": 0.3058579574909279, + "grad_norm": 0.5965785384178162, + "kl": 0.740234375, + "learning_rate": 2.999972667137661e-06, + "loss": 0.2412, + "reward": 0.9750000387430191, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.10000000149011612, + "rewards/format_reward": 0.8750000298023224, + "step": 295 + }, + { + "completion_length": 161.44999980926514, + "epoch": 0.3068947641264904, + "grad_norm": 0.45877698063850403, + "kl": 0.638671875, + "learning_rate": 2.9999606407308272e-06, + "loss": 0.1932, + "reward": 1.3750000596046448, + "reward_std": 0.3913118876516819, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.9250000268220901, + "step": 296 + }, + { + "completion_length": 230.45000648498535, + "epoch": 0.30793157076205285, + "grad_norm": 0.774716854095459, + "kl": 0.755859375, + "learning_rate": 2.999946427746007e-06, + "loss": 0.2691, + "reward": 1.2750000357627869, + "reward_std": 0.472340852022171, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.8750000149011612, + "step": 297 + }, + { + "completion_length": 90.85000038146973, + "epoch": 0.30896837739761535, + "grad_norm": 0.41126278042793274, + "kl": 0.732421875, + "learning_rate": 2.99993002820392e-06, + "loss": 0.0062, + "reward": 1.1500000059604645, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.17500000447034836, + "rewards/format_reward": 0.9750000089406967, + "step": 298 + }, + { + "completion_length": 37.85000038146973, + "epoch": 0.31000518403317784, + "grad_norm": 1.6783168315887451, + "kl": 0.7119140625, + "learning_rate": 2.999911442128473e-06, + "loss": 0.0434, + "reward": 1.2500000298023224, + "reward_std": 0.1928323321044445, + "rewards/accuracy_reward": 0.24999999813735485, + "rewards/format_reward": 1.0, + "step": 299 + }, + { + "completion_length": 91.00000095367432, + "epoch": 0.3110419906687403, + "grad_norm": 0.8122076988220215, + "kl": 0.76318359375, + "learning_rate": 2.9998906695467587e-06, + "loss": 0.1001, + "reward": 1.475000023841858, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 300 + }, + { + "completion_length": 55.82500076293945, + "epoch": 0.31207879730430277, + "grad_norm": 0.7224457859992981, + "kl": 0.87890625, + "learning_rate": 2.9998677104890597e-06, + "loss": 0.1167, + "reward": 1.5750000476837158, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 0.9500000178813934, + "step": 301 + }, + { + "completion_length": 138.60000133514404, + "epoch": 0.3131156039398652, + "grad_norm": 0.742752730846405, + "kl": 0.798828125, + "learning_rate": 2.9998425649888448e-06, + "loss": 0.1063, + "reward": 1.0750000178813934, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.15000000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 302 + }, + { + "completion_length": 148.55000114440918, + "epoch": 0.3141524105754277, + "grad_norm": 1.1690900325775146, + "kl": 0.955078125, + "learning_rate": 2.9998152330827693e-06, + "loss": 0.1856, + "reward": 1.4000000357627869, + "reward_std": 0.4723408482968807, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.8750000149011612, + "step": 303 + }, + { + "completion_length": 225.50000381469727, + "epoch": 0.31518921721099014, + "grad_norm": 0.7336009740829468, + "kl": 0.8759765625, + "learning_rate": 2.9997857148106767e-06, + "loss": 0.2877, + "reward": 1.0750000476837158, + "reward_std": 0.3479737937450409, + "rewards/accuracy_reward": 0.22500000149011612, + "rewards/format_reward": 0.8500000238418579, + "step": 304 + }, + { + "completion_length": 199.55000400543213, + "epoch": 0.31622602384655263, + "grad_norm": 1.3255078792572021, + "kl": 1.025390625, + "learning_rate": 2.9997540102155977e-06, + "loss": 0.2364, + "reward": 1.3000000417232513, + "reward_std": 0.4723408408463001, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.8750000149011612, + "step": 305 + }, + { + "completion_length": 237.2249984741211, + "epoch": 0.31726283048211507, + "grad_norm": 0.7050570845603943, + "kl": 0.7607421875, + "learning_rate": 2.9997201193437498e-06, + "loss": 0.2609, + "reward": 1.3750000596046448, + "reward_std": 0.5481655970215797, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8250000178813934, + "step": 306 + }, + { + "completion_length": 232.12500858306885, + "epoch": 0.31829963711767756, + "grad_norm": 0.6234104037284851, + "kl": 0.64306640625, + "learning_rate": 2.9996840422445373e-06, + "loss": 0.2367, + "reward": 1.0250000208616257, + "reward_std": 0.3804604709148407, + "rewards/accuracy_reward": 0.15000000223517418, + "rewards/format_reward": 0.8750000149011612, + "step": 307 + }, + { + "completion_length": 286.7750015258789, + "epoch": 0.31933644375324, + "grad_norm": 0.5603132247924805, + "kl": 0.68310546875, + "learning_rate": 2.9996457789705527e-06, + "loss": 0.3039, + "reward": 1.1500000208616257, + "reward_std": 0.38409100100398064, + "rewards/accuracy_reward": 0.2750000022351742, + "rewards/format_reward": 0.8750000149011612, + "step": 308 + }, + { + "completion_length": 97.02500247955322, + "epoch": 0.3203732503888025, + "grad_norm": 0.44136732816696167, + "kl": 0.67578125, + "learning_rate": 2.9996053295775733e-06, + "loss": 0.0828, + "reward": 1.25, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 309 + }, + { + "completion_length": 253.27499961853027, + "epoch": 0.321410057024365, + "grad_norm": 0.6165421009063721, + "kl": 0.64892578125, + "learning_rate": 2.999562694124565e-06, + "loss": 0.2532, + "reward": 1.350000023841858, + "reward_std": 0.347973819822073, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.8750000298023224, + "step": 310 + }, + { + "completion_length": 67.55000019073486, + "epoch": 0.3224468636599274, + "grad_norm": 0.5345252752304077, + "kl": 0.552734375, + "learning_rate": 2.999517872673681e-06, + "loss": 0.0778, + "reward": 1.5250000208616257, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 311 + }, + { + "completion_length": 140.20000171661377, + "epoch": 0.3234836702954899, + "grad_norm": 1.1492671966552734, + "kl": 0.56787109375, + "learning_rate": 2.9994708652902586e-06, + "loss": 0.1224, + "reward": 1.2000000029802322, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.25, + "rewards/format_reward": 0.9500000178813934, + "step": 312 + }, + { + "completion_length": 95.85000228881836, + "epoch": 0.32452047693105235, + "grad_norm": 0.9709307551383972, + "kl": 0.71728515625, + "learning_rate": 2.999421672042824e-06, + "loss": 0.0839, + "reward": 1.449999988079071, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 313 + }, + { + "completion_length": 92.50000190734863, + "epoch": 0.32555728356661484, + "grad_norm": 0.989802896976471, + "kl": 0.6923828125, + "learning_rate": 2.999370293003089e-06, + "loss": 0.0873, + "reward": 1.4250000268220901, + "reward_std": 0.18026872351765633, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 314 + }, + { + "completion_length": 140.47500038146973, + "epoch": 0.3265940902021773, + "grad_norm": 0.6137426495552063, + "kl": 0.791015625, + "learning_rate": 2.9993167282459508e-06, + "loss": 0.1567, + "reward": 1.5750000178813934, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9500000178813934, + "step": 315 + }, + { + "completion_length": 105.0000057220459, + "epoch": 0.32763089683773977, + "grad_norm": 0.6631371974945068, + "kl": 0.58154296875, + "learning_rate": 2.9992609778494945e-06, + "loss": 0.0668, + "reward": 1.3250000178813934, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.9500000178813934, + "step": 316 + }, + { + "completion_length": 57.30000114440918, + "epoch": 0.3286677034733022, + "grad_norm": 0.542193591594696, + "kl": 0.53125, + "learning_rate": 2.999203041894991e-06, + "loss": 0.0198, + "reward": 1.475000023841858, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 1.0, + "step": 317 + }, + { + "completion_length": 110.09999942779541, + "epoch": 0.3297045101088647, + "grad_norm": 0.8033139705657959, + "kl": 0.7060546875, + "learning_rate": 2.999142920466896e-06, + "loss": 0.0931, + "reward": 1.3750000298023224, + "reward_std": 0.3354102112352848, + "rewards/accuracy_reward": 0.4250000026077032, + "rewards/format_reward": 0.9500000029802322, + "step": 318 + }, + { + "completion_length": 155.02500247955322, + "epoch": 0.33074131674442714, + "grad_norm": 0.7720692753791809, + "kl": 0.59521484375, + "learning_rate": 2.999080613652853e-06, + "loss": 0.102, + "reward": 1.600000023841858, + "reward_std": 0.25609345361590385, + "rewards/accuracy_reward": 0.6500000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 319 + }, + { + "completion_length": 125.42500495910645, + "epoch": 0.33177812337998963, + "grad_norm": 0.43371641635894775, + "kl": 0.7421875, + "learning_rate": 2.9990161215436888e-06, + "loss": 0.1467, + "reward": 1.0500000268220901, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.10000000335276127, + "rewards/format_reward": 0.9500000178813934, + "step": 320 + }, + { + "completion_length": 63.07500076293945, + "epoch": 0.3328149300155521, + "grad_norm": 1.1089128255844116, + "kl": 0.56591796875, + "learning_rate": 2.9989494442334186e-06, + "loss": 0.0028, + "reward": 1.375, + "reward_std": 0.27228759974241257, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.9750000089406967, + "step": 321 + }, + { + "completion_length": 69.57500171661377, + "epoch": 0.33385173665111456, + "grad_norm": 0.5004400610923767, + "kl": 0.697265625, + "learning_rate": 2.998880581819241e-06, + "loss": 0.0053, + "reward": 1.5250000357627869, + "reward_std": 0.13693063706159592, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 1.0, + "step": 322 + }, + { + "completion_length": 85.10000228881836, + "epoch": 0.33488854328667705, + "grad_norm": 0.7260141968727112, + "kl": 0.4267578125, + "learning_rate": 2.9988095344015403e-06, + "loss": 0.0783, + "reward": 1.5499999821186066, + "reward_std": 0.279508501291275, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 323 + }, + { + "completion_length": 179.37500381469727, + "epoch": 0.3359253499222395, + "grad_norm": 0.7410564422607422, + "kl": 0.70703125, + "learning_rate": 2.9987363020838867e-06, + "loss": 0.1431, + "reward": 1.25, + "reward_std": 0.3856646940112114, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.925000011920929, + "step": 324 + }, + { + "completion_length": 128.05000114440918, + "epoch": 0.336962156557802, + "grad_norm": 0.2647928297519684, + "kl": 0.49267578125, + "learning_rate": 2.9986608849730356e-06, + "loss": 0.0786, + "reward": 1.2500000298023224, + "reward_std": 0.18026870489120483, + "rewards/accuracy_reward": 0.2750000022351742, + "rewards/format_reward": 0.9750000089406967, + "step": 325 + }, + { + "completion_length": 139.60000228881836, + "epoch": 0.3379989631933644, + "grad_norm": 0.6439674496650696, + "kl": 0.62255859375, + "learning_rate": 2.998583283178926e-06, + "loss": 0.113, + "reward": 1.3750000298023224, + "reward_std": 0.31733787432312965, + "rewards/accuracy_reward": 0.4250000100582838, + "rewards/format_reward": 0.9500000178813934, + "step": 326 + }, + { + "completion_length": 122.85000038146973, + "epoch": 0.3390357698289269, + "grad_norm": 0.4776754677295685, + "kl": 0.53173828125, + "learning_rate": 2.9985034968146832e-06, + "loss": 0.0487, + "reward": 1.7000000178813934, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 327 + }, + { + "completion_length": 324.1000061035156, + "epoch": 0.34007257646448935, + "grad_norm": 0.7620232701301575, + "kl": 0.4736328125, + "learning_rate": 2.998421525996616e-06, + "loss": 0.2654, + "reward": 1.2750000357627869, + "reward_std": 0.5517961122095585, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 0.8750000298023224, + "step": 328 + }, + { + "completion_length": 468.9500198364258, + "epoch": 0.34110938310005184, + "grad_norm": 0.35654211044311523, + "kl": 0.5458984375, + "learning_rate": 2.9983373708442182e-06, + "loss": 0.3329, + "reward": 1.3500000089406967, + "reward_std": 0.4958944171667099, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.7750000059604645, + "step": 329 + }, + { + "completion_length": 370.1750030517578, + "epoch": 0.3421461897356143, + "grad_norm": 0.5573723316192627, + "kl": 0.564453125, + "learning_rate": 2.9982510314801665e-06, + "loss": 0.2937, + "reward": 1.2500000149011612, + "reward_std": 0.6527480855584145, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.8500000238418579, + "step": 330 + }, + { + "completion_length": 277.975004196167, + "epoch": 0.3431829963711768, + "grad_norm": 0.5659462213516235, + "kl": 0.5029296875, + "learning_rate": 2.998162508030324e-06, + "loss": 0.2449, + "reward": 1.5000000298023224, + "reward_std": 0.5031153149902821, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9000000208616257, + "step": 331 + }, + { + "completion_length": 175.72500610351562, + "epoch": 0.34421980300673927, + "grad_norm": 0.402981698513031, + "kl": 0.49267578125, + "learning_rate": 2.998071800623736e-06, + "loss": 0.132, + "reward": 1.550000011920929, + "reward_std": 0.15685366839170456, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 332 + }, + { + "completion_length": 442.42501068115234, + "epoch": 0.3452566096423017, + "grad_norm": 0.40613406896591187, + "kl": 0.59130859375, + "learning_rate": 2.997978909392631e-06, + "loss": 0.3856, + "reward": 1.3750000298023224, + "reward_std": 0.7138539552688599, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.7750000059604645, + "step": 333 + }, + { + "completion_length": 437.3250045776367, + "epoch": 0.3462934162778642, + "grad_norm": 0.4869440793991089, + "kl": 0.6865234375, + "learning_rate": 2.997883834472423e-06, + "loss": 0.2613, + "reward": 1.1000000089406967, + "reward_std": 0.596707858145237, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.7750000208616257, + "step": 334 + }, + { + "completion_length": 175.32500648498535, + "epoch": 0.34733022291342663, + "grad_norm": 0.45390427112579346, + "kl": 0.51025390625, + "learning_rate": 2.997786576001708e-06, + "loss": 0.1233, + "reward": 1.600000023841858, + "reward_std": 0.3245587684214115, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9500000178813934, + "step": 335 + }, + { + "completion_length": 263.725004196167, + "epoch": 0.3483670295489891, + "grad_norm": 0.42588937282562256, + "kl": 0.72509765625, + "learning_rate": 2.9976871341222644e-06, + "loss": 0.2371, + "reward": 1.3250000178813934, + "reward_std": 0.3913118802011013, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9000000208616257, + "step": 336 + }, + { + "completion_length": 228.55000686645508, + "epoch": 0.34940383618455156, + "grad_norm": 0.6435961127281189, + "kl": 0.6455078125, + "learning_rate": 2.997585508979055e-06, + "loss": 0.1453, + "reward": 1.425000011920929, + "reward_std": 0.3479738272726536, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 337 + }, + { + "completion_length": 381.4500026702881, + "epoch": 0.35044064282011406, + "grad_norm": 0.48273399472236633, + "kl": 0.6640625, + "learning_rate": 2.9974817007202243e-06, + "loss": 0.2484, + "reward": 1.350000038743019, + "reward_std": 0.48553743213415146, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.8250000029802322, + "step": 338 + }, + { + "completion_length": 181.50000381469727, + "epoch": 0.3514774494556765, + "grad_norm": 0.4054751694202423, + "kl": 0.6533203125, + "learning_rate": 2.9973757094970992e-06, + "loss": 0.1321, + "reward": 1.050000011920929, + "reward_std": 0.2812207043170929, + "rewards/accuracy_reward": 0.15000000223517418, + "rewards/format_reward": 0.9000000059604645, + "step": 339 + }, + { + "completion_length": 564.7500076293945, + "epoch": 0.352514256091239, + "grad_norm": 0.4397445321083069, + "kl": 0.755859375, + "learning_rate": 2.9972675354641895e-06, + "loss": 0.2461, + "reward": 1.0500000268220901, + "reward_std": 0.49746809527277946, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.7250000089406967, + "step": 340 + }, + { + "completion_length": 295.7500057220459, + "epoch": 0.3535510627268014, + "grad_norm": 0.6863256096839905, + "kl": 0.6005859375, + "learning_rate": 2.9971571787791866e-06, + "loss": 0.0904, + "reward": 1.550000011920929, + "reward_std": 0.5533697679638863, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 341 + }, + { + "completion_length": 529.0750045776367, + "epoch": 0.3545878693623639, + "grad_norm": 0.5725196003913879, + "kl": 0.5673828125, + "learning_rate": 2.9970446396029643e-06, + "loss": 0.3386, + "reward": 0.9500000178813934, + "reward_std": 0.5535083040595055, + "rewards/accuracy_reward": 0.1750000026077032, + "rewards/format_reward": 0.7750000208616257, + "step": 342 + }, + { + "completion_length": 258.2249984741211, + "epoch": 0.3556246759979264, + "grad_norm": 0.4096214771270752, + "kl": 0.51904296875, + "learning_rate": 2.9969299180995763e-06, + "loss": 0.2192, + "reward": 1.2750000059604645, + "reward_std": 0.3913118839263916, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.875, + "step": 343 + }, + { + "completion_length": 363.0500030517578, + "epoch": 0.35666148263348885, + "grad_norm": 0.6106461882591248, + "kl": 0.5458984375, + "learning_rate": 2.996813014436259e-06, + "loss": 0.2965, + "reward": 1.4000000357627869, + "reward_std": 0.6172638833522797, + "rewards/accuracy_reward": 0.5750000141561031, + "rewards/format_reward": 0.8250000178813934, + "step": 344 + }, + { + "completion_length": 123.2500057220459, + "epoch": 0.35769828926905134, + "grad_norm": 0.5532033443450928, + "kl": 0.46533203125, + "learning_rate": 2.9966939287834297e-06, + "loss": -0.0141, + "reward": 1.5, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 345 + }, + { + "completion_length": 330.5750045776367, + "epoch": 0.3587350959046138, + "grad_norm": 0.5643976330757141, + "kl": 0.4521484375, + "learning_rate": 2.9965726613146864e-06, + "loss": 0.2691, + "reward": 0.9000000208616257, + "reward_std": 0.5625799112021923, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/format_reward": 0.7750000059604645, + "step": 346 + }, + { + "completion_length": 296.6499996185303, + "epoch": 0.35977190254017627, + "grad_norm": 0.5962963104248047, + "kl": 0.5673828125, + "learning_rate": 2.9964492122068074e-06, + "loss": 0.1955, + "reward": 1.2750000357627869, + "reward_std": 0.5660719126462936, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.875, + "step": 347 + }, + { + "completion_length": 210.375, + "epoch": 0.3608087091757387, + "grad_norm": 0.4225082993507385, + "kl": 0.61572265625, + "learning_rate": 2.9963235816397516e-06, + "loss": 0.1628, + "reward": 1.350000023841858, + "reward_std": 0.29270509257912636, + "rewards/accuracy_reward": 0.4250000100582838, + "rewards/format_reward": 0.925000011920929, + "step": 348 + }, + { + "completion_length": 552.0000076293945, + "epoch": 0.3618455158113012, + "grad_norm": 0.7490342855453491, + "kl": 0.5478515625, + "learning_rate": 2.996195769796658e-06, + "loss": 0.2469, + "reward": 1.1250000447034836, + "reward_std": 0.5973408445715904, + "rewards/accuracy_reward": 0.3500000052154064, + "rewards/format_reward": 0.7750000208616257, + "step": 349 + }, + { + "completion_length": 236.82500839233398, + "epoch": 0.36288232244686364, + "grad_norm": 0.4927626848220825, + "kl": 0.4287109375, + "learning_rate": 2.996065776863845e-06, + "loss": 0.0864, + "reward": 1.475000023841858, + "reward_std": 0.3966546319425106, + "rewards/accuracy_reward": 0.5500000156462193, + "rewards/format_reward": 0.925000011920929, + "step": 350 + }, + { + "completion_length": 193.45000457763672, + "epoch": 0.36391912908242613, + "grad_norm": 0.5020635724067688, + "kl": 0.38720703125, + "learning_rate": 2.9959336030308104e-06, + "loss": 0.0558, + "reward": 1.2500000298023224, + "reward_std": 0.3299014940857887, + "rewards/accuracy_reward": 0.30000000819563866, + "rewards/format_reward": 0.9500000178813934, + "step": 351 + }, + { + "completion_length": 222.27500534057617, + "epoch": 0.3649559357179886, + "grad_norm": 0.4954478442668915, + "kl": 0.48486328125, + "learning_rate": 2.995799248490232e-06, + "loss": 0.1715, + "reward": 1.2000000178813934, + "reward_std": 0.36053742095828056, + "rewards/accuracy_reward": 0.27500000409781933, + "rewards/format_reward": 0.9250000268220901, + "step": 352 + }, + { + "completion_length": 213.42500686645508, + "epoch": 0.36599274235355106, + "grad_norm": 0.42243900895118713, + "kl": 0.34814453125, + "learning_rate": 2.995662713437966e-06, + "loss": 0.1177, + "reward": 1.5500000417232513, + "reward_std": 0.4290027320384979, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 353 + }, + { + "completion_length": 118.6500015258789, + "epoch": 0.36702954898911355, + "grad_norm": 0.5565142035484314, + "kl": 0.290283203125, + "learning_rate": 2.995523998073047e-06, + "loss": 0.0399, + "reward": 1.675000011920929, + "reward_std": 0.31719936057925224, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 1.0, + "step": 354 + }, + { + "completion_length": 167.67500495910645, + "epoch": 0.368066355624676, + "grad_norm": 0.425595223903656, + "kl": 0.333984375, + "learning_rate": 2.9953831025976885e-06, + "loss": 0.0722, + "reward": 1.5750000178813934, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.5999999940395355, + "rewards/format_reward": 0.9750000089406967, + "step": 355 + }, + { + "completion_length": 263.0250053405762, + "epoch": 0.3691031622602385, + "grad_norm": 0.34887924790382385, + "kl": 0.301025390625, + "learning_rate": 2.9952400272172815e-06, + "loss": 0.1951, + "reward": 1.4250000268220901, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 356 + }, + { + "completion_length": 161.350004196167, + "epoch": 0.3701399688958009, + "grad_norm": 0.5152186155319214, + "kl": 0.34130859375, + "learning_rate": 2.995094772140396e-06, + "loss": 0.0896, + "reward": 1.4000000357627869, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.4249999988824129, + "rewards/format_reward": 0.9750000089406967, + "step": 357 + }, + { + "completion_length": 126.57500076293945, + "epoch": 0.3711767755313634, + "grad_norm": 0.5196745991706848, + "kl": 0.33740234375, + "learning_rate": 2.9949473375787776e-06, + "loss": -0.0047, + "reward": 1.2750000059604645, + "reward_std": 0.16770508512854576, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 0.9750000089406967, + "step": 358 + }, + { + "completion_length": 160.05000686645508, + "epoch": 0.37221358216692585, + "grad_norm": 0.6606730222702026, + "kl": 0.4091796875, + "learning_rate": 2.994797723747351e-06, + "loss": 0.1742, + "reward": 1.5750000178813934, + "reward_std": 0.4363621734082699, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9500000178813934, + "step": 359 + }, + { + "completion_length": 119.82500267028809, + "epoch": 0.37325038880248834, + "grad_norm": 0.3371819257736206, + "kl": 0.364990234375, + "learning_rate": 2.9946459308642155e-06, + "loss": -0.0122, + "reward": 1.6250000298023224, + "reward_std": 0.13693063706159592, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 1.0, + "step": 360 + }, + { + "completion_length": 221.07500076293945, + "epoch": 0.3742871954380508, + "grad_norm": 0.536543607711792, + "kl": 0.4248046875, + "learning_rate": 2.9944919591506495e-06, + "loss": 0.1191, + "reward": 1.1500000208616257, + "reward_std": 0.25609344989061356, + "rewards/accuracy_reward": 0.20000000670552254, + "rewards/format_reward": 0.9500000029802322, + "step": 361 + }, + { + "completion_length": 203.3250026702881, + "epoch": 0.37532400207361327, + "grad_norm": 0.3392835855484009, + "kl": 0.365234375, + "learning_rate": 2.994335808831106e-06, + "loss": 0.0678, + "reward": 1.5750000178813934, + "reward_std": 0.19346532225608826, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000029802322, + "step": 362 + }, + { + "completion_length": 148.77499961853027, + "epoch": 0.37636080870917576, + "grad_norm": 0.15861853957176208, + "kl": 0.36328125, + "learning_rate": 2.994177480133214e-06, + "loss": 0.0677, + "reward": 1.3499999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9750000089406967, + "step": 363 + }, + { + "completion_length": 403.47501373291016, + "epoch": 0.3773976153447382, + "grad_norm": 0.5314847230911255, + "kl": 0.5068359375, + "learning_rate": 2.994016973287778e-06, + "loss": 0.2355, + "reward": 1.4000000059604645, + "reward_std": 0.5173911042511463, + "rewards/accuracy_reward": 0.5500000156462193, + "rewards/format_reward": 0.8500000238418579, + "step": 364 + }, + { + "completion_length": 160.77500534057617, + "epoch": 0.3784344219803007, + "grad_norm": 0.3340126574039459, + "kl": 0.288330078125, + "learning_rate": 2.9938542885287786e-06, + "loss": 0.0289, + "reward": 1.350000023841858, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 365 + }, + { + "completion_length": 215.65000534057617, + "epoch": 0.37947122861586313, + "grad_norm": 0.35049688816070557, + "kl": 0.376953125, + "learning_rate": 2.99368942609337e-06, + "loss": 0.1184, + "reward": 1.3000000268220901, + "reward_std": 0.22894952073693275, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.9500000178813934, + "step": 366 + }, + { + "completion_length": 192.12500381469727, + "epoch": 0.3805080352514256, + "grad_norm": 0.4274624288082123, + "kl": 0.3583984375, + "learning_rate": 2.993522386221882e-06, + "loss": -0.0248, + "reward": 1.5999999940395355, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9500000029802322, + "step": 367 + }, + { + "completion_length": 148.375, + "epoch": 0.38154484188698806, + "grad_norm": 0.6505801677703857, + "kl": 0.35107421875, + "learning_rate": 2.9933531691578178e-06, + "loss": 0.026, + "reward": 1.550000011920929, + "reward_std": 0.26865705102682114, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9750000089406967, + "step": 368 + }, + { + "completion_length": 293.8000030517578, + "epoch": 0.38258164852255055, + "grad_norm": 0.30225127935409546, + "kl": 0.44970703125, + "learning_rate": 2.9931817751478547e-06, + "loss": 0.2183, + "reward": 1.4250000268220901, + "reward_std": 0.25609344989061356, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 0.9000000059604645, + "step": 369 + }, + { + "completion_length": 151.32499885559082, + "epoch": 0.383618455158113, + "grad_norm": 0.5463283658027649, + "kl": 0.296875, + "learning_rate": 2.9930082044418442e-06, + "loss": 0.0759, + "reward": 1.350000023841858, + "reward_std": 0.24873404577374458, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 370 + }, + { + "completion_length": 120.22500228881836, + "epoch": 0.3846552617936755, + "grad_norm": 0.6592673063278198, + "kl": 0.37451171875, + "learning_rate": 2.99283245729281e-06, + "loss": 0.083, + "reward": 1.4000000357627869, + "reward_std": 0.37310105934739113, + "rewards/accuracy_reward": 0.4000000152736902, + "rewards/format_reward": 1.0, + "step": 371 + }, + { + "completion_length": 150.79999923706055, + "epoch": 0.3856920684292379, + "grad_norm": 0.5108554363250732, + "kl": 0.326171875, + "learning_rate": 2.9926545339569476e-06, + "loss": 0.0444, + "reward": 1.4500000178813934, + "reward_std": 0.30463575199246407, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 372 + }, + { + "completion_length": 87.57500267028809, + "epoch": 0.3867288750648004, + "grad_norm": 0.5240278840065002, + "kl": 0.3642578125, + "learning_rate": 2.992474434693627e-06, + "loss": -0.0313, + "reward": 1.600000023841858, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.6000000033527613, + "rewards/format_reward": 1.0, + "step": 373 + }, + { + "completion_length": 108.4000015258789, + "epoch": 0.3877656817003629, + "grad_norm": 0.46822282671928406, + "kl": 0.33447265625, + "learning_rate": 2.9922921597653893e-06, + "loss": -0.0006, + "reward": 1.5750000178813934, + "reward_std": 0.1928323470056057, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 1.0, + "step": 374 + }, + { + "completion_length": 137.70000457763672, + "epoch": 0.38880248833592534, + "grad_norm": 0.554831326007843, + "kl": 0.4365234375, + "learning_rate": 2.9921077094379466e-06, + "loss": 0.0776, + "reward": 1.375, + "reward_std": 0.2812207005918026, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.9750000089406967, + "step": 375 + }, + { + "completion_length": 239.45000839233398, + "epoch": 0.38983929497148784, + "grad_norm": 0.5239160060882568, + "kl": 0.412841796875, + "learning_rate": 2.9919210839801827e-06, + "loss": 0.1685, + "reward": 1.475000023841858, + "reward_std": 0.452556349337101, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.925000011920929, + "step": 376 + }, + { + "completion_length": 87.90000343322754, + "epoch": 0.3908761016070503, + "grad_norm": 0.6395469307899475, + "kl": 0.35498046875, + "learning_rate": 2.991732283664152e-06, + "loss": 0.0022, + "reward": 1.5250000357627869, + "reward_std": 0.3046357184648514, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 1.0, + "step": 377 + }, + { + "completion_length": 200.35000038146973, + "epoch": 0.39191290824261277, + "grad_norm": 0.6615502834320068, + "kl": 0.3955078125, + "learning_rate": 2.99154130876508e-06, + "loss": 0.1602, + "reward": 1.425000011920929, + "reward_std": 0.504966028034687, + "rewards/accuracy_reward": 0.5250000152736902, + "rewards/format_reward": 0.9000000357627869, + "step": 378 + }, + { + "completion_length": 243.87500190734863, + "epoch": 0.3929497148781752, + "grad_norm": 0.588063657283783, + "kl": 0.41064453125, + "learning_rate": 2.9913481595613613e-06, + "loss": 0.1196, + "reward": 1.2500000149011612, + "reward_std": 0.4181513413786888, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.9000000059604645, + "step": 379 + }, + { + "completion_length": 92.82500076293945, + "epoch": 0.3939865215137377, + "grad_norm": 0.3841590881347656, + "kl": 0.41552734375, + "learning_rate": 2.99115283633456e-06, + "loss": 0.022, + "reward": 1.7000000178813934, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.7000000067055225, + "rewards/format_reward": 1.0, + "step": 380 + }, + { + "completion_length": 167.8250026702881, + "epoch": 0.39502332814930013, + "grad_norm": 0.6007952094078064, + "kl": 0.5205078125, + "learning_rate": 2.9909553393694107e-06, + "loss": 0.1088, + "reward": 1.2999999970197678, + "reward_std": 0.3245587833225727, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9000000059604645, + "step": 381 + }, + { + "completion_length": 150.350004196167, + "epoch": 0.3960601347848626, + "grad_norm": 0.4360502064228058, + "kl": 0.29931640625, + "learning_rate": 2.9907556689538158e-06, + "loss": 0.0512, + "reward": 1.4500000178813934, + "reward_std": 0.22894952446222305, + "rewards/accuracy_reward": 0.4749999940395355, + "rewards/format_reward": 0.9750000089406967, + "step": 382 + }, + { + "completion_length": 145.3750057220459, + "epoch": 0.39709694142042506, + "grad_norm": 0.7903898358345032, + "kl": 0.355712890625, + "learning_rate": 2.990553825378846e-06, + "loss": 0.127, + "reward": 1.6250000298023224, + "reward_std": 0.41643914207816124, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9750000089406967, + "step": 383 + }, + { + "completion_length": 194.225004196167, + "epoch": 0.39813374805598756, + "grad_norm": 0.7693355083465576, + "kl": 0.4150390625, + "learning_rate": 2.99034980893874e-06, + "loss": 0.0364, + "reward": 1.7000000476837158, + "reward_std": 0.3479738086462021, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 384 + }, + { + "completion_length": 145.20000076293945, + "epoch": 0.39917055469155005, + "grad_norm": 0.5979889631271362, + "kl": 0.472900390625, + "learning_rate": 2.9901436199309044e-06, + "loss": 0.0716, + "reward": 1.7000000178813934, + "reward_std": 0.30463574081659317, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 385 + }, + { + "completion_length": 244.1250114440918, + "epoch": 0.4002073613271125, + "grad_norm": 0.4943735897541046, + "kl": 0.5654296875, + "learning_rate": 2.9899352586559133e-06, + "loss": 0.1302, + "reward": 1.2000000476837158, + "reward_std": 0.3856646865606308, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 386 + }, + { + "completion_length": 99.10000228881836, + "epoch": 0.401244167962675, + "grad_norm": 0.41970863938331604, + "kl": 0.36328125, + "learning_rate": 2.9897247254175052e-06, + "loss": -0.0189, + "reward": 1.5750000178813934, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.5749999973922968, + "rewards/format_reward": 1.0, + "step": 387 + }, + { + "completion_length": 187.52499961853027, + "epoch": 0.4022809745982374, + "grad_norm": 0.3286798894405365, + "kl": 0.505859375, + "learning_rate": 2.9895120205225883e-06, + "loss": 0.1248, + "reward": 1.175000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.22500000894069672, + "rewards/format_reward": 0.9500000178813934, + "step": 388 + }, + { + "completion_length": 209.52500534057617, + "epoch": 0.4033177812337999, + "grad_norm": 0.5088163018226624, + "kl": 0.3583984375, + "learning_rate": 2.9892971442812333e-06, + "loss": 0.1461, + "reward": 1.2500000298023224, + "reward_std": 0.31719937175512314, + "rewards/accuracy_reward": 0.29999999701976776, + "rewards/format_reward": 0.9500000178813934, + "step": 389 + }, + { + "completion_length": 158.02499961853027, + "epoch": 0.40435458786936235, + "grad_norm": 0.3768276274204254, + "kl": 0.33251953125, + "learning_rate": 2.989080097006678e-06, + "loss": 0.0693, + "reward": 1.2999999970197678, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 390 + }, + { + "completion_length": 263.57500076293945, + "epoch": 0.40539139450492484, + "grad_norm": 12.056117057800293, + "kl": 1.38525390625, + "learning_rate": 2.988860879015324e-06, + "loss": 0.2372, + "reward": 1.475000023841858, + "reward_std": 0.4363621547818184, + "rewards/accuracy_reward": 0.5500000156462193, + "rewards/format_reward": 0.925000011920929, + "step": 391 + }, + { + "completion_length": 165.475004196167, + "epoch": 0.4064282011404873, + "grad_norm": 0.36441394686698914, + "kl": 0.35595703125, + "learning_rate": 2.9886394906267387e-06, + "loss": 0.0168, + "reward": 1.25, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.2750000022351742, + "rewards/format_reward": 0.9750000089406967, + "step": 392 + }, + { + "completion_length": 254.17500495910645, + "epoch": 0.40746500777604977, + "grad_norm": 0.32492274045944214, + "kl": 0.48095703125, + "learning_rate": 2.988415932163652e-06, + "loss": 0.128, + "reward": 1.125, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.20000000298023224, + "rewards/format_reward": 0.925000011920929, + "step": 393 + }, + { + "completion_length": 247.7250099182129, + "epoch": 0.4085018144116122, + "grad_norm": 425.0500183105469, + "kl": 15.10888671875, + "learning_rate": 2.988190203951958e-06, + "loss": 1.26, + "reward": 1.4750000536441803, + "reward_std": 0.5084580183029175, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 394 + }, + { + "completion_length": 204.92500114440918, + "epoch": 0.4095386210471747, + "grad_norm": 0.4983512759208679, + "kl": 0.404052734375, + "learning_rate": 2.9879623063207136e-06, + "loss": 0.0576, + "reward": 1.4250000417232513, + "reward_std": 0.3354102037847042, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.9500000178813934, + "step": 395 + }, + { + "completion_length": 252.95000648498535, + "epoch": 0.4105754276827372, + "grad_norm": 0.6655735373497009, + "kl": 0.510986328125, + "learning_rate": 2.987732239602138e-06, + "loss": 0.059, + "reward": 1.2499999850988388, + "reward_std": 0.26129766926169395, + "rewards/accuracy_reward": 0.32500001043081284, + "rewards/format_reward": 0.9249999970197678, + "step": 396 + }, + { + "completion_length": 258.30000495910645, + "epoch": 0.41161223431829963, + "grad_norm": 17.00373077392578, + "kl": 0.52880859375, + "learning_rate": 2.9875000041316126e-06, + "loss": 0.2363, + "reward": 1.5500000417232513, + "reward_std": 0.46511994302272797, + "rewards/accuracy_reward": 0.6250000037252903, + "rewards/format_reward": 0.9250000268220901, + "step": 397 + }, + { + "completion_length": 305.8000087738037, + "epoch": 0.4126490409538621, + "grad_norm": 5.655200004577637, + "kl": 1.62890625, + "learning_rate": 2.98726560024768e-06, + "loss": 0.3134, + "reward": 1.475000023841858, + "reward_std": 0.5571388304233551, + "rewards/accuracy_reward": 0.600000012665987, + "rewards/format_reward": 0.8750000149011612, + "step": 398 + }, + { + "completion_length": 186.90000343322754, + "epoch": 0.41368584758942456, + "grad_norm": 67.16045379638672, + "kl": 10.04833984375, + "learning_rate": 2.987029028292045e-06, + "loss": 0.84, + "reward": 1.3000000268220901, + "reward_std": 0.3245587907731533, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.9500000029802322, + "step": 399 + }, + { + "completion_length": 168.5250015258789, + "epoch": 0.41472265422498705, + "grad_norm": 17.609403610229492, + "kl": 3.233154296875, + "learning_rate": 2.986790288609571e-06, + "loss": 0.265, + "reward": 1.6250000298023224, + "reward_std": 0.3533165529370308, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 400 + }, + { + "completion_length": 184.3000030517578, + "epoch": 0.4157594608605495, + "grad_norm": 1.3612682819366455, + "kl": 0.52490234375, + "learning_rate": 2.9865493815482827e-06, + "loss": 0.122, + "reward": 1.6250000298023224, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9250000268220901, + "step": 401 + }, + { + "completion_length": 267.40000343322754, + "epoch": 0.416796267496112, + "grad_norm": 31.001413345336914, + "kl": 4.72412109375, + "learning_rate": 2.986306307459364e-06, + "loss": 0.5375, + "reward": 1.4500000029802322, + "reward_std": 0.43636219948530197, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 0.8500000089406967, + "step": 402 + }, + { + "completion_length": 211.35000038146973, + "epoch": 0.4178330741316744, + "grad_norm": 1.4517061710357666, + "kl": 1.05419921875, + "learning_rate": 2.986061066697158e-06, + "loss": 0.0987, + "reward": 1.2500000149011612, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.925000011920929, + "step": 403 + }, + { + "completion_length": 142.9250030517578, + "epoch": 0.4188698807672369, + "grad_norm": 1.8906280994415283, + "kl": 1.05908203125, + "learning_rate": 2.9858136596191662e-06, + "loss": 0.1521, + "reward": 1.3250000327825546, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.3999999985098839, + "rewards/format_reward": 0.9250000268220901, + "step": 404 + }, + { + "completion_length": 141.87500190734863, + "epoch": 0.4199066874027994, + "grad_norm": 23.721681594848633, + "kl": 3.887939453125, + "learning_rate": 2.985564086586048e-06, + "loss": 0.3364, + "reward": 1.475000023841858, + "reward_std": 0.4740530550479889, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9000000059604645, + "step": 405 + }, + { + "completion_length": 162.39999961853027, + "epoch": 0.42094349403836184, + "grad_norm": 65.63410949707031, + "kl": 7.84228515625, + "learning_rate": 2.9853123479616196e-06, + "loss": 0.5849, + "reward": 1.5500000268220901, + "reward_std": 0.3479738086462021, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000208616257, + "step": 406 + }, + { + "completion_length": 247.4749984741211, + "epoch": 0.42198030067392434, + "grad_norm": 10.55413818359375, + "kl": 4.9853515625, + "learning_rate": 2.9850584441128557e-06, + "loss": 0.4714, + "reward": 1.475000023841858, + "reward_std": 0.4472135789692402, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8500000238418579, + "step": 407 + }, + { + "completion_length": 220.52500534057617, + "epoch": 0.4230171073094868, + "grad_norm": 0.9092906713485718, + "kl": 0.403076171875, + "learning_rate": 2.984802375409885e-06, + "loss": 0.0505, + "reward": 1.2750000059604645, + "reward_std": 0.4495588019490242, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 408 + }, + { + "completion_length": 199.35000038146973, + "epoch": 0.42405391394504927, + "grad_norm": 1.2738308906555176, + "kl": 0.4384765625, + "learning_rate": 2.984544142225995e-06, + "loss": 0.1137, + "reward": 1.4000000059604645, + "reward_std": 0.3046357296407223, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.9500000178813934, + "step": 409 + }, + { + "completion_length": 305.950008392334, + "epoch": 0.4250907205806117, + "grad_norm": 2.3068294525146484, + "kl": 0.871826171875, + "learning_rate": 2.9842837449376264e-06, + "loss": 0.2054, + "reward": 1.550000011920929, + "reward_std": 0.4290027618408203, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.8750000298023224, + "step": 410 + }, + { + "completion_length": 237.6500015258789, + "epoch": 0.4261275272161742, + "grad_norm": 2.3890562057495117, + "kl": 1.326171875, + "learning_rate": 2.9840211839243747e-06, + "loss": 0.2206, + "reward": 1.5, + "reward_std": 0.47247934713959694, + "rewards/accuracy_reward": 0.5999999940395355, + "rewards/format_reward": 0.9000000357627869, + "step": 411 + }, + { + "completion_length": 491.37500953674316, + "epoch": 0.42716433385173663, + "grad_norm": 0.7049680948257446, + "kl": 0.55810546875, + "learning_rate": 2.9837564595689896e-06, + "loss": 0.0784, + "reward": 1.3500000536441803, + "reward_std": 0.3913119025528431, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 0.7500000149011612, + "step": 412 + }, + { + "completion_length": 219.30000114440918, + "epoch": 0.4282011404872991, + "grad_norm": 5.027242183685303, + "kl": 1.90966796875, + "learning_rate": 2.983489572257376e-06, + "loss": 0.1836, + "reward": 1.2000000178813934, + "reward_std": 0.27228760719299316, + "rewards/accuracy_reward": 0.4000000096857548, + "rewards/format_reward": 0.800000011920929, + "step": 413 + }, + { + "completion_length": 601.2500057220459, + "epoch": 0.42923794712286156, + "grad_norm": 33.8704948425293, + "kl": 11.8896484375, + "learning_rate": 2.9832205223785905e-06, + "loss": 1.1407, + "reward": 1.1500000059604645, + "reward_std": 0.5894869565963745, + "rewards/accuracy_reward": 0.4999999962747097, + "rewards/format_reward": 0.6499999985098839, + "step": 414 + }, + { + "completion_length": 697.1750144958496, + "epoch": 0.43027475375842406, + "grad_norm": 0.6441704034805298, + "kl": 0.45263671875, + "learning_rate": 2.9829493103248416e-06, + "loss": 0.169, + "reward": 1.0000000149011612, + "reward_std": 0.4237985797226429, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.6500000208616257, + "step": 415 + }, + { + "completion_length": 626.6500091552734, + "epoch": 0.43131156039398655, + "grad_norm": 1.1363885402679443, + "kl": 1.099609375, + "learning_rate": 2.9826759364914915e-06, + "loss": 0.283, + "reward": 1.0250000059604645, + "reward_std": 0.504827506840229, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.6500000134110451, + "step": 416 + }, + { + "completion_length": 217.35000610351562, + "epoch": 0.432348367029549, + "grad_norm": 0.9580169916152954, + "kl": 0.4140625, + "learning_rate": 2.9824004012770523e-06, + "loss": 0.1531, + "reward": 1.2750000059604645, + "reward_std": 0.3533165343105793, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9500000178813934, + "step": 417 + }, + { + "completion_length": 687.4500045776367, + "epoch": 0.4333851736651115, + "grad_norm": 0.6423575282096863, + "kl": 0.49609375, + "learning_rate": 2.9821227050831874e-06, + "loss": 0.1532, + "reward": 0.9749999940395355, + "reward_std": 0.41643914207816124, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.6250000074505806, + "step": 418 + }, + { + "completion_length": 339.0999984741211, + "epoch": 0.4344219803006739, + "grad_norm": 1.1070829629898071, + "kl": 1.319580078125, + "learning_rate": 2.9818428483147104e-06, + "loss": 0.259, + "reward": 1.1750000417232513, + "reward_std": 0.5048274733126163, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.8250000029802322, + "step": 419 + }, + { + "completion_length": 391.54999923706055, + "epoch": 0.4354587869362364, + "grad_norm": 0.9474948048591614, + "kl": 0.658203125, + "learning_rate": 2.9815608313795845e-06, + "loss": 0.1309, + "reward": 1.2500000447034836, + "reward_std": 0.44219936802983284, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.7750000059604645, + "step": 420 + }, + { + "completion_length": 418.125, + "epoch": 0.43649559357179885, + "grad_norm": 35.806827545166016, + "kl": 12.6572265625, + "learning_rate": 2.981276654688922e-06, + "loss": 0.7257, + "reward": 1.0750000029802322, + "reward_std": 0.4723408445715904, + "rewards/accuracy_reward": 0.30000000819563866, + "rewards/format_reward": 0.7750000208616257, + "step": 421 + }, + { + "completion_length": 406.25000762939453, + "epoch": 0.43753240020736134, + "grad_norm": 4.112382411956787, + "kl": 2.720703125, + "learning_rate": 2.9809903186569826e-06, + "loss": 0.4188, + "reward": 1.1750000268220901, + "reward_std": 0.5968463569879532, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.8250000178813934, + "step": 422 + }, + { + "completion_length": 389.7249984741211, + "epoch": 0.4385692068429238, + "grad_norm": 3.500101327896118, + "kl": 4.51220703125, + "learning_rate": 2.9807018237011756e-06, + "loss": 0.4401, + "reward": 1.2500000149011612, + "reward_std": 0.3858031928539276, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.8500000238418579, + "step": 423 + }, + { + "completion_length": 309.550012588501, + "epoch": 0.43960601347848627, + "grad_norm": 3.486119031906128, + "kl": 4.0400390625, + "learning_rate": 2.980411170242056e-06, + "loss": 0.4344, + "reward": 1.3250000178813934, + "reward_std": 0.5156789012253284, + "rewards/accuracy_reward": 0.4500000160187483, + "rewards/format_reward": 0.8750000149011612, + "step": 424 + }, + { + "completion_length": 142.0750026702881, + "epoch": 0.4406428201140487, + "grad_norm": 0.783508837223053, + "kl": 1.0380859375, + "learning_rate": 2.9801183587033266e-06, + "loss": 0.1069, + "reward": 1.5250000357627869, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.9750000089406967, + "step": 425 + }, + { + "completion_length": 118.95000076293945, + "epoch": 0.4416796267496112, + "grad_norm": 0.49163949489593506, + "kl": 0.404296875, + "learning_rate": 2.9798233895118343e-06, + "loss": 0.0385, + "reward": 1.350000023841858, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000029802322, + "step": 426 + }, + { + "completion_length": 482.90000915527344, + "epoch": 0.4427164333851737, + "grad_norm": 1.6972235441207886, + "kl": 1.5712890625, + "learning_rate": 2.9795262630975732e-06, + "loss": 0.3296, + "reward": 0.9750000387430191, + "reward_std": 0.5535082966089249, + "rewards/accuracy_reward": 0.20000000298023224, + "rewards/format_reward": 0.7750000059604645, + "step": 427 + }, + { + "completion_length": 404.5250015258789, + "epoch": 0.44375324002073613, + "grad_norm": 1.4435909986495972, + "kl": 2.01611328125, + "learning_rate": 2.979226979893681e-06, + "loss": 0.3348, + "reward": 1.2500000298023224, + "reward_std": 0.4349784702062607, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.8500000238418579, + "step": 428 + }, + { + "completion_length": 241.77500343322754, + "epoch": 0.4447900466562986, + "grad_norm": 1.073678970336914, + "kl": 2.5791015625, + "learning_rate": 2.978925540336441e-06, + "loss": 0.2655, + "reward": 1.3250000327825546, + "reward_std": 0.40387552604079247, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 429 + }, + { + "completion_length": 259.87500762939453, + "epoch": 0.44582685329186106, + "grad_norm": 3.7204360961914062, + "kl": 5.47900390625, + "learning_rate": 2.978621944865277e-06, + "loss": 0.3859, + "reward": 1.3250000327825546, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9000000208616257, + "step": 430 + }, + { + "completion_length": 210.2000026702881, + "epoch": 0.44686365992742355, + "grad_norm": 0.5499123930931091, + "kl": 0.99951171875, + "learning_rate": 2.9783161939227587e-06, + "loss": 0.1771, + "reward": 1.0999999940395355, + "reward_std": 0.22360678017139435, + "rewards/accuracy_reward": 0.15000000596046448, + "rewards/format_reward": 0.9500000029802322, + "step": 431 + }, + { + "completion_length": 254.32501220703125, + "epoch": 0.447900466562986, + "grad_norm": 1.4711023569107056, + "kl": 1.126220703125, + "learning_rate": 2.9780082879545958e-06, + "loss": 0.2227, + "reward": 1.4500000029802322, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 432 + }, + { + "completion_length": 149.57500076293945, + "epoch": 0.4489372731985485, + "grad_norm": 0.3505478799343109, + "kl": 0.44189453125, + "learning_rate": 2.9776982274096416e-06, + "loss": 0.0277, + "reward": 1.600000023841858, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9750000089406967, + "step": 433 + }, + { + "completion_length": 118.62500381469727, + "epoch": 0.4499740798341109, + "grad_norm": 1.0018848180770874, + "kl": 0.70654296875, + "learning_rate": 2.9773860127398872e-06, + "loss": 0.0473, + "reward": 1.550000011920929, + "reward_std": 0.25609345361590385, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9500000178813934, + "step": 434 + }, + { + "completion_length": 149.22500610351562, + "epoch": 0.4510108864696734, + "grad_norm": 0.4854688048362732, + "kl": 1.08154296875, + "learning_rate": 2.9770716444004673e-06, + "loss": 0.0993, + "reward": 1.300000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 435 + }, + { + "completion_length": 225.75, + "epoch": 0.45204769310523585, + "grad_norm": 1.2092115879058838, + "kl": 1.08251953125, + "learning_rate": 2.9767551228496537e-06, + "loss": 0.0811, + "reward": 1.300000011920929, + "reward_std": 0.22894952446222305, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 436 + }, + { + "completion_length": 191.77499961853027, + "epoch": 0.45308449974079834, + "grad_norm": 2.3160505294799805, + "kl": 1.91552734375, + "learning_rate": 2.9764364485488584e-06, + "loss": 0.1791, + "reward": 1.2000000029802322, + "reward_std": 0.1604842059314251, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 437 + }, + { + "completion_length": 445.32500648498535, + "epoch": 0.45412130637636083, + "grad_norm": 1.1174695491790771, + "kl": 2.077392578125, + "learning_rate": 2.9761156219626305e-06, + "loss": 0.3718, + "reward": 1.4749999940395355, + "reward_std": 0.521021630614996, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.8250000029802322, + "step": 438 + }, + { + "completion_length": 422.75, + "epoch": 0.45515811301192327, + "grad_norm": 1.358939528465271, + "kl": 2.649658203125, + "learning_rate": 2.9757926435586572e-06, + "loss": 0.1542, + "reward": 1.1000000089406967, + "reward_std": 0.37310105934739113, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.7750000059604645, + "step": 439 + }, + { + "completion_length": 638.4000091552734, + "epoch": 0.45619491964748576, + "grad_norm": 1.3631545305252075, + "kl": 2.658203125, + "learning_rate": 2.975467513807763e-06, + "loss": 0.4623, + "reward": 1.0000000447034836, + "reward_std": 0.7521778345108032, + "rewards/accuracy_reward": 0.37500000186264515, + "rewards/format_reward": 0.6250000149011612, + "step": 440 + }, + { + "completion_length": 549.6250038146973, + "epoch": 0.4572317262830482, + "grad_norm": 1.7535611391067505, + "kl": 1.56591796875, + "learning_rate": 2.975140233183908e-06, + "loss": 0.3238, + "reward": 1.1500000059604645, + "reward_std": 0.5408061631023884, + "rewards/accuracy_reward": 0.44999999552965164, + "rewards/format_reward": 0.7000000029802322, + "step": 441 + }, + { + "completion_length": 441.25000762939453, + "epoch": 0.4582685329186107, + "grad_norm": 2.6060733795166016, + "kl": 1.5341796875, + "learning_rate": 2.9748108021641875e-06, + "loss": 0.3068, + "reward": 1.3250000178813934, + "reward_std": 0.6076978109776974, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.8250000029802322, + "step": 442 + }, + { + "completion_length": 449.1750068664551, + "epoch": 0.45930533955417313, + "grad_norm": 2.323462963104248, + "kl": 3.5185546875, + "learning_rate": 2.974479221228831e-06, + "loss": 0.5005, + "reward": 1.1499999910593033, + "reward_std": 0.4981010742485523, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.7749999910593033, + "step": 443 + }, + { + "completion_length": 369.0999984741211, + "epoch": 0.4603421461897356, + "grad_norm": 9.374835968017578, + "kl": 4.626953125, + "learning_rate": 2.9741454908612047e-06, + "loss": 0.4378, + "reward": 1.1250000149011612, + "reward_std": 0.4525563307106495, + "rewards/accuracy_reward": 0.3000000026077032, + "rewards/format_reward": 0.8250000178813934, + "step": 444 + }, + { + "completion_length": 254.1500072479248, + "epoch": 0.46137895282529806, + "grad_norm": 1.3832933902740479, + "kl": 1.93603515625, + "learning_rate": 2.9738096115478055e-06, + "loss": 0.2301, + "reward": 1.3000000268220901, + "reward_std": 0.3479738235473633, + "rewards/accuracy_reward": 0.40000000409781933, + "rewards/format_reward": 0.9000000208616257, + "step": 445 + }, + { + "completion_length": 344.5, + "epoch": 0.46241575946086055, + "grad_norm": 1.9132006168365479, + "kl": 2.30419921875, + "learning_rate": 2.9734715837782635e-06, + "loss": 0.2175, + "reward": 1.225000038743019, + "reward_std": 0.40387552976608276, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.8750000149011612, + "step": 446 + }, + { + "completion_length": 238.82500648498535, + "epoch": 0.463452566096423, + "grad_norm": 1.1751809120178223, + "kl": 1.783203125, + "learning_rate": 2.973131408045341e-06, + "loss": 0.3002, + "reward": 1.5750000476837158, + "reward_std": 0.3913118988275528, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 447 + }, + { + "completion_length": 186.3499984741211, + "epoch": 0.4644893727319855, + "grad_norm": 0.9299616813659668, + "kl": 0.9111328125, + "learning_rate": 2.972789084844932e-06, + "loss": 0.1332, + "reward": 1.5750000178813934, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000178813934, + "step": 448 + }, + { + "completion_length": 307.1250171661377, + "epoch": 0.465526179367548, + "grad_norm": 1.2203192710876465, + "kl": 2.68505859375, + "learning_rate": 2.97244461467606e-06, + "loss": 0.3509, + "reward": 1.550000011920929, + "reward_std": 0.36053743958473206, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 449 + }, + { + "completion_length": 197.77500534057617, + "epoch": 0.4665629860031104, + "grad_norm": 0.5328684449195862, + "kl": 1.17578125, + "learning_rate": 2.9720979980408795e-06, + "loss": 0.173, + "reward": 1.4749999940395355, + "reward_std": 0.14429005607962608, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 0.9500000178813934, + "step": 450 + }, + { + "completion_length": 183.67500495910645, + "epoch": 0.4675997926386729, + "grad_norm": 2.1329684257507324, + "kl": 1.76220703125, + "learning_rate": 2.9717492354446727e-06, + "loss": 0.1922, + "reward": 1.2750000059604645, + "reward_std": 0.2795085087418556, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 451 + }, + { + "completion_length": 97.95000076293945, + "epoch": 0.46863659927423534, + "grad_norm": 0.40684643387794495, + "kl": 0.35595703125, + "learning_rate": 2.9713983273958507e-06, + "loss": 0.027, + "reward": 1.574999988079071, + "reward_std": 0.18026871606707573, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 452 + }, + { + "completion_length": 141.92500019073486, + "epoch": 0.46967340590979784, + "grad_norm": 3.87530517578125, + "kl": 2.384521484375, + "learning_rate": 2.971045274405952e-06, + "loss": 0.2588, + "reward": 1.600000023841858, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 453 + }, + { + "completion_length": 138.90000343322754, + "epoch": 0.4707102125453603, + "grad_norm": 4.4843430519104, + "kl": 2.71875, + "learning_rate": 2.9706900769896434e-06, + "loss": 0.1234, + "reward": 1.5500000417232513, + "reward_std": 0.36789682134985924, + "rewards/accuracy_reward": 0.650000000372529, + "rewards/format_reward": 0.9000000059604645, + "step": 454 + }, + { + "completion_length": 337.850004196167, + "epoch": 0.47174701918092277, + "grad_norm": 25.936668395996094, + "kl": 4.15185546875, + "learning_rate": 2.970332735664714e-06, + "loss": 0.3841, + "reward": 1.350000023841858, + "reward_std": 0.3856646902859211, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.8750000149011612, + "step": 455 + }, + { + "completion_length": 245.5, + "epoch": 0.4727838258164852, + "grad_norm": 1.4629117250442505, + "kl": 0.405517578125, + "learning_rate": 2.969973250952083e-06, + "loss": 0.1739, + "reward": 1.100000023841858, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.25, + "rewards/format_reward": 0.8500000089406967, + "step": 456 + }, + { + "completion_length": 551.0750122070312, + "epoch": 0.4738206324520477, + "grad_norm": 2.0260560512542725, + "kl": 0.8857421875, + "learning_rate": 2.96961162337579e-06, + "loss": 0.2795, + "reward": 1.050000011920929, + "reward_std": 0.4651199281215668, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 0.7500000149011612, + "step": 457 + }, + { + "completion_length": 735.4250030517578, + "epoch": 0.47485743908761013, + "grad_norm": 5.954936981201172, + "kl": 0.9501953125, + "learning_rate": 2.969247853463e-06, + "loss": 0.424, + "reward": 0.8750000149011612, + "reward_std": 0.582404587417841, + "rewards/accuracy_reward": 0.24999999813735485, + "rewards/format_reward": 0.6250000186264515, + "step": 458 + }, + { + "completion_length": 592.0750122070312, + "epoch": 0.4758942457231726, + "grad_norm": 4.934417247772217, + "kl": 0.63671875, + "learning_rate": 2.968881941744003e-06, + "loss": 0.2356, + "reward": 1.1500000208616257, + "reward_std": 0.5342182517051697, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.7000000029802322, + "step": 459 + }, + { + "completion_length": 586.4000091552734, + "epoch": 0.4769310523587351, + "grad_norm": 2.2492358684539795, + "kl": 0.421875, + "learning_rate": 2.968513888752207e-06, + "loss": 0.3741, + "reward": 1.2500000298023224, + "reward_std": 0.8159334361553192, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.6749999970197678, + "step": 460 + }, + { + "completion_length": 455.5500030517578, + "epoch": 0.47796785899429756, + "grad_norm": 2.0860514640808105, + "kl": 0.35400390625, + "learning_rate": 2.9681436950241457e-06, + "loss": 0.2959, + "reward": 1.2500000149011612, + "reward_std": 0.6224680915474892, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.7750000208616257, + "step": 461 + }, + { + "completion_length": 485.87500762939453, + "epoch": 0.47900466562986005, + "grad_norm": 68.13333892822266, + "kl": 13.9140625, + "learning_rate": 2.9677713610994704e-06, + "loss": 1.3759, + "reward": 1.1500000208616257, + "reward_std": 0.5533697977662086, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.7500000149011612, + "step": 462 + }, + { + "completion_length": 181.725004196167, + "epoch": 0.4800414722654225, + "grad_norm": 9.417370796203613, + "kl": 1.84912109375, + "learning_rate": 2.9673968875209545e-06, + "loss": 0.1592, + "reward": 1.4500000178813934, + "reward_std": 0.5253834910690784, + "rewards/accuracy_reward": 0.600000012665987, + "rewards/format_reward": 0.8500000238418579, + "step": 463 + }, + { + "completion_length": 575.8250045776367, + "epoch": 0.481078278900985, + "grad_norm": 2.923924446105957, + "kl": 0.75537109375, + "learning_rate": 2.967020274834488e-06, + "loss": 0.2154, + "reward": 1.2000000178813934, + "reward_std": 0.658585287630558, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.7250000238418579, + "step": 464 + }, + { + "completion_length": 235.97500228881836, + "epoch": 0.4821150855365474, + "grad_norm": 27.21836280822754, + "kl": 8.6328125, + "learning_rate": 2.9666415235890823e-06, + "loss": 0.814, + "reward": 1.3500000089406967, + "reward_std": 0.3913119100034237, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9000000208616257, + "step": 465 + }, + { + "completion_length": 180.10000038146973, + "epoch": 0.4831518921721099, + "grad_norm": 1.1835033893585205, + "kl": 0.5048828125, + "learning_rate": 2.966260634336863e-06, + "loss": 0.0695, + "reward": 1.350000023841858, + "reward_std": 0.26129766553640366, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000029802322, + "step": 466 + }, + { + "completion_length": 141.75, + "epoch": 0.48418869880767235, + "grad_norm": 0.5516549348831177, + "kl": 0.375244140625, + "learning_rate": 2.965877607633075e-06, + "loss": 0.0181, + "reward": 1.5999999940395355, + "reward_std": 0.33353202417492867, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.925000011920929, + "step": 467 + }, + { + "completion_length": 151.57499885559082, + "epoch": 0.48522550544323484, + "grad_norm": 2.3915843963623047, + "kl": 0.5625, + "learning_rate": 2.965492444036078e-06, + "loss": 0.0569, + "reward": 1.3499999940395355, + "reward_std": 0.281220693141222, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 468 + }, + { + "completion_length": 89.47500228881836, + "epoch": 0.48626231207879733, + "grad_norm": 0.3632679879665375, + "kl": 0.4560546875, + "learning_rate": 2.965105144107347e-06, + "loss": 0.0197, + "reward": 1.2750000059604645, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 1.0, + "step": 469 + }, + { + "completion_length": 104.25000381469727, + "epoch": 0.48729911871435977, + "grad_norm": 2.3217339515686035, + "kl": 0.4345703125, + "learning_rate": 2.9647157084114704e-06, + "loss": -0.0065, + "reward": 1.5, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.9750000089406967, + "step": 470 + }, + { + "completion_length": 157.50000381469727, + "epoch": 0.48833592534992226, + "grad_norm": 1.2729434967041016, + "kl": 0.7998046875, + "learning_rate": 2.9643241375161517e-06, + "loss": 0.1192, + "reward": 1.375, + "reward_std": 0.5100317001342773, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.875, + "step": 471 + }, + { + "completion_length": 656.1500015258789, + "epoch": 0.4893727319854847, + "grad_norm": 1.7226147651672363, + "kl": 0.68115234375, + "learning_rate": 2.963930431992206e-06, + "loss": 0.4477, + "reward": 0.8500000238418579, + "reward_std": 0.5660719275474548, + "rewards/accuracy_reward": 0.1750000026077032, + "rewards/format_reward": 0.6749999970197678, + "step": 472 + }, + { + "completion_length": 406.3750057220459, + "epoch": 0.4904095386210472, + "grad_norm": 0.951695442199707, + "kl": 0.376953125, + "learning_rate": 2.96353459241356e-06, + "loss": 0.2062, + "reward": 1.225000023841858, + "reward_std": 0.49088017642498016, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.7750000208616257, + "step": 473 + }, + { + "completion_length": 170.22500228881836, + "epoch": 0.49144634525660963, + "grad_norm": 0.9786409735679626, + "kl": 0.43505859375, + "learning_rate": 2.963136619357251e-06, + "loss": 0.0848, + "reward": 1.425000011920929, + "reward_std": 0.4797002226114273, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.8750000149011612, + "step": 474 + }, + { + "completion_length": 180.82500076293945, + "epoch": 0.4924831518921721, + "grad_norm": 0.526511013507843, + "kl": 0.50146484375, + "learning_rate": 2.9627365134034276e-06, + "loss": 0.0805, + "reward": 1.2999999970197678, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9249999970197678, + "step": 475 + }, + { + "completion_length": 179.5000057220459, + "epoch": 0.49351995852773456, + "grad_norm": 1.3897082805633545, + "kl": 0.3935546875, + "learning_rate": 2.9623342751353473e-06, + "loss": 0.1006, + "reward": 1.2500000298023224, + "reward_std": 0.23617039993405342, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.9000000208616257, + "step": 476 + }, + { + "completion_length": 104.20000267028809, + "epoch": 0.49455676516329705, + "grad_norm": 1.2519752979278564, + "kl": 0.62939453125, + "learning_rate": 2.961929905139375e-06, + "loss": 0.0275, + "reward": 1.5750000178813934, + "reward_std": 0.3856646902859211, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 1.0, + "step": 477 + }, + { + "completion_length": 136.77500534057617, + "epoch": 0.4955935717988595, + "grad_norm": 32.357139587402344, + "kl": 8.801513671875, + "learning_rate": 2.9615234040049845e-06, + "loss": 0.5752, + "reward": 1.1250000149011612, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.17500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 478 + }, + { + "completion_length": 151.99999809265137, + "epoch": 0.496630378434422, + "grad_norm": 4.595073699951172, + "kl": 1.439453125, + "learning_rate": 2.9611147723247555e-06, + "loss": 0.154, + "reward": 1.5250000059604645, + "reward_std": 0.28485121950507164, + "rewards/accuracy_reward": 0.5749999992549419, + "rewards/format_reward": 0.9500000029802322, + "step": 479 + }, + { + "completion_length": 102.72500038146973, + "epoch": 0.4976671850699845, + "grad_norm": 0.591180682182312, + "kl": 0.371337890625, + "learning_rate": 2.9607040106943733e-06, + "loss": 0.0109, + "reward": 1.4250000268220901, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 480 + }, + { + "completion_length": 152.975004196167, + "epoch": 0.4987039917055469, + "grad_norm": 1.7282302379608154, + "kl": 1.240234375, + "learning_rate": 2.9602911197126286e-06, + "loss": 0.1355, + "reward": 1.2750000208616257, + "reward_std": 0.26129766181111336, + "rewards/accuracy_reward": 0.30000000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 481 + }, + { + "completion_length": 119.20000076293945, + "epoch": 0.4997407983411094, + "grad_norm": 1.2713792324066162, + "kl": 1.33203125, + "learning_rate": 2.9598760999814165e-06, + "loss": 0.0979, + "reward": 1.5000000298023224, + "reward_std": 0.2795084938406944, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 482 + }, + { + "completion_length": 133.9250030517578, + "epoch": 0.5007776049766719, + "grad_norm": 0.8651389479637146, + "kl": 0.416259765625, + "learning_rate": 2.9594589521057342e-06, + "loss": 0.0718, + "reward": 1.4000000208616257, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.9500000178813934, + "step": 483 + }, + { + "completion_length": 134.850004196167, + "epoch": 0.5018144116122343, + "grad_norm": 1.2900584936141968, + "kl": 1.51318359375, + "learning_rate": 2.9590396766936826e-06, + "loss": 0.1183, + "reward": 1.350000038743019, + "reward_std": 0.3479738235473633, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.9500000178813934, + "step": 484 + }, + { + "completion_length": 86.97500133514404, + "epoch": 0.5028512182477968, + "grad_norm": 0.4113655388355255, + "kl": 0.3232421875, + "learning_rate": 2.9586182743564627e-06, + "loss": 0.0003, + "reward": 1.675000011920929, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 1.0, + "step": 485 + }, + { + "completion_length": 138.35000038146973, + "epoch": 0.5038880248833593, + "grad_norm": 3.6755096912384033, + "kl": 2.095703125, + "learning_rate": 2.9581947457083773e-06, + "loss": 0.1498, + "reward": 1.550000011920929, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 486 + }, + { + "completion_length": 87.00000381469727, + "epoch": 0.5049248315189218, + "grad_norm": 0.35109928250312805, + "kl": 0.46728515625, + "learning_rate": 2.957769091366828e-06, + "loss": 0.0177, + "reward": 1.550000011920929, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 1.0, + "step": 487 + }, + { + "completion_length": 77.22500228881836, + "epoch": 0.5059616381544841, + "grad_norm": 1.4628422260284424, + "kl": 0.9501953125, + "learning_rate": 2.9573413119523146e-06, + "loss": 0.0246, + "reward": 1.5750000178813934, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 1.0, + "step": 488 + }, + { + "completion_length": 168.70000457763672, + "epoch": 0.5069984447900466, + "grad_norm": 6.089077472686768, + "kl": 2.49169921875, + "learning_rate": 2.9569114080884365e-06, + "loss": 0.2892, + "reward": 1.7000000476837158, + "reward_std": 0.3354102000594139, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9250000268220901, + "step": 489 + }, + { + "completion_length": 93.82500457763672, + "epoch": 0.5080352514256091, + "grad_norm": 1.7852115631103516, + "kl": 0.6474609375, + "learning_rate": 2.9564793804018885e-06, + "loss": 0.0774, + "reward": 1.5250000059604645, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.9750000089406967, + "step": 490 + }, + { + "completion_length": 70.42500114440918, + "epoch": 0.5090720580611716, + "grad_norm": 0.8128455281257629, + "kl": 0.35009765625, + "learning_rate": 2.9560452295224624e-06, + "loss": 0.0278, + "reward": 1.5000000298023224, + "reward_std": 0.31719935312867165, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 1.0, + "step": 491 + }, + { + "completion_length": 125.45000076293945, + "epoch": 0.5101088646967341, + "grad_norm": 0.7844669222831726, + "kl": 0.4482421875, + "learning_rate": 2.955608956083044e-06, + "loss": 0.05, + "reward": 1.550000011920929, + "reward_std": 0.15685366839170456, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 492 + }, + { + "completion_length": 127.45000457763672, + "epoch": 0.5111456713322965, + "grad_norm": 1.1734215021133423, + "kl": 1.258544921875, + "learning_rate": 2.9551705607196147e-06, + "loss": 0.0648, + "reward": 1.3750000298023224, + "reward_std": 0.3407529294490814, + "rewards/accuracy_reward": 0.4500000048428774, + "rewards/format_reward": 0.925000011920929, + "step": 493 + }, + { + "completion_length": 180.07500457763672, + "epoch": 0.512182477967859, + "grad_norm": 0.663034975528717, + "kl": 0.40966796875, + "learning_rate": 2.9547300440712477e-06, + "loss": 0.0733, + "reward": 1.5250000059604645, + "reward_std": 0.17304783314466476, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000029802322, + "step": 494 + }, + { + "completion_length": 179.37500667572021, + "epoch": 0.5132192846034215, + "grad_norm": 0.9060681462287903, + "kl": 0.76513671875, + "learning_rate": 2.9542874067801094e-06, + "loss": 0.1252, + "reward": 1.5249999910593033, + "reward_std": 0.2686570920050144, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9000000059604645, + "step": 495 + }, + { + "completion_length": 197.75000667572021, + "epoch": 0.514256091238984, + "grad_norm": 1.4791067838668823, + "kl": 1.224609375, + "learning_rate": 2.9538426494914574e-06, + "loss": 0.1744, + "reward": 1.475000023841858, + "reward_std": 0.3913119062781334, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.925000011920929, + "step": 496 + }, + { + "completion_length": 82.92500305175781, + "epoch": 0.5152928978745464, + "grad_norm": 22.87507438659668, + "kl": 4.554931640625, + "learning_rate": 2.9533957728536397e-06, + "loss": 0.3108, + "reward": 1.6250000298023224, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.6500000208616257, + "rewards/format_reward": 0.9750000089406967, + "step": 497 + }, + { + "completion_length": 97.22500038146973, + "epoch": 0.5163297045101088, + "grad_norm": 2.5906031131744385, + "kl": 0.286376953125, + "learning_rate": 2.9529467775180935e-06, + "loss": 0.0659, + "reward": 1.5000000298023224, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.9750000089406967, + "step": 498 + }, + { + "completion_length": 160.65000343322754, + "epoch": 0.5173665111456713, + "grad_norm": 1.3695307970046997, + "kl": 1.130859375, + "learning_rate": 2.952495664139346e-06, + "loss": 0.1592, + "reward": 1.475000023841858, + "reward_std": 0.4723408408463001, + "rewards/accuracy_reward": 0.5750000141561031, + "rewards/format_reward": 0.9000000208616257, + "step": 499 + }, + { + "completion_length": 145.62500667572021, + "epoch": 0.5184033177812338, + "grad_norm": 3.733125925064087, + "kl": 4.13330078125, + "learning_rate": 2.952042433375009e-06, + "loss": 0.1987, + "reward": 1.425000011920929, + "reward_std": 0.28485122323036194, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 500 + }, + { + "completion_length": 125.32500457763672, + "epoch": 0.5194401244167963, + "grad_norm": 0.9280956983566284, + "kl": 0.7236328125, + "learning_rate": 2.9515870858857846e-06, + "loss": 0.0983, + "reward": 1.3250000178813934, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9500000178813934, + "step": 501 + }, + { + "completion_length": 75.125, + "epoch": 0.5204769310523587, + "grad_norm": 0.5787338614463806, + "kl": 0.39306640625, + "learning_rate": 2.9511296223354573e-06, + "loss": 0.0172, + "reward": 1.300000011920929, + "reward_std": 0.06846531480550766, + "rewards/accuracy_reward": 0.30000000074505806, + "rewards/format_reward": 1.0, + "step": 502 + }, + { + "completion_length": 75.7750015258789, + "epoch": 0.5215137376879212, + "grad_norm": 0.7337321639060974, + "kl": 0.386962890625, + "learning_rate": 2.9506700433908988e-06, + "loss": 0.0384, + "reward": 1.375, + "reward_std": 0.26129766181111336, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 1.0, + "step": 503 + }, + { + "completion_length": 67.75000095367432, + "epoch": 0.5225505443234837, + "grad_norm": 0.9252765774726868, + "kl": 0.3251953125, + "learning_rate": 2.9502083497220632e-06, + "loss": -0.0029, + "reward": 1.675000011920929, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 1.0, + "step": 504 + }, + { + "completion_length": 69.30000114440918, + "epoch": 0.5235873509590462, + "grad_norm": 0.8446915745735168, + "kl": 0.34375, + "learning_rate": 2.949744542001987e-06, + "loss": 0.0156, + "reward": 1.3750000298023224, + "reward_std": 0.32818931341171265, + "rewards/accuracy_reward": 0.40000000409781933, + "rewards/format_reward": 0.9750000089406967, + "step": 505 + }, + { + "completion_length": 127.27500343322754, + "epoch": 0.5246241575946086, + "grad_norm": 2.3860890865325928, + "kl": 1.30126953125, + "learning_rate": 2.94927862090679e-06, + "loss": 0.1343, + "reward": 1.4749999940395355, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9750000089406967, + "step": 506 + }, + { + "completion_length": 71.9250020980835, + "epoch": 0.5256609642301711, + "grad_norm": 0.4810761511325836, + "kl": 0.458984375, + "learning_rate": 2.9488105871156717e-06, + "loss": 0.063, + "reward": 1.3750000149011612, + "reward_std": 0.20019174739718437, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 0.9500000029802322, + "step": 507 + }, + { + "completion_length": 117.42500686645508, + "epoch": 0.5266977708657335, + "grad_norm": 1.6127690076828003, + "kl": 0.67724609375, + "learning_rate": 2.948340441310911e-06, + "loss": 0.0579, + "reward": 1.3750000447034836, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 508 + }, + { + "completion_length": 67.17500305175781, + "epoch": 0.527734577501296, + "grad_norm": 1.0923641920089722, + "kl": 0.5478515625, + "learning_rate": 2.9478681841778667e-06, + "loss": 0.0588, + "reward": 1.5000000298023224, + "reward_std": 0.292072132229805, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.9750000089406967, + "step": 509 + }, + { + "completion_length": 91.72500228881836, + "epoch": 0.5287713841368584, + "grad_norm": 1.9580719470977783, + "kl": 0.9287109375, + "learning_rate": 2.9473938164049744e-06, + "loss": 0.0741, + "reward": 1.4250000417232513, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.9500000178813934, + "step": 510 + }, + { + "completion_length": 74.10000133514404, + "epoch": 0.5298081907724209, + "grad_norm": 0.6495768427848816, + "kl": 0.37744140625, + "learning_rate": 2.946917338683748e-06, + "loss": 0.0301, + "reward": 1.125, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.15000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 511 + }, + { + "completion_length": 121.17500114440918, + "epoch": 0.5308449974079834, + "grad_norm": 0.9497795104980469, + "kl": 0.634765625, + "learning_rate": 2.946438751708775e-06, + "loss": 0.0806, + "reward": 1.4000000059604645, + "reward_std": 0.37310105189681053, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9000000208616257, + "step": 512 + }, + { + "completion_length": 71.57499980926514, + "epoch": 0.5318818040435459, + "grad_norm": 0.5596094131469727, + "kl": 0.39501953125, + "learning_rate": 2.94595805617772e-06, + "loss": 0.0342, + "reward": 1.2000000029802322, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.22500000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 513 + }, + { + "completion_length": 174.45000171661377, + "epoch": 0.5329186106791084, + "grad_norm": 5.549030303955078, + "kl": 0.6337890625, + "learning_rate": 2.9454752527913194e-06, + "loss": 0.1269, + "reward": 1.1750000268220901, + "reward_std": 0.2920721061527729, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.9500000178813934, + "step": 514 + }, + { + "completion_length": 148.6500072479248, + "epoch": 0.5339554173146708, + "grad_norm": 36.280643463134766, + "kl": 4.50537109375, + "learning_rate": 2.9449903422533836e-06, + "loss": 0.3704, + "reward": 1.600000023841858, + "reward_std": 0.347973819822073, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 515 + }, + { + "completion_length": 205.25000762939453, + "epoch": 0.5349922239502333, + "grad_norm": 97.75532531738281, + "kl": 7.87744140625, + "learning_rate": 2.9445033252707943e-06, + "loss": 0.7113, + "reward": 1.2249999940395355, + "reward_std": 0.26143617182970047, + "rewards/accuracy_reward": 0.30000001192092896, + "rewards/format_reward": 0.925000011920929, + "step": 516 + }, + { + "completion_length": 83.45000267028809, + "epoch": 0.5360290305857958, + "grad_norm": 0.8470296263694763, + "kl": 0.443359375, + "learning_rate": 2.944014202553503e-06, + "loss": -0.0235, + "reward": 1.600000023841858, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6000000033527613, + "rewards/format_reward": 1.0, + "step": 517 + }, + { + "completion_length": 80.52500057220459, + "epoch": 0.5370658372213583, + "grad_norm": 4.682936191558838, + "kl": 0.51123046875, + "learning_rate": 2.943522974814533e-06, + "loss": 0.0104, + "reward": 1.7000000178813934, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 1.0, + "step": 518 + }, + { + "completion_length": 80.22500228881836, + "epoch": 0.5381026438569206, + "grad_norm": 0.6545246243476868, + "kl": 0.56591796875, + "learning_rate": 2.9430296427699743e-06, + "loss": 0.0316, + "reward": 1.550000011920929, + "reward_std": 0.21275537461042404, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 519 + }, + { + "completion_length": 139.82500457763672, + "epoch": 0.5391394504924831, + "grad_norm": 0.5691772699356079, + "kl": 0.54736328125, + "learning_rate": 2.9425342071389853e-06, + "loss": 0.0921, + "reward": 1.2750000357627869, + "reward_std": 0.1928323432803154, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9500000029802322, + "step": 520 + }, + { + "completion_length": 185.52500534057617, + "epoch": 0.5401762571280456, + "grad_norm": 1.905286192893982, + "kl": 0.724609375, + "learning_rate": 2.94203666864379e-06, + "loss": 0.1483, + "reward": 1.475000023841858, + "reward_std": 0.4290027618408203, + "rewards/accuracy_reward": 0.5249999947845936, + "rewards/format_reward": 0.9500000178813934, + "step": 521 + }, + { + "completion_length": 80.97500038146973, + "epoch": 0.5412130637636081, + "grad_norm": 0.35716575384140015, + "kl": 0.343994140625, + "learning_rate": 2.9415370280096797e-06, + "loss": 0.0305, + "reward": 1.3499999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9750000089406967, + "step": 522 + }, + { + "completion_length": 81.00000190734863, + "epoch": 0.5422498703991706, + "grad_norm": 0.46804380416870117, + "kl": 0.41943359375, + "learning_rate": 2.941035285965008e-06, + "loss": 0.0485, + "reward": 1.449999988079071, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9500000178813934, + "step": 523 + }, + { + "completion_length": 241.47500610351562, + "epoch": 0.543286677034733, + "grad_norm": 150.73275756835938, + "kl": 35.86083984375, + "learning_rate": 2.9405314432411933e-06, + "loss": 2.8672, + "reward": 1.350000023841858, + "reward_std": 0.30477424710989, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.925000011920929, + "step": 524 + }, + { + "completion_length": 128.45000457763672, + "epoch": 0.5443234836702955, + "grad_norm": 1.8508985042572021, + "kl": 1.2958984375, + "learning_rate": 2.9400255005727162e-06, + "loss": 0.1372, + "reward": 1.550000011920929, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 525 + }, + { + "completion_length": 245.07500839233398, + "epoch": 0.545360290305858, + "grad_norm": 1146.21435546875, + "kl": 151.87255859375, + "learning_rate": 2.9395174586971174e-06, + "loss": 11.6084, + "reward": 1.4000000059604645, + "reward_std": 0.3913118802011013, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9000000208616257, + "step": 526 + }, + { + "completion_length": 86.67500114440918, + "epoch": 0.5463970969414205, + "grad_norm": 0.6502148509025574, + "kl": 0.30712890625, + "learning_rate": 2.939007318354999e-06, + "loss": -0.0365, + "reward": 1.6500000357627869, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 1.0, + "step": 527 + }, + { + "completion_length": 163.04999923706055, + "epoch": 0.5474339035769828, + "grad_norm": 1.801808476448059, + "kl": 0.300537109375, + "learning_rate": 2.938495080290021e-06, + "loss": 0.0618, + "reward": 1.0250000357627869, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.07500000111758709, + "rewards/format_reward": 0.9500000178813934, + "step": 528 + }, + { + "completion_length": 288.6750011444092, + "epoch": 0.5484707102125453, + "grad_norm": 6.281752109527588, + "kl": 0.3994140625, + "learning_rate": 2.937980745248902e-06, + "loss": 0.1594, + "reward": 1.7000000476837158, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.8000000193715096, + "rewards/format_reward": 0.9000000208616257, + "step": 529 + }, + { + "completion_length": 155.25000381469727, + "epoch": 0.5495075168481078, + "grad_norm": 0.6555303931236267, + "kl": 0.35302734375, + "learning_rate": 2.9374643139814184e-06, + "loss": 0.0639, + "reward": 1.300000011920929, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 530 + }, + { + "completion_length": 214.6250114440918, + "epoch": 0.5505443234836703, + "grad_norm": 2.5419232845306396, + "kl": 0.55859375, + "learning_rate": 2.936945787240401e-06, + "loss": 0.1867, + "reward": 1.3250000476837158, + "reward_std": 0.44892582297325134, + "rewards/accuracy_reward": 0.3999999985098839, + "rewards/format_reward": 0.925000011920929, + "step": 531 + }, + { + "completion_length": 100.6500015258789, + "epoch": 0.5515811301192327, + "grad_norm": 0.7034846544265747, + "kl": 0.30908203125, + "learning_rate": 2.936425165781735e-06, + "loss": 0.0153, + "reward": 1.5500000417232513, + "reward_std": 0.31719936802983284, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 1.0, + "step": 532 + }, + { + "completion_length": 256.25, + "epoch": 0.5526179367547952, + "grad_norm": 4.066845417022705, + "kl": 0.75, + "learning_rate": 2.935902450364361e-06, + "loss": 0.1729, + "reward": 1.6000000536441803, + "reward_std": 0.2487340271472931, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9000000059604645, + "step": 533 + }, + { + "completion_length": 94.05000305175781, + "epoch": 0.5536547433903577, + "grad_norm": 0.6132490634918213, + "kl": 0.25537109375, + "learning_rate": 2.9353776417502704e-06, + "loss": 0.0136, + "reward": 1.550000011920929, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 1.0, + "step": 534 + }, + { + "completion_length": 213.6500015258789, + "epoch": 0.5546915500259202, + "grad_norm": 11.384795188903809, + "kl": 2.087890625, + "learning_rate": 2.934850740704506e-06, + "loss": 0.3464, + "reward": 1.6000000536441803, + "reward_std": 0.5031152740120888, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9000000357627869, + "step": 535 + }, + { + "completion_length": 102.1500015258789, + "epoch": 0.5557283566614827, + "grad_norm": 1.437371850013733, + "kl": 0.63671875, + "learning_rate": 2.934321747995163e-06, + "loss": 0.0334, + "reward": 1.7250000536441803, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 536 + }, + { + "completion_length": 164.0500030517578, + "epoch": 0.5567651632970451, + "grad_norm": 0.8429715037345886, + "kl": 0.400634765625, + "learning_rate": 2.933790664393383e-06, + "loss": 0.093, + "reward": 1.425000011920929, + "reward_std": 0.3354102075099945, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.9500000178813934, + "step": 537 + }, + { + "completion_length": 169.07500076293945, + "epoch": 0.5578019699326076, + "grad_norm": 0.981852650642395, + "kl": 0.619384765625, + "learning_rate": 2.9332574906733574e-06, + "loss": 0.1237, + "reward": 1.7000000476837158, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9500000178813934, + "step": 538 + }, + { + "completion_length": 343.6750011444092, + "epoch": 0.55883877656817, + "grad_norm": 1.502892255783081, + "kl": 0.77099609375, + "learning_rate": 2.9327222276123232e-06, + "loss": 0.2786, + "reward": 1.5000000149011612, + "reward_std": 0.3533165603876114, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.8750000149011612, + "step": 539 + }, + { + "completion_length": 167.00000190734863, + "epoch": 0.5598755832037325, + "grad_norm": 0.9482361674308777, + "kl": 0.635009765625, + "learning_rate": 2.9321848759905643e-06, + "loss": 0.08, + "reward": 1.725000023841858, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 540 + }, + { + "completion_length": 113.20000267028809, + "epoch": 0.5609123898392949, + "grad_norm": 0.9624785780906677, + "kl": 0.711669921875, + "learning_rate": 2.931645436591409e-06, + "loss": 0.1214, + "reward": 1.4500000178813934, + "reward_std": 0.2614361569285393, + "rewards/accuracy_reward": 0.5000000018626451, + "rewards/format_reward": 0.9500000178813934, + "step": 541 + }, + { + "completion_length": 353.399995803833, + "epoch": 0.5619491964748574, + "grad_norm": 4.333626747131348, + "kl": 2.579345703125, + "learning_rate": 2.9311039102012277e-06, + "loss": 0.2335, + "reward": 1.2000000178813934, + "reward_std": 0.28485121577978134, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.8500000089406967, + "step": 542 + }, + { + "completion_length": 177.00000381469727, + "epoch": 0.5629860031104199, + "grad_norm": 1.3422383069992065, + "kl": 0.70458984375, + "learning_rate": 2.9305602976094354e-06, + "loss": 0.1408, + "reward": 1.3750000149011612, + "reward_std": 0.279508501291275, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9250000268220901, + "step": 543 + }, + { + "completion_length": 181.97500610351562, + "epoch": 0.5640228097459824, + "grad_norm": 1.8206522464752197, + "kl": 1.82177734375, + "learning_rate": 2.9300145996084865e-06, + "loss": 0.2003, + "reward": 1.4000000059604645, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9500000178813934, + "step": 544 + }, + { + "completion_length": 340.52500915527344, + "epoch": 0.5650596163815449, + "grad_norm": 77.96588134765625, + "kl": 18.46630859375, + "learning_rate": 2.929466816993877e-06, + "loss": 1.4838, + "reward": 1.4250000268220901, + "reward_std": 0.38109346479177475, + "rewards/accuracy_reward": 0.5500000063329935, + "rewards/format_reward": 0.8750000149011612, + "step": 545 + }, + { + "completion_length": 107.55000114440918, + "epoch": 0.5660964230171073, + "grad_norm": 0.47127431631088257, + "kl": 0.373779296875, + "learning_rate": 2.928916950564139e-06, + "loss": 0.0424, + "reward": 1.2000000327825546, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.9750000089406967, + "step": 546 + }, + { + "completion_length": 187.20000076293945, + "epoch": 0.5671332296526698, + "grad_norm": 4.727897644042969, + "kl": 3.77197265625, + "learning_rate": 2.928365001120846e-06, + "loss": 0.2137, + "reward": 1.5000000298023224, + "reward_std": 0.4723408445715904, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9000000208616257, + "step": 547 + }, + { + "completion_length": 212.32500457763672, + "epoch": 0.5681700362882323, + "grad_norm": 1.5274919271469116, + "kl": 2.5712890625, + "learning_rate": 2.927810969468605e-06, + "loss": 0.3168, + "reward": 1.3000000268220901, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.9250000268220901, + "step": 548 + }, + { + "completion_length": 250.85000038146973, + "epoch": 0.5692068429237948, + "grad_norm": 1.4735158681869507, + "kl": 1.9208984375, + "learning_rate": 2.9272548564150593e-06, + "loss": 0.2983, + "reward": 1.5, + "reward_std": 0.3913119100034237, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 549 + }, + { + "completion_length": 102.10000228881836, + "epoch": 0.5702436495593571, + "grad_norm": 0.5658621191978455, + "kl": 0.34375, + "learning_rate": 2.926696662770887e-06, + "loss": -0.0251, + "reward": 1.350000023841858, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 1.0, + "step": 550 + }, + { + "completion_length": 118.80000495910645, + "epoch": 0.5712804561949196, + "grad_norm": 0.3587953448295593, + "kl": 0.31005859375, + "learning_rate": 2.9261363893497988e-06, + "loss": 0.0136, + "reward": 1.425000011920929, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 1.0, + "step": 551 + }, + { + "completion_length": 147.57500076293945, + "epoch": 0.5723172628304821, + "grad_norm": 0.8031432032585144, + "kl": 0.4482421875, + "learning_rate": 2.9255740369685367e-06, + "loss": 0.016, + "reward": 1.3500000089406967, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 552 + }, + { + "completion_length": 164.05000495910645, + "epoch": 0.5733540694660446, + "grad_norm": 1.5201157331466675, + "kl": 1.3935546875, + "learning_rate": 2.9250096064468745e-06, + "loss": 0.1377, + "reward": 1.3500000536441803, + "reward_std": 0.37310105562210083, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 553 + }, + { + "completion_length": 101.75000190734863, + "epoch": 0.574390876101607, + "grad_norm": 0.4400118887424469, + "kl": 0.295166015625, + "learning_rate": 2.9244430986076146e-06, + "loss": -0.0034, + "reward": 1.550000011920929, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 1.0, + "step": 554 + }, + { + "completion_length": 202.8499984741211, + "epoch": 0.5754276827371695, + "grad_norm": 2.103760004043579, + "kl": 2.682373046875, + "learning_rate": 2.923874514276587e-06, + "loss": 0.2481, + "reward": 1.300000011920929, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9250000268220901, + "step": 555 + }, + { + "completion_length": 103.17500114440918, + "epoch": 0.576464489372732, + "grad_norm": 0.5067854523658752, + "kl": 0.31201171875, + "learning_rate": 2.9233038542826496e-06, + "loss": 0.0438, + "reward": 1.5250000059604645, + "reward_std": 0.1928323432803154, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 1.0, + "step": 556 + }, + { + "completion_length": 172.92499923706055, + "epoch": 0.5775012960082945, + "grad_norm": 1.1114473342895508, + "kl": 0.310791015625, + "learning_rate": 2.9227311194576865e-06, + "loss": 0.0272, + "reward": 1.4500000178813934, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.4749999977648258, + "rewards/format_reward": 0.9750000089406967, + "step": 557 + }, + { + "completion_length": 156.7500057220459, + "epoch": 0.578538102643857, + "grad_norm": 1.3353294134140015, + "kl": 1.7294921875, + "learning_rate": 2.9221563106366057e-06, + "loss": 0.1717, + "reward": 1.225000023841858, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 558 + }, + { + "completion_length": 175.80000686645508, + "epoch": 0.5795749092794193, + "grad_norm": 3.629734516143799, + "kl": 1.093505859375, + "learning_rate": 2.9215794286573383e-06, + "loss": 0.1307, + "reward": 1.275000050663948, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.9500000178813934, + "step": 559 + }, + { + "completion_length": 111.82500267028809, + "epoch": 0.5806117159149818, + "grad_norm": 0.19621193408966064, + "kl": 0.2568359375, + "learning_rate": 2.921000474360838e-06, + "loss": 0.0142, + "reward": 1.4749999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 1.0, + "step": 560 + }, + { + "completion_length": 121.42500305175781, + "epoch": 0.5816485225505443, + "grad_norm": 0.4136292040348053, + "kl": 0.21484375, + "learning_rate": 2.9204194485910798e-06, + "loss": -0.0062, + "reward": 1.600000023841858, + "reward_std": 0.1928323358297348, + "rewards/accuracy_reward": 0.5999999940395355, + "rewards/format_reward": 1.0, + "step": 561 + }, + { + "completion_length": 131.62500190734863, + "epoch": 0.5826853291861068, + "grad_norm": 0.49414509534835815, + "kl": 0.26416015625, + "learning_rate": 2.9198363521950583e-06, + "loss": 0.0075, + "reward": 1.4000000357627869, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 1.0, + "step": 562 + }, + { + "completion_length": 173.89999961853027, + "epoch": 0.5837221358216692, + "grad_norm": 0.8812211751937866, + "kl": 0.484375, + "learning_rate": 2.9192511860227855e-06, + "loss": 0.0233, + "reward": 1.3500000089406967, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 563 + }, + { + "completion_length": 175.225004196167, + "epoch": 0.5847589424572317, + "grad_norm": 1.1985092163085938, + "kl": 1.24365234375, + "learning_rate": 2.9186639509272923e-06, + "loss": 0.0881, + "reward": 1.5000000298023224, + "reward_std": 0.2795084938406944, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9500000178813934, + "step": 564 + }, + { + "completion_length": 134.4250030517578, + "epoch": 0.5857957490927942, + "grad_norm": 0.4528398811817169, + "kl": 0.299072265625, + "learning_rate": 2.918074647764624e-06, + "loss": 0.0062, + "reward": 1.699999988079071, + "reward_std": 0.18026871606707573, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 1.0, + "step": 565 + }, + { + "completion_length": 120.1500015258789, + "epoch": 0.5868325557283567, + "grad_norm": 0.4748910069465637, + "kl": 0.2451171875, + "learning_rate": 2.9174832773938413e-06, + "loss": 0.026, + "reward": 1.725000023841858, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.725000012665987, + "rewards/format_reward": 1.0, + "step": 566 + }, + { + "completion_length": 180.22500610351562, + "epoch": 0.5878693623639192, + "grad_norm": 0.43165019154548645, + "kl": 0.501708984375, + "learning_rate": 2.916889840677019e-06, + "loss": 0.0587, + "reward": 1.350000023841858, + "reward_std": 0.248734038323164, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 567 + }, + { + "completion_length": 176.225004196167, + "epoch": 0.5889061689994816, + "grad_norm": 0.5352876782417297, + "kl": 0.711669921875, + "learning_rate": 2.9162943384792435e-06, + "loss": 0.0419, + "reward": 1.6500000059604645, + "reward_std": 0.21275537088513374, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9750000089406967, + "step": 568 + }, + { + "completion_length": 193.17500495910645, + "epoch": 0.589942975635044, + "grad_norm": 0.6368687152862549, + "kl": 0.36376953125, + "learning_rate": 2.915696771668612e-06, + "loss": 0.0535, + "reward": 1.5750000476837158, + "reward_std": 0.36053743958473206, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 569 + }, + { + "completion_length": 132.32500457763672, + "epoch": 0.5909797822706065, + "grad_norm": 0.36829519271850586, + "kl": 0.236328125, + "learning_rate": 2.9150971411162315e-06, + "loss": -0.0057, + "reward": 1.4500000178813934, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.44999999925494194, + "rewards/format_reward": 1.0, + "step": 570 + }, + { + "completion_length": 152.65000343322754, + "epoch": 0.592016588906169, + "grad_norm": 0.4644145965576172, + "kl": 0.241943359375, + "learning_rate": 2.914495447696218e-06, + "loss": -0.0219, + "reward": 1.625, + "reward_std": 0.24873404577374458, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 1.0, + "step": 571 + }, + { + "completion_length": 190.15000915527344, + "epoch": 0.5930533955417314, + "grad_norm": 0.4808287024497986, + "kl": 0.278564453125, + "learning_rate": 2.9138916922856946e-06, + "loss": 0.0605, + "reward": 1.5250000059604645, + "reward_std": 0.21275536343455315, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9750000089406967, + "step": 572 + }, + { + "completion_length": 203.225004196167, + "epoch": 0.5940902021772939, + "grad_norm": 9.010258674621582, + "kl": 5.313232421875, + "learning_rate": 2.9132858757647884e-06, + "loss": 0.4188, + "reward": 1.6500000357627869, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 573 + }, + { + "completion_length": 142.100004196167, + "epoch": 0.5951270088128564, + "grad_norm": 0.4201050400733948, + "kl": 0.2431640625, + "learning_rate": 2.912677999016634e-06, + "loss": 0.0162, + "reward": 1.350000023841858, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 1.0, + "step": 574 + }, + { + "completion_length": 174.1750030517578, + "epoch": 0.5961638154484189, + "grad_norm": 0.3380955755710602, + "kl": 0.2294921875, + "learning_rate": 2.9120680629273677e-06, + "loss": -0.0121, + "reward": 1.4000000357627869, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9750000089406967, + "step": 575 + }, + { + "completion_length": 153.12500381469727, + "epoch": 0.5972006220839814, + "grad_norm": 0.44647836685180664, + "kl": 0.2392578125, + "learning_rate": 2.9114560683861275e-06, + "loss": 0.0226, + "reward": 1.5250000357627869, + "reward_std": 0.24873404577374458, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 1.0, + "step": 576 + }, + { + "completion_length": 262.6500015258789, + "epoch": 0.5982374287195438, + "grad_norm": 7.015067100524902, + "kl": 0.88427734375, + "learning_rate": 2.910842016285053e-06, + "loss": 0.1761, + "reward": 1.475000023841858, + "reward_std": 0.3354101851582527, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.925000011920929, + "step": 577 + }, + { + "completion_length": 132.9000015258789, + "epoch": 0.5992742353551063, + "grad_norm": 0.2848128080368042, + "kl": 0.248046875, + "learning_rate": 2.910225907519283e-06, + "loss": -0.002, + "reward": 1.2750000059604645, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 1.0, + "step": 578 + }, + { + "completion_length": 303.50000190734863, + "epoch": 0.6003110419906688, + "grad_norm": 96.8685302734375, + "kl": 25.11279296875, + "learning_rate": 2.9096077429869543e-06, + "loss": 1.9498, + "reward": 1.4500000178813934, + "reward_std": 0.2920721024274826, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.9000000059604645, + "step": 579 + }, + { + "completion_length": 245.5250072479248, + "epoch": 0.6013478486262313, + "grad_norm": 1.2218270301818848, + "kl": 0.70263671875, + "learning_rate": 2.9089875235892005e-06, + "loss": 0.1386, + "reward": 1.5250000208616257, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 580 + }, + { + "completion_length": 188.2000026702881, + "epoch": 0.6023846552617936, + "grad_norm": 4.220678806304932, + "kl": 2.01708984375, + "learning_rate": 2.9083652502301506e-06, + "loss": 0.1683, + "reward": 1.600000023841858, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 581 + }, + { + "completion_length": 269.4500045776367, + "epoch": 0.6034214618973561, + "grad_norm": 5.021411895751953, + "kl": 4.0146484375, + "learning_rate": 2.907740923816928e-06, + "loss": 0.3696, + "reward": 1.2500000298023224, + "reward_std": 0.4038754962384701, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.9250000268220901, + "step": 582 + }, + { + "completion_length": 183.97500038146973, + "epoch": 0.6044582685329186, + "grad_norm": 0.7388213872909546, + "kl": 0.39697265625, + "learning_rate": 2.907114545259649e-06, + "loss": 0.1162, + "reward": 1.350000038743019, + "reward_std": 0.3354102112352848, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.9000000208616257, + "step": 583 + }, + { + "completion_length": 198.9749984741211, + "epoch": 0.6054950751684811, + "grad_norm": 6.0927934646606445, + "kl": 1.3525390625, + "learning_rate": 2.9064861154714206e-06, + "loss": 0.108, + "reward": 1.3750000149011612, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 584 + }, + { + "completion_length": 350.6000099182129, + "epoch": 0.6065318818040435, + "grad_norm": 3.432065010070801, + "kl": 1.06787109375, + "learning_rate": 2.9058556353683413e-06, + "loss": 0.1571, + "reward": 1.6500000059604645, + "reward_std": 0.5715806148946285, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.8750000149011612, + "step": 585 + }, + { + "completion_length": 186.05000495910645, + "epoch": 0.607568688439606, + "grad_norm": 1.2789217233657837, + "kl": 1.263916015625, + "learning_rate": 2.9052231058694975e-06, + "loss": 0.1736, + "reward": 1.550000011920929, + "reward_std": 0.38409099355340004, + "rewards/accuracy_reward": 0.6500000152736902, + "rewards/format_reward": 0.9000000059604645, + "step": 586 + }, + { + "completion_length": 140.97500228881836, + "epoch": 0.6086054950751685, + "grad_norm": 0.31413471698760986, + "kl": 0.222900390625, + "learning_rate": 2.9045885278969635e-06, + "loss": -0.0107, + "reward": 1.675000011920929, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 587 + }, + { + "completion_length": 130.9000015258789, + "epoch": 0.609642301710731, + "grad_norm": 0.4696264863014221, + "kl": 0.25244140625, + "learning_rate": 2.9039519023758004e-06, + "loss": 0.0824, + "reward": 1.4000000059604645, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 1.0, + "step": 588 + }, + { + "completion_length": 196.32500839233398, + "epoch": 0.6106791083462935, + "grad_norm": 4.557334899902344, + "kl": 0.811767578125, + "learning_rate": 2.9033132302340523e-06, + "loss": 0.1282, + "reward": 1.5250000655651093, + "reward_std": 0.36053743958473206, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 589 + }, + { + "completion_length": 292.9749984741211, + "epoch": 0.6117159149818558, + "grad_norm": 9.806172370910645, + "kl": 5.443359375, + "learning_rate": 2.902672512402749e-06, + "loss": 0.2563, + "reward": 1.4000000059604645, + "reward_std": 0.2974148504436016, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 590 + }, + { + "completion_length": 274.25, + "epoch": 0.6127527216174183, + "grad_norm": 2.256058692932129, + "kl": 1.713134765625, + "learning_rate": 2.9020297498159004e-06, + "loss": 0.2101, + "reward": 1.5750000476837158, + "reward_std": 0.36588016152381897, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 591 + }, + { + "completion_length": 241.0500030517578, + "epoch": 0.6137895282529808, + "grad_norm": 1.3324028253555298, + "kl": 0.608154296875, + "learning_rate": 2.901384943410499e-06, + "loss": 0.1396, + "reward": 1.5750000476837158, + "reward_std": 0.403875507414341, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 592 + }, + { + "completion_length": 247.00000953674316, + "epoch": 0.6148263348885433, + "grad_norm": 6.3438873291015625, + "kl": 0.499267578125, + "learning_rate": 2.900738094126515e-06, + "loss": 0.1196, + "reward": 1.600000023841858, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 593 + }, + { + "completion_length": 296.42499923706055, + "epoch": 0.6158631415241057, + "grad_norm": 1.7976075410842896, + "kl": 0.67138671875, + "learning_rate": 2.9000892029068974e-06, + "loss": 0.0767, + "reward": 1.3000000268220901, + "reward_std": 0.3297629915177822, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.9249999970197678, + "step": 594 + }, + { + "completion_length": 152.9000015258789, + "epoch": 0.6168999481596682, + "grad_norm": 0.41897207498550415, + "kl": 0.201171875, + "learning_rate": 2.899438270697573e-06, + "loss": 0.0164, + "reward": 1.4000000059604645, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.4000000115483999, + "rewards/format_reward": 1.0, + "step": 595 + }, + { + "completion_length": 224.95000457763672, + "epoch": 0.6179367547952307, + "grad_norm": 1.1412527561187744, + "kl": 1.325439453125, + "learning_rate": 2.8987852984474413e-06, + "loss": 0.1502, + "reward": 1.4000000357627869, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9750000089406967, + "step": 596 + }, + { + "completion_length": 170.4000015258789, + "epoch": 0.6189735614307932, + "grad_norm": 0.8881285190582275, + "kl": 0.442626953125, + "learning_rate": 2.898130287108378e-06, + "loss": 0.0808, + "reward": 1.2500000447034836, + "reward_std": 0.1928323321044445, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 597 + }, + { + "completion_length": 282.5500030517578, + "epoch": 0.6200103680663557, + "grad_norm": 11.336788177490234, + "kl": 6.169921875, + "learning_rate": 2.8974732376352302e-06, + "loss": 0.6068, + "reward": 1.5250000357627869, + "reward_std": 0.347973819822073, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9250000268220901, + "step": 598 + }, + { + "completion_length": 203.20000648498535, + "epoch": 0.6210471747019181, + "grad_norm": 1.8857403993606567, + "kl": 1.327392578125, + "learning_rate": 2.8968141509858157e-06, + "loss": 0.1394, + "reward": 1.3250000178813934, + "reward_std": 0.21275536715984344, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.9750000089406967, + "step": 599 + }, + { + "completion_length": 165.30000114440918, + "epoch": 0.6220839813374806, + "grad_norm": 0.3913152515888214, + "kl": 0.234619140625, + "learning_rate": 2.896153028120924e-06, + "loss": -0.0227, + "reward": 1.625, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 1.0, + "step": 600 + }, + { + "completion_length": 190.225004196167, + "epoch": 0.623120787973043, + "grad_norm": 2.7854464054107666, + "kl": 0.409423828125, + "learning_rate": 2.89548987000431e-06, + "loss": 0.0567, + "reward": 1.2000000178813934, + "reward_std": 0.28485122323036194, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.9750000089406967, + "step": 601 + }, + { + "completion_length": 253.82500076293945, + "epoch": 0.6241575946086055, + "grad_norm": 5.791003704071045, + "kl": 0.375244140625, + "learning_rate": 2.8948246776026985e-06, + "loss": 0.1521, + "reward": 1.4500000327825546, + "reward_std": 0.31733787432312965, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 602 + }, + { + "completion_length": 136.20000076293945, + "epoch": 0.6251944012441679, + "grad_norm": 0.45052769780158997, + "kl": 0.23486328125, + "learning_rate": 2.8941574518857763e-06, + "loss": -0.0226, + "reward": 1.449999988079071, + "reward_std": 0.18026872351765633, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 1.0, + "step": 603 + }, + { + "completion_length": 189.35000610351562, + "epoch": 0.6262312078797304, + "grad_norm": 1.993778109550476, + "kl": 0.494140625, + "learning_rate": 2.8934881938261987e-06, + "loss": 0.0845, + "reward": 1.425000011920929, + "reward_std": 0.21275537461042404, + "rewards/accuracy_reward": 0.4499999973922968, + "rewards/format_reward": 0.9750000089406967, + "step": 604 + }, + { + "completion_length": 299.3500099182129, + "epoch": 0.6272680145152929, + "grad_norm": 3.556016683578491, + "kl": 1.59912109375, + "learning_rate": 2.8928169043995793e-06, + "loss": 0.2569, + "reward": 1.2749999910593033, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9000000357627869, + "step": 605 + }, + { + "completion_length": 132.07500076293945, + "epoch": 0.6283048211508554, + "grad_norm": 0.5948020219802856, + "kl": 0.26513671875, + "learning_rate": 2.8921435845844963e-06, + "loss": 0.0789, + "reward": 1.7000000178813934, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 1.0, + "step": 606 + }, + { + "completion_length": 239.55000495910645, + "epoch": 0.6293416277864178, + "grad_norm": 11.134238243103027, + "kl": 3.626220703125, + "learning_rate": 2.8914682353624855e-06, + "loss": 0.3149, + "reward": 1.3750000596046448, + "reward_std": 0.36053745448589325, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9500000178813934, + "step": 607 + }, + { + "completion_length": 217.1500015258789, + "epoch": 0.6303784344219803, + "grad_norm": 0.6335292458534241, + "kl": 0.396484375, + "learning_rate": 2.890790857718042e-06, + "loss": 0.0686, + "reward": 1.7000000178813934, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 608 + }, + { + "completion_length": 252.00000381469727, + "epoch": 0.6314152410575428, + "grad_norm": 45.83848190307617, + "kl": 8.284423828125, + "learning_rate": 2.8901114526386176e-06, + "loss": 0.6874, + "reward": 1.5250000357627869, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.925000011920929, + "step": 609 + }, + { + "completion_length": 224.375, + "epoch": 0.6324520476931053, + "grad_norm": 6.116477012634277, + "kl": 2.39501953125, + "learning_rate": 2.8894300211146207e-06, + "loss": 0.2235, + "reward": 1.375, + "reward_std": 0.22360679879784584, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 610 + }, + { + "completion_length": 194.2000026702881, + "epoch": 0.6334888543286677, + "grad_norm": 0.7082158923149109, + "kl": 0.6552734375, + "learning_rate": 2.8887465641394114e-06, + "loss": 0.1211, + "reward": 1.425000011920929, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 611 + }, + { + "completion_length": 181.80000686645508, + "epoch": 0.6345256609642301, + "grad_norm": 0.7598231434822083, + "kl": 1.06689453125, + "learning_rate": 2.8880610827093045e-06, + "loss": 0.0976, + "reward": 1.4000000059604645, + "reward_std": 0.21638592705130577, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9750000089406967, + "step": 612 + }, + { + "completion_length": 165.0500030517578, + "epoch": 0.6355624675997926, + "grad_norm": 0.45146992802619934, + "kl": 0.185302734375, + "learning_rate": 2.8873735778235657e-06, + "loss": 0.0104, + "reward": 1.5, + "reward_std": 0.3354102000594139, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.9750000089406967, + "step": 613 + }, + { + "completion_length": 252.3750114440918, + "epoch": 0.6365992742353551, + "grad_norm": 0.7813171148300171, + "kl": 0.745849609375, + "learning_rate": 2.8866840504844094e-06, + "loss": 0.1231, + "reward": 1.2500000298023224, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.925000011920929, + "step": 614 + }, + { + "completion_length": 289.6500015258789, + "epoch": 0.6376360808709176, + "grad_norm": 3.395939588546753, + "kl": 3.01611328125, + "learning_rate": 2.8859925016969995e-06, + "loss": 0.2634, + "reward": 1.4249999821186066, + "reward_std": 0.3737340532243252, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.8750000149011612, + "step": 615 + }, + { + "completion_length": 462.9249954223633, + "epoch": 0.63867288750648, + "grad_norm": 2.6913440227508545, + "kl": 3.1015625, + "learning_rate": 2.885298932469445e-06, + "loss": 0.4213, + "reward": 1.3250000476837158, + "reward_std": 0.7158706709742546, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.8250000029802322, + "step": 616 + }, + { + "completion_length": 311.7750129699707, + "epoch": 0.6397096941420425, + "grad_norm": 1.5979684591293335, + "kl": 2.26220703125, + "learning_rate": 2.884603343812802e-06, + "loss": 0.1652, + "reward": 1.2500000149011612, + "reward_std": 0.39728759974241257, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.9000000208616257, + "step": 617 + }, + { + "completion_length": 327.32501220703125, + "epoch": 0.640746500777605, + "grad_norm": 12.20130443572998, + "kl": 6.74072265625, + "learning_rate": 2.8839057367410696e-06, + "loss": 0.6414, + "reward": 1.600000023841858, + "reward_std": 0.4237985163927078, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9000000208616257, + "step": 618 + }, + { + "completion_length": 207.82500076293945, + "epoch": 0.6417833074131675, + "grad_norm": 1.5905780792236328, + "kl": 0.2890625, + "learning_rate": 2.883206112271189e-06, + "loss": 0.0787, + "reward": 1.375, + "reward_std": 0.3354102000594139, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.9750000089406967, + "step": 619 + }, + { + "completion_length": 218.95000076293945, + "epoch": 0.64282011404873, + "grad_norm": 3.3236258029937744, + "kl": 1.61669921875, + "learning_rate": 2.8825044714230427e-06, + "loss": 0.1564, + "reward": 1.5250000208616257, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 620 + }, + { + "completion_length": 233.95000076293945, + "epoch": 0.6438569206842923, + "grad_norm": 2.439260721206665, + "kl": 0.500732421875, + "learning_rate": 2.881800815219453e-06, + "loss": 0.1286, + "reward": 1.3250000178813934, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.37500000186264515, + "rewards/format_reward": 0.9500000178813934, + "step": 621 + }, + { + "completion_length": 200.4000015258789, + "epoch": 0.6448937273198548, + "grad_norm": 0.662738025188446, + "kl": 0.79296875, + "learning_rate": 2.881095144686179e-06, + "loss": 0.0613, + "reward": 1.4500000178813934, + "reward_std": 0.3533165529370308, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9500000029802322, + "step": 622 + }, + { + "completion_length": 300.60000228881836, + "epoch": 0.6459305339554173, + "grad_norm": 1.7969317436218262, + "kl": 0.51513671875, + "learning_rate": 2.880387460851917e-06, + "loss": 0.108, + "reward": 1.225000023841858, + "reward_std": 0.6021890863776207, + "rewards/accuracy_reward": 0.34999999962747097, + "rewards/format_reward": 0.8750000298023224, + "step": 623 + }, + { + "completion_length": 242.57500076293945, + "epoch": 0.6469673405909798, + "grad_norm": 0.5022680759429932, + "kl": 0.31103515625, + "learning_rate": 2.879677764748298e-06, + "loss": 0.0101, + "reward": 1.475000023841858, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 0.9500000029802322, + "step": 624 + }, + { + "completion_length": 245.57500076293945, + "epoch": 0.6480041472265422, + "grad_norm": 0.5517757534980774, + "kl": 0.272705078125, + "learning_rate": 2.8789660574098863e-06, + "loss": 0.0851, + "reward": 1.25, + "reward_std": 0.3696090541779995, + "rewards/accuracy_reward": 0.30000000819563866, + "rewards/format_reward": 0.9500000029802322, + "step": 625 + }, + { + "completion_length": 160.2750015258789, + "epoch": 0.6490409538621047, + "grad_norm": 0.43617773056030273, + "kl": 0.24365234375, + "learning_rate": 2.8782523398741783e-06, + "loss": -0.0277, + "reward": 1.3250000327825546, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 626 + }, + { + "completion_length": 194.95000457763672, + "epoch": 0.6500777604976672, + "grad_norm": 0.6225988268852234, + "kl": 0.365234375, + "learning_rate": 2.8775366131816006e-06, + "loss": 0.0594, + "reward": 1.4250000268220901, + "reward_std": 0.37323956564068794, + "rewards/accuracy_reward": 0.5250000040978193, + "rewards/format_reward": 0.9000000208616257, + "step": 627 + }, + { + "completion_length": 320.20000076293945, + "epoch": 0.6511145671332297, + "grad_norm": 5.313282489776611, + "kl": 1.0943603515625, + "learning_rate": 2.876818878375509e-06, + "loss": 0.2469, + "reward": 1.2750000208616257, + "reward_std": 0.4415663853287697, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9000000059604645, + "step": 628 + }, + { + "completion_length": 197.725004196167, + "epoch": 0.6521513737687922, + "grad_norm": 2.7158896923065186, + "kl": 1.244873046875, + "learning_rate": 2.8760991365021856e-06, + "loss": 0.2859, + "reward": 1.4250000268220901, + "reward_std": 0.5048274546861649, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.925000011920929, + "step": 629 + }, + { + "completion_length": 232.45000457763672, + "epoch": 0.6531881804043546, + "grad_norm": 16.022092819213867, + "kl": 4.732421875, + "learning_rate": 2.8753773886108405e-06, + "loss": 0.3514, + "reward": 1.6500000059604645, + "reward_std": 0.44533541798591614, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 630 + }, + { + "completion_length": 307.2250061035156, + "epoch": 0.654224987039917, + "grad_norm": 62.19873809814453, + "kl": 17.984130859375, + "learning_rate": 2.874653635753605e-06, + "loss": 1.3583, + "reward": 1.4250000417232513, + "reward_std": 0.4525563195347786, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 631 + }, + { + "completion_length": 149.2000026702881, + "epoch": 0.6552617936754795, + "grad_norm": 0.5755558013916016, + "kl": 0.265869140625, + "learning_rate": 2.8739278789855362e-06, + "loss": -0.005, + "reward": 1.3250000327825546, + "reward_std": 0.40387552231550217, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 632 + }, + { + "completion_length": 168.62499809265137, + "epoch": 0.656298600311042, + "grad_norm": 0.7181000113487244, + "kl": 0.38720703125, + "learning_rate": 2.8732001193646104e-06, + "loss": 0.078, + "reward": 1.300000011920929, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.32500001043081284, + "rewards/format_reward": 0.9750000089406967, + "step": 633 + }, + { + "completion_length": 253.22500610351562, + "epoch": 0.6573354069466044, + "grad_norm": 77.6446304321289, + "kl": 20.80859375, + "learning_rate": 2.872470357951724e-06, + "loss": 1.654, + "reward": 1.225000038743019, + "reward_std": 0.3245587572455406, + "rewards/accuracy_reward": 0.2999999988824129, + "rewards/format_reward": 0.925000011920929, + "step": 634 + }, + { + "completion_length": 251.20000076293945, + "epoch": 0.6583722135821669, + "grad_norm": 1.6625399589538574, + "kl": 1.81103515625, + "learning_rate": 2.871738595810692e-06, + "loss": 0.212, + "reward": 1.3999999910593033, + "reward_std": 0.2686570808291435, + "rewards/accuracy_reward": 0.47500001825392246, + "rewards/format_reward": 0.925000011920929, + "step": 635 + }, + { + "completion_length": 499.15001678466797, + "epoch": 0.6594090202177294, + "grad_norm": 8.252129554748535, + "kl": 2.3154296875, + "learning_rate": 2.871004834008246e-06, + "loss": 0.3376, + "reward": 1.0500000417232513, + "reward_std": 0.597340852022171, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.7750000208616257, + "step": 636 + }, + { + "completion_length": 207.90000534057617, + "epoch": 0.6604458268532919, + "grad_norm": 2.2362051010131836, + "kl": 0.253662109375, + "learning_rate": 2.8702690736140318e-06, + "loss": 0.0666, + "reward": 1.550000011920929, + "reward_std": 0.43636220321059227, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 637 + }, + { + "completion_length": 230.37500381469727, + "epoch": 0.6614826334888543, + "grad_norm": 1.240039587020874, + "kl": 0.55322265625, + "learning_rate": 2.8695313157006097e-06, + "loss": 0.1479, + "reward": 1.3250000029802322, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.8750000149011612, + "step": 638 + }, + { + "completion_length": 283.0250053405762, + "epoch": 0.6625194401244168, + "grad_norm": 3.555227041244507, + "kl": 0.6796875, + "learning_rate": 2.868791561343451e-06, + "loss": 0.16, + "reward": 1.475000023841858, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9000000059604645, + "step": 639 + }, + { + "completion_length": 114.42500305175781, + "epoch": 0.6635562467599793, + "grad_norm": 0.4519474506378174, + "kl": 0.255126953125, + "learning_rate": 2.868049811620938e-06, + "loss": 0.0198, + "reward": 1.6500000059604645, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6500000022351742, + "rewards/format_reward": 1.0, + "step": 640 + }, + { + "completion_length": 236.2750015258789, + "epoch": 0.6645930533955418, + "grad_norm": 7.132613182067871, + "kl": 4.499267578125, + "learning_rate": 2.8673060676143616e-06, + "loss": 0.4182, + "reward": 1.4000000059604645, + "reward_std": 0.620261438190937, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8500000238418579, + "step": 641 + }, + { + "completion_length": 224.92499923706055, + "epoch": 0.6656298600311042, + "grad_norm": 40.20530319213867, + "kl": 15.702392578125, + "learning_rate": 2.8665603304079197e-06, + "loss": 1.1686, + "reward": 1.0250000059604645, + "reward_std": 0.3731010742485523, + "rewards/accuracy_reward": 0.20000000298023224, + "rewards/format_reward": 0.8250000029802322, + "step": 642 + }, + { + "completion_length": 186.72500038146973, + "epoch": 0.6666666666666666, + "grad_norm": 0.7823323011398315, + "kl": 0.713623046875, + "learning_rate": 2.865812601088715e-06, + "loss": 0.154, + "reward": 1.5500000417232513, + "reward_std": 0.41643913835287094, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000208616257, + "step": 643 + }, + { + "completion_length": 192.20000457763672, + "epoch": 0.6677034733022291, + "grad_norm": 1.6138575077056885, + "kl": 1.10498046875, + "learning_rate": 2.865062880746756e-06, + "loss": 0.1178, + "reward": 1.5750000476837158, + "reward_std": 0.2937843129038811, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 644 + }, + { + "completion_length": 202.75000381469727, + "epoch": 0.6687402799377916, + "grad_norm": 1.0543968677520752, + "kl": 1.532470703125, + "learning_rate": 2.864311170474953e-06, + "loss": 0.1468, + "reward": 1.350000023841858, + "reward_std": 0.29270511120557785, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.925000011920929, + "step": 645 + }, + { + "completion_length": 143.85000038146973, + "epoch": 0.6697770865733541, + "grad_norm": 0.39155247807502747, + "kl": 0.27880859375, + "learning_rate": 2.8635574713691158e-06, + "loss": 0.017, + "reward": 1.350000023841858, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.37500000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 646 + }, + { + "completion_length": 149.47500228881836, + "epoch": 0.6708138932089165, + "grad_norm": 5.471457481384277, + "kl": 2.15234375, + "learning_rate": 2.8628017845279553e-06, + "loss": 0.1969, + "reward": 1.4250000268220901, + "reward_std": 0.4415663853287697, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.9000000208616257, + "step": 647 + }, + { + "completion_length": 180.02500534057617, + "epoch": 0.671850699844479, + "grad_norm": 1.4774538278579712, + "kl": 1.03564453125, + "learning_rate": 2.862044111053079e-06, + "loss": 0.1221, + "reward": 1.350000038743019, + "reward_std": 0.4291412830352783, + "rewards/accuracy_reward": 0.4500000048428774, + "rewards/format_reward": 0.9000000208616257, + "step": 648 + }, + { + "completion_length": 247.97500228881836, + "epoch": 0.6728875064800415, + "grad_norm": 0.93613600730896, + "kl": 0.880126953125, + "learning_rate": 2.8612844520489902e-06, + "loss": 0.1469, + "reward": 1.2000000178813934, + "reward_std": 0.4129471369087696, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.9000000208616257, + "step": 649 + }, + { + "completion_length": 251.92500686645508, + "epoch": 0.673924313115604, + "grad_norm": 2.6798095703125, + "kl": 1.1048583984375, + "learning_rate": 2.8605228086230877e-06, + "loss": 0.211, + "reward": 1.475000023841858, + "reward_std": 0.391311913728714, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9250000268220901, + "step": 650 + }, + { + "completion_length": 218.125, + "epoch": 0.6749611197511665, + "grad_norm": 2.4933316707611084, + "kl": 1.405029296875, + "learning_rate": 2.8597591818856625e-06, + "loss": 0.18, + "reward": 1.4500000178813934, + "reward_std": 0.45977719500660896, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.875, + "step": 651 + }, + { + "completion_length": 183.3750057220459, + "epoch": 0.6759979263867288, + "grad_norm": 0.8898698091506958, + "kl": 1.068603515625, + "learning_rate": 2.858993572949896e-06, + "loss": 0.1626, + "reward": 1.6250000298023224, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 652 + }, + { + "completion_length": 262.07501220703125, + "epoch": 0.6770347330222913, + "grad_norm": 2.555324077606201, + "kl": 3.16259765625, + "learning_rate": 2.8582259829318604e-06, + "loss": 0.3425, + "reward": 1.5000000149011612, + "reward_std": 0.36053744331002235, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 0.8750000149011612, + "step": 653 + }, + { + "completion_length": 177.27500534057617, + "epoch": 0.6780715396578538, + "grad_norm": 1.2724261283874512, + "kl": 2.198974609375, + "learning_rate": 2.8574564129505154e-06, + "loss": 0.2648, + "reward": 1.3250000178813934, + "reward_std": 0.452556312084198, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 0.9250000268220901, + "step": 654 + }, + { + "completion_length": 144.27500343322754, + "epoch": 0.6791083462934163, + "grad_norm": 0.3718305826187134, + "kl": 0.251220703125, + "learning_rate": 2.856684864127707e-06, + "loss": 0.0359, + "reward": 1.5000000298023224, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 1.0, + "step": 655 + }, + { + "completion_length": 239.875, + "epoch": 0.6801451529289787, + "grad_norm": 1.6971384286880493, + "kl": 2.00830078125, + "learning_rate": 2.8559113375881657e-06, + "loss": 0.298, + "reward": 1.0250000357627869, + "reward_std": 0.31199515238404274, + "rewards/accuracy_reward": 0.15000000223517418, + "rewards/format_reward": 0.8750000149011612, + "step": 656 + }, + { + "completion_length": 223.1500015258789, + "epoch": 0.6811819595645412, + "grad_norm": 1.9796103239059448, + "kl": 3.091796875, + "learning_rate": 2.855135834459505e-06, + "loss": 0.23, + "reward": 1.5000000298023224, + "reward_std": 0.546148881316185, + "rewards/accuracy_reward": 0.5999999940395355, + "rewards/format_reward": 0.9000000208616257, + "step": 657 + }, + { + "completion_length": 201.97500228881836, + "epoch": 0.6822187662001037, + "grad_norm": 1.6794363260269165, + "kl": 1.33544921875, + "learning_rate": 2.85435835587222e-06, + "loss": 0.1351, + "reward": 1.675000011920929, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 658 + }, + { + "completion_length": 144.45000076293945, + "epoch": 0.6832555728356662, + "grad_norm": 0.49781835079193115, + "kl": 0.26416015625, + "learning_rate": 2.8535789029596854e-06, + "loss": -0.0649, + "reward": 1.324999988079071, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 1.0, + "step": 659 + }, + { + "completion_length": 242.22500038146973, + "epoch": 0.6842923794712286, + "grad_norm": 0.9583486318588257, + "kl": 1.5, + "learning_rate": 2.8527974768581547e-06, + "loss": 0.1847, + "reward": 1.4250000417232513, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9250000268220901, + "step": 660 + }, + { + "completion_length": 209.17499923706055, + "epoch": 0.685329186106791, + "grad_norm": 1.6011143922805786, + "kl": 0.8896484375, + "learning_rate": 2.8520140787067567e-06, + "loss": 0.1899, + "reward": 1.475000038743019, + "reward_std": 0.4399927146732807, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9000000357627869, + "step": 661 + }, + { + "completion_length": 249.95000076293945, + "epoch": 0.6863659927423535, + "grad_norm": 1.6479218006134033, + "kl": 0.660888671875, + "learning_rate": 2.8512287096474955e-06, + "loss": 0.0907, + "reward": 1.375, + "reward_std": 0.3731010630726814, + "rewards/accuracy_reward": 0.42500000819563866, + "rewards/format_reward": 0.9500000178813934, + "step": 662 + }, + { + "completion_length": 138.65000343322754, + "epoch": 0.687402799377916, + "grad_norm": 0.9126381278038025, + "kl": 0.72998046875, + "learning_rate": 2.8504413708252483e-06, + "loss": 0.0053, + "reward": 1.3750000149011612, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.4000000152736902, + "rewards/format_reward": 0.9750000089406967, + "step": 663 + }, + { + "completion_length": 162.7750015258789, + "epoch": 0.6884396060134785, + "grad_norm": 0.6816839575767517, + "kl": 1.11083984375, + "learning_rate": 2.8496520633877635e-06, + "loss": 0.1068, + "reward": 1.600000023841858, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9750000089406967, + "step": 664 + }, + { + "completion_length": 163.40000534057617, + "epoch": 0.6894764126490409, + "grad_norm": 0.30915480852127075, + "kl": 0.21044921875, + "learning_rate": 2.84886078848566e-06, + "loss": 0.0071, + "reward": 1.574999988079071, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 665 + }, + { + "completion_length": 138.15000534057617, + "epoch": 0.6905132192846034, + "grad_norm": 0.2813793420791626, + "kl": 0.23046875, + "learning_rate": 2.8480675472724236e-06, + "loss": 0.0338, + "reward": 1.6250000298023224, + "reward_std": 0.1369306445121765, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 1.0, + "step": 666 + }, + { + "completion_length": 244.97500228881836, + "epoch": 0.6915500259201659, + "grad_norm": 8.316036224365234, + "kl": 6.7216796875, + "learning_rate": 2.847272340904408e-06, + "loss": 0.6203, + "reward": 1.4500000476837158, + "reward_std": 0.5163118988275528, + "rewards/accuracy_reward": 0.5750000141561031, + "rewards/format_reward": 0.8750000298023224, + "step": 667 + }, + { + "completion_length": 216.9250030517578, + "epoch": 0.6925868325557284, + "grad_norm": 0.797091007232666, + "kl": 1.13037109375, + "learning_rate": 2.8464751705408294e-06, + "loss": 0.1587, + "reward": 1.2500000149011612, + "reward_std": 0.2795085124671459, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.9500000178813934, + "step": 668 + }, + { + "completion_length": 279.1749973297119, + "epoch": 0.6936236391912908, + "grad_norm": 4.413926601409912, + "kl": 4.05224609375, + "learning_rate": 2.84567603734377e-06, + "loss": 0.4332, + "reward": 1.4500000178813934, + "reward_std": 0.4525563158094883, + "rewards/accuracy_reward": 0.5250000171363354, + "rewards/format_reward": 0.9250000268220901, + "step": 669 + }, + { + "completion_length": 322.7249984741211, + "epoch": 0.6946604458268533, + "grad_norm": 2.120075225830078, + "kl": 5.30810546875, + "learning_rate": 2.8448749424781704e-06, + "loss": 0.3886, + "reward": 1.2750000208616257, + "reward_std": 0.4958944171667099, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8500000089406967, + "step": 670 + }, + { + "completion_length": 299.50000762939453, + "epoch": 0.6956972524624158, + "grad_norm": 0.918604850769043, + "kl": 0.921630859375, + "learning_rate": 2.844071887111833e-06, + "loss": 0.196, + "reward": 1.725000023841858, + "reward_std": 0.3354102149605751, + "rewards/accuracy_reward": 0.8000000193715096, + "rewards/format_reward": 0.9250000268220901, + "step": 671 + }, + { + "completion_length": 267.4250068664551, + "epoch": 0.6967340590979783, + "grad_norm": 1.9089648723602295, + "kl": 1.540283203125, + "learning_rate": 2.843266872415417e-06, + "loss": 0.193, + "reward": 1.4500000178813934, + "reward_std": 0.3354101926088333, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 0.925000011920929, + "step": 672 + }, + { + "completion_length": 206.4250030517578, + "epoch": 0.6977708657335407, + "grad_norm": 0.4940345883369446, + "kl": 0.331787109375, + "learning_rate": 2.8424598995624386e-06, + "loss": 0.0692, + "reward": 1.4250000417232513, + "reward_std": 0.30463574826717377, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 673 + }, + { + "completion_length": 187.12500381469727, + "epoch": 0.6988076723691031, + "grad_norm": 0.8684625029563904, + "kl": 0.30908203125, + "learning_rate": 2.8416509697292676e-06, + "loss": 0.1091, + "reward": 1.5000000298023224, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 674 + }, + { + "completion_length": 191.79999923706055, + "epoch": 0.6998444790046656, + "grad_norm": 1.0280557870864868, + "kl": 1.33935546875, + "learning_rate": 2.8408400840951266e-06, + "loss": 0.1251, + "reward": 1.6250000298023224, + "reward_std": 0.3479738049209118, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9500000178813934, + "step": 675 + }, + { + "completion_length": 267.15000915527344, + "epoch": 0.7008812856402281, + "grad_norm": 0.3398536741733551, + "kl": 0.2470703125, + "learning_rate": 2.8400272438420907e-06, + "loss": 0.033, + "reward": 1.2750000208616257, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.925000011920929, + "step": 676 + }, + { + "completion_length": 391.2000045776367, + "epoch": 0.7019180922757906, + "grad_norm": 1.9886372089385986, + "kl": 1.83837890625, + "learning_rate": 2.8392124501550828e-06, + "loss": 0.3499, + "reward": 1.1500000208616257, + "reward_std": 0.5283810868859291, + "rewards/accuracy_reward": 0.2749999985098839, + "rewards/format_reward": 0.8750000298023224, + "step": 677 + }, + { + "completion_length": 174.3000030517578, + "epoch": 0.702954898911353, + "grad_norm": 0.7306681275367737, + "kl": 0.58935546875, + "learning_rate": 2.838395704221874e-06, + "loss": 0.0423, + "reward": 1.7249999940395355, + "reward_std": 0.3533165641129017, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 678 + }, + { + "completion_length": 258.9750099182129, + "epoch": 0.7039917055469155, + "grad_norm": 10.755302429199219, + "kl": 6.359130859375, + "learning_rate": 2.8375770072330817e-06, + "loss": 0.3311, + "reward": 1.3250000029802322, + "reward_std": 0.2722875848412514, + "rewards/accuracy_reward": 0.4500000011175871, + "rewards/format_reward": 0.875, + "step": 679 + }, + { + "completion_length": 271.7000102996826, + "epoch": 0.705028512182478, + "grad_norm": 1.0601376295089722, + "kl": 1.66357421875, + "learning_rate": 2.8367563603821675e-06, + "loss": 0.2495, + "reward": 1.4750000536441803, + "reward_std": 0.3354101851582527, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.9250000268220901, + "step": 680 + }, + { + "completion_length": 313.95000076293945, + "epoch": 0.7060653188180405, + "grad_norm": 1.3850581645965576, + "kl": 1.41650390625, + "learning_rate": 2.8359337648654334e-06, + "loss": 0.219, + "reward": 1.3750000298023224, + "reward_std": 0.3913118839263916, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.9000000208616257, + "step": 681 + }, + { + "completion_length": 259.2250061035156, + "epoch": 0.7071021254536028, + "grad_norm": 1.035261631011963, + "kl": 1.794677734375, + "learning_rate": 2.8351092218820252e-06, + "loss": 0.2137, + "reward": 1.625, + "reward_std": 0.4290027618408203, + "rewards/accuracy_reward": 0.6750000193715096, + "rewards/format_reward": 0.9500000029802322, + "step": 682 + }, + { + "completion_length": 289.79999923706055, + "epoch": 0.7081389320891653, + "grad_norm": 2.4445176124572754, + "kl": 0.818115234375, + "learning_rate": 2.8342827326339255e-06, + "loss": 0.0812, + "reward": 1.175000011920929, + "reward_std": 0.31733785569667816, + "rewards/accuracy_reward": 0.2500000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 683 + }, + { + "completion_length": 278.17499923706055, + "epoch": 0.7091757387247278, + "grad_norm": 0.5835633873939514, + "kl": 0.353759765625, + "learning_rate": 2.833454298325955e-06, + "loss": 0.0398, + "reward": 1.550000011920929, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9500000029802322, + "step": 684 + }, + { + "completion_length": 190.80000686645508, + "epoch": 0.7102125453602903, + "grad_norm": 1.9967930316925049, + "kl": 0.5185546875, + "learning_rate": 2.832623920165769e-06, + "loss": 0.1104, + "reward": 1.7750000357627869, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 685 + }, + { + "completion_length": 240.6250114440918, + "epoch": 0.7112493519958528, + "grad_norm": 2.839909315109253, + "kl": 0.76611328125, + "learning_rate": 2.831791599363858e-06, + "loss": 0.2053, + "reward": 1.2750000208616257, + "reward_std": 0.3245587721467018, + "rewards/accuracy_reward": 0.32499999925494194, + "rewards/format_reward": 0.9500000029802322, + "step": 686 + }, + { + "completion_length": 175.7750015258789, + "epoch": 0.7122861586314152, + "grad_norm": 0.2983464002609253, + "kl": 0.295166015625, + "learning_rate": 2.8309573371335426e-06, + "loss": 0.0193, + "reward": 1.675000011920929, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.6750000007450581, + "rewards/format_reward": 1.0, + "step": 687 + }, + { + "completion_length": 150.92499923706055, + "epoch": 0.7133229652669777, + "grad_norm": 0.7637765407562256, + "kl": 0.322998046875, + "learning_rate": 2.830121134690974e-06, + "loss": 0.0776, + "reward": 1.3250000178813934, + "reward_std": 0.1928323358297348, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.9750000089406967, + "step": 688 + }, + { + "completion_length": 268.8000011444092, + "epoch": 0.7143597719025402, + "grad_norm": 0.7986348271369934, + "kl": 1.36376953125, + "learning_rate": 2.8292829932551333e-06, + "loss": 0.2448, + "reward": 1.375, + "reward_std": 0.2812207192182541, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.925000011920929, + "step": 689 + }, + { + "completion_length": 245.72500228881836, + "epoch": 0.7153965785381027, + "grad_norm": 0.7799951434135437, + "kl": 0.99072265625, + "learning_rate": 2.8284429140478255e-06, + "loss": 0.1347, + "reward": 1.475000023841858, + "reward_std": 0.47234082594513893, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 690 + }, + { + "completion_length": 282.9250068664551, + "epoch": 0.7164333851736651, + "grad_norm": 0.9315149784088135, + "kl": 1.8046875, + "learning_rate": 2.827600898293683e-06, + "loss": 0.2766, + "reward": 1.2500000149011612, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.9000000208616257, + "step": 691 + }, + { + "completion_length": 182.22500228881836, + "epoch": 0.7174701918092276, + "grad_norm": 24.62877655029297, + "kl": 7.6640625, + "learning_rate": 2.82675694722016e-06, + "loss": 0.6286, + "reward": 1.3250000029802322, + "reward_std": 0.20019175857305527, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.9500000178813934, + "step": 692 + }, + { + "completion_length": 183.7250099182129, + "epoch": 0.71850699844479, + "grad_norm": 0.7584772109985352, + "kl": 1.05224609375, + "learning_rate": 2.8259110620575313e-06, + "loss": 0.1729, + "reward": 1.3250000178813934, + "reward_std": 0.3245587646961212, + "rewards/accuracy_reward": 0.37499999813735485, + "rewards/format_reward": 0.9500000178813934, + "step": 693 + }, + { + "completion_length": 123.92500305175781, + "epoch": 0.7195438050803525, + "grad_norm": 0.2979358434677124, + "kl": 0.254150390625, + "learning_rate": 2.8250632440388927e-06, + "loss": -0.0232, + "reward": 1.7000000178813934, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 1.0, + "step": 694 + }, + { + "completion_length": 223.82501220703125, + "epoch": 0.720580611715915, + "grad_norm": 1.2255027294158936, + "kl": 1.716552734375, + "learning_rate": 2.8242134944001558e-06, + "loss": 0.2352, + "reward": 1.625, + "reward_std": 0.2795085050165653, + "rewards/accuracy_reward": 0.6750000193715096, + "rewards/format_reward": 0.9500000178813934, + "step": 695 + }, + { + "completion_length": 152.20000076293945, + "epoch": 0.7216174183514774, + "grad_norm": 0.9140526652336121, + "kl": 0.26806640625, + "learning_rate": 2.82336181438005e-06, + "loss": 0.0913, + "reward": 1.675000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 696 + }, + { + "completion_length": 134.87500381469727, + "epoch": 0.7226542249870399, + "grad_norm": 0.4066285192966461, + "kl": 0.310791015625, + "learning_rate": 2.822508205220117e-06, + "loss": -0.0062, + "reward": 1.5750000476837158, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 1.0, + "step": 697 + }, + { + "completion_length": 226.60000038146973, + "epoch": 0.7236910316226024, + "grad_norm": 0.5627696514129639, + "kl": 1.79150390625, + "learning_rate": 2.8216526681647118e-06, + "loss": 0.0035, + "reward": 1.8250000178813934, + "reward_std": 0.2487340308725834, + "rewards/accuracy_reward": 0.8750000149011612, + "rewards/format_reward": 0.9500000029802322, + "step": 698 + }, + { + "completion_length": 133.60000038146973, + "epoch": 0.7247278382581649, + "grad_norm": 0.7156115174293518, + "kl": 0.302001953125, + "learning_rate": 2.820795204460999e-06, + "loss": 0.0222, + "reward": 1.4749999940395355, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 699 + }, + { + "completion_length": 123.40000343322754, + "epoch": 0.7257646448937273, + "grad_norm": 0.44991451501846313, + "kl": 0.252685546875, + "learning_rate": 2.819935815358953e-06, + "loss": -0.0084, + "reward": 1.425000011920929, + "reward_std": 0.18026870489120483, + "rewards/accuracy_reward": 0.42499999701976776, + "rewards/format_reward": 1.0, + "step": 700 + }, + { + "completion_length": 330.52499771118164, + "epoch": 0.7268014515292898, + "grad_norm": 3.5027003288269043, + "kl": 1.857177734375, + "learning_rate": 2.819074502111354e-06, + "loss": 0.2354, + "reward": 1.5000000149011612, + "reward_std": 0.41643912717700005, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.875, + "step": 701 + }, + { + "completion_length": 294.82500076293945, + "epoch": 0.7278382581648523, + "grad_norm": 34.1741828918457, + "kl": 10.48681640625, + "learning_rate": 2.818211265973787e-06, + "loss": 0.4339, + "reward": 1.2000000178813934, + "reward_std": 0.32818929105997086, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.8750000149011612, + "step": 702 + }, + { + "completion_length": 178.725004196167, + "epoch": 0.7288750648004148, + "grad_norm": 0.9051024913787842, + "kl": 0.39794921875, + "learning_rate": 2.8173461082046407e-06, + "loss": 0.0751, + "reward": 1.3000000417232513, + "reward_std": 0.24151314049959183, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 703 + }, + { + "completion_length": 159.60000610351562, + "epoch": 0.7299118714359772, + "grad_norm": 1.2098740339279175, + "kl": 0.4189453125, + "learning_rate": 2.8164790300651042e-06, + "loss": 0.0657, + "reward": 1.5000000298023224, + "reward_std": 0.41643914952874184, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.9750000089406967, + "step": 704 + }, + { + "completion_length": 143.62500190734863, + "epoch": 0.7309486780715396, + "grad_norm": 0.858209490776062, + "kl": 1.26513671875, + "learning_rate": 2.815610032819168e-06, + "loss": 0.1172, + "reward": 1.475000023841858, + "reward_std": 0.31199515610933304, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 705 + }, + { + "completion_length": 216.0250072479248, + "epoch": 0.7319854847071021, + "grad_norm": 0.6318058967590332, + "kl": 0.768798828125, + "learning_rate": 2.8147391177336176e-06, + "loss": 0.0993, + "reward": 1.7000000029802322, + "reward_std": 0.1604842059314251, + "rewards/accuracy_reward": 0.7500000093132257, + "rewards/format_reward": 0.9500000029802322, + "step": 706 + }, + { + "completion_length": 224.87500381469727, + "epoch": 0.7330222913426646, + "grad_norm": 518.7442016601562, + "kl": 119.13671875, + "learning_rate": 2.8138662860780356e-06, + "loss": 4.7663, + "reward": 1.2500000149011612, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.925000011920929, + "step": 707 + }, + { + "completion_length": 115.35000038146973, + "epoch": 0.7340590979782271, + "grad_norm": 0.6044578552246094, + "kl": 0.365234375, + "learning_rate": 2.8129915391247988e-06, + "loss": 0.0126, + "reward": 1.4749999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 1.0, + "step": 708 + }, + { + "completion_length": 186.9250087738037, + "epoch": 0.7350959046137895, + "grad_norm": 2.9357755184173584, + "kl": 1.806884765625, + "learning_rate": 2.8121148781490754e-06, + "loss": 0.1944, + "reward": 1.4250000268220901, + "reward_std": 0.22894952073693275, + "rewards/accuracy_reward": 0.4749999940395355, + "rewards/format_reward": 0.9500000178813934, + "step": 709 + }, + { + "completion_length": 138.32500076293945, + "epoch": 0.736132711249352, + "grad_norm": 0.2896471321582794, + "kl": 0.189453125, + "learning_rate": 2.8112363044288243e-06, + "loss": 0.0259, + "reward": 1.5250000059604645, + "reward_std": 0.12436702102422714, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.9750000089406967, + "step": 710 + }, + { + "completion_length": 258.32500648498535, + "epoch": 0.7371695178849145, + "grad_norm": 18.005449295043945, + "kl": 4.0224609375, + "learning_rate": 2.810355819244792e-06, + "loss": 0.4517, + "reward": 1.350000023841858, + "reward_std": 0.3407529331743717, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.925000011920929, + "step": 711 + }, + { + "completion_length": 114.47500228881836, + "epoch": 0.738206324520477, + "grad_norm": 0.36921098828315735, + "kl": 0.2666015625, + "learning_rate": 2.809473423880512e-06, + "loss": 0.0209, + "reward": 1.675000011920929, + "reward_std": 0.12436702102422714, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 1.0, + "step": 712 + }, + { + "completion_length": 199.05000686645508, + "epoch": 0.7392431311560393, + "grad_norm": 0.9610119462013245, + "kl": 1.69140625, + "learning_rate": 2.8085891196223025e-06, + "loss": 0.1894, + "reward": 1.1500000208616257, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.20000000670552254, + "rewards/format_reward": 0.9500000178813934, + "step": 713 + }, + { + "completion_length": 154.62500381469727, + "epoch": 0.7402799377916018, + "grad_norm": 1.1751526594161987, + "kl": 0.3271484375, + "learning_rate": 2.8077029077592633e-06, + "loss": 0.0186, + "reward": 1.225000023841858, + "reward_std": 0.30463574081659317, + "rewards/accuracy_reward": 0.27500000409781933, + "rewards/format_reward": 0.9500000178813934, + "step": 714 + }, + { + "completion_length": 209.70000839233398, + "epoch": 0.7413167444271643, + "grad_norm": 1.0980820655822754, + "kl": 0.487548828125, + "learning_rate": 2.806814789583277e-06, + "loss": 0.1487, + "reward": 1.5250000059604645, + "reward_std": 0.2487340308725834, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 715 + }, + { + "completion_length": 227.0000114440918, + "epoch": 0.7423535510627268, + "grad_norm": 0.5780226588249207, + "kl": 1.57666015625, + "learning_rate": 2.8059247663890032e-06, + "loss": 0.2161, + "reward": 1.5250000059604645, + "reward_std": 0.18026871606707573, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9249999970197678, + "step": 716 + }, + { + "completion_length": 280.20000076293945, + "epoch": 0.7433903576982893, + "grad_norm": 1.4871811866760254, + "kl": 1.12109375, + "learning_rate": 2.8050328394738803e-06, + "loss": 0.2866, + "reward": 1.4250000417232513, + "reward_std": 0.4729737937450409, + "rewards/accuracy_reward": 0.5500000081956387, + "rewards/format_reward": 0.8750000149011612, + "step": 717 + }, + { + "completion_length": 310.9500045776367, + "epoch": 0.7444271643338517, + "grad_norm": 1.3770198822021484, + "kl": 0.87255859375, + "learning_rate": 2.8041390101381195e-06, + "loss": 0.229, + "reward": 1.4000000059604645, + "reward_std": 0.6291945092380047, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.8750000149011612, + "step": 718 + }, + { + "completion_length": 225.77500343322754, + "epoch": 0.7454639709694142, + "grad_norm": 0.8252406120300293, + "kl": 1.41943359375, + "learning_rate": 2.8032432796847076e-06, + "loss": 0.2017, + "reward": 1.4249999970197678, + "reward_std": 0.21275536343455315, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.925000011920929, + "step": 719 + }, + { + "completion_length": 256.90000343322754, + "epoch": 0.7465007776049767, + "grad_norm": 0.9165952205657959, + "kl": 1.5322265625, + "learning_rate": 2.802345649419402e-06, + "loss": 0.2268, + "reward": 1.375, + "reward_std": 0.329762976616621, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.8750000149011612, + "step": 720 + }, + { + "completion_length": 112.55000305175781, + "epoch": 0.7475375842405392, + "grad_norm": 0.8553647398948669, + "kl": 0.322265625, + "learning_rate": 2.8014461206507292e-06, + "loss": -0.0239, + "reward": 1.5250000357627869, + "reward_std": 0.30463573709130287, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 721 + }, + { + "completion_length": 308.075008392334, + "epoch": 0.7485743908761016, + "grad_norm": 2.7667524814605713, + "kl": 3.8203125, + "learning_rate": 2.800544694689983e-06, + "loss": 0.3563, + "reward": 1.2750000208616257, + "reward_std": 0.4604102000594139, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.8750000149011612, + "step": 722 + }, + { + "completion_length": 280.6499996185303, + "epoch": 0.749611197511664, + "grad_norm": 6.384510517120361, + "kl": 3.0322265625, + "learning_rate": 2.7996413728512236e-06, + "loss": 0.3557, + "reward": 1.4500000029802322, + "reward_std": 0.26129764690995216, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9000000059604645, + "step": 723 + }, + { + "completion_length": 279.8000068664551, + "epoch": 0.7506480041472265, + "grad_norm": 7.1993584632873535, + "kl": 2.51171875, + "learning_rate": 2.7987361564512757e-06, + "loss": 0.354, + "reward": 1.3250000178813934, + "reward_std": 0.38409097865223885, + "rewards/accuracy_reward": 0.4249999988824129, + "rewards/format_reward": 0.9000000357627869, + "step": 724 + }, + { + "completion_length": 149.2750015258789, + "epoch": 0.751684810782789, + "grad_norm": 3.0475800037384033, + "kl": 2.4169921875, + "learning_rate": 2.797829046809723e-06, + "loss": 0.1504, + "reward": 1.300000011920929, + "reward_std": 0.36053744331002235, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8750000149011612, + "step": 725 + }, + { + "completion_length": 177.97500228881836, + "epoch": 0.7527216174183515, + "grad_norm": 0.7727159857749939, + "kl": 0.75732421875, + "learning_rate": 2.7969200452489117e-06, + "loss": 0.1334, + "reward": 1.574999988079071, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9750000089406967, + "step": 726 + }, + { + "completion_length": 162.05000495910645, + "epoch": 0.7537584240539139, + "grad_norm": 1.6836689710617065, + "kl": 1.5498046875, + "learning_rate": 2.7960091530939453e-06, + "loss": 0.1498, + "reward": 1.375, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 727 + }, + { + "completion_length": 108.67500305175781, + "epoch": 0.7547952306894764, + "grad_norm": 0.8589237332344055, + "kl": 0.76123046875, + "learning_rate": 2.7950963716726833e-06, + "loss": 0.0888, + "reward": 1.3750000298023224, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9500000178813934, + "step": 728 + }, + { + "completion_length": 119.17500114440918, + "epoch": 0.7558320373250389, + "grad_norm": 0.9010735750198364, + "kl": 0.44287109375, + "learning_rate": 2.7941817023157385e-06, + "loss": 0.0313, + "reward": 1.600000023841858, + "reward_std": 0.1928323470056057, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9750000089406967, + "step": 729 + }, + { + "completion_length": 121.82500076293945, + "epoch": 0.7568688439606014, + "grad_norm": 2.7615885734558105, + "kl": 1.18115234375, + "learning_rate": 2.7932651463564763e-06, + "loss": 0.1171, + "reward": 1.300000011920929, + "reward_std": 0.1604841984808445, + "rewards/accuracy_reward": 0.3249999973922968, + "rewards/format_reward": 0.9750000089406967, + "step": 730 + }, + { + "completion_length": 102.30000305175781, + "epoch": 0.7579056505961638, + "grad_norm": 0.4340980350971222, + "kl": 0.39697265625, + "learning_rate": 2.792346705131013e-06, + "loss": 0.0185, + "reward": 1.2000000178813934, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.2000000048428774, + "rewards/format_reward": 1.0, + "step": 731 + }, + { + "completion_length": 118.50000381469727, + "epoch": 0.7589424572317263, + "grad_norm": 0.6814145445823669, + "kl": 0.67236328125, + "learning_rate": 2.791426379978213e-06, + "loss": 0.0499, + "reward": 1.5250000357627869, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 732 + }, + { + "completion_length": 139.77500534057617, + "epoch": 0.7599792638672888, + "grad_norm": 3.444451093673706, + "kl": 2.74609375, + "learning_rate": 2.7905041722396854e-06, + "loss": 0.1425, + "reward": 1.4000000059604645, + "reward_std": 0.18026872351765633, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9500000178813934, + "step": 733 + }, + { + "completion_length": 429.12499809265137, + "epoch": 0.7610160705028512, + "grad_norm": 10.602200508117676, + "kl": 9.31494140625, + "learning_rate": 2.7895800832597865e-06, + "loss": 0.7934, + "reward": 1.3249999955296516, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.8250000104308128, + "step": 734 + }, + { + "completion_length": 126.12500381469727, + "epoch": 0.7620528771384136, + "grad_norm": 0.5543495416641235, + "kl": 0.70166015625, + "learning_rate": 2.788654114385612e-06, + "loss": -0.0408, + "reward": 1.4750000536441803, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 735 + }, + { + "completion_length": 106.92499923706055, + "epoch": 0.7630896837739761, + "grad_norm": 0.33099156618118286, + "kl": 0.531982421875, + "learning_rate": 2.787726266967e-06, + "loss": -0.0037, + "reward": 1.675000011920929, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 736 + }, + { + "completion_length": 266.87500381469727, + "epoch": 0.7641264904095386, + "grad_norm": 1.9715381860733032, + "kl": 1.3359375, + "learning_rate": 2.7867965423565273e-06, + "loss": 0.2879, + "reward": 1.3000000268220901, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.4000000096857548, + "rewards/format_reward": 0.9000000357627869, + "step": 737 + }, + { + "completion_length": 204.62499809265137, + "epoch": 0.7651632970451011, + "grad_norm": 0.3641264736652374, + "kl": 0.6787109375, + "learning_rate": 2.785864941909505e-06, + "loss": 0.0829, + "reward": 1.4750000089406967, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 0.9500000029802322, + "step": 738 + }, + { + "completion_length": 233.37500381469727, + "epoch": 0.7662001036806636, + "grad_norm": 1.9150782823562622, + "kl": 1.84033203125, + "learning_rate": 2.784931466983981e-06, + "loss": 0.2618, + "reward": 1.2750000208616257, + "reward_std": 0.4038755036890507, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 739 + }, + { + "completion_length": 168.02499961853027, + "epoch": 0.767236910316226, + "grad_norm": 23.784427642822266, + "kl": 9.75, + "learning_rate": 2.783996118940734e-06, + "loss": 0.6841, + "reward": 1.225000038743019, + "reward_std": 0.3245587609708309, + "rewards/accuracy_reward": 0.30000001192092896, + "rewards/format_reward": 0.9250000268220901, + "step": 740 + }, + { + "completion_length": 413.52500915527344, + "epoch": 0.7682737169517885, + "grad_norm": 28.01406478881836, + "kl": 10.80078125, + "learning_rate": 2.7830588991432745e-06, + "loss": 0.8084, + "reward": 1.200000062584877, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.8500000238418579, + "step": 741 + }, + { + "completion_length": 134.600004196167, + "epoch": 0.769310523587351, + "grad_norm": 1.1930382251739502, + "kl": 1.346923828125, + "learning_rate": 2.7821198089578417e-06, + "loss": 0.1408, + "reward": 1.375, + "reward_std": 0.1604842059314251, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 742 + }, + { + "completion_length": 269.3500061035156, + "epoch": 0.7703473302229135, + "grad_norm": 3.615288496017456, + "kl": 4.735107421875, + "learning_rate": 2.7811788497533998e-06, + "loss": 0.4038, + "reward": 1.2500000298023224, + "reward_std": 0.36053742095828056, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 0.8500000089406967, + "step": 743 + }, + { + "completion_length": 355.05001068115234, + "epoch": 0.7713841368584758, + "grad_norm": 23.248554229736328, + "kl": 9.390625, + "learning_rate": 2.780236022901639e-06, + "loss": 0.8676, + "reward": 1.225000038743019, + "reward_std": 0.632825031876564, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.800000011920929, + "step": 744 + }, + { + "completion_length": 301.225004196167, + "epoch": 0.7724209434940383, + "grad_norm": 1.500929594039917, + "kl": 1.21484375, + "learning_rate": 2.7792913297769723e-06, + "loss": 0.1665, + "reward": 1.2750000208616257, + "reward_std": 0.4604101926088333, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.875, + "step": 745 + }, + { + "completion_length": 242.7750015258789, + "epoch": 0.7734577501296008, + "grad_norm": 0.9008773565292358, + "kl": 1.644775390625, + "learning_rate": 2.7783447717565313e-06, + "loss": 0.2006, + "reward": 1.600000023841858, + "reward_std": 0.2795084938406944, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9000000059604645, + "step": 746 + }, + { + "completion_length": 321.52500915527344, + "epoch": 0.7744945567651633, + "grad_norm": 4.788297176361084, + "kl": 4.009765625, + "learning_rate": 2.777396350220169e-06, + "loss": 0.436, + "reward": 1.1500000059604645, + "reward_std": 0.3534550555050373, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/format_reward": 0.9000000357627869, + "step": 747 + }, + { + "completion_length": 274.0750045776367, + "epoch": 0.7755313634007258, + "grad_norm": 2.297362804412842, + "kl": 1.6220703125, + "learning_rate": 2.7764460665504522e-06, + "loss": 0.245, + "reward": 1.4250000268220901, + "reward_std": 0.3245587684214115, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 748 + }, + { + "completion_length": 185.92500495910645, + "epoch": 0.7765681700362882, + "grad_norm": 1.0662373304367065, + "kl": 1.704345703125, + "learning_rate": 2.775493922132664e-06, + "loss": 0.16, + "reward": 1.3750000298023224, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9500000029802322, + "step": 749 + }, + { + "completion_length": 183.27500343322754, + "epoch": 0.7776049766718507, + "grad_norm": 0.667378306388855, + "kl": 0.505615234375, + "learning_rate": 2.7745399183548e-06, + "loss": 0.0602, + "reward": 1.5499999970197678, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 750 + }, + { + "completion_length": 165.1500015258789, + "epoch": 0.7786417833074132, + "grad_norm": 0.5699002146720886, + "kl": 0.941162109375, + "learning_rate": 2.7735840566075646e-06, + "loss": 0.0966, + "reward": 1.4500000029802322, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 751 + }, + { + "completion_length": 170.72500228881836, + "epoch": 0.7796785899429757, + "grad_norm": 3.4902076721191406, + "kl": 1.697265625, + "learning_rate": 2.7726263382843726e-06, + "loss": 0.1494, + "reward": 1.5999999940395355, + "reward_std": 0.25609344989061356, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 752 + }, + { + "completion_length": 129.47500228881836, + "epoch": 0.7807153965785381, + "grad_norm": 0.3129628598690033, + "kl": 0.272705078125, + "learning_rate": 2.771666764781345e-06, + "loss": 0.0141, + "reward": 1.425000011920929, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.4250000063329935, + "rewards/format_reward": 1.0, + "step": 753 + }, + { + "completion_length": 171.75000381469727, + "epoch": 0.7817522032141005, + "grad_norm": 0.5923978090286255, + "kl": 0.968505859375, + "learning_rate": 2.7707053374973055e-06, + "loss": 0.0773, + "reward": 1.4000000059604645, + "reward_std": 0.21638592705130577, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9249999970197678, + "step": 754 + }, + { + "completion_length": 132.92500114440918, + "epoch": 0.782789009849663, + "grad_norm": 0.7645208835601807, + "kl": 0.856201171875, + "learning_rate": 2.7697420578337823e-06, + "loss": 0.0276, + "reward": 1.5750000178813934, + "reward_std": 0.21275537833571434, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 755 + }, + { + "completion_length": 174.9250030517578, + "epoch": 0.7838258164852255, + "grad_norm": 2.4929704666137695, + "kl": 0.317626953125, + "learning_rate": 2.768776927195003e-06, + "loss": 0.0762, + "reward": 1.600000023841858, + "reward_std": 0.281220693141222, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 756 + }, + { + "completion_length": 185.57500076293945, + "epoch": 0.784862623120788, + "grad_norm": 1.6370223760604858, + "kl": 0.39794921875, + "learning_rate": 2.7678099469878927e-06, + "loss": 0.0027, + "reward": 1.5250000655651093, + "reward_std": 0.36053744703531265, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 757 + }, + { + "completion_length": 108.42500114440918, + "epoch": 0.7858994297563504, + "grad_norm": 0.4571923613548279, + "kl": 0.2900390625, + "learning_rate": 2.766841118622074e-06, + "loss": -0.0458, + "reward": 1.3750000298023224, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 1.0, + "step": 758 + }, + { + "completion_length": 200.82500648498535, + "epoch": 0.7869362363919129, + "grad_norm": 2.9244139194488525, + "kl": 3.072265625, + "learning_rate": 2.765870443509863e-06, + "loss": 0.3235, + "reward": 1.4000000059604645, + "reward_std": 0.4472135864198208, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9000000059604645, + "step": 759 + }, + { + "completion_length": 134.95000457763672, + "epoch": 0.7879730430274754, + "grad_norm": 1.5931257009506226, + "kl": 0.43212890625, + "learning_rate": 2.7648979230662682e-06, + "loss": 0.0302, + "reward": 1.5, + "reward_std": 0.31199514865875244, + "rewards/accuracy_reward": 0.5500000212341547, + "rewards/format_reward": 0.9500000178813934, + "step": 760 + }, + { + "completion_length": 267.575008392334, + "epoch": 0.7890098496630379, + "grad_norm": 2.634326934814453, + "kl": 3.15771484375, + "learning_rate": 2.763923558708988e-06, + "loss": 0.3409, + "reward": 1.2999999970197678, + "reward_std": 0.2795084938406944, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9250000268220901, + "step": 761 + }, + { + "completion_length": 128.4250030517578, + "epoch": 0.7900466562986003, + "grad_norm": 0.6787009835243225, + "kl": 0.467041015625, + "learning_rate": 2.7629473518584088e-06, + "loss": 0.0092, + "reward": 1.350000023841858, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 762 + }, + { + "completion_length": 264.8250026702881, + "epoch": 0.7910834629341628, + "grad_norm": 0.48455703258514404, + "kl": 1.8251953125, + "learning_rate": 2.7619693039376025e-06, + "loss": 0.2773, + "reward": 1.2000000029802322, + "reward_std": 0.15685366839170456, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 763 + }, + { + "completion_length": 204.0750026702881, + "epoch": 0.7921202695697253, + "grad_norm": 0.7012991309165955, + "kl": 0.8583984375, + "learning_rate": 2.7609894163723257e-06, + "loss": 0.1481, + "reward": 1.600000023841858, + "reward_std": 0.42178187519311905, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.925000011920929, + "step": 764 + }, + { + "completion_length": 263.72500228881836, + "epoch": 0.7931570762052877, + "grad_norm": 1.760319709777832, + "kl": 1.53173828125, + "learning_rate": 2.760007690591016e-06, + "loss": 0.1753, + "reward": 1.2000000327825546, + "reward_std": 0.3371223732829094, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.8750000149011612, + "step": 765 + }, + { + "completion_length": 131.42500495910645, + "epoch": 0.7941938828408501, + "grad_norm": 0.4826253056526184, + "kl": 0.946044921875, + "learning_rate": 2.7590241280247906e-06, + "loss": 0.1077, + "reward": 1.5, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.5250000022351742, + "rewards/format_reward": 0.9750000089406967, + "step": 766 + }, + { + "completion_length": 117.82500076293945, + "epoch": 0.7952306894764126, + "grad_norm": 0.8102556467056274, + "kl": 0.491455078125, + "learning_rate": 2.758038730107446e-06, + "loss": 0.0176, + "reward": 1.550000011920929, + "reward_std": 0.21275536715984344, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 767 + }, + { + "completion_length": 131.2750015258789, + "epoch": 0.7962674961119751, + "grad_norm": 0.7947399020195007, + "kl": 1.2119140625, + "learning_rate": 2.7570514982754507e-06, + "loss": 0.061, + "reward": 1.1250000149011612, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.17500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 768 + }, + { + "completion_length": 215.7250099182129, + "epoch": 0.7973043027475376, + "grad_norm": 3.324897527694702, + "kl": 1.559814453125, + "learning_rate": 2.7560624339679505e-06, + "loss": 0.2325, + "reward": 1.475000023841858, + "reward_std": 0.3245587795972824, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 769 + }, + { + "completion_length": 254.65000915527344, + "epoch": 0.7983411093831001, + "grad_norm": 2.051250696182251, + "kl": 3.601806640625, + "learning_rate": 2.75507153862676e-06, + "loss": 0.2329, + "reward": 1.300000011920929, + "reward_std": 0.27228758856654167, + "rewards/accuracy_reward": 0.42499999701976776, + "rewards/format_reward": 0.8750000149011612, + "step": 770 + }, + { + "completion_length": 113.37500381469727, + "epoch": 0.7993779160186625, + "grad_norm": 0.8960034847259521, + "kl": 1.621826171875, + "learning_rate": 2.7540788136963637e-06, + "loss": 0.1835, + "reward": 1.350000023841858, + "reward_std": 0.40387552231550217, + "rewards/accuracy_reward": 0.4250000063329935, + "rewards/format_reward": 0.925000011920929, + "step": 771 + }, + { + "completion_length": 134.89999961853027, + "epoch": 0.800414722654225, + "grad_norm": 0.7624129056930542, + "kl": 0.291259765625, + "learning_rate": 2.7530842606239135e-06, + "loss": 0.088, + "reward": 1.2750000059604645, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.35000000707805157, + "rewards/format_reward": 0.9250000268220901, + "step": 772 + }, + { + "completion_length": 127.10000419616699, + "epoch": 0.8014515292897875, + "grad_norm": 0.4995887279510498, + "kl": 0.360107421875, + "learning_rate": 2.7520878808592255e-06, + "loss": 0.0129, + "reward": 1.6500000357627869, + "reward_std": 0.3479738347232342, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 773 + }, + { + "completion_length": 192.8000030517578, + "epoch": 0.80248833592535, + "grad_norm": 1.0516695976257324, + "kl": 1.542236328125, + "learning_rate": 2.7510896758547796e-06, + "loss": 0.1974, + "reward": 1.3250000029802322, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.4499999973922968, + "rewards/format_reward": 0.8750000149011612, + "step": 774 + }, + { + "completion_length": 286.1500053405762, + "epoch": 0.8035251425609123, + "grad_norm": 1.4060876369476318, + "kl": 2.7529296875, + "learning_rate": 2.7500896470657166e-06, + "loss": 0.3356, + "reward": 1.200000062584877, + "reward_std": 0.5842827521264553, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.8500000238418579, + "step": 775 + }, + { + "completion_length": 212.77499961853027, + "epoch": 0.8045619491964748, + "grad_norm": 1.4797383546829224, + "kl": 1.840576171875, + "learning_rate": 2.749087795949834e-06, + "loss": 0.2744, + "reward": 1.1000000536441803, + "reward_std": 0.3913118988275528, + "rewards/accuracy_reward": 0.20000000298023224, + "rewards/format_reward": 0.9000000357627869, + "step": 776 + }, + { + "completion_length": 171.00000190734863, + "epoch": 0.8055987558320373, + "grad_norm": 8.790672302246094, + "kl": 3.203125, + "learning_rate": 2.7480841239675883e-06, + "loss": 0.2483, + "reward": 1.1250000298023224, + "reward_std": 0.4814124368131161, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/format_reward": 0.8750000298023224, + "step": 777 + }, + { + "completion_length": 208.57500648498535, + "epoch": 0.8066355624675998, + "grad_norm": 0.8135401606559753, + "kl": 1.341064453125, + "learning_rate": 2.747078632582089e-06, + "loss": 0.0399, + "reward": 1.3750000596046448, + "reward_std": 0.26129765436053276, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9500000029802322, + "step": 778 + }, + { + "completion_length": 237.90000343322754, + "epoch": 0.8076723691031623, + "grad_norm": 1.9624338150024414, + "kl": 1.96484375, + "learning_rate": 2.7460713232590973e-06, + "loss": 0.3619, + "reward": 1.4000000357627869, + "reward_std": 0.45977723225951195, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.8750000149011612, + "step": 779 + }, + { + "completion_length": 91.25, + "epoch": 0.8087091757387247, + "grad_norm": 2.1631221771240234, + "kl": 5.6884765625, + "learning_rate": 2.745062197467027e-06, + "loss": -0.0101, + "reward": 1.3250000029802322, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9500000178813934, + "step": 780 + }, + { + "completion_length": 204.02500915527344, + "epoch": 0.8097459823742872, + "grad_norm": 2.164458990097046, + "kl": 2.19189453125, + "learning_rate": 2.7440512566769368e-06, + "loss": 0.2447, + "reward": 1.8500000536441803, + "reward_std": 0.292072094976902, + "rewards/accuracy_reward": 0.9000000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 781 + }, + { + "completion_length": 107.25000190734863, + "epoch": 0.8107827890098497, + "grad_norm": 0.8715801239013672, + "kl": 0.63037109375, + "learning_rate": 2.7430385023625334e-06, + "loss": 0.073, + "reward": 1.2500000298023224, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.2750000022351742, + "rewards/format_reward": 0.9750000089406967, + "step": 782 + }, + { + "completion_length": 149.97500228881836, + "epoch": 0.8118195956454122, + "grad_norm": 2.8529815673828125, + "kl": 2.78369140625, + "learning_rate": 2.7420239360001654e-06, + "loss": 0.258, + "reward": 1.8250000178813934, + "reward_std": 0.3354101926088333, + "rewards/accuracy_reward": 0.875, + "rewards/format_reward": 0.9500000178813934, + "step": 783 + }, + { + "completion_length": 98.1500015258789, + "epoch": 0.8128564022809746, + "grad_norm": 1.0965875387191772, + "kl": 0.5849609375, + "learning_rate": 2.741007559068825e-06, + "loss": 0.0432, + "reward": 1.375, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 784 + }, + { + "completion_length": 200.9000072479248, + "epoch": 0.813893208916537, + "grad_norm": 0.8102331757545471, + "kl": 1.67041015625, + "learning_rate": 2.739989373050142e-06, + "loss": 0.196, + "reward": 1.1749999970197678, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.22500000149011612, + "rewards/format_reward": 0.9500000178813934, + "step": 785 + }, + { + "completion_length": 275.60000228881836, + "epoch": 0.8149300155520995, + "grad_norm": 10.27814769744873, + "kl": 7.51904296875, + "learning_rate": 2.7389693794283847e-06, + "loss": 0.7619, + "reward": 1.3750000298023224, + "reward_std": 0.45977722853422165, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.9000000208616257, + "step": 786 + }, + { + "completion_length": 141.5000057220459, + "epoch": 0.815966822187662, + "grad_norm": 1.1177572011947632, + "kl": 1.534423828125, + "learning_rate": 2.7379475796904555e-06, + "loss": 0.1921, + "reward": 1.4500000327825546, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 787 + }, + { + "completion_length": 159.07500076293945, + "epoch": 0.8170036288232244, + "grad_norm": 0.926374614238739, + "kl": 0.93505859375, + "learning_rate": 2.73692397532589e-06, + "loss": 0.1362, + "reward": 1.449999988079071, + "reward_std": 0.36053742468357086, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 788 + }, + { + "completion_length": 151.77500343322754, + "epoch": 0.8180404354587869, + "grad_norm": 0.6936078071594238, + "kl": 0.8388671875, + "learning_rate": 2.7358985678268544e-06, + "loss": 0.0432, + "reward": 1.3499999940395355, + "reward_std": 0.28485121950507164, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 789 + }, + { + "completion_length": 166.9000072479248, + "epoch": 0.8190772420943494, + "grad_norm": 0.9093707799911499, + "kl": 0.895263671875, + "learning_rate": 2.734871358688144e-06, + "loss": 0.1818, + "reward": 1.5750000178813934, + "reward_std": 0.33712238073349, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000029802322, + "step": 790 + }, + { + "completion_length": 286.17500495910645, + "epoch": 0.8201140487299119, + "grad_norm": 11.58954906463623, + "kl": 5.29541015625, + "learning_rate": 2.7338423494071795e-06, + "loss": 0.6152, + "reward": 1.4249999970197678, + "reward_std": 0.29270510002970695, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000208616257, + "step": 791 + }, + { + "completion_length": 293.9500026702881, + "epoch": 0.8211508553654744, + "grad_norm": 2.5993452072143555, + "kl": 2.10888671875, + "learning_rate": 2.7328115414840066e-06, + "loss": 0.3068, + "reward": 1.2249999940395355, + "reward_std": 0.3047742620110512, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9000000208616257, + "step": 792 + }, + { + "completion_length": 179.12500381469727, + "epoch": 0.8221876620010368, + "grad_norm": 43.453060150146484, + "kl": 9.94287109375, + "learning_rate": 2.7317789364212927e-06, + "loss": 0.847, + "reward": 1.300000011920929, + "reward_std": 0.30114371702075005, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 793 + }, + { + "completion_length": 198.8250026702881, + "epoch": 0.8232244686365993, + "grad_norm": 1.3790888786315918, + "kl": 1.91845703125, + "learning_rate": 2.730744535724325e-06, + "loss": 0.1889, + "reward": 1.3250000178813934, + "reward_std": 0.4652584567666054, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.925000011920929, + "step": 794 + }, + { + "completion_length": 166.50000190734863, + "epoch": 0.8242612752721618, + "grad_norm": 13.034732818603516, + "kl": 8.81005859375, + "learning_rate": 2.729708340901008e-06, + "loss": 0.7422, + "reward": 1.550000011920929, + "reward_std": 0.25609344989061356, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 795 + }, + { + "completion_length": 134.90000534057617, + "epoch": 0.8252980819077242, + "grad_norm": 1.2326252460479736, + "kl": 0.88623046875, + "learning_rate": 2.7286703534618622e-06, + "loss": 0.0926, + "reward": 1.6000000536441803, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 796 + }, + { + "completion_length": 133.35000228881836, + "epoch": 0.8263348885432866, + "grad_norm": 1.0299975872039795, + "kl": 1.529296875, + "learning_rate": 2.7276305749200206e-06, + "loss": 0.1607, + "reward": 1.475000023841858, + "reward_std": 0.3966546133160591, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.9500000178813934, + "step": 797 + }, + { + "completion_length": 201.5750026702881, + "epoch": 0.8273716951788491, + "grad_norm": 1.292431354522705, + "kl": 0.6396484375, + "learning_rate": 2.7265890067912275e-06, + "loss": 0.0736, + "reward": 1.600000023841858, + "reward_std": 0.324558787047863, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9500000178813934, + "step": 798 + }, + { + "completion_length": 197.82500839233398, + "epoch": 0.8284085018144116, + "grad_norm": 1.056061863899231, + "kl": 0.56640625, + "learning_rate": 2.725545650593837e-06, + "loss": 0.0988, + "reward": 1.4749999940395355, + "reward_std": 0.25609343126416206, + "rewards/accuracy_reward": 0.5250000152736902, + "rewards/format_reward": 0.9500000178813934, + "step": 799 + }, + { + "completion_length": 151.5750026702881, + "epoch": 0.8294453084499741, + "grad_norm": 2.8053700923919678, + "kl": 2.60009765625, + "learning_rate": 2.7245005078488077e-06, + "loss": 0.2199, + "reward": 1.324999988079071, + "reward_std": 0.46511994674801826, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9500000178813934, + "step": 800 + }, + { + "completion_length": 137.5250015258789, + "epoch": 0.8304821150855366, + "grad_norm": 0.9269716739654541, + "kl": 1.56103515625, + "learning_rate": 2.7234535800797037e-06, + "loss": 0.1335, + "reward": 1.0999999940395355, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 801 + }, + { + "completion_length": 195.5750026702881, + "epoch": 0.831518921721099, + "grad_norm": 5.310616970062256, + "kl": 3.5379638671875, + "learning_rate": 2.7224048688126915e-06, + "loss": 0.3573, + "reward": 1.7500000596046448, + "reward_std": 0.41643913835287094, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 802 + }, + { + "completion_length": 243.20000839233398, + "epoch": 0.8325557283566615, + "grad_norm": 11.0709810256958, + "kl": 6.59619140625, + "learning_rate": 2.7213543755765377e-06, + "loss": 0.6267, + "reward": 1.225000023841858, + "reward_std": 0.40387551486492157, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.9250000268220901, + "step": 803 + }, + { + "completion_length": 199.4000072479248, + "epoch": 0.833592534992224, + "grad_norm": 1.7671570777893066, + "kl": 2.40234375, + "learning_rate": 2.720302101902605e-06, + "loss": 0.2168, + "reward": 1.5250000059604645, + "reward_std": 0.4363621585071087, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 804 + }, + { + "completion_length": 160.1750030517578, + "epoch": 0.8346293416277865, + "grad_norm": 0.5831762552261353, + "kl": 1.02880859375, + "learning_rate": 2.719248049324853e-06, + "loss": 0.1246, + "reward": 1.7750000357627869, + "reward_std": 0.3407529145479202, + "rewards/accuracy_reward": 0.7999999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 805 + }, + { + "completion_length": 110.62500190734863, + "epoch": 0.8356661482633488, + "grad_norm": 0.35738223791122437, + "kl": 0.25, + "learning_rate": 2.718192219379835e-06, + "loss": -0.0206, + "reward": 1.350000023841858, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.9750000089406967, + "step": 806 + }, + { + "completion_length": 156.70000457763672, + "epoch": 0.8367029548989113, + "grad_norm": 1.1867387294769287, + "kl": 0.779296875, + "learning_rate": 2.7171346136066936e-06, + "loss": 0.0683, + "reward": 1.5750000178813934, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.6000000052154064, + "rewards/format_reward": 0.9750000089406967, + "step": 807 + }, + { + "completion_length": 196.1250057220459, + "epoch": 0.8377397615344738, + "grad_norm": 0.2853090763092041, + "kl": 0.412841796875, + "learning_rate": 2.716075233547161e-06, + "loss": 0.0884, + "reward": 1.2999999970197678, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 808 + }, + { + "completion_length": 203.225004196167, + "epoch": 0.8387765681700363, + "grad_norm": 1.5570887327194214, + "kl": 1.113525390625, + "learning_rate": 2.7150140807455555e-06, + "loss": 0.1719, + "reward": 1.449999988079071, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9500000178813934, + "step": 809 + }, + { + "completion_length": 269.4750099182129, + "epoch": 0.8398133748055988, + "grad_norm": 1.8617844581604004, + "kl": 2.31640625, + "learning_rate": 2.7139511567487806e-06, + "loss": 0.3669, + "reward": 1.4500000476837158, + "reward_std": 0.3913119025528431, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9000000357627869, + "step": 810 + }, + { + "completion_length": 152.3750057220459, + "epoch": 0.8408501814411612, + "grad_norm": 0.6547606587409973, + "kl": 0.7275390625, + "learning_rate": 2.712886463106321e-06, + "loss": 0.0751, + "reward": 1.5250000059604645, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 811 + }, + { + "completion_length": 165.82500648498535, + "epoch": 0.8418869880767237, + "grad_norm": 0.41356778144836426, + "kl": 0.31201171875, + "learning_rate": 2.7118200013702407e-06, + "loss": 0.0653, + "reward": 1.675000011920929, + "reward_std": 0.30463574826717377, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 812 + }, + { + "completion_length": 276.02500915527344, + "epoch": 0.8429237947122862, + "grad_norm": 0.9744496941566467, + "kl": 2.72900390625, + "learning_rate": 2.7107517730951818e-06, + "loss": 0.3531, + "reward": 1.400000050663948, + "reward_std": 0.4723408408463001, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9250000268220901, + "step": 813 + }, + { + "completion_length": 187.32500457763672, + "epoch": 0.8439606013478487, + "grad_norm": 1.1766014099121094, + "kl": 0.46630859375, + "learning_rate": 2.709681779838362e-06, + "loss": 0.0853, + "reward": 1.225000023841858, + "reward_std": 0.3046357333660126, + "rewards/accuracy_reward": 0.2750000096857548, + "rewards/format_reward": 0.9500000178813934, + "step": 814 + }, + { + "completion_length": 262.5499973297119, + "epoch": 0.844997407983411, + "grad_norm": 0.8816485404968262, + "kl": 1.880126953125, + "learning_rate": 2.7086100231595707e-06, + "loss": 0.2727, + "reward": 1.1250000298023224, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.20000000298023224, + "rewards/format_reward": 0.925000011920929, + "step": 815 + }, + { + "completion_length": 181.67500114440918, + "epoch": 0.8460342146189735, + "grad_norm": 0.4687977731227875, + "kl": 1.150634765625, + "learning_rate": 2.7075365046211684e-06, + "loss": 0.0978, + "reward": 1.2500000298023224, + "reward_std": 0.2487340308725834, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 816 + }, + { + "completion_length": 228.07500839233398, + "epoch": 0.847071021254536, + "grad_norm": 2.606320381164551, + "kl": 3.34521484375, + "learning_rate": 2.706461225788085e-06, + "loss": 0.3432, + "reward": 1.5250000059604645, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9500000029802322, + "step": 817 + }, + { + "completion_length": 156.0500030517578, + "epoch": 0.8481078278900985, + "grad_norm": 2.9354310035705566, + "kl": 2.510498046875, + "learning_rate": 2.7053841882278156e-06, + "loss": 0.239, + "reward": 1.1000000089406967, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 818 + }, + { + "completion_length": 190.42499923706055, + "epoch": 0.8491446345256609, + "grad_norm": 0.6598667502403259, + "kl": 0.934814453125, + "learning_rate": 2.7043053935104183e-06, + "loss": 0.0775, + "reward": 1.375, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 819 + }, + { + "completion_length": 182.6250057220459, + "epoch": 0.8501814411612234, + "grad_norm": 0.46700766682624817, + "kl": 1.113525390625, + "learning_rate": 2.7032248432085143e-06, + "loss": 0.141, + "reward": 1.3250000327825546, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 820 + }, + { + "completion_length": 221.8000087738037, + "epoch": 0.8512182477967859, + "grad_norm": 3.7838664054870605, + "kl": 0.30810546875, + "learning_rate": 2.702142538897283e-06, + "loss": 0.1173, + "reward": 1.3250000178813934, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 821 + }, + { + "completion_length": 353.32501220703125, + "epoch": 0.8522550544323484, + "grad_norm": 1.2203960418701172, + "kl": 2.563232421875, + "learning_rate": 2.701058482154461e-06, + "loss": 0.3616, + "reward": 1.3750000298023224, + "reward_std": 0.48141248524188995, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.8750000149011612, + "step": 822 + }, + { + "completion_length": 303.4500045776367, + "epoch": 0.8532918610679109, + "grad_norm": 6.787600994110107, + "kl": 6.1015625, + "learning_rate": 2.6999726745603396e-06, + "loss": 0.6308, + "reward": 1.475000038743019, + "reward_std": 0.4237985461950302, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9000000208616257, + "step": 823 + }, + { + "completion_length": 325.97500801086426, + "epoch": 0.8543286677034733, + "grad_norm": 22.804424285888672, + "kl": 9.36083984375, + "learning_rate": 2.698885117697762e-06, + "loss": 0.8731, + "reward": 1.2750000208616257, + "reward_std": 0.3840910233557224, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9000000208616257, + "step": 824 + }, + { + "completion_length": 269.49999618530273, + "epoch": 0.8553654743390358, + "grad_norm": 1.0976574420928955, + "kl": 1.403564453125, + "learning_rate": 2.697795813152122e-06, + "loss": 0.0994, + "reward": 1.475000023841858, + "reward_std": 0.31719936430454254, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.925000011920929, + "step": 825 + }, + { + "completion_length": 226.20000457763672, + "epoch": 0.8564022809745983, + "grad_norm": 1.3571267127990723, + "kl": 1.186279296875, + "learning_rate": 2.6967047625113606e-06, + "loss": 0.1751, + "reward": 1.3250000327825546, + "reward_std": 0.31199514865875244, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.9500000178813934, + "step": 826 + }, + { + "completion_length": 199.25000381469727, + "epoch": 0.8574390876101607, + "grad_norm": 1.5436522960662842, + "kl": 0.467529296875, + "learning_rate": 2.6956119673659652e-06, + "loss": 0.0601, + "reward": 1.7249999940395355, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 827 + }, + { + "completion_length": 180.52500915527344, + "epoch": 0.8584758942457231, + "grad_norm": 2.2522006034851074, + "kl": 0.473388671875, + "learning_rate": 2.6945174293089644e-06, + "loss": 0.0962, + "reward": 1.4000000357627869, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9750000089406967, + "step": 828 + }, + { + "completion_length": 210.47500610351562, + "epoch": 0.8595127008812856, + "grad_norm": 0.6633756756782532, + "kl": 1.012451171875, + "learning_rate": 2.6934211499359293e-06, + "loss": 0.1798, + "reward": 1.1500000208616257, + "reward_std": 0.1928323470056057, + "rewards/accuracy_reward": 0.20000000670552254, + "rewards/format_reward": 0.9500000029802322, + "step": 829 + }, + { + "completion_length": 273.35000228881836, + "epoch": 0.8605495075168481, + "grad_norm": 0.7028055191040039, + "kl": 0.66015625, + "learning_rate": 2.6923231308449686e-06, + "loss": 0.159, + "reward": 1.425000011920929, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.5000000018626451, + "rewards/format_reward": 0.925000011920929, + "step": 830 + }, + { + "completion_length": 220.07500839233398, + "epoch": 0.8615863141524106, + "grad_norm": 1.8734394311904907, + "kl": 1.123291015625, + "learning_rate": 2.6912233736367267e-06, + "loss": 0.1689, + "reward": 1.4500000327825546, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9500000178813934, + "step": 831 + }, + { + "completion_length": 316.2000026702881, + "epoch": 0.8626231207879731, + "grad_norm": 1.5336312055587769, + "kl": 1.52294921875, + "learning_rate": 2.6901218799143837e-06, + "loss": 0.1906, + "reward": 1.3250000178813934, + "reward_std": 0.36053743585944176, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9000000059604645, + "step": 832 + }, + { + "completion_length": 132.975004196167, + "epoch": 0.8636599274235355, + "grad_norm": 0.27653202414512634, + "kl": 0.23974609375, + "learning_rate": 2.689018651283648e-06, + "loss": 0.0261, + "reward": 1.6500000059604645, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 1.0, + "step": 833 + }, + { + "completion_length": 267.3249988555908, + "epoch": 0.864696734059098, + "grad_norm": 1.0966119766235352, + "kl": 1.613037109375, + "learning_rate": 2.6879136893527593e-06, + "loss": 0.2441, + "reward": 1.4500000476837158, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.925000011920929, + "step": 834 + }, + { + "completion_length": 174.17500114440918, + "epoch": 0.8657335406946605, + "grad_norm": 0.47363966703414917, + "kl": 0.385986328125, + "learning_rate": 2.686806995732484e-06, + "loss": 0.0541, + "reward": 1.425000011920929, + "reward_std": 0.26129766926169395, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 835 + }, + { + "completion_length": 234.4000015258789, + "epoch": 0.866770347330223, + "grad_norm": 2.3432672023773193, + "kl": 3.037841796875, + "learning_rate": 2.6856985720361114e-06, + "loss": 0.3162, + "reward": 1.4000000357627869, + "reward_std": 0.40387555956840515, + "rewards/accuracy_reward": 0.4500000048428774, + "rewards/format_reward": 0.9500000178813934, + "step": 836 + }, + { + "completion_length": 188.27500343322754, + "epoch": 0.8678071539657853, + "grad_norm": 0.5459396243095398, + "kl": 0.52294921875, + "learning_rate": 2.684588419879455e-06, + "loss": 0.0788, + "reward": 1.5250000059604645, + "reward_std": 0.3966546431183815, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.9750000089406967, + "step": 837 + }, + { + "completion_length": 330.22500801086426, + "epoch": 0.8688439606013478, + "grad_norm": 3.8449041843414307, + "kl": 3.85595703125, + "learning_rate": 2.683476540880846e-06, + "loss": 0.4513, + "reward": 1.4750000536441803, + "reward_std": 0.46511993557214737, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.9000000208616257, + "step": 838 + }, + { + "completion_length": 366.2750053405762, + "epoch": 0.8698807672369103, + "grad_norm": 6.319846153259277, + "kl": 7.76025390625, + "learning_rate": 2.682362936661134e-06, + "loss": 0.6544, + "reward": 1.5, + "reward_std": 0.40450848266482353, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8500000089406967, + "step": 839 + }, + { + "completion_length": 122.9000015258789, + "epoch": 0.8709175738724728, + "grad_norm": 0.4149312376976013, + "kl": 0.285400390625, + "learning_rate": 2.6812476088436827e-06, + "loss": 0.0377, + "reward": 1.4000000059604645, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 1.0, + "step": 840 + }, + { + "completion_length": 126.32500076293945, + "epoch": 0.8719543805080352, + "grad_norm": 0.42162078619003296, + "kl": 0.322998046875, + "learning_rate": 2.6801305590543697e-06, + "loss": 0.0095, + "reward": 1.3750000298023224, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.37499999813735485, + "rewards/format_reward": 1.0, + "step": 841 + }, + { + "completion_length": 267.50000381469727, + "epoch": 0.8729911871435977, + "grad_norm": 2.6304068565368652, + "kl": 1.82666015625, + "learning_rate": 2.6790117889215812e-06, + "loss": 0.3007, + "reward": 1.125, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.22500000149011612, + "rewards/format_reward": 0.9000000208616257, + "step": 842 + }, + { + "completion_length": 169.42500686645508, + "epoch": 0.8740279937791602, + "grad_norm": 1.3741490840911865, + "kl": 0.783203125, + "learning_rate": 2.677891300076213e-06, + "loss": 0.0916, + "reward": 1.375, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 843 + }, + { + "completion_length": 173.32500457763672, + "epoch": 0.8750648004147227, + "grad_norm": 1.253536343574524, + "kl": 0.6121826171875, + "learning_rate": 2.6767690941516653e-06, + "loss": 0.0845, + "reward": 1.7249999940395355, + "reward_std": 0.3479738272726536, + "rewards/accuracy_reward": 0.7500000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 844 + }, + { + "completion_length": 263.67500495910645, + "epoch": 0.8761016070502852, + "grad_norm": 1.1694815158843994, + "kl": 1.185546875, + "learning_rate": 2.6756451727838407e-06, + "loss": 0.1564, + "reward": 1.3750000149011612, + "reward_std": 0.3299015015363693, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.875, + "step": 845 + }, + { + "completion_length": 181.50000190734863, + "epoch": 0.8771384136858476, + "grad_norm": 0.7774064540863037, + "kl": 0.954833984375, + "learning_rate": 2.674519537611144e-06, + "loss": 0.0741, + "reward": 1.5750000476837158, + "reward_std": 0.36053744703531265, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 846 + }, + { + "completion_length": 183.3000030517578, + "epoch": 0.87817522032141, + "grad_norm": 0.5851902961730957, + "kl": 1.0263671875, + "learning_rate": 2.673392190274478e-06, + "loss": 0.1204, + "reward": 1.550000011920929, + "reward_std": 0.21275537833571434, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 847 + }, + { + "completion_length": 326.75000381469727, + "epoch": 0.8792120269569725, + "grad_norm": 6.058239459991455, + "kl": 1.504150390625, + "learning_rate": 2.6722631324172407e-06, + "loss": 0.2304, + "reward": 1.3250000327825546, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.8750000149011612, + "step": 848 + }, + { + "completion_length": 246.55000114440918, + "epoch": 0.880248833592535, + "grad_norm": 1.6069053411483765, + "kl": 0.45849609375, + "learning_rate": 2.6711323656853235e-06, + "loss": 0.0939, + "reward": 1.6500000357627869, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.7250000033527613, + "rewards/format_reward": 0.925000011920929, + "step": 849 + }, + { + "completion_length": 203.32500076293945, + "epoch": 0.8812856402280974, + "grad_norm": 0.8683740496635437, + "kl": 0.555908203125, + "learning_rate": 2.6699998917271096e-06, + "loss": 0.0142, + "reward": 1.4749999940395355, + "reward_std": 0.3611704334616661, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000029802322, + "step": 850 + }, + { + "completion_length": 176.20000076293945, + "epoch": 0.8823224468636599, + "grad_norm": 2.5231356620788574, + "kl": 2.1685791015625, + "learning_rate": 2.6688657121934713e-06, + "loss": 0.2105, + "reward": 1.6250000298023224, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.6500000208616257, + "rewards/format_reward": 0.9750000089406967, + "step": 851 + }, + { + "completion_length": 313.75000381469727, + "epoch": 0.8833592534992224, + "grad_norm": 26.584848403930664, + "kl": 10.794189453125, + "learning_rate": 2.6677298287377666e-06, + "loss": 0.9613, + "reward": 1.350000023841858, + "reward_std": 0.535601943731308, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9000000208616257, + "step": 852 + }, + { + "completion_length": 376.1000118255615, + "epoch": 0.8843960601347849, + "grad_norm": 12.684377670288086, + "kl": 10.703125, + "learning_rate": 2.666592243015837e-06, + "loss": 0.9707, + "reward": 1.0750000327825546, + "reward_std": 0.542072132229805, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/format_reward": 0.8250000178813934, + "step": 853 + }, + { + "completion_length": 372.6500129699707, + "epoch": 0.8854328667703474, + "grad_norm": 1.3346270322799683, + "kl": 3.55712890625, + "learning_rate": 2.665452956686006e-06, + "loss": 0.3998, + "reward": 1.4500000178813934, + "reward_std": 0.38409099727869034, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.8750000149011612, + "step": 854 + }, + { + "completion_length": 209.50000762939453, + "epoch": 0.8864696734059098, + "grad_norm": 6.27949333190918, + "kl": 3.934814453125, + "learning_rate": 2.664311971409076e-06, + "loss": 0.3364, + "reward": 1.3500000536441803, + "reward_std": 0.46511994302272797, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.925000011920929, + "step": 855 + }, + { + "completion_length": 294.7000026702881, + "epoch": 0.8875064800414723, + "grad_norm": 1.2558188438415527, + "kl": 2.42041015625, + "learning_rate": 2.663169288848327e-06, + "loss": 0.2752, + "reward": 1.375, + "reward_std": 0.49746809154748917, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9000000059604645, + "step": 856 + }, + { + "completion_length": 430.9500026702881, + "epoch": 0.8885432866770347, + "grad_norm": 1.8420369625091553, + "kl": 3.364501953125, + "learning_rate": 2.662024910669511e-06, + "loss": 0.3637, + "reward": 1.4250000268220901, + "reward_std": 0.582266092300415, + "rewards/accuracy_reward": 0.5999999977648258, + "rewards/format_reward": 0.8250000029802322, + "step": 857 + }, + { + "completion_length": 131.52500534057617, + "epoch": 0.8895800933125972, + "grad_norm": 0.38641226291656494, + "kl": 0.2255859375, + "learning_rate": 2.6608788385408546e-06, + "loss": -0.0141, + "reward": 1.3499999940395355, + "reward_std": 0.1928323470056057, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 1.0, + "step": 858 + }, + { + "completion_length": 321.15000915527344, + "epoch": 0.8906168999481596, + "grad_norm": 1.2833192348480225, + "kl": 1.921875, + "learning_rate": 2.6597310741330515e-06, + "loss": 0.2401, + "reward": 1.2750000059604645, + "reward_std": 0.4976065903902054, + "rewards/accuracy_reward": 0.4250000063329935, + "rewards/format_reward": 0.8500000238418579, + "step": 859 + }, + { + "completion_length": 118.60000419616699, + "epoch": 0.8916537065837221, + "grad_norm": 0.39673030376434326, + "kl": 0.205078125, + "learning_rate": 2.6585816191192637e-06, + "loss": 0.0051, + "reward": 1.5, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 1.0, + "step": 860 + }, + { + "completion_length": 176.8750057220459, + "epoch": 0.8926905132192846, + "grad_norm": 2.3243377208709717, + "kl": 0.39306640625, + "learning_rate": 2.657430475175117e-06, + "loss": 0.0705, + "reward": 1.5000000298023224, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9500000178813934, + "step": 861 + }, + { + "completion_length": 292.87500381469727, + "epoch": 0.8937273198548471, + "grad_norm": 3.260263442993164, + "kl": 1.212890625, + "learning_rate": 2.656277643978699e-06, + "loss": 0.1044, + "reward": 1.550000011920929, + "reward_std": 0.34448180347681046, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9250000268220901, + "step": 862 + }, + { + "completion_length": 292.2000141143799, + "epoch": 0.8947641264904095, + "grad_norm": 1.0654324293136597, + "kl": 0.982666015625, + "learning_rate": 2.655123127210558e-06, + "loss": 0.0851, + "reward": 1.5500000268220901, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 863 + }, + { + "completion_length": 318.0000114440918, + "epoch": 0.895800933125972, + "grad_norm": 146.21029663085938, + "kl": 1.025390625, + "learning_rate": 2.6539669265536987e-06, + "loss": 0.1239, + "reward": 1.2000000178813934, + "reward_std": 0.46511993184685707, + "rewards/accuracy_reward": 0.3000000100582838, + "rewards/format_reward": 0.9000000208616257, + "step": 864 + }, + { + "completion_length": 179.82500076293945, + "epoch": 0.8968377397615345, + "grad_norm": 3.667255163192749, + "kl": 0.69873046875, + "learning_rate": 2.6528090436935803e-06, + "loss": 0.0528, + "reward": 1.425000011920929, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.4499999973922968, + "rewards/format_reward": 0.9750000089406967, + "step": 865 + }, + { + "completion_length": 241.0500030517578, + "epoch": 0.897874546397097, + "grad_norm": 0.824424684047699, + "kl": 0.658203125, + "learning_rate": 2.6516494803181147e-06, + "loss": 0.0801, + "reward": 1.550000011920929, + "reward_std": 0.4849044717848301, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 866 + }, + { + "completion_length": 235.8499984741211, + "epoch": 0.8989113530326595, + "grad_norm": 0.9227349162101746, + "kl": 0.5498046875, + "learning_rate": 2.650488238117663e-06, + "loss": 0.1104, + "reward": 1.5250000059604645, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9500000029802322, + "step": 867 + }, + { + "completion_length": 333.700008392334, + "epoch": 0.8999481596682218, + "grad_norm": 1.584023356437683, + "kl": 1.18994140625, + "learning_rate": 2.6493253187850347e-06, + "loss": 0.1872, + "reward": 1.2249999940395355, + "reward_std": 0.45318927988409996, + "rewards/accuracy_reward": 0.35000001080334187, + "rewards/format_reward": 0.8750000149011612, + "step": 868 + }, + { + "completion_length": 330.27500343322754, + "epoch": 0.9009849663037843, + "grad_norm": 3.394610643386841, + "kl": 3.2109375, + "learning_rate": 2.6481607240154826e-06, + "loss": 0.3783, + "reward": 1.3750000298023224, + "reward_std": 0.3533165380358696, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.9000000208616257, + "step": 869 + }, + { + "completion_length": 196.25, + "epoch": 0.9020217729393468, + "grad_norm": 0.48655158281326294, + "kl": 0.21875, + "learning_rate": 2.646994455506704e-06, + "loss": 0.0491, + "reward": 1.625, + "reward_std": 0.39302411302924156, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9750000089406967, + "step": 870 + }, + { + "completion_length": 240.25000190734863, + "epoch": 0.9030585795749093, + "grad_norm": 2.112935781478882, + "kl": 1.31494140625, + "learning_rate": 2.6458265149588345e-06, + "loss": 0.1163, + "reward": 1.3250000327825546, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.4000000096857548, + "rewards/format_reward": 0.9250000268220901, + "step": 871 + }, + { + "completion_length": 249.52500915527344, + "epoch": 0.9040953862104717, + "grad_norm": 1.491579294204712, + "kl": 1.40673828125, + "learning_rate": 2.6446569040744464e-06, + "loss": 0.1719, + "reward": 1.5250000357627869, + "reward_std": 0.3299014940857887, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000029802322, + "step": 872 + }, + { + "completion_length": 239.32500076293945, + "epoch": 0.9051321928460342, + "grad_norm": 0.9075613021850586, + "kl": 0.900390625, + "learning_rate": 2.6434856245585493e-06, + "loss": 0.1707, + "reward": 1.6500000357627869, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 873 + }, + { + "completion_length": 213.05000495910645, + "epoch": 0.9061689994815967, + "grad_norm": 1.415359616279602, + "kl": 1.92919921875, + "learning_rate": 2.642312678118584e-06, + "loss": 0.217, + "reward": 1.3750000298023224, + "reward_std": 0.27228760346770287, + "rewards/accuracy_reward": 0.42500000819563866, + "rewards/format_reward": 0.9500000178813934, + "step": 874 + }, + { + "completion_length": 283.8000068664551, + "epoch": 0.9072058061171592, + "grad_norm": 40.05736541748047, + "kl": 9.90576171875, + "learning_rate": 2.6411380664644208e-06, + "loss": 0.8472, + "reward": 1.2500000149011612, + "reward_std": 0.30463574826717377, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.925000011920929, + "step": 875 + }, + { + "completion_length": 227.15000534057617, + "epoch": 0.9082426127527217, + "grad_norm": 1.3082458972930908, + "kl": 2.181884765625, + "learning_rate": 2.6399617913083574e-06, + "loss": 0.0988, + "reward": 1.225000023841858, + "reward_std": 0.41643914207816124, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.9000000208616257, + "step": 876 + }, + { + "completion_length": 118.55000114440918, + "epoch": 0.909279419388284, + "grad_norm": 0.44255509972572327, + "kl": 0.2578125, + "learning_rate": 2.6387838543651183e-06, + "loss": 0.0397, + "reward": 1.5499999821186066, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 877 + }, + { + "completion_length": 211.2750015258789, + "epoch": 0.9103162260238465, + "grad_norm": 3.5066232681274414, + "kl": 2.357421875, + "learning_rate": 2.6376042573518482e-06, + "loss": 0.2314, + "reward": 1.5000000298023224, + "reward_std": 0.2253189980983734, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 878 + }, + { + "completion_length": 240.5250015258789, + "epoch": 0.911353032659409, + "grad_norm": 2.016002655029297, + "kl": 2.30078125, + "learning_rate": 2.6364230019881132e-06, + "loss": 0.354, + "reward": 1.4250000417232513, + "reward_std": 0.4723408557474613, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.9000000208616257, + "step": 879 + }, + { + "completion_length": 308.575008392334, + "epoch": 0.9123898392949715, + "grad_norm": 3.611039876937866, + "kl": 2.6015625, + "learning_rate": 2.6352400899958967e-06, + "loss": 0.3464, + "reward": 1.4249999970197678, + "reward_std": 0.36053743585944176, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 880 + }, + { + "completion_length": 282.22500228881836, + "epoch": 0.9134266459305339, + "grad_norm": 2.582071542739868, + "kl": 2.25634765625, + "learning_rate": 2.6340555230995963e-06, + "loss": 0.152, + "reward": 1.300000011920929, + "reward_std": 0.4092182442545891, + "rewards/accuracy_reward": 0.3750000149011612, + "rewards/format_reward": 0.925000011920929, + "step": 881 + }, + { + "completion_length": 237.97501182556152, + "epoch": 0.9144634525660964, + "grad_norm": 1.9722750186920166, + "kl": 1.4404296875, + "learning_rate": 2.6328693030260233e-06, + "loss": 0.1863, + "reward": 1.4249999970197678, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 882 + }, + { + "completion_length": 262.6999988555908, + "epoch": 0.9155002592016589, + "grad_norm": 3.5364909172058105, + "kl": 1.622314453125, + "learning_rate": 2.631681431504397e-06, + "loss": 0.2271, + "reward": 1.4000000357627869, + "reward_std": 0.3533165268599987, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.925000011920929, + "step": 883 + }, + { + "completion_length": 308.45000076293945, + "epoch": 0.9165370658372214, + "grad_norm": 2.106987237930298, + "kl": 1.511474609375, + "learning_rate": 2.6304919102663467e-06, + "loss": 0.2561, + "reward": 1.2750000208616257, + "reward_std": 0.31719936802983284, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 884 + }, + { + "completion_length": 103.62500190734863, + "epoch": 0.9175738724727839, + "grad_norm": 0.7540234923362732, + "kl": 0.2861328125, + "learning_rate": 2.629300741045904e-06, + "loss": 0.0162, + "reward": 1.5500000417232513, + "reward_std": 0.31719936057925224, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 1.0, + "step": 885 + }, + { + "completion_length": 272.4250011444092, + "epoch": 0.9186106791083463, + "grad_norm": 5.264670372009277, + "kl": 3.08740234375, + "learning_rate": 2.628107925579505e-06, + "loss": 0.3337, + "reward": 1.5750000178813934, + "reward_std": 0.31199517846107483, + "rewards/accuracy_reward": 0.6500000022351742, + "rewards/format_reward": 0.925000011920929, + "step": 886 + }, + { + "completion_length": 229.125, + "epoch": 0.9196474857439088, + "grad_norm": 1.4002385139465332, + "kl": 0.61328125, + "learning_rate": 2.626913465605985e-06, + "loss": 0.0328, + "reward": 1.4250000417232513, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9500000029802322, + "step": 887 + }, + { + "completion_length": 177.32500457763672, + "epoch": 0.9206842923794712, + "grad_norm": 8.91387939453125, + "kl": 3.0546875, + "learning_rate": 2.6257173628665746e-06, + "loss": 0.237, + "reward": 1.2000000178813934, + "reward_std": 0.22894951701164246, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/format_reward": 0.9500000029802322, + "step": 888 + }, + { + "completion_length": 123.5, + "epoch": 0.9217210990150337, + "grad_norm": 0.1785755157470703, + "kl": 0.228515625, + "learning_rate": 2.6245196191049022e-06, + "loss": -0.0007, + "reward": 1.300000011920929, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.30000000074505806, + "rewards/format_reward": 1.0, + "step": 889 + }, + { + "completion_length": 254.17500686645508, + "epoch": 0.9227579056505961, + "grad_norm": 3.5954160690307617, + "kl": 2.435546875, + "learning_rate": 2.6233202360669864e-06, + "loss": 0.334, + "reward": 1.275000050663948, + "reward_std": 0.4472136162221432, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9000000357627869, + "step": 890 + }, + { + "completion_length": 260.72500228881836, + "epoch": 0.9237947122861586, + "grad_norm": 14.043997764587402, + "kl": 6.834228515625, + "learning_rate": 2.6221192155012366e-06, + "loss": 0.6215, + "reward": 1.6250000298023224, + "reward_std": 0.3245587609708309, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 891 + }, + { + "completion_length": 350.0750045776367, + "epoch": 0.9248315189217211, + "grad_norm": 92.7138442993164, + "kl": 18.8623046875, + "learning_rate": 2.6209165591584487e-06, + "loss": 1.5923, + "reward": 1.0750000178813934, + "reward_std": 0.35394952073693275, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.8500000089406967, + "step": 892 + }, + { + "completion_length": 241.80001068115234, + "epoch": 0.9258683255572836, + "grad_norm": 42.686927795410156, + "kl": 10.860107421875, + "learning_rate": 2.619712268791803e-06, + "loss": 0.9162, + "reward": 1.4000000357627869, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 0.9500000178813934, + "step": 893 + }, + { + "completion_length": 269.225004196167, + "epoch": 0.926905132192846, + "grad_norm": 125.65151977539062, + "kl": 40.829833984375, + "learning_rate": 2.6185063461568624e-06, + "loss": 3.204, + "reward": 1.3250000476837158, + "reward_std": 0.3804604783654213, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 894 + }, + { + "completion_length": 165.17500495910645, + "epoch": 0.9279419388284085, + "grad_norm": 0.945558488368988, + "kl": 0.5986328125, + "learning_rate": 2.617298793011569e-06, + "loss": 0.0603, + "reward": 1.5000000298023224, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.5250000040978193, + "rewards/format_reward": 0.9750000089406967, + "step": 895 + }, + { + "completion_length": 142.74999809265137, + "epoch": 0.928978745463971, + "grad_norm": 0.27745455503463745, + "kl": 0.215576171875, + "learning_rate": 2.6160896111162412e-06, + "loss": 0.0277, + "reward": 1.550000011920929, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 1.0, + "step": 896 + }, + { + "completion_length": 269.32501220703125, + "epoch": 0.9300155520995335, + "grad_norm": 3.364424705505371, + "kl": 1.71484375, + "learning_rate": 2.614878802233573e-06, + "loss": 0.2486, + "reward": 1.3750000298023224, + "reward_std": 0.5173911079764366, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9250000268220901, + "step": 897 + }, + { + "completion_length": 307.37500381469727, + "epoch": 0.931052358735096, + "grad_norm": 8.091796875, + "kl": 0.7021484375, + "learning_rate": 2.613666368128629e-06, + "loss": 0.1984, + "reward": 1.5, + "reward_std": 0.5715806074440479, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9000000208616257, + "step": 898 + }, + { + "completion_length": 216.17500114440918, + "epoch": 0.9320891653706583, + "grad_norm": 3.7539572715759277, + "kl": 0.489990234375, + "learning_rate": 2.612452310568843e-06, + "loss": 0.166, + "reward": 1.4500000476837158, + "reward_std": 0.347973819822073, + "rewards/accuracy_reward": 0.5000000018626451, + "rewards/format_reward": 0.9500000178813934, + "step": 899 + }, + { + "completion_length": 306.25000381469727, + "epoch": 0.9331259720062208, + "grad_norm": 19.283233642578125, + "kl": 12.60498046875, + "learning_rate": 2.611236631324017e-06, + "loss": 0.5753, + "reward": 1.3000000268220901, + "reward_std": 0.4038755036890507, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.9000000208616257, + "step": 900 + }, + { + "completion_length": 301.6750068664551, + "epoch": 0.9341627786417833, + "grad_norm": 3.5665500164031982, + "kl": 3.050048828125, + "learning_rate": 2.6100193321663147e-06, + "loss": 0.3779, + "reward": 1.0750000327825546, + "reward_std": 0.4019973427057266, + "rewards/accuracy_reward": 0.17500000447034836, + "rewards/format_reward": 0.9000000208616257, + "step": 901 + }, + { + "completion_length": 171.35000801086426, + "epoch": 0.9351995852773458, + "grad_norm": 1.0143321752548218, + "kl": 0.2734375, + "learning_rate": 2.608800414870263e-06, + "loss": 0.0389, + "reward": 1.5250000357627869, + "reward_std": 0.30463574826717377, + "rewards/accuracy_reward": 0.5500000081956387, + "rewards/format_reward": 0.9750000089406967, + "step": 902 + }, + { + "completion_length": 174.27500343322754, + "epoch": 0.9362363919129082, + "grad_norm": 1.870166301727295, + "kl": 0.5362548828125, + "learning_rate": 2.6075798812127463e-06, + "loss": 0.0717, + "reward": 1.3250000327825546, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.9750000089406967, + "step": 903 + }, + { + "completion_length": 270.29999923706055, + "epoch": 0.9372731985484707, + "grad_norm": 1.5089102983474731, + "kl": 1.107421875, + "learning_rate": 2.6063577329730072e-06, + "loss": 0.2227, + "reward": 1.4000000357627869, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.9000000059604645, + "step": 904 + }, + { + "completion_length": 157.52499771118164, + "epoch": 0.9383100051840332, + "grad_norm": 1.0162603855133057, + "kl": 0.883056640625, + "learning_rate": 2.605133971932639e-06, + "loss": 0.1052, + "reward": 1.8000000417232513, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 905 + }, + { + "completion_length": 117.125, + "epoch": 0.9393468118195957, + "grad_norm": 0.2638833224773407, + "kl": 0.188232421875, + "learning_rate": 2.6039085998755895e-06, + "loss": -0.0455, + "reward": 1.5, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 1.0, + "step": 906 + }, + { + "completion_length": 220.10000228881836, + "epoch": 0.9403836184551582, + "grad_norm": 0.5877365469932556, + "kl": 2.642822265625, + "learning_rate": 2.602681618588152e-06, + "loss": 0.2276, + "reward": 1.3250000327825546, + "reward_std": 0.31199513375759125, + "rewards/accuracy_reward": 0.4000000152736902, + "rewards/format_reward": 0.9250000268220901, + "step": 907 + }, + { + "completion_length": 260.37500762939453, + "epoch": 0.9414204250907205, + "grad_norm": 1.3724843263626099, + "kl": 3.5703125, + "learning_rate": 2.6014530298589674e-06, + "loss": 0.3882, + "reward": 1.3000000268220901, + "reward_std": 0.3479737974703312, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.925000011920929, + "step": 908 + }, + { + "completion_length": 233.57500457763672, + "epoch": 0.942457231726283, + "grad_norm": 1.3091270923614502, + "kl": 1.9521484375, + "learning_rate": 2.60022283547902e-06, + "loss": 0.1282, + "reward": 1.375, + "reward_std": 0.4415663853287697, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 909 + }, + { + "completion_length": 172.9249973297119, + "epoch": 0.9434940383618455, + "grad_norm": 0.9247521758079529, + "kl": 0.97021484375, + "learning_rate": 2.5989910372416335e-06, + "loss": 0.114, + "reward": 1.350000023841858, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 910 + }, + { + "completion_length": 195.77500343322754, + "epoch": 0.944530844997408, + "grad_norm": 1.2640548944473267, + "kl": 1.4638671875, + "learning_rate": 2.59775763694247e-06, + "loss": 0.1241, + "reward": 1.225000023841858, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 0.9500000029802322, + "step": 911 + }, + { + "completion_length": 269.72500228881836, + "epoch": 0.9455676516329704, + "grad_norm": 0.822454035282135, + "kl": 0.965576171875, + "learning_rate": 2.596522636379528e-06, + "loss": 0.2064, + "reward": 1.5000000298023224, + "reward_std": 0.30463575199246407, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.925000011920929, + "step": 912 + }, + { + "completion_length": 119.125, + "epoch": 0.9466044582685329, + "grad_norm": 0.47956565022468567, + "kl": 0.22607421875, + "learning_rate": 2.595286037353138e-06, + "loss": 0.018, + "reward": 1.675000011920929, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 1.0, + "step": 913 + }, + { + "completion_length": 257.4500026702881, + "epoch": 0.9476412649040954, + "grad_norm": 1.073453664779663, + "kl": 2.339599609375, + "learning_rate": 2.5940478416659605e-06, + "loss": 0.2984, + "reward": 1.4000000059604645, + "reward_std": 0.3407529331743717, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.925000011920929, + "step": 914 + }, + { + "completion_length": 169.75000190734863, + "epoch": 0.9486780715396579, + "grad_norm": 0.3584200441837311, + "kl": 0.40869140625, + "learning_rate": 2.5928080511229837e-06, + "loss": 0.059, + "reward": 1.699999988079071, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 915 + }, + { + "completion_length": 268.6500053405762, + "epoch": 0.9497148781752203, + "grad_norm": 0.72416752576828, + "kl": 1.800537109375, + "learning_rate": 2.591566667531522e-06, + "loss": 0.2042, + "reward": 1.5250000059604645, + "reward_std": 0.34860679507255554, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 0.9000000208616257, + "step": 916 + }, + { + "completion_length": 141.25, + "epoch": 0.9507516848107828, + "grad_norm": 0.3592819273471832, + "kl": 0.32470703125, + "learning_rate": 2.590323692701209e-06, + "loss": 0.0149, + "reward": 1.4000000059604645, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 1.0, + "step": 917 + }, + { + "completion_length": 224.8500099182129, + "epoch": 0.9517884914463453, + "grad_norm": 1.1734187602996826, + "kl": 0.6982421875, + "learning_rate": 2.589079128444001e-06, + "loss": 0.1334, + "reward": 1.225000023841858, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 918 + }, + { + "completion_length": 230.95000457763672, + "epoch": 0.9528252980819077, + "grad_norm": 1.34982430934906, + "kl": 0.759033203125, + "learning_rate": 2.58783297657417e-06, + "loss": 0.1402, + "reward": 1.5500000417232513, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 919 + }, + { + "completion_length": 417.5250015258789, + "epoch": 0.9538621047174702, + "grad_norm": 1.6819778680801392, + "kl": 1.3720703125, + "learning_rate": 2.586585238908303e-06, + "loss": 0.2049, + "reward": 1.4500000476837158, + "reward_std": 0.5282425507903099, + "rewards/accuracy_reward": 0.6250000223517418, + "rewards/format_reward": 0.8250000178813934, + "step": 920 + }, + { + "completion_length": 179.52500343322754, + "epoch": 0.9548989113530326, + "grad_norm": 0.4821496307849884, + "kl": 0.600830078125, + "learning_rate": 2.585335917265297e-06, + "loss": 0.0838, + "reward": 1.4000000208616257, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.42499999701976776, + "rewards/format_reward": 0.9750000089406967, + "step": 921 + }, + { + "completion_length": 266.1250057220459, + "epoch": 0.9559357179885951, + "grad_norm": 1.7604442834854126, + "kl": 1.4638671875, + "learning_rate": 2.5840850134663604e-06, + "loss": 0.1657, + "reward": 1.2500000298023224, + "reward_std": 0.30463574826717377, + "rewards/accuracy_reward": 0.325000012293458, + "rewards/format_reward": 0.925000011920929, + "step": 922 + }, + { + "completion_length": 157.10000038146973, + "epoch": 0.9569725246241576, + "grad_norm": 0.7505897879600525, + "kl": 0.798583984375, + "learning_rate": 2.5828325293350064e-06, + "loss": 0.1055, + "reward": 1.600000023841858, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 923 + }, + { + "completion_length": 172.25000190734863, + "epoch": 0.9580093312597201, + "grad_norm": 0.8031712174415588, + "kl": 1.382080078125, + "learning_rate": 2.581578466697052e-06, + "loss": 0.1606, + "reward": 1.3250000327825546, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 924 + }, + { + "completion_length": 115.32500267028809, + "epoch": 0.9590461378952825, + "grad_norm": 0.28106948733329773, + "kl": 0.29638671875, + "learning_rate": 2.580322827380616e-06, + "loss": 0.0158, + "reward": 1.6500000357627869, + "reward_std": 0.1369306445121765, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 1.0, + "step": 925 + }, + { + "completion_length": 245.8000030517578, + "epoch": 0.960082944530845, + "grad_norm": 1.201001763343811, + "kl": 1.44873046875, + "learning_rate": 2.5790656132161155e-06, + "loss": 0.284, + "reward": 1.425000011920929, + "reward_std": 0.3678968548774719, + "rewards/accuracy_reward": 0.5000000167638063, + "rewards/format_reward": 0.925000011920929, + "step": 926 + }, + { + "completion_length": 117.50000190734863, + "epoch": 0.9611197511664075, + "grad_norm": 0.5040531158447266, + "kl": 0.27978515625, + "learning_rate": 2.577806826036263e-06, + "loss": -0.001, + "reward": 1.7250000536441803, + "reward_std": 0.20539596676826477, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 1.0, + "step": 927 + }, + { + "completion_length": 106.87500381469727, + "epoch": 0.96215655780197, + "grad_norm": 0.43804383277893066, + "kl": 0.2802734375, + "learning_rate": 2.576546467676064e-06, + "loss": 0.0252, + "reward": 1.5999999940395355, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 1.0, + "step": 928 + }, + { + "completion_length": 106.35000038146973, + "epoch": 0.9631933644375325, + "grad_norm": 0.4537351429462433, + "kl": 0.33935546875, + "learning_rate": 2.575284539972815e-06, + "loss": 0.0106, + "reward": 1.5750000178813934, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 1.0, + "step": 929 + }, + { + "completion_length": 242.62500190734863, + "epoch": 0.9642301710730948, + "grad_norm": 47.74652099609375, + "kl": 13.046875, + "learning_rate": 2.5740210447660993e-06, + "loss": 1.1571, + "reward": 1.5, + "reward_std": 0.3913119062781334, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9250000268220901, + "step": 930 + }, + { + "completion_length": 146.2500057220459, + "epoch": 0.9652669777086573, + "grad_norm": 24.46285057067871, + "kl": 5.71484375, + "learning_rate": 2.572755983897786e-06, + "loss": 0.4407, + "reward": 1.625, + "reward_std": 0.36053745076060295, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 931 + }, + { + "completion_length": 257.9250144958496, + "epoch": 0.9663037843442198, + "grad_norm": 3.5134782791137695, + "kl": 3.75927734375, + "learning_rate": 2.5714893592120267e-06, + "loss": 0.1903, + "reward": 1.550000011920929, + "reward_std": 0.17304782569408417, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 932 + }, + { + "completion_length": 159.45000457763672, + "epoch": 0.9673405909797823, + "grad_norm": 2.399360418319702, + "kl": 2.16796875, + "learning_rate": 2.570221172555252e-06, + "loss": 0.1898, + "reward": 1.3500000089406967, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9750000089406967, + "step": 933 + }, + { + "completion_length": 180.92499923706055, + "epoch": 0.9683773976153447, + "grad_norm": 0.9570638537406921, + "kl": 1.00927734375, + "learning_rate": 2.5689514257761695e-06, + "loss": 0.1518, + "reward": 1.600000023841858, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 934 + }, + { + "completion_length": 147.29999923706055, + "epoch": 0.9694142042509072, + "grad_norm": 10.051190376281738, + "kl": 3.88720703125, + "learning_rate": 2.5676801207257614e-06, + "loss": 0.2523, + "reward": 1.350000023841858, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 935 + }, + { + "completion_length": 183.37499809265137, + "epoch": 0.9704510108864697, + "grad_norm": 1.349984884262085, + "kl": 2.6650390625, + "learning_rate": 2.5664072592572823e-06, + "loss": 0.2796, + "reward": 1.550000011920929, + "reward_std": 0.5247505605220795, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 936 + }, + { + "completion_length": 158.97500228881836, + "epoch": 0.9714878175220322, + "grad_norm": 0.5514269471168518, + "kl": 0.19921875, + "learning_rate": 2.565132843226253e-06, + "loss": 0.0189, + "reward": 1.4500000178813934, + "reward_std": 0.3371223993599415, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 937 + }, + { + "completion_length": 192.82500076293945, + "epoch": 0.9725246241575947, + "grad_norm": 22.37244987487793, + "kl": 6.8203125, + "learning_rate": 2.563856874490465e-06, + "loss": 0.634, + "reward": 1.5000000149011612, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9500000029802322, + "step": 938 + }, + { + "completion_length": 110.50000190734863, + "epoch": 0.973561430793157, + "grad_norm": 0.673575222492218, + "kl": 0.44140625, + "learning_rate": 2.5625793549099676e-06, + "loss": 0.003, + "reward": 1.5250000059604645, + "reward_std": 0.49746808037161827, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9750000089406967, + "step": 939 + }, + { + "completion_length": 97.17500114440918, + "epoch": 0.9745982374287195, + "grad_norm": 0.3349624276161194, + "kl": 0.352783203125, + "learning_rate": 2.561300286347075e-06, + "loss": 0.0131, + "reward": 1.699999988079071, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 1.0, + "step": 940 + }, + { + "completion_length": 245.49999809265137, + "epoch": 0.975635044064282, + "grad_norm": 2.73366641998291, + "kl": 1.84619140625, + "learning_rate": 2.560019670666359e-06, + "loss": 0.2749, + "reward": 1.4500000178813934, + "reward_std": 0.30463574826717377, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9000000059604645, + "step": 941 + }, + { + "completion_length": 138.72500610351562, + "epoch": 0.9766718506998445, + "grad_norm": 0.5809561610221863, + "kl": 0.8857421875, + "learning_rate": 2.558737509734645e-06, + "loss": 0.0642, + "reward": 1.4500000178813934, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 942 + }, + { + "completion_length": 132.2750015258789, + "epoch": 0.9777086573354069, + "grad_norm": 0.9825214147567749, + "kl": 0.572998046875, + "learning_rate": 2.5574538054210123e-06, + "loss": 0.0908, + "reward": 1.5750000178813934, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 943 + }, + { + "completion_length": 172.37500381469727, + "epoch": 0.9787454639709694, + "grad_norm": 1.4355342388153076, + "kl": 0.579833984375, + "learning_rate": 2.5561685595967895e-06, + "loss": 0.1356, + "reward": 1.4500000327825546, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.5250000208616257, + "rewards/format_reward": 0.9250000268220901, + "step": 944 + }, + { + "completion_length": 162.45000076293945, + "epoch": 0.9797822706065319, + "grad_norm": 0.6745811104774475, + "kl": 0.871826171875, + "learning_rate": 2.5548817741355533e-06, + "loss": 0.1104, + "reward": 1.324999988079071, + "reward_std": 0.20019176974892616, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9500000178813934, + "step": 945 + }, + { + "completion_length": 151.02500534057617, + "epoch": 0.9808190772420944, + "grad_norm": 0.9925585985183716, + "kl": 0.726318359375, + "learning_rate": 2.5535934509131233e-06, + "loss": 0.075, + "reward": 1.5999999940395355, + "reward_std": 0.3533165454864502, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 946 + }, + { + "completion_length": 163.75000381469727, + "epoch": 0.9818558838776568, + "grad_norm": 0.7139672040939331, + "kl": 0.64013671875, + "learning_rate": 2.5523035918075613e-06, + "loss": 0.064, + "reward": 1.25, + "reward_std": 0.20019175857305527, + "rewards/accuracy_reward": 0.2750000096857548, + "rewards/format_reward": 0.9750000089406967, + "step": 947 + }, + { + "completion_length": 152.00000381469727, + "epoch": 0.9828926905132193, + "grad_norm": 2.130598306655884, + "kl": 0.46142578125, + "learning_rate": 2.551012198699169e-06, + "loss": 0.0743, + "reward": 1.7000000178813934, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 948 + }, + { + "completion_length": 196.75000190734863, + "epoch": 0.9839294971487818, + "grad_norm": 1.8375300168991089, + "kl": 0.7880859375, + "learning_rate": 2.549719273470484e-06, + "loss": 0.1423, + "reward": 1.725000023841858, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 949 + }, + { + "completion_length": 222.3250026702881, + "epoch": 0.9849663037843442, + "grad_norm": 5.316202640533447, + "kl": 4.0380859375, + "learning_rate": 2.548424818006276e-06, + "loss": 0.4561, + "reward": 1.675000011920929, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 950 + }, + { + "completion_length": 212.09999656677246, + "epoch": 0.9860031104199067, + "grad_norm": 0.7121568918228149, + "kl": 0.4951171875, + "learning_rate": 2.547128834193547e-06, + "loss": 0.0836, + "reward": 1.3500000089406967, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.42499999701976776, + "rewards/format_reward": 0.9250000268220901, + "step": 951 + }, + { + "completion_length": 155.22500038146973, + "epoch": 0.9870399170554691, + "grad_norm": 2.3164560794830322, + "kl": 0.57177734375, + "learning_rate": 2.5458313239215266e-06, + "loss": 0.0801, + "reward": 1.5250000059604645, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 952 + }, + { + "completion_length": 197.80000495910645, + "epoch": 0.9880767236910316, + "grad_norm": 1.507447361946106, + "kl": 2.1240234375, + "learning_rate": 2.5445322890816685e-06, + "loss": 0.2326, + "reward": 1.6500000357627869, + "reward_std": 0.4723408557474613, + "rewards/accuracy_reward": 0.7250000238418579, + "rewards/format_reward": 0.925000011920929, + "step": 953 + }, + { + "completion_length": 174.45000076293945, + "epoch": 0.9891135303265941, + "grad_norm": 8.216100692749023, + "kl": 0.30615234375, + "learning_rate": 2.54323173156765e-06, + "loss": 0.0657, + "reward": 1.5250000059604645, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.9750000089406967, + "step": 954 + }, + { + "completion_length": 121.99999809265137, + "epoch": 0.9901503369621566, + "grad_norm": 0.4198542833328247, + "kl": 0.197021484375, + "learning_rate": 2.5419296532753684e-06, + "loss": 0.0393, + "reward": 1.5999999940395355, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.6000000108033419, + "rewards/format_reward": 1.0, + "step": 955 + }, + { + "completion_length": 268.40000915527344, + "epoch": 0.991187143597719, + "grad_norm": 14.27199935913086, + "kl": 7.554931640625, + "learning_rate": 2.5406260561029365e-06, + "loss": 0.7576, + "reward": 1.425000011920929, + "reward_std": 0.4363621771335602, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 956 + }, + { + "completion_length": 219.4249973297119, + "epoch": 0.9922239502332815, + "grad_norm": 8.010781288146973, + "kl": 5.810546875, + "learning_rate": 2.539320941950683e-06, + "loss": 0.423, + "reward": 1.5000000298023224, + "reward_std": 0.4472135826945305, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.925000011920929, + "step": 957 + }, + { + "completion_length": 205.37499809265137, + "epoch": 0.993260756868844, + "grad_norm": 6.571266174316406, + "kl": 5.18212890625, + "learning_rate": 2.5380143127211464e-06, + "loss": 0.4746, + "reward": 1.6250000298023224, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9500000178813934, + "step": 958 + }, + { + "completion_length": 168.30000686645508, + "epoch": 0.9942975635044065, + "grad_norm": 8.582088470458984, + "kl": 3.159423828125, + "learning_rate": 2.5367061703190743e-06, + "loss": 0.239, + "reward": 1.475000023841858, + "reward_std": 0.30463573709130287, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.9750000089406967, + "step": 959 + }, + { + "completion_length": 225.42501068115234, + "epoch": 0.995334370139969, + "grad_norm": 1.5330493450164795, + "kl": 2.22802734375, + "learning_rate": 2.535396516651421e-06, + "loss": 0.2924, + "reward": 1.4500000178813934, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.5000000018626451, + "rewards/format_reward": 0.9500000178813934, + "step": 960 + }, + { + "completion_length": 221.0999984741211, + "epoch": 0.9963711767755313, + "grad_norm": 3.0697624683380127, + "kl": 2.22119140625, + "learning_rate": 2.534085353627343e-06, + "loss": 0.2233, + "reward": 1.2750000208616257, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 961 + }, + { + "completion_length": 362.57499504089355, + "epoch": 0.9974079834110938, + "grad_norm": 3.0965917110443115, + "kl": 0.933349609375, + "learning_rate": 2.5327726831581968e-06, + "loss": 0.2787, + "reward": 1.6500000059604645, + "reward_std": 0.41643914580345154, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.8750000149011612, + "step": 962 + }, + { + "completion_length": 171.12499809265137, + "epoch": 0.9984447900466563, + "grad_norm": 1.9162588119506836, + "kl": 0.46533203125, + "learning_rate": 2.5314585071575373e-06, + "loss": 0.031, + "reward": 1.6500000357627869, + "reward_std": 0.3731010779738426, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 963 + }, + { + "completion_length": 260.2000102996826, + "epoch": 0.9994815966822188, + "grad_norm": 3.7568962574005127, + "kl": 2.827392578125, + "learning_rate": 2.530142827541113e-06, + "loss": 0.3048, + "reward": 1.4000000059604645, + "reward_std": 0.25609344989061356, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.925000011920929, + "step": 964 + }, + { + "completion_length": 416.1000213623047, + "epoch": 1.0, + "grad_norm": 3.7568962574005127, + "kl": 3.8359375, + "learning_rate": 2.5288256462268662e-06, + "loss": 0.1403, + "reward": 1.5000000596046448, + "reward_std": 0.5590169876813889, + "rewards/accuracy_reward": 0.6500000208616257, + "rewards/format_reward": 0.8500000238418579, + "step": 965 + }, + { + "completion_length": 213.12500953674316, + "epoch": 1.0010368066355624, + "grad_norm": 3.9320881366729736, + "kl": 4.014892578125, + "learning_rate": 2.5275069651349257e-06, + "loss": 0.3534, + "reward": 1.6250000298023224, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.6750000268220901, + "rewards/format_reward": 0.9500000178813934, + "step": 966 + }, + { + "completion_length": 305.67501068115234, + "epoch": 1.002073613271125, + "grad_norm": 7.879758358001709, + "kl": 0.9921875, + "learning_rate": 2.526186786187609e-06, + "loss": 0.2681, + "reward": 1.5, + "reward_std": 0.5210216678678989, + "rewards/accuracy_reward": 0.5999999940395355, + "rewards/format_reward": 0.9000000357627869, + "step": 967 + }, + { + "completion_length": 266.3250026702881, + "epoch": 1.0031104199066874, + "grad_norm": 4.553547382354736, + "kl": 0.9305419921875, + "learning_rate": 2.524865111309415e-06, + "loss": 0.2502, + "reward": 1.4500000029802322, + "reward_std": 0.3354101814329624, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9000000059604645, + "step": 968 + }, + { + "completion_length": 224.07500648498535, + "epoch": 1.00414722654225, + "grad_norm": 4.466383934020996, + "kl": 0.649169921875, + "learning_rate": 2.523541942427026e-06, + "loss": 0.1327, + "reward": 1.6500000357627869, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 969 + }, + { + "completion_length": 271.1000118255615, + "epoch": 1.0051840331778124, + "grad_norm": 1.9565415382385254, + "kl": 1.4482421875, + "learning_rate": 2.522217281469299e-06, + "loss": 0.1794, + "reward": 1.5250000357627869, + "reward_std": 0.5048274882137775, + "rewards/accuracy_reward": 0.6500000208616257, + "rewards/format_reward": 0.8750000149011612, + "step": 970 + }, + { + "completion_length": 273.3500099182129, + "epoch": 1.0062208398133747, + "grad_norm": 2.1400437355041504, + "kl": 3.146484375, + "learning_rate": 2.5208911303672685e-06, + "loss": 0.343, + "reward": 1.425000011920929, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.5250000022351742, + "rewards/format_reward": 0.9000000208616257, + "step": 971 + }, + { + "completion_length": 232.50000762939453, + "epoch": 1.0072576464489373, + "grad_norm": 4.056196689605713, + "kl": 0.76904296875, + "learning_rate": 2.5195634910541404e-06, + "loss": 0.1956, + "reward": 1.7750000357627869, + "reward_std": 0.40387551859021187, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 972 + }, + { + "completion_length": 279.9250087738037, + "epoch": 1.0082944530844997, + "grad_norm": 1.1603353023529053, + "kl": 0.69775390625, + "learning_rate": 2.5182343654652895e-06, + "loss": 0.0603, + "reward": 1.5500000417232513, + "reward_std": 0.5408061668276787, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9000000208616257, + "step": 973 + }, + { + "completion_length": 361.125, + "epoch": 1.009331259720062, + "grad_norm": 0.9591207504272461, + "kl": 2.771484375, + "learning_rate": 2.5169037555382583e-06, + "loss": 0.3757, + "reward": 1.4500000178813934, + "reward_std": 0.36053745076060295, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.875, + "step": 974 + }, + { + "completion_length": 314.2750015258789, + "epoch": 1.0103680663556247, + "grad_norm": 3.0426323413848877, + "kl": 3.90234375, + "learning_rate": 2.515571663212753e-06, + "loss": 0.4432, + "reward": 1.3750000298023224, + "reward_std": 0.45977723225951195, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9000000357627869, + "step": 975 + }, + { + "completion_length": 282.5999984741211, + "epoch": 1.011404872991187, + "grad_norm": 2.4559974670410156, + "kl": 1.0947265625, + "learning_rate": 2.514238090430639e-06, + "loss": 0.1671, + "reward": 1.5999999940395355, + "reward_std": 0.29270510002970695, + "rewards/accuracy_reward": 0.6750000212341547, + "rewards/format_reward": 0.9249999970197678, + "step": 976 + }, + { + "completion_length": 169.12500190734863, + "epoch": 1.0124416796267497, + "grad_norm": 0.7503567934036255, + "kl": 1.152587890625, + "learning_rate": 2.5129030391359427e-06, + "loss": 0.1039, + "reward": 1.8250000178813934, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 977 + }, + { + "completion_length": 222.95000648498535, + "epoch": 1.013478486262312, + "grad_norm": 0.9791756868362427, + "kl": 0.797607421875, + "learning_rate": 2.5115665112748435e-06, + "loss": 0.1764, + "reward": 1.4250000417232513, + "reward_std": 0.3354102112352848, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 978 + }, + { + "completion_length": 191.25000381469727, + "epoch": 1.0145152928978745, + "grad_norm": 0.6412131786346436, + "kl": 0.7979736328125, + "learning_rate": 2.510228508795675e-06, + "loss": 0.078, + "reward": 1.475000023841858, + "reward_std": 0.3913118988275528, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 979 + }, + { + "completion_length": 298.9750099182129, + "epoch": 1.015552099533437, + "grad_norm": 0.37992602586746216, + "kl": 1.24951171875, + "learning_rate": 2.5088890336489185e-06, + "loss": 0.1824, + "reward": 1.5250000357627869, + "reward_std": 0.4415663778781891, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9000000059604645, + "step": 980 + }, + { + "completion_length": 235.97500610351562, + "epoch": 1.0165889061689994, + "grad_norm": 0.8093326091766357, + "kl": 1.566162109375, + "learning_rate": 2.5075480877872034e-06, + "loss": 0.2168, + "reward": 1.5250000357627869, + "reward_std": 0.4525563232600689, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9250000268220901, + "step": 981 + }, + { + "completion_length": 243.54999923706055, + "epoch": 1.017625712804562, + "grad_norm": 3.8579392433166504, + "kl": 2.362548828125, + "learning_rate": 2.5062056731653035e-06, + "loss": 0.2871, + "reward": 1.550000011920929, + "reward_std": 0.2795085087418556, + "rewards/accuracy_reward": 0.6000000108033419, + "rewards/format_reward": 0.9500000029802322, + "step": 982 + }, + { + "completion_length": 270.3000087738037, + "epoch": 1.0186625194401244, + "grad_norm": 0.3653315007686615, + "kl": 1.9560546875, + "learning_rate": 2.5048617917401322e-06, + "loss": 0.1349, + "reward": 1.2750000655651093, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 983 + }, + { + "completion_length": 185.5499973297119, + "epoch": 1.0196993260756868, + "grad_norm": 0.9105724096298218, + "kl": 1.087158203125, + "learning_rate": 2.5035164454707425e-06, + "loss": 0.1579, + "reward": 1.5750000476837158, + "reward_std": 0.3804604485630989, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 984 + }, + { + "completion_length": 339.25000762939453, + "epoch": 1.0207361327112494, + "grad_norm": 3.0632193088531494, + "kl": 5.13818359375, + "learning_rate": 2.502169636318322e-06, + "loss": 0.3792, + "reward": 1.225000038743019, + "reward_std": 0.3856646940112114, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.8750000149011612, + "step": 985 + }, + { + "completion_length": 172.12500190734863, + "epoch": 1.0217729393468118, + "grad_norm": 4.367365837097168, + "kl": 1.84619140625, + "learning_rate": 2.5008213662461906e-06, + "loss": 0.1867, + "reward": 1.1500000059604645, + "reward_std": 0.3047742396593094, + "rewards/accuracy_reward": 0.22500000707805157, + "rewards/format_reward": 0.925000011920929, + "step": 986 + }, + { + "completion_length": 369.50000381469727, + "epoch": 1.0228097459823744, + "grad_norm": 0.5143448710441589, + "kl": 4.598876953125, + "learning_rate": 2.4994716372197986e-06, + "loss": 0.4032, + "reward": 1.3750000447034836, + "reward_std": 0.3966546170413494, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.8750000149011612, + "step": 987 + }, + { + "completion_length": 241.3250026702881, + "epoch": 1.0238465526179368, + "grad_norm": 6.717979431152344, + "kl": 2.7890625, + "learning_rate": 2.4981204512067235e-06, + "loss": 0.2272, + "reward": 1.2750000357627869, + "reward_std": 0.46162791550159454, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.8750000298023224, + "step": 988 + }, + { + "completion_length": 271.475004196167, + "epoch": 1.0248833592534992, + "grad_norm": 2.0469918251037598, + "kl": 2.169921875, + "learning_rate": 2.4967678101766655e-06, + "loss": 0.2312, + "reward": 1.350000023841858, + "reward_std": 0.44533543661236763, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9000000059604645, + "step": 989 + }, + { + "completion_length": 235.65000534057617, + "epoch": 1.0259201658890618, + "grad_norm": 2.4610843658447266, + "kl": 0.60595703125, + "learning_rate": 2.495413716101446e-06, + "loss": 0.1438, + "reward": 1.5250000357627869, + "reward_std": 0.2686570733785629, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 990 + }, + { + "completion_length": 177.72500038146973, + "epoch": 1.0269569725246241, + "grad_norm": 1.2966387271881104, + "kl": 0.51416015625, + "learning_rate": 2.494058170955005e-06, + "loss": 0.068, + "reward": 1.699999988079071, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 991 + }, + { + "completion_length": 297.5000057220459, + "epoch": 1.0279937791601865, + "grad_norm": 5.457717418670654, + "kl": 9.78857421875, + "learning_rate": 2.4927011767133986e-06, + "loss": 0.8263, + "reward": 1.7000000178813934, + "reward_std": 0.4604101702570915, + "rewards/accuracy_reward": 0.8250000029802322, + "rewards/format_reward": 0.8750000149011612, + "step": 992 + }, + { + "completion_length": 134.30000495910645, + "epoch": 1.0290305857957491, + "grad_norm": 26.234420776367188, + "kl": 0.30419921875, + "learning_rate": 2.4913427353547945e-06, + "loss": -0.0281, + "reward": 1.5, + "reward_std": 0.30463575199246407, + "rewards/accuracy_reward": 0.5000000167638063, + "rewards/format_reward": 1.0, + "step": 993 + }, + { + "completion_length": 119.55000305175781, + "epoch": 1.0300673924313115, + "grad_norm": 0.37444260716438293, + "kl": 0.25048828125, + "learning_rate": 2.489982848859469e-06, + "loss": 0.0221, + "reward": 1.675000011920929, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.6750000007450581, + "rewards/format_reward": 1.0, + "step": 994 + }, + { + "completion_length": 256.825008392334, + "epoch": 1.0311041990668741, + "grad_norm": 1.2725344896316528, + "kl": 3.269775390625, + "learning_rate": 2.4886215192098067e-06, + "loss": 0.4284, + "reward": 1.5250000059604645, + "reward_std": 0.492263849824667, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.8750000149011612, + "step": 995 + }, + { + "completion_length": 197.6500072479248, + "epoch": 1.0321410057024365, + "grad_norm": 2.2583558559417725, + "kl": 1.155517578125, + "learning_rate": 2.487258748390295e-06, + "loss": 0.1296, + "reward": 1.425000011920929, + "reward_std": 0.32818929105997086, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 0.9000000357627869, + "step": 996 + }, + { + "completion_length": 436.72500228881836, + "epoch": 1.0331778123379989, + "grad_norm": 1.6040884256362915, + "kl": 2.791015625, + "learning_rate": 2.485894538387523e-06, + "loss": 0.3845, + "reward": 1.475000023841858, + "reward_std": 0.6526095606386662, + "rewards/accuracy_reward": 0.6750000268220901, + "rewards/format_reward": 0.7999999970197678, + "step": 997 + }, + { + "completion_length": 271.0500068664551, + "epoch": 1.0342146189735615, + "grad_norm": 2.792837142944336, + "kl": 1.82421875, + "learning_rate": 2.4845288911901767e-06, + "loss": 0.2269, + "reward": 1.300000011920929, + "reward_std": 0.31719936802983284, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.9000000059604645, + "step": 998 + }, + { + "completion_length": 281.075008392334, + "epoch": 1.0352514256091239, + "grad_norm": 1.2233655452728271, + "kl": 1.7705078125, + "learning_rate": 2.483161808789038e-06, + "loss": 0.2829, + "reward": 1.550000011920929, + "reward_std": 0.32818929851055145, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000208616257, + "step": 999 + }, + { + "completion_length": 294.1000061035156, + "epoch": 1.0362882322446865, + "grad_norm": 3.1707940101623535, + "kl": 1.6826171875, + "learning_rate": 2.481793293176981e-06, + "loss": 0.2716, + "reward": 1.2000000178813934, + "reward_std": 0.5084580183029175, + "rewards/accuracy_reward": 0.3000000026077032, + "rewards/format_reward": 0.9000000357627869, + "step": 1000 + }, + { + "completion_length": 174.37500381469727, + "epoch": 1.0373250388802489, + "grad_norm": 2.2555198669433594, + "kl": 0.905029296875, + "learning_rate": 2.480423346348968e-06, + "loss": 0.1053, + "reward": 1.7750000357627869, + "reward_std": 0.3479738011956215, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 1001 + }, + { + "completion_length": 344.47500801086426, + "epoch": 1.0383618455158112, + "grad_norm": 1.215617299079895, + "kl": 2.71875, + "learning_rate": 2.479051970302049e-06, + "loss": 0.3774, + "reward": 1.2000000327825546, + "reward_std": 0.2487340271472931, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.8750000149011612, + "step": 1002 + }, + { + "completion_length": 194.17500495910645, + "epoch": 1.0393986521513738, + "grad_norm": 4.0535383224487305, + "kl": 3.351318359375, + "learning_rate": 2.4776791670353576e-06, + "loss": 0.1693, + "reward": 1.5250000059604645, + "reward_std": 0.36053745448589325, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8750000149011612, + "step": 1003 + }, + { + "completion_length": 387.32501220703125, + "epoch": 1.0404354587869362, + "grad_norm": 2.0389881134033203, + "kl": 3.008056640625, + "learning_rate": 2.476304938550106e-06, + "loss": 0.3585, + "reward": 1.5250000357627869, + "reward_std": 0.4614894278347492, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8500000089406967, + "step": 1004 + }, + { + "completion_length": 176.37500381469727, + "epoch": 1.0414722654224986, + "grad_norm": 5.974527835845947, + "kl": 0.7509765625, + "learning_rate": 2.4749292868495854e-06, + "loss": 0.092, + "reward": 1.875, + "reward_std": 0.1604842059314251, + "rewards/accuracy_reward": 0.9000000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1005 + }, + { + "completion_length": 245.35000038146973, + "epoch": 1.0425090720580612, + "grad_norm": 2.9274890422821045, + "kl": 1.8115234375, + "learning_rate": 2.4735522139391623e-06, + "loss": 0.2333, + "reward": 1.7500000298023224, + "reward_std": 0.27950847521424294, + "rewards/accuracy_reward": 0.8250000104308128, + "rewards/format_reward": 0.9250000268220901, + "step": 1006 + }, + { + "completion_length": 184.90000343322754, + "epoch": 1.0435458786936236, + "grad_norm": 0.9111028909683228, + "kl": 1.2724609375, + "learning_rate": 2.4721737218262735e-06, + "loss": 0.0843, + "reward": 1.5250000059604645, + "reward_std": 0.3047742620110512, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9500000029802322, + "step": 1007 + }, + { + "completion_length": 184.0250072479248, + "epoch": 1.0445826853291862, + "grad_norm": 28.965343475341797, + "kl": 0.28125, + "learning_rate": 2.4707938125204265e-06, + "loss": 0.0264, + "reward": 1.5750000178813934, + "reward_std": 0.40387551113963127, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1008 + }, + { + "completion_length": 173.77500534057617, + "epoch": 1.0456194919647486, + "grad_norm": 0.49043765664100647, + "kl": 0.2275390625, + "learning_rate": 2.4694124880331923e-06, + "loss": 0.0912, + "reward": 1.625, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.650000024586916, + "rewards/format_reward": 0.9750000089406967, + "step": 1009 + }, + { + "completion_length": 174.9250030517578, + "epoch": 1.046656298600311, + "grad_norm": 0.7365868091583252, + "kl": 0.93701171875, + "learning_rate": 2.468029750378206e-06, + "loss": 0.1141, + "reward": 1.5750000178813934, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.6000000108033419, + "rewards/format_reward": 0.9750000089406967, + "step": 1010 + }, + { + "completion_length": 385.7500114440918, + "epoch": 1.0476931052358736, + "grad_norm": 16.921222686767578, + "kl": 8.4140625, + "learning_rate": 2.4666456015711653e-06, + "loss": 0.8527, + "reward": 1.3000000417232513, + "reward_std": 0.5373543128371239, + "rewards/accuracy_reward": 0.44999999925494194, + "rewards/format_reward": 0.8500000238418579, + "step": 1011 + }, + { + "completion_length": 270.9250144958496, + "epoch": 1.048729911871436, + "grad_norm": 5.205394744873047, + "kl": 3.22900390625, + "learning_rate": 2.4652600436298206e-06, + "loss": 0.3294, + "reward": 1.5250000357627869, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.925000011920929, + "step": 1012 + }, + { + "completion_length": 210.22500610351562, + "epoch": 1.0497667185069985, + "grad_norm": 5.2738037109375, + "kl": 0.7490234375, + "learning_rate": 2.4638730785739785e-06, + "loss": 0.1077, + "reward": 1.675000011920929, + "reward_std": 0.3479738235473633, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1013 + }, + { + "completion_length": 175.07500648498535, + "epoch": 1.050803525142561, + "grad_norm": 0.3870754837989807, + "kl": 1.836669921875, + "learning_rate": 2.4624847084254983e-06, + "loss": 0.1884, + "reward": 1.3750000298023224, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 0.9750000089406967, + "step": 1014 + }, + { + "completion_length": 139.60000228881836, + "epoch": 1.0518403317781233, + "grad_norm": 6.656829357147217, + "kl": 0.237548828125, + "learning_rate": 2.461094935208285e-06, + "loss": -0.0181, + "reward": 1.2750000059604645, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.27500000409781933, + "rewards/format_reward": 1.0, + "step": 1015 + }, + { + "completion_length": 228.225004196167, + "epoch": 1.052877138413686, + "grad_norm": 0.4164898991584778, + "kl": 0.665283203125, + "learning_rate": 2.4597037609482902e-06, + "loss": 0.1427, + "reward": 1.375, + "reward_std": 0.3354101963341236, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9500000029802322, + "step": 1016 + }, + { + "completion_length": 218.77499771118164, + "epoch": 1.0539139450492483, + "grad_norm": 3.3085732460021973, + "kl": 1.551025390625, + "learning_rate": 2.4583111876735088e-06, + "loss": 0.1379, + "reward": 1.2750000059604645, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.9250000268220901, + "step": 1017 + }, + { + "completion_length": 140.6750030517578, + "epoch": 1.0549507516848107, + "grad_norm": 0.5721271634101868, + "kl": 0.28271484375, + "learning_rate": 2.456917217413973e-06, + "loss": -0.0236, + "reward": 1.6500000059604645, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.6500000096857548, + "rewards/format_reward": 1.0, + "step": 1018 + }, + { + "completion_length": 271.0500030517578, + "epoch": 1.0559875583203733, + "grad_norm": 1.24607253074646, + "kl": 0.8017578125, + "learning_rate": 2.455521852201753e-06, + "loss": 0.161, + "reward": 1.1750000268220901, + "reward_std": 0.3354101963341236, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 1019 + }, + { + "completion_length": 275.5000057220459, + "epoch": 1.0570243649559357, + "grad_norm": 1.0766854286193848, + "kl": 2.4658203125, + "learning_rate": 2.4541250940709523e-06, + "loss": 0.2699, + "reward": 1.3500000089406967, + "reward_std": 0.4472135938704014, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.8750000298023224, + "step": 1020 + }, + { + "completion_length": 129.25000190734863, + "epoch": 1.0580611715914983, + "grad_norm": 1.8902692794799805, + "kl": 1.03955078125, + "learning_rate": 2.452726945057704e-06, + "loss": 0.0865, + "reward": 1.6000000089406967, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1021 + }, + { + "completion_length": 120.00000190734863, + "epoch": 1.0590979782270606, + "grad_norm": 0.7355002164840698, + "kl": 0.20166015625, + "learning_rate": 2.4513274072001697e-06, + "loss": -0.0098, + "reward": 1.675000011920929, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 1.0, + "step": 1022 + }, + { + "completion_length": 170.75, + "epoch": 1.060134784862623, + "grad_norm": 1.266566514968872, + "kl": 1.27099609375, + "learning_rate": 2.4499264825385365e-06, + "loss": 0.0977, + "reward": 1.3250000178813934, + "reward_std": 0.3966546505689621, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9500000178813934, + "step": 1023 + }, + { + "completion_length": 144.475004196167, + "epoch": 1.0611715914981856, + "grad_norm": 1.2526934146881104, + "kl": 0.265625, + "learning_rate": 2.4485241731150104e-06, + "loss": 0.028, + "reward": 1.5250000357627869, + "reward_std": 0.2253189980983734, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1024 + }, + { + "completion_length": 165.47500801086426, + "epoch": 1.062208398133748, + "grad_norm": 3.019862651824951, + "kl": 1.36767578125, + "learning_rate": 2.447120480973819e-06, + "loss": 0.1348, + "reward": 1.5250000357627869, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1025 + }, + { + "completion_length": 129.42500495910645, + "epoch": 1.0632452047693106, + "grad_norm": 1.9610161781311035, + "kl": 0.24755859375, + "learning_rate": 2.4457154081612032e-06, + "loss": 0.0116, + "reward": 1.7750000059604645, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 1.0, + "step": 1026 + }, + { + "completion_length": 135.72500038146973, + "epoch": 1.064282011404873, + "grad_norm": 3.3690032958984375, + "kl": 2.0792236328125, + "learning_rate": 2.444308956725418e-06, + "loss": 0.077, + "reward": 1.3250000178813934, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1027 + }, + { + "completion_length": 142.1250057220459, + "epoch": 1.0653188180404354, + "grad_norm": 0.37863561511039734, + "kl": 0.21630859375, + "learning_rate": 2.4429011287167277e-06, + "loss": -0.0329, + "reward": 1.4750000536441803, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.47500000707805157, + "rewards/format_reward": 1.0, + "step": 1028 + }, + { + "completion_length": 162.57500457763672, + "epoch": 1.066355624675998, + "grad_norm": 2.589862585067749, + "kl": 0.984130859375, + "learning_rate": 2.441491926187403e-06, + "loss": 0.1099, + "reward": 1.4500000178813934, + "reward_std": 0.24524202942848206, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1029 + }, + { + "completion_length": 175.65000534057617, + "epoch": 1.0673924313115604, + "grad_norm": 0.28419941663742065, + "kl": 1.521728515625, + "learning_rate": 2.4400813511917195e-06, + "loss": 0.1385, + "reward": 1.625, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1030 + }, + { + "completion_length": 239.27499771118164, + "epoch": 1.068429237947123, + "grad_norm": 5.503000259399414, + "kl": 1.56494140625, + "learning_rate": 2.4386694057859516e-06, + "loss": 0.2001, + "reward": 1.4250000417232513, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1031 + }, + { + "completion_length": 217.3750057220459, + "epoch": 1.0694660445826853, + "grad_norm": 7.886969566345215, + "kl": 2.300048828125, + "learning_rate": 2.4372560920283734e-06, + "loss": 0.2426, + "reward": 1.7500000298023224, + "reward_std": 0.22360678017139435, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1032 + }, + { + "completion_length": 219.3750057220459, + "epoch": 1.0705028512182477, + "grad_norm": 9.453092575073242, + "kl": 2.71875, + "learning_rate": 2.435841411979252e-06, + "loss": 0.234, + "reward": 1.5, + "reward_std": 0.3407529331743717, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 1033 + }, + { + "completion_length": 231.45000076293945, + "epoch": 1.0715396578538103, + "grad_norm": 1.6650032997131348, + "kl": 0.6064453125, + "learning_rate": 2.434425367700847e-06, + "loss": 0.1066, + "reward": 1.5750000178813934, + "reward_std": 0.21638590842485428, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000178813934, + "step": 1034 + }, + { + "completion_length": 231.65000534057617, + "epoch": 1.0725764644893727, + "grad_norm": 3.1586148738861084, + "kl": 0.628662109375, + "learning_rate": 2.4330079612574076e-06, + "loss": 0.1026, + "reward": 1.4000000208616257, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9500000178813934, + "step": 1035 + }, + { + "completion_length": 231.07500648498535, + "epoch": 1.073613271124935, + "grad_norm": 3.5182924270629883, + "kl": 2.221923828125, + "learning_rate": 2.431589194715167e-06, + "loss": 0.2519, + "reward": 1.600000023841858, + "reward_std": 0.34448178485035896, + "rewards/accuracy_reward": 0.6750000156462193, + "rewards/format_reward": 0.925000011920929, + "step": 1036 + }, + { + "completion_length": 135.10000228881836, + "epoch": 1.0746500777604977, + "grad_norm": 2.1421992778778076, + "kl": 0.193359375, + "learning_rate": 2.4301690701423425e-06, + "loss": -0.0087, + "reward": 1.5750000476837158, + "reward_std": 0.2487340271472931, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 1.0, + "step": 1037 + }, + { + "completion_length": 225.4749984741211, + "epoch": 1.07568688439606, + "grad_norm": 0.7044557929039001, + "kl": 1.645263671875, + "learning_rate": 2.4287475896091296e-06, + "loss": 0.2239, + "reward": 1.475000038743019, + "reward_std": 0.42900275811553, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1038 + }, + { + "completion_length": 179.27500534057617, + "epoch": 1.0767236910316227, + "grad_norm": 0.8215576410293579, + "kl": 2.30810546875, + "learning_rate": 2.4273247551877027e-06, + "loss": 0.2102, + "reward": 1.5000000298023224, + "reward_std": 0.3479738272726536, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1039 + }, + { + "completion_length": 173.82500076293945, + "epoch": 1.077760497667185, + "grad_norm": 7.556845664978027, + "kl": 2.841796875, + "learning_rate": 2.4259005689522076e-06, + "loss": 0.2651, + "reward": 1.6250000149011612, + "reward_std": 0.2795084938406944, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 1040 + }, + { + "completion_length": 179.82500076293945, + "epoch": 1.0787973043027475, + "grad_norm": 15.503252983093262, + "kl": 2.53857421875, + "learning_rate": 2.4244750329787617e-06, + "loss": 0.2299, + "reward": 1.5250000059604645, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.5500000137835741, + "rewards/format_reward": 0.9750000089406967, + "step": 1041 + }, + { + "completion_length": 164.42499923706055, + "epoch": 1.07983411093831, + "grad_norm": 0.4848686456680298, + "kl": 0.856689453125, + "learning_rate": 2.4230481493454505e-06, + "loss": 0.0877, + "reward": 1.4749999940395355, + "reward_std": 0.25609345361590385, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 1042 + }, + { + "completion_length": 171.25000190734863, + "epoch": 1.0808709175738724, + "grad_norm": 0.8640466928482056, + "kl": 1.219482421875, + "learning_rate": 2.4216199201323223e-06, + "loss": 0.1252, + "reward": 1.675000011920929, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 1043 + }, + { + "completion_length": 258.52499771118164, + "epoch": 1.081907724209435, + "grad_norm": 1.0331501960754395, + "kl": 2.021484375, + "learning_rate": 2.4201903474213887e-06, + "loss": 0.2789, + "reward": 1.5250000655651093, + "reward_std": 0.4472136050462723, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9000000357627869, + "step": 1044 + }, + { + "completion_length": 217.97500801086426, + "epoch": 1.0829445308449974, + "grad_norm": 4.569601058959961, + "kl": 1.484130859375, + "learning_rate": 2.4187594332966205e-06, + "loss": 0.2315, + "reward": 1.675000011920929, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 1045 + }, + { + "completion_length": 284.87500381469727, + "epoch": 1.0839813374805598, + "grad_norm": 3.5887093544006348, + "kl": 1.0830078125, + "learning_rate": 2.4173271798439406e-06, + "loss": 0.2006, + "reward": 1.4500000476837158, + "reward_std": 0.5408061631023884, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9000000208616257, + "step": 1046 + }, + { + "completion_length": 229.8000030517578, + "epoch": 1.0850181441161224, + "grad_norm": 1.2264657020568848, + "kl": 3.2431640625, + "learning_rate": 2.415893589151227e-06, + "loss": 0.3404, + "reward": 1.4750000089406967, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1047 + }, + { + "completion_length": 237.8000030517578, + "epoch": 1.0860549507516848, + "grad_norm": 10.850797653198242, + "kl": 1.75244140625, + "learning_rate": 2.414458663308307e-06, + "loss": 0.175, + "reward": 1.425000011920929, + "reward_std": 0.29270510748028755, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 0.9000000208616257, + "step": 1048 + }, + { + "completion_length": 126.72500038146973, + "epoch": 1.0870917573872472, + "grad_norm": 2.8704419136047363, + "kl": 0.30712890625, + "learning_rate": 2.4130224044069536e-06, + "loss": -0.014, + "reward": 1.600000023841858, + "reward_std": 0.27228759601712227, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1049 + }, + { + "completion_length": 211.30001068115234, + "epoch": 1.0881285640228098, + "grad_norm": 1.047118067741394, + "kl": 0.555419921875, + "learning_rate": 2.4115848145408823e-06, + "loss": 0.1497, + "reward": 1.425000011920929, + "reward_std": 0.3245587982237339, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.9500000178813934, + "step": 1050 + }, + { + "completion_length": 194.82499885559082, + "epoch": 1.0891653706583722, + "grad_norm": 2.3370983600616455, + "kl": 1.25830078125, + "learning_rate": 2.4101458958057504e-06, + "loss": 0.2197, + "reward": 1.6250000298023224, + "reward_std": 0.3407529182732105, + "rewards/accuracy_reward": 0.7000000067055225, + "rewards/format_reward": 0.9250000268220901, + "step": 1051 + }, + { + "completion_length": 247.9250030517578, + "epoch": 1.0902021772939348, + "grad_norm": 1.1062910556793213, + "kl": 4.09375, + "learning_rate": 2.408705650299152e-06, + "loss": 0.3605, + "reward": 1.5750000178813934, + "reward_std": 0.42178187519311905, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.8750000149011612, + "step": 1052 + }, + { + "completion_length": 178.850004196167, + "epoch": 1.0912389839294971, + "grad_norm": 3.615429639816284, + "kl": 1.240478515625, + "learning_rate": 2.4072640801206136e-06, + "loss": 0.1906, + "reward": 1.2000000327825546, + "reward_std": 0.26129766181111336, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 1053 + }, + { + "completion_length": 136.82500457763672, + "epoch": 1.0922757905650595, + "grad_norm": 1.0081489086151123, + "kl": 1.263671875, + "learning_rate": 2.4058211873715953e-06, + "loss": 0.0754, + "reward": 1.2500000298023224, + "reward_std": 0.30463573709130287, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.925000011920929, + "step": 1054 + }, + { + "completion_length": 243.55000495910645, + "epoch": 1.0933125972006221, + "grad_norm": 0.662563681602478, + "kl": 3.501220703125, + "learning_rate": 2.4043769741554834e-06, + "loss": 0.3914, + "reward": 1.6250000298023224, + "reward_std": 0.4525563269853592, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9000000059604645, + "step": 1055 + }, + { + "completion_length": 130.2750015258789, + "epoch": 1.0943494038361845, + "grad_norm": 3.082343816757202, + "kl": 1.136962890625, + "learning_rate": 2.40293144257759e-06, + "loss": 0.0961, + "reward": 1.7000000178813934, + "reward_std": 0.30463574081659317, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1056 + }, + { + "completion_length": 430.7750129699707, + "epoch": 1.0953862104717471, + "grad_norm": 10.758148193359375, + "kl": 5.751953125, + "learning_rate": 2.4014845947451485e-06, + "loss": 0.6337, + "reward": 1.3250000029802322, + "reward_std": 0.4290027618408203, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.8000000268220901, + "step": 1057 + }, + { + "completion_length": 181.2750015258789, + "epoch": 1.0964230171073095, + "grad_norm": 1.5399192571640015, + "kl": 1.5126953125, + "learning_rate": 2.4000364327673114e-06, + "loss": 0.1621, + "reward": 1.6500000059604645, + "reward_std": 0.4472136050462723, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 0.9000000208616257, + "step": 1058 + }, + { + "completion_length": 208.05000686645508, + "epoch": 1.0974598237428719, + "grad_norm": 1.2210434675216675, + "kl": 1.3642578125, + "learning_rate": 2.3985869587551465e-06, + "loss": 0.1137, + "reward": 1.475000023841858, + "reward_std": 0.3804604709148407, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1059 + }, + { + "completion_length": 127.70000457763672, + "epoch": 1.0984966303784345, + "grad_norm": 1.9639644622802734, + "kl": 1.4990234375, + "learning_rate": 2.3971361748216347e-06, + "loss": 0.0412, + "reward": 1.5750000178813934, + "reward_std": 0.36588017642498016, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.925000011920929, + "step": 1060 + }, + { + "completion_length": 206.90000343322754, + "epoch": 1.0995334370139969, + "grad_norm": 1.0273288488388062, + "kl": 1.095703125, + "learning_rate": 2.395684083081667e-06, + "loss": 0.1791, + "reward": 1.6000000536441803, + "reward_std": 0.40387552604079247, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9500000178813934, + "step": 1061 + }, + { + "completion_length": 175.100004196167, + "epoch": 1.1005702436495595, + "grad_norm": 1.2841686010360718, + "kl": 0.399658203125, + "learning_rate": 2.3942306856520393e-06, + "loss": 0.092, + "reward": 1.675000011920929, + "reward_std": 0.18026872351765633, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 0.9249999970197678, + "step": 1062 + }, + { + "completion_length": 292.4250030517578, + "epoch": 1.1016070502851218, + "grad_norm": 6.283333778381348, + "kl": 3.5751953125, + "learning_rate": 2.3927759846514525e-06, + "loss": 0.4198, + "reward": 1.425000011920929, + "reward_std": 0.46511995792388916, + "rewards/accuracy_reward": 0.5249999910593033, + "rewards/format_reward": 0.9000000208616257, + "step": 1063 + }, + { + "completion_length": 161.7000026702881, + "epoch": 1.1026438569206842, + "grad_norm": 0.9103049635887146, + "kl": 2.02099609375, + "learning_rate": 2.391319982200507e-06, + "loss": 0.1582, + "reward": 1.5500000417232513, + "reward_std": 0.3046357333660126, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 1064 + }, + { + "completion_length": 203.75000190734863, + "epoch": 1.1036806635562468, + "grad_norm": 2.299445152282715, + "kl": 1.322021484375, + "learning_rate": 2.389862680421699e-06, + "loss": 0.2158, + "reward": 1.5500000268220901, + "reward_std": 0.28485123440623283, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1065 + }, + { + "completion_length": 305.350004196167, + "epoch": 1.1047174701918092, + "grad_norm": 2.727337121963501, + "kl": 9.0703125, + "learning_rate": 2.388404081439422e-06, + "loss": 0.8764, + "reward": 1.600000023841858, + "reward_std": 0.4958943910896778, + "rewards/accuracy_reward": 0.7250000238418579, + "rewards/format_reward": 0.8750000298023224, + "step": 1066 + }, + { + "completion_length": 212.52500534057617, + "epoch": 1.1057542768273716, + "grad_norm": 26.0318660736084, + "kl": 1.458984375, + "learning_rate": 2.3869441873799577e-06, + "loss": 0.143, + "reward": 1.350000023841858, + "reward_std": 0.4706011861562729, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.8750000149011612, + "step": 1067 + }, + { + "completion_length": 149.62500190734863, + "epoch": 1.1067910834629342, + "grad_norm": 1.5431420803070068, + "kl": 1.5517578125, + "learning_rate": 2.3854830003714773e-06, + "loss": 0.0428, + "reward": 1.7750000059604645, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1068 + }, + { + "completion_length": 111.67500305175781, + "epoch": 1.1078278900984966, + "grad_norm": 1.1982154846191406, + "kl": 0.256591796875, + "learning_rate": 2.384020522544036e-06, + "loss": 0.0384, + "reward": 1.6250000298023224, + "reward_std": 0.13693063706159592, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 1.0, + "step": 1069 + }, + { + "completion_length": 161.32500076293945, + "epoch": 1.1088646967340592, + "grad_norm": 5.154061317443848, + "kl": 3.11767578125, + "learning_rate": 2.382556756029571e-06, + "loss": 0.2601, + "reward": 1.550000011920929, + "reward_std": 0.40387552231550217, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9500000178813934, + "step": 1070 + }, + { + "completion_length": 109.50000190734863, + "epoch": 1.1099015033696216, + "grad_norm": 1.1652772426605225, + "kl": 0.291015625, + "learning_rate": 2.3810917029618975e-06, + "loss": 0.018, + "reward": 1.3000000417232513, + "reward_std": 0.248734038323164, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 1.0, + "step": 1071 + }, + { + "completion_length": 156.07499885559082, + "epoch": 1.110938310005184, + "grad_norm": 0.9779496788978577, + "kl": 0.75146484375, + "learning_rate": 2.379625365476708e-06, + "loss": 0.0488, + "reward": 1.375, + "reward_std": 0.27228761091828346, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1072 + }, + { + "completion_length": 154.07500076293945, + "epoch": 1.1119751166407466, + "grad_norm": 14.546243667602539, + "kl": 1.85107421875, + "learning_rate": 2.378157745711565e-06, + "loss": 0.1481, + "reward": 1.4500000178813934, + "reward_std": 0.1928323432803154, + "rewards/accuracy_reward": 0.4749999977648258, + "rewards/format_reward": 0.9750000089406967, + "step": 1073 + }, + { + "completion_length": 252.77500343322754, + "epoch": 1.113011923276309, + "grad_norm": 0.485859751701355, + "kl": 1.296875, + "learning_rate": 2.3766888458059024e-06, + "loss": 0.2356, + "reward": 1.5000000596046448, + "reward_std": 0.4290027692914009, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9000000059604645, + "step": 1074 + }, + { + "completion_length": 206.87499809265137, + "epoch": 1.1140487299118715, + "grad_norm": 1.4165109395980835, + "kl": 10.2841796875, + "learning_rate": 2.375218667901018e-06, + "loss": 0.6785, + "reward": 1.4249999970197678, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.925000011920929, + "step": 1075 + }, + { + "completion_length": 155.850004196167, + "epoch": 1.115085536547434, + "grad_norm": 33.7458381652832, + "kl": 2.77001953125, + "learning_rate": 2.373747214140074e-06, + "loss": 0.2534, + "reward": 1.3750000447034836, + "reward_std": 0.347973819822073, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 0.925000011920929, + "step": 1076 + }, + { + "completion_length": 113.37500190734863, + "epoch": 1.1161223431829963, + "grad_norm": 1.9779350757598877, + "kl": 0.39306640625, + "learning_rate": 2.3722744866680927e-06, + "loss": 0.0276, + "reward": 1.5250000059604645, + "reward_std": 0.21275536343455315, + "rewards/accuracy_reward": 0.5750000011175871, + "rewards/format_reward": 0.9500000178813934, + "step": 1077 + }, + { + "completion_length": 96.97500228881836, + "epoch": 1.117159149818559, + "grad_norm": 0.3221043050289154, + "kl": 0.2822265625, + "learning_rate": 2.370800487631953e-06, + "loss": 0.0201, + "reward": 1.800000011920929, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.8000000044703484, + "rewards/format_reward": 1.0, + "step": 1078 + }, + { + "completion_length": 168.4500026702881, + "epoch": 1.1181959564541213, + "grad_norm": 1.1919106245040894, + "kl": 0.35498046875, + "learning_rate": 2.3693252191803867e-06, + "loss": 0.0311, + "reward": 1.6250000298023224, + "reward_std": 0.3479738049209118, + "rewards/accuracy_reward": 0.7000000216066837, + "rewards/format_reward": 0.925000011920929, + "step": 1079 + }, + { + "completion_length": 105.07500076293945, + "epoch": 1.1192327630896837, + "grad_norm": 0.6131631731987, + "kl": 0.6875, + "learning_rate": 2.3678486834639764e-06, + "loss": -0.0015, + "reward": 1.550000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 1080 + }, + { + "completion_length": 163.0, + "epoch": 1.1202695697252463, + "grad_norm": 0.5884708166122437, + "kl": 6.12939453125, + "learning_rate": 2.3663708826351527e-06, + "loss": 0.5169, + "reward": 1.1500000208616257, + "reward_std": 0.4562852270901203, + "rewards/accuracy_reward": 0.2750000078231096, + "rewards/format_reward": 0.8750000149011612, + "step": 1081 + }, + { + "completion_length": 144.90000534057617, + "epoch": 1.1213063763608087, + "grad_norm": 16.477420806884766, + "kl": 2.0087890625, + "learning_rate": 2.3648918188481898e-06, + "loss": 0.1819, + "reward": 1.225000023841858, + "reward_std": 0.4525563269853592, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.9000000208616257, + "step": 1082 + }, + { + "completion_length": 154.07500076293945, + "epoch": 1.1223431829963713, + "grad_norm": 2.045259952545166, + "kl": 1.214111328125, + "learning_rate": 2.363411494259203e-06, + "loss": 0.1302, + "reward": 1.7250000536441803, + "reward_std": 0.3533165380358696, + "rewards/accuracy_reward": 0.7750000134110451, + "rewards/format_reward": 0.9500000178813934, + "step": 1083 + }, + { + "completion_length": 104.54999923706055, + "epoch": 1.1233799896319336, + "grad_norm": 1.0909665822982788, + "kl": 0.363525390625, + "learning_rate": 2.361929911026146e-06, + "loss": 0.0156, + "reward": 1.6500000059604645, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1084 + }, + { + "completion_length": 104.95000076293945, + "epoch": 1.124416796267496, + "grad_norm": 0.852575957775116, + "kl": 0.97265625, + "learning_rate": 2.3604470713088075e-06, + "loss": 0.0517, + "reward": 1.6250000298023224, + "reward_std": 0.1369306445121765, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9750000089406967, + "step": 1085 + }, + { + "completion_length": 148.725004196167, + "epoch": 1.1254536029030586, + "grad_norm": 0.9597511291503906, + "kl": 0.377685546875, + "learning_rate": 2.3589629772688055e-06, + "loss": 0.0358, + "reward": 1.5999999940395355, + "reward_std": 0.2795085087418556, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1086 + }, + { + "completion_length": 170.22500038146973, + "epoch": 1.126490409538621, + "grad_norm": 0.8441897034645081, + "kl": 2.5185546875, + "learning_rate": 2.3574776310695903e-06, + "loss": 0.2241, + "reward": 1.5, + "reward_std": 0.28858011215925217, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.925000011920929, + "step": 1087 + }, + { + "completion_length": 212.350004196167, + "epoch": 1.1275272161741836, + "grad_norm": 1.9597944021224976, + "kl": 3.5634765625, + "learning_rate": 2.3559910348764346e-06, + "loss": 0.3404, + "reward": 1.4250000417232513, + "reward_std": 0.5049660131335258, + "rewards/accuracy_reward": 0.5500000212341547, + "rewards/format_reward": 0.8750000298023224, + "step": 1088 + }, + { + "completion_length": 152.79999923706055, + "epoch": 1.128564022809746, + "grad_norm": 1.717069387435913, + "kl": 1.9423828125, + "learning_rate": 2.354503190856434e-06, + "loss": 0.1411, + "reward": 1.4000000357627869, + "reward_std": 0.49226388335227966, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.925000011920929, + "step": 1089 + }, + { + "completion_length": 206.45000648498535, + "epoch": 1.1296008294453084, + "grad_norm": 1.356911301612854, + "kl": 1.68505859375, + "learning_rate": 2.3530141011785045e-06, + "loss": 0.2599, + "reward": 1.2500000298023224, + "reward_std": 0.5049659982323647, + "rewards/accuracy_reward": 0.4000000096857548, + "rewards/format_reward": 0.8500000089406967, + "step": 1090 + }, + { + "completion_length": 152.62500190734863, + "epoch": 1.130637636080871, + "grad_norm": 4.397830486297607, + "kl": 1.978515625, + "learning_rate": 2.3515237680133764e-06, + "loss": 0.2519, + "reward": 1.3750000149011612, + "reward_std": 0.43277180567383766, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.8750000149011612, + "step": 1091 + }, + { + "completion_length": 267.5249996185303, + "epoch": 1.1316744427164334, + "grad_norm": 1.6721715927124023, + "kl": 1.498291015625, + "learning_rate": 2.3500321935335924e-06, + "loss": 0.2189, + "reward": 1.3500000089406967, + "reward_std": 0.40450848639011383, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.8750000149011612, + "step": 1092 + }, + { + "completion_length": 179.25000190734863, + "epoch": 1.132711249351996, + "grad_norm": 5.060147762298584, + "kl": 5.09765625, + "learning_rate": 2.3485393799135067e-06, + "loss": 0.4689, + "reward": 1.5000000149011612, + "reward_std": 0.3479738272726536, + "rewards/accuracy_reward": 0.6000000033527613, + "rewards/format_reward": 0.9000000208616257, + "step": 1093 + }, + { + "completion_length": 179.57499885559082, + "epoch": 1.1337480559875583, + "grad_norm": 8.466435432434082, + "kl": 2.309814453125, + "learning_rate": 2.3470453293292774e-06, + "loss": 0.2717, + "reward": 1.5500000417232513, + "reward_std": 0.31719936430454254, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 1094 + }, + { + "completion_length": 311.2500057220459, + "epoch": 1.1347848626231207, + "grad_norm": 7.621610164642334, + "kl": 5.3955078125, + "learning_rate": 2.345550043958868e-06, + "loss": 0.4766, + "reward": 1.2250000089406967, + "reward_std": 0.49088017642498016, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.7999999970197678, + "step": 1095 + }, + { + "completion_length": 263.1749973297119, + "epoch": 1.1358216692586833, + "grad_norm": 1.0027433633804321, + "kl": 0.97802734375, + "learning_rate": 2.344053525982041e-06, + "loss": 0.1937, + "reward": 1.574999988079071, + "reward_std": 0.4525563083589077, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.8750000149011612, + "step": 1096 + }, + { + "completion_length": 341.5750026702881, + "epoch": 1.1368584758942457, + "grad_norm": 2.4278464317321777, + "kl": 2.37744140625, + "learning_rate": 2.3425557775803554e-06, + "loss": 0.2962, + "reward": 1.2750000208616257, + "reward_std": 0.5533697977662086, + "rewards/accuracy_reward": 0.45000001788139343, + "rewards/format_reward": 0.8250000029802322, + "step": 1097 + }, + { + "completion_length": 266.7000045776367, + "epoch": 1.137895282529808, + "grad_norm": 18.119483947753906, + "kl": 7.452392578125, + "learning_rate": 2.3410568009371644e-06, + "loss": 0.7244, + "reward": 1.675000011920929, + "reward_std": 0.36588015779852867, + "rewards/accuracy_reward": 0.8000000193715096, + "rewards/format_reward": 0.8750000149011612, + "step": 1098 + }, + { + "completion_length": 217.1250057220459, + "epoch": 1.1389320891653707, + "grad_norm": 1.204911231994629, + "kl": 1.53857421875, + "learning_rate": 2.3395565982376117e-06, + "loss": 0.167, + "reward": 1.425000011920929, + "reward_std": 0.3371223956346512, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1099 + }, + { + "completion_length": 203.77500534057617, + "epoch": 1.139968895800933, + "grad_norm": 0.6387304663658142, + "kl": 1.47216796875, + "learning_rate": 2.3380551716686286e-06, + "loss": 0.235, + "reward": 1.6500000059604645, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1100 + }, + { + "completion_length": 252.6750087738037, + "epoch": 1.1410057024364957, + "grad_norm": 2.940162420272827, + "kl": 1.165283203125, + "learning_rate": 2.3365525234189294e-06, + "loss": 0.1856, + "reward": 1.1250000447034836, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.2000000048428774, + "rewards/format_reward": 0.925000011920929, + "step": 1101 + }, + { + "completion_length": 341.80001068115234, + "epoch": 1.142042509072058, + "grad_norm": 0.830411434173584, + "kl": 2.815673828125, + "learning_rate": 2.3350486556790096e-06, + "loss": 0.445, + "reward": 1.2999999970197678, + "reward_std": 0.44219936802983284, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.8500000089406967, + "step": 1102 + }, + { + "completion_length": 150.82500076293945, + "epoch": 1.1430793157076204, + "grad_norm": 38.25456237792969, + "kl": 0.4033203125, + "learning_rate": 2.333543570641144e-06, + "loss": 0.036, + "reward": 1.550000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 1103 + }, + { + "completion_length": 160.10000610351562, + "epoch": 1.144116122343183, + "grad_norm": 1.2388648986816406, + "kl": 1.23974609375, + "learning_rate": 2.33203727049938e-06, + "loss": 0.1099, + "reward": 1.6500000357627869, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9750000089406967, + "step": 1104 + }, + { + "completion_length": 128.52500343322754, + "epoch": 1.1451529289787454, + "grad_norm": 0.07461313158273697, + "kl": 0.1943359375, + "learning_rate": 2.3305297574495363e-06, + "loss": 0.0078, + "reward": 1.375, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 1.0, + "step": 1105 + }, + { + "completion_length": 266.99999618530273, + "epoch": 1.1461897356143078, + "grad_norm": 0.3935621678829193, + "kl": 2.285888671875, + "learning_rate": 2.329021033689202e-06, + "loss": 0.1073, + "reward": 1.3250000029802322, + "reward_std": 0.32519175484776497, + "rewards/accuracy_reward": 0.4250000026077032, + "rewards/format_reward": 0.9000000059604645, + "step": 1106 + }, + { + "completion_length": 140.55000495910645, + "epoch": 1.1472265422498704, + "grad_norm": 2.322416067123413, + "kl": 0.85546875, + "learning_rate": 2.327511101417729e-06, + "loss": 0.0957, + "reward": 1.7750000357627869, + "reward_std": 0.13693063706159592, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1107 + }, + { + "completion_length": 182.12500762939453, + "epoch": 1.1482633488854328, + "grad_norm": 1.7401773929595947, + "kl": 1.87109375, + "learning_rate": 2.3259999628362302e-06, + "loss": 0.228, + "reward": 1.5250000059604645, + "reward_std": 0.34448183327913284, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1108 + }, + { + "completion_length": 159.72500610351562, + "epoch": 1.1493001555209954, + "grad_norm": 1.2455532550811768, + "kl": 0.74462890625, + "learning_rate": 2.32448762014758e-06, + "loss": 0.094, + "reward": 1.725000023841858, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1109 + }, + { + "completion_length": 208.05000495910645, + "epoch": 1.1503369621565578, + "grad_norm": 28.125736236572266, + "kl": 7.759521484375, + "learning_rate": 2.3229740755564063e-06, + "loss": 0.6062, + "reward": 1.425000011920929, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1110 + }, + { + "completion_length": 111.25000190734863, + "epoch": 1.1513737687921202, + "grad_norm": 0.134598970413208, + "kl": 0.265869140625, + "learning_rate": 2.3214593312690885e-06, + "loss": 0.0104, + "reward": 1.75, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.75, + "rewards/format_reward": 1.0, + "step": 1111 + }, + { + "completion_length": 316.625, + "epoch": 1.1524105754276828, + "grad_norm": 1.7135318517684937, + "kl": 1.057861328125, + "learning_rate": 2.3199433894937556e-06, + "loss": 0.2495, + "reward": 1.3750000298023224, + "reward_std": 0.45977722480893135, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.9000000208616257, + "step": 1112 + }, + { + "completion_length": 100.82500076293945, + "epoch": 1.1534473820632452, + "grad_norm": 1.3429611921310425, + "kl": 0.369873046875, + "learning_rate": 2.3184262524402823e-06, + "loss": 0.0181, + "reward": 1.675000011920929, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 1.0, + "step": 1113 + }, + { + "completion_length": 112.92500114440918, + "epoch": 1.1544841886988078, + "grad_norm": 0.9468095302581787, + "kl": 0.24267578125, + "learning_rate": 2.316907922320286e-06, + "loss": -0.0086, + "reward": 1.4000000059604645, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 1.0, + "step": 1114 + }, + { + "completion_length": 125.9749984741211, + "epoch": 1.1555209953343701, + "grad_norm": 0.4861811399459839, + "kl": 0.239501953125, + "learning_rate": 2.315388401347122e-06, + "loss": 0.004, + "reward": 1.3250000178813934, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.32500001043081284, + "rewards/format_reward": 1.0, + "step": 1115 + }, + { + "completion_length": 163.12500190734863, + "epoch": 1.1565578019699325, + "grad_norm": 1.0500177145004272, + "kl": 1.072509765625, + "learning_rate": 2.313867691735884e-06, + "loss": 0.1148, + "reward": 1.300000011920929, + "reward_std": 0.15685366839170456, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.9500000029802322, + "step": 1116 + }, + { + "completion_length": 163.45000457763672, + "epoch": 1.1575946086054951, + "grad_norm": 0.3982069194316864, + "kl": 0.38818359375, + "learning_rate": 2.312345795703395e-06, + "loss": 0.0074, + "reward": 1.3750000298023224, + "reward_std": 0.3371223844587803, + "rewards/accuracy_reward": 0.42500001564621925, + "rewards/format_reward": 0.9500000178813934, + "step": 1117 + }, + { + "completion_length": 279.8500061035156, + "epoch": 1.1586314152410575, + "grad_norm": 6.1592888832092285, + "kl": 6.94091796875, + "learning_rate": 2.310822715468211e-06, + "loss": 0.6748, + "reward": 1.225000023841858, + "reward_std": 0.4092182442545891, + "rewards/accuracy_reward": 0.3500000052154064, + "rewards/format_reward": 0.8750000149011612, + "step": 1118 + }, + { + "completion_length": 139.72500228881836, + "epoch": 1.15966822187662, + "grad_norm": 74.81144714355469, + "kl": 19.76904296875, + "learning_rate": 2.309298453250612e-06, + "loss": 1.0662, + "reward": 1.5000000298023224, + "reward_std": 0.304635725915432, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1119 + }, + { + "completion_length": 122.17499923706055, + "epoch": 1.1607050285121825, + "grad_norm": 0.7593364715576172, + "kl": 0.323974609375, + "learning_rate": 2.307773011272602e-06, + "loss": -0.0027, + "reward": 1.3499999940395355, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 1.0, + "step": 1120 + }, + { + "completion_length": 264.75000381469727, + "epoch": 1.1617418351477449, + "grad_norm": 0.7510677576065063, + "kl": 2.908203125, + "learning_rate": 2.3062463917579046e-06, + "loss": 0.3516, + "reward": 1.2500000149011612, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.32500001043081284, + "rewards/format_reward": 0.925000011920929, + "step": 1121 + }, + { + "completion_length": 206.10000038146973, + "epoch": 1.1627786417833075, + "grad_norm": 3.3034536838531494, + "kl": 0.33837890625, + "learning_rate": 2.3047185969319605e-06, + "loss": 0.1074, + "reward": 1.625, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1122 + }, + { + "completion_length": 227.97500228881836, + "epoch": 1.1638154484188699, + "grad_norm": 1.0775312185287476, + "kl": 1.544921875, + "learning_rate": 2.303189629021922e-06, + "loss": 0.207, + "reward": 1.4000000208616257, + "reward_std": 0.32818931341171265, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9500000178813934, + "step": 1123 + }, + { + "completion_length": 166.17500114440918, + "epoch": 1.1648522550544325, + "grad_norm": 0.43752148747444153, + "kl": 0.21826171875, + "learning_rate": 2.3016594902566532e-06, + "loss": 0.0541, + "reward": 1.5, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1124 + }, + { + "completion_length": 135.1750030517578, + "epoch": 1.1658890616899948, + "grad_norm": 0.3818998634815216, + "kl": 0.21142578125, + "learning_rate": 2.300128182866725e-06, + "loss": -0.0078, + "reward": 1.6500000059604645, + "reward_std": 0.30463575199246407, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 1.0, + "step": 1125 + }, + { + "completion_length": 214.30000686645508, + "epoch": 1.1669258683255572, + "grad_norm": 1.9454764127731323, + "kl": 0.822998046875, + "learning_rate": 2.29859570908441e-06, + "loss": 0.1374, + "reward": 1.600000023841858, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9500000178813934, + "step": 1126 + }, + { + "completion_length": 265.1500053405762, + "epoch": 1.1679626749611198, + "grad_norm": 3.658550500869751, + "kl": 1.494140625, + "learning_rate": 2.2970620711436836e-06, + "loss": 0.2519, + "reward": 1.3250000178813934, + "reward_std": 0.4290027394890785, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9000000208616257, + "step": 1127 + }, + { + "completion_length": 141.5250015258789, + "epoch": 1.1689994815966822, + "grad_norm": 1.1074241399765015, + "kl": 0.46044921875, + "learning_rate": 2.295527271280217e-06, + "loss": 0.0418, + "reward": 1.350000023841858, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 1128 + }, + { + "completion_length": 285.9250068664551, + "epoch": 1.1700362882322446, + "grad_norm": 0.3166654706001282, + "kl": 2.0283203125, + "learning_rate": 2.293991311731375e-06, + "loss": 0.2676, + "reward": 1.425000011920929, + "reward_std": 0.25609345734119415, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 1129 + }, + { + "completion_length": 207.27500343322754, + "epoch": 1.1710730948678072, + "grad_norm": 3.2603349685668945, + "kl": 2.12890625, + "learning_rate": 2.2924541947362132e-06, + "loss": 0.2093, + "reward": 1.5250000357627869, + "reward_std": 0.40387551859021187, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1130 + }, + { + "completion_length": 257.24999809265137, + "epoch": 1.1721099015033696, + "grad_norm": 1.6190482378005981, + "kl": 1.931396484375, + "learning_rate": 2.2909159225354742e-06, + "loss": 0.2589, + "reward": 1.4000000357627869, + "reward_std": 0.43999266624450684, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.925000011920929, + "step": 1131 + }, + { + "completion_length": 309.35000801086426, + "epoch": 1.1731467081389322, + "grad_norm": 1.865369439125061, + "kl": 1.885009765625, + "learning_rate": 2.2893764973715864e-06, + "loss": 0.3008, + "reward": 1.4749999940395355, + "reward_std": 0.30477428063750267, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9000000208616257, + "step": 1132 + }, + { + "completion_length": 263.5000057220459, + "epoch": 1.1741835147744946, + "grad_norm": 3.0509140491485596, + "kl": 2.080078125, + "learning_rate": 2.2878359214886555e-06, + "loss": 0.1884, + "reward": 1.3500000089406967, + "reward_std": 0.3768700994551182, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9000000208616257, + "step": 1133 + }, + { + "completion_length": 460.1749973297119, + "epoch": 1.175220321410057, + "grad_norm": 1.640738606452942, + "kl": 5.125, + "learning_rate": 2.2862941971324675e-06, + "loss": 0.6654, + "reward": 1.25, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8250000029802322, + "step": 1134 + }, + { + "completion_length": 222.4750099182129, + "epoch": 1.1762571280456195, + "grad_norm": 20.201419830322266, + "kl": 7.2978515625, + "learning_rate": 2.2847513265504824e-06, + "loss": 0.7116, + "reward": 1.600000023841858, + "reward_std": 0.4525563381612301, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 1135 + }, + { + "completion_length": 154.95000457763672, + "epoch": 1.177293934681182, + "grad_norm": 1.1167094707489014, + "kl": 0.981689453125, + "learning_rate": 2.2832073119918294e-06, + "loss": 0.0693, + "reward": 1.4500000178813934, + "reward_std": 0.15685367584228516, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1136 + }, + { + "completion_length": 162.9500026702881, + "epoch": 1.1783307413167443, + "grad_norm": 6.627492427825928, + "kl": 3.576904296875, + "learning_rate": 2.281662155707307e-06, + "loss": 0.2332, + "reward": 1.475000023841858, + "reward_std": 0.4723408557474613, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.925000011920929, + "step": 1137 + }, + { + "completion_length": 171.6500015258789, + "epoch": 1.179367547952307, + "grad_norm": 1.2426679134368896, + "kl": 1.3828125, + "learning_rate": 2.280115859949377e-06, + "loss": 0.1562, + "reward": 1.5000000298023224, + "reward_std": 0.32455874979496, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9000000059604645, + "step": 1138 + }, + { + "completion_length": 156.2000026702881, + "epoch": 1.1804043545878693, + "grad_norm": 0.8162038326263428, + "kl": 1.317138671875, + "learning_rate": 2.2785684269721627e-06, + "loss": 0.1348, + "reward": 1.6500000059604645, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 1139 + }, + { + "completion_length": 100.6500015258789, + "epoch": 1.181441161223432, + "grad_norm": 0.6516849994659424, + "kl": 1.010498046875, + "learning_rate": 2.2770198590314462e-06, + "loss": -0.038, + "reward": 1.600000023841858, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.6250000186264515, + "rewards/format_reward": 0.9750000089406967, + "step": 1140 + }, + { + "completion_length": 265.700008392334, + "epoch": 1.1824779678589943, + "grad_norm": 8.137269020080566, + "kl": 3.6572265625, + "learning_rate": 2.2754701583846622e-06, + "loss": 0.4887, + "reward": 1.4500000178813934, + "reward_std": 0.6761630997061729, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.8750000298023224, + "step": 1141 + }, + { + "completion_length": 164.90000534057617, + "epoch": 1.1835147744945567, + "grad_norm": 3.577707529067993, + "kl": 2.4912109375, + "learning_rate": 2.2739193272908977e-06, + "loss": 0.1773, + "reward": 1.3250000178813934, + "reward_std": 0.20019173622131348, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1142 + }, + { + "completion_length": 198.350004196167, + "epoch": 1.1845515811301193, + "grad_norm": 3.3634819984436035, + "kl": 1.25439453125, + "learning_rate": 2.2723673680108876e-06, + "loss": 0.1872, + "reward": 1.7000000178813934, + "reward_std": 0.33353203535079956, + "rewards/accuracy_reward": 0.7749999910593033, + "rewards/format_reward": 0.925000011920929, + "step": 1143 + }, + { + "completion_length": 262.3000068664551, + "epoch": 1.1855883877656817, + "grad_norm": 1.0182909965515137, + "kl": 2.65087890625, + "learning_rate": 2.2708142828070105e-06, + "loss": 0.2845, + "reward": 1.5250000357627869, + "reward_std": 0.3913118988275528, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1144 + }, + { + "completion_length": 362.57501220703125, + "epoch": 1.1866251944012443, + "grad_norm": 7.052261829376221, + "kl": 6.25390625, + "learning_rate": 2.2692600739432884e-06, + "loss": 0.7407, + "reward": 1.3000000417232513, + "reward_std": 0.6345372498035431, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.8250000178813934, + "step": 1145 + }, + { + "completion_length": 296.1500053405762, + "epoch": 1.1876620010368066, + "grad_norm": 11.791178703308105, + "kl": 5.19189453125, + "learning_rate": 2.267704743685379e-06, + "loss": 0.4376, + "reward": 1.4500000476837158, + "reward_std": 0.4291412830352783, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.8500000089406967, + "step": 1146 + }, + { + "completion_length": 250.85000610351562, + "epoch": 1.188698807672369, + "grad_norm": 5.4525885581970215, + "kl": 2.895263671875, + "learning_rate": 2.2661482943005763e-06, + "loss": 0.2821, + "reward": 1.4000000208616257, + "reward_std": 0.40558773651719093, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9000000208616257, + "step": 1147 + }, + { + "completion_length": 203.62499809265137, + "epoch": 1.1897356143079316, + "grad_norm": 0.6782277822494507, + "kl": 1.380859375, + "learning_rate": 2.2645907280578046e-06, + "loss": 0.171, + "reward": 1.350000038743019, + "reward_std": 0.37310105189681053, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1148 + }, + { + "completion_length": 499.77501678466797, + "epoch": 1.190772420943494, + "grad_norm": 2.9671342372894287, + "kl": 2.22021484375, + "learning_rate": 2.2630320472276173e-06, + "loss": 0.3943, + "reward": 1.225000038743019, + "reward_std": 0.5967078506946564, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.7750000208616257, + "step": 1149 + }, + { + "completion_length": 508.0750045776367, + "epoch": 1.1918092275790566, + "grad_norm": 2.0540173053741455, + "kl": 4.39453125, + "learning_rate": 2.261472254082192e-06, + "loss": 0.3224, + "reward": 1.2000000029802322, + "reward_std": 0.4987340494990349, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.7500000149011612, + "step": 1150 + }, + { + "completion_length": 269.67500495910645, + "epoch": 1.192846034214619, + "grad_norm": 2.222440242767334, + "kl": 2.26806640625, + "learning_rate": 2.259911350895328e-06, + "loss": 0.2526, + "reward": 1.4749999940395355, + "reward_std": 0.45977720618247986, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.8500000089406967, + "step": 1151 + }, + { + "completion_length": 279.2750053405762, + "epoch": 1.1938828408501814, + "grad_norm": 1.225080132484436, + "kl": 3.06201171875, + "learning_rate": 2.2583493399424427e-06, + "loss": 0.4461, + "reward": 1.2000000178813934, + "reward_std": 0.478316567838192, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.8750000149011612, + "step": 1152 + }, + { + "completion_length": 504.17501068115234, + "epoch": 1.194919647485744, + "grad_norm": 2.603325366973877, + "kl": 3.29296875, + "learning_rate": 2.256786223500568e-06, + "loss": 0.4565, + "reward": 1.2000000178813934, + "reward_std": 0.8058728724718094, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.7250000089406967, + "step": 1153 + }, + { + "completion_length": 355.2750015258789, + "epoch": 1.1959564541213064, + "grad_norm": 4.560014247894287, + "kl": 3.18359375, + "learning_rate": 2.2552220038483483e-06, + "loss": 0.4786, + "reward": 1.425000011920929, + "reward_std": 0.7879664748907089, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.8250000178813934, + "step": 1154 + }, + { + "completion_length": 498.1250228881836, + "epoch": 1.196993260756869, + "grad_norm": 2.23262882232666, + "kl": 5.962890625, + "learning_rate": 2.253656683266035e-06, + "loss": 0.6254, + "reward": 1.050000011920929, + "reward_std": 0.6675183773040771, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/format_reward": 0.6750000268220901, + "step": 1155 + }, + { + "completion_length": 533.9000091552734, + "epoch": 1.1980300673924313, + "grad_norm": 8.015848159790039, + "kl": 3.220703125, + "learning_rate": 2.2520902640354854e-06, + "loss": 0.4371, + "reward": 1.2000000029802322, + "reward_std": 0.5498777888715267, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.7250000238418579, + "step": 1156 + }, + { + "completion_length": 398.05001068115234, + "epoch": 1.1990668740279937, + "grad_norm": 2.3568358421325684, + "kl": 3.130859375, + "learning_rate": 2.2505227484401575e-06, + "loss": 0.4058, + "reward": 1.4500000178813934, + "reward_std": 0.6388007551431656, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.7750000059604645, + "step": 1157 + }, + { + "completion_length": 509.17501068115234, + "epoch": 1.2001036806635563, + "grad_norm": 2.682429075241089, + "kl": 3.63671875, + "learning_rate": 2.248954138765109e-06, + "loss": 0.5159, + "reward": 0.8500000238418579, + "reward_std": 0.4651199206709862, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/format_reward": 0.7250000089406967, + "step": 1158 + }, + { + "completion_length": 699.5250091552734, + "epoch": 1.2011404872991187, + "grad_norm": 4.163193225860596, + "kl": 7.796875, + "learning_rate": 2.2473844372969906e-06, + "loss": 0.7881, + "reward": 1.2000000476837158, + "reward_std": 0.7290672734379768, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.6500000208616257, + "step": 1159 + }, + { + "completion_length": 333.4000053405762, + "epoch": 1.202177293934681, + "grad_norm": 1.7668286561965942, + "kl": 1.573974609375, + "learning_rate": 2.2458136463240455e-06, + "loss": 0.3326, + "reward": 1.3750000298023224, + "reward_std": 0.5335852652788162, + "rewards/accuracy_reward": 0.5000000111758709, + "rewards/format_reward": 0.8750000298023224, + "step": 1160 + }, + { + "completion_length": 240.45000839233398, + "epoch": 1.2032141005702437, + "grad_norm": 1.1698496341705322, + "kl": 2.04541015625, + "learning_rate": 2.244241768136105e-06, + "loss": 0.2676, + "reward": 1.3750000149011612, + "reward_std": 0.3856646865606308, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.8500000238418579, + "step": 1161 + }, + { + "completion_length": 489.1750068664551, + "epoch": 1.204250907205806, + "grad_norm": 1.149490237236023, + "kl": 2.4130859375, + "learning_rate": 2.242668805024586e-06, + "loss": 0.4075, + "reward": 1.2000000178813934, + "reward_std": 0.5356019288301468, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.75, + "step": 1162 + }, + { + "completion_length": 261.7750053405762, + "epoch": 1.2052877138413687, + "grad_norm": 6.335738658905029, + "kl": 2.89794921875, + "learning_rate": 2.241094759282486e-06, + "loss": 0.3619, + "reward": 1.600000023841858, + "reward_std": 0.5288755036890507, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.8250000178813934, + "step": 1163 + }, + { + "completion_length": 449.8250160217285, + "epoch": 1.206324520476931, + "grad_norm": 2.405794620513916, + "kl": 3.873046875, + "learning_rate": 2.239519633204381e-06, + "loss": 0.4872, + "reward": 1.100000023841858, + "reward_std": 0.6381677836179733, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.7250000089406967, + "step": 1164 + }, + { + "completion_length": 300.4500102996826, + "epoch": 1.2073613271124934, + "grad_norm": 1.931716799736023, + "kl": 1.939208984375, + "learning_rate": 2.2379434290864216e-06, + "loss": 0.3559, + "reward": 1.5000000298023224, + "reward_std": 0.5156788975000381, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.8750000149011612, + "step": 1165 + }, + { + "completion_length": 504.9000072479248, + "epoch": 1.208398133748056, + "grad_norm": 2.2452728748321533, + "kl": 2.67333984375, + "learning_rate": 2.2363661492263312e-06, + "loss": 0.3062, + "reward": 1.2500000298023224, + "reward_std": 0.4415663778781891, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.7250000089406967, + "step": 1166 + }, + { + "completion_length": 245.92500114440918, + "epoch": 1.2094349403836184, + "grad_norm": 3.8794145584106445, + "kl": 2.95263671875, + "learning_rate": 2.2347877959234e-06, + "loss": 0.2499, + "reward": 1.3750000298023224, + "reward_std": 0.46511995792388916, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.9000000208616257, + "step": 1167 + }, + { + "completion_length": 343.25000762939453, + "epoch": 1.2104717470191808, + "grad_norm": 6.801175117492676, + "kl": 3.704345703125, + "learning_rate": 2.233208371478482e-06, + "loss": 0.4683, + "reward": 1.350000038743019, + "reward_std": 0.3354101814329624, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.8750000149011612, + "step": 1168 + }, + { + "completion_length": 398.4750061035156, + "epoch": 1.2115085536547434, + "grad_norm": 2.9643471240997314, + "kl": 5.6796875, + "learning_rate": 2.231627878193997e-06, + "loss": 0.5255, + "reward": 1.350000023841858, + "reward_std": 0.4958943761885166, + "rewards/accuracy_reward": 0.5000000111758709, + "rewards/format_reward": 0.8500000238418579, + "step": 1169 + }, + { + "completion_length": 163.37500190734863, + "epoch": 1.2125453602903058, + "grad_norm": 11.943863868713379, + "kl": 2.26806640625, + "learning_rate": 2.230046318373917e-06, + "loss": 0.097, + "reward": 1.425000011920929, + "reward_std": 0.42963574826717377, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.925000011920929, + "step": 1170 + }, + { + "completion_length": 203.0, + "epoch": 1.2135821669258684, + "grad_norm": 1.9657776355743408, + "kl": 1.68310546875, + "learning_rate": 2.228463694323774e-06, + "loss": 0.1682, + "reward": 1.4500000178813934, + "reward_std": 0.2974148504436016, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 1171 + }, + { + "completion_length": 242.9749984741211, + "epoch": 1.2146189735614308, + "grad_norm": 1.2443591356277466, + "kl": 2.547607421875, + "learning_rate": 2.2268800083506476e-06, + "loss": 0.1818, + "reward": 1.475000023841858, + "reward_std": 0.4849044680595398, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.875, + "step": 1172 + }, + { + "completion_length": 162.95000076293945, + "epoch": 1.2156557801969932, + "grad_norm": 1.3411865234375, + "kl": 1.81201171875, + "learning_rate": 2.225295262763168e-06, + "loss": 0.167, + "reward": 1.4000000357627869, + "reward_std": 0.45977720245718956, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1173 + }, + { + "completion_length": 223.49999809265137, + "epoch": 1.2166925868325558, + "grad_norm": 0.7742236852645874, + "kl": 2.1669921875, + "learning_rate": 2.2237094598715085e-06, + "loss": 0.2562, + "reward": 1.5750000178813934, + "reward_std": 0.5408061631023884, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.875, + "step": 1174 + }, + { + "completion_length": 250.45000457763672, + "epoch": 1.2177293934681181, + "grad_norm": 2.619572639465332, + "kl": 1.431640625, + "learning_rate": 2.222122601987384e-06, + "loss": 0.2507, + "reward": 1.5000000298023224, + "reward_std": 0.423798531293869, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9000000208616257, + "step": 1175 + }, + { + "completion_length": 248.00001335144043, + "epoch": 1.2187662001036808, + "grad_norm": 1.728539228439331, + "kl": 2.7685546875, + "learning_rate": 2.2205346914240487e-06, + "loss": 0.3616, + "reward": 1.5250000208616257, + "reward_std": 0.4472136050462723, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1176 + }, + { + "completion_length": 254.57500839233398, + "epoch": 1.2198030067392431, + "grad_norm": 1.1736640930175781, + "kl": 2.730224609375, + "learning_rate": 2.2189457304962886e-06, + "loss": 0.3847, + "reward": 1.5, + "reward_std": 0.4797002375125885, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 1177 + }, + { + "completion_length": 165.75000190734863, + "epoch": 1.2208398133748055, + "grad_norm": 0.7769097685813904, + "kl": 0.267822265625, + "learning_rate": 2.2173557215204233e-06, + "loss": 0.0461, + "reward": 1.6500000059604645, + "reward_std": 0.21275537833571434, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1178 + }, + { + "completion_length": 153.87500381469727, + "epoch": 1.2218766200103681, + "grad_norm": 0.6941525936126709, + "kl": 3.181640625, + "learning_rate": 2.2157646668142998e-06, + "loss": 0.2306, + "reward": 1.649999976158142, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 1179 + }, + { + "completion_length": 207.65000915527344, + "epoch": 1.2229134266459305, + "grad_norm": 12.102463722229004, + "kl": 3.49609375, + "learning_rate": 2.214172568697289e-06, + "loss": 0.3298, + "reward": 1.5750000178813934, + "reward_std": 0.3354102112352848, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9250000268220901, + "step": 1180 + }, + { + "completion_length": 355.87500381469727, + "epoch": 1.223950233281493, + "grad_norm": 2.0645124912261963, + "kl": 4.75, + "learning_rate": 2.2125794294902818e-06, + "loss": 0.5486, + "reward": 1.100000023841858, + "reward_std": 0.5769233554601669, + "rewards/accuracy_reward": 0.27500000409781933, + "rewards/format_reward": 0.8250000029802322, + "step": 1181 + }, + { + "completion_length": 188.42500495910645, + "epoch": 1.2249870399170555, + "grad_norm": 4.254176139831543, + "kl": 2.12890625, + "learning_rate": 2.2109852515156885e-06, + "loss": 0.3345, + "reward": 1.475000023841858, + "reward_std": 0.40387551859021187, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.9000000208616257, + "step": 1182 + }, + { + "completion_length": 107.5250015258789, + "epoch": 1.2260238465526179, + "grad_norm": 0.7304279208183289, + "kl": 0.62451171875, + "learning_rate": 2.209390037097434e-06, + "loss": 0.0252, + "reward": 1.8250000178813934, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1183 + }, + { + "completion_length": 200.14999961853027, + "epoch": 1.2270606531881805, + "grad_norm": 0.655267596244812, + "kl": 1.7373046875, + "learning_rate": 2.207793788560952e-06, + "loss": 0.1922, + "reward": 1.6250000298023224, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 1184 + }, + { + "completion_length": 148.1750030517578, + "epoch": 1.2280974598237429, + "grad_norm": 0.9158124923706055, + "kl": 0.294921875, + "learning_rate": 2.206196508233185e-06, + "loss": 0.0711, + "reward": 1.8750000298023224, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.9000000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1185 + }, + { + "completion_length": 247.2000026702881, + "epoch": 1.2291342664593055, + "grad_norm": 1.4075510501861572, + "kl": 1.9541015625, + "learning_rate": 2.20459819844258e-06, + "loss": 0.13, + "reward": 1.4000000059604645, + "reward_std": 0.3840910233557224, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9000000059604645, + "step": 1186 + }, + { + "completion_length": 171.60000228881836, + "epoch": 1.2301710730948678, + "grad_norm": 0.9474394917488098, + "kl": 0.662841796875, + "learning_rate": 2.202998861519083e-06, + "loss": 0.1273, + "reward": 1.4000000059604645, + "reward_std": 0.28485123440623283, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.9249999970197678, + "step": 1187 + }, + { + "completion_length": 190.0250015258789, + "epoch": 1.2312078797304302, + "grad_norm": 2.5202767848968506, + "kl": 0.23974609375, + "learning_rate": 2.2013984997941395e-06, + "loss": 0.1328, + "reward": 1.6500000357627869, + "reward_std": 0.3354102037847042, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9500000029802322, + "step": 1188 + }, + { + "completion_length": 269.8500061035156, + "epoch": 1.2322446863659928, + "grad_norm": 2.44858455657959, + "kl": 1.68701171875, + "learning_rate": 2.1997971156006877e-06, + "loss": 0.2688, + "reward": 1.2000000327825546, + "reward_std": 0.335410188883543, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.9000000208616257, + "step": 1189 + }, + { + "completion_length": 348.62500762939453, + "epoch": 1.2332814930015552, + "grad_norm": 1.9599698781967163, + "kl": 3.48486328125, + "learning_rate": 2.198194711273156e-06, + "loss": 0.4033, + "reward": 1.2250000089406967, + "reward_std": 0.4688488282263279, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 0.8250000178813934, + "step": 1190 + }, + { + "completion_length": 258.1250057220459, + "epoch": 1.2343182996371176, + "grad_norm": 4.182277679443359, + "kl": 1.91015625, + "learning_rate": 2.196591289147461e-06, + "loss": 0.1094, + "reward": 1.4500000476837158, + "reward_std": 0.4290027618408203, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.925000011920929, + "step": 1191 + }, + { + "completion_length": 348.00000762939453, + "epoch": 1.2353551062726802, + "grad_norm": 5.709355354309082, + "kl": 4.673828125, + "learning_rate": 2.1949868515610018e-06, + "loss": 0.5129, + "reward": 1.5000000298023224, + "reward_std": 0.6381677538156509, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8500000238418579, + "step": 1192 + }, + { + "completion_length": 251.725004196167, + "epoch": 1.2363919129082426, + "grad_norm": 4.844602584838867, + "kl": 0.80517578125, + "learning_rate": 2.1933814008526593e-06, + "loss": 0.1768, + "reward": 1.5750000476837158, + "reward_std": 0.2937843129038811, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 1193 + }, + { + "completion_length": 297.0750102996826, + "epoch": 1.237428719543805, + "grad_norm": 0.785921573638916, + "kl": 2.92822265625, + "learning_rate": 2.191774939362789e-06, + "loss": 0.408, + "reward": 1.350000023841858, + "reward_std": 0.6149187088012695, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9000000208616257, + "step": 1194 + }, + { + "completion_length": 202.75, + "epoch": 1.2384655261793676, + "grad_norm": 2.0941972732543945, + "kl": 1.6484375, + "learning_rate": 2.1901674694332225e-06, + "loss": 0.2486, + "reward": 1.6500000357627869, + "reward_std": 0.31199512630701065, + "rewards/accuracy_reward": 0.7250000163912773, + "rewards/format_reward": 0.9250000268220901, + "step": 1195 + }, + { + "completion_length": 256.6250057220459, + "epoch": 1.23950233281493, + "grad_norm": 3.958770990371704, + "kl": 1.618896484375, + "learning_rate": 2.1885589934072593e-06, + "loss": 0.1736, + "reward": 1.4250000417232513, + "reward_std": 0.4415663927793503, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1196 + }, + { + "completion_length": 257.82500648498535, + "epoch": 1.2405391394504925, + "grad_norm": 0.5299058556556702, + "kl": 0.7626953125, + "learning_rate": 2.1869495136296666e-06, + "loss": 0.0838, + "reward": 1.550000011920929, + "reward_std": 0.36053745821118355, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1197 + }, + { + "completion_length": 193.32500457763672, + "epoch": 1.241575946086055, + "grad_norm": 1.3431684970855713, + "kl": 1.329345703125, + "learning_rate": 2.185339032446675e-06, + "loss": 0.1783, + "reward": 1.5500000417232513, + "reward_std": 0.28485122323036194, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 1198 + }, + { + "completion_length": 199.8250026702881, + "epoch": 1.2426127527216173, + "grad_norm": 1.8247193098068237, + "kl": 1.277587890625, + "learning_rate": 2.1837275522059746e-06, + "loss": 0.1962, + "reward": 1.5500000417232513, + "reward_std": 0.3479738011956215, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 1199 + }, + { + "completion_length": 335.6750068664551, + "epoch": 1.24364955935718, + "grad_norm": 2.906721830368042, + "kl": 1.59765625, + "learning_rate": 2.1821150752567115e-06, + "loss": 0.2166, + "reward": 1.4500000327825546, + "reward_std": 0.4038755036890507, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.8750000149011612, + "step": 1200 + }, + { + "completion_length": 151.32500076293945, + "epoch": 1.2446863659927423, + "grad_norm": 0.7558800578117371, + "kl": 0.267822265625, + "learning_rate": 2.180501603949485e-06, + "loss": 0.0495, + "reward": 1.4500000327825546, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1201 + }, + { + "completion_length": 167.72500610351562, + "epoch": 1.245723172628305, + "grad_norm": 0.3539299964904785, + "kl": 3.014892578125, + "learning_rate": 2.1788871406363445e-06, + "loss": 0.2357, + "reward": 1.5750000178813934, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 1202 + }, + { + "completion_length": 254.00000762939453, + "epoch": 1.2467599792638673, + "grad_norm": 5.103694438934326, + "kl": 1.91455078125, + "learning_rate": 2.1772716876707855e-06, + "loss": 0.2527, + "reward": 1.3750000298023224, + "reward_std": 0.3245587609708309, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 0.9250000268220901, + "step": 1203 + }, + { + "completion_length": 210.57500648498535, + "epoch": 1.2477967858994297, + "grad_norm": 0.8210768699645996, + "kl": 0.304443359375, + "learning_rate": 2.1756552474077445e-06, + "loss": 0.1541, + "reward": 1.7500000298023224, + "reward_std": 0.3354102149605751, + "rewards/accuracy_reward": 0.8000000268220901, + "rewards/format_reward": 0.9500000178813934, + "step": 1204 + }, + { + "completion_length": 103.35000228881836, + "epoch": 1.2488335925349923, + "grad_norm": 1.1031365394592285, + "kl": 0.89453125, + "learning_rate": 2.174037822203599e-06, + "loss": 0.0896, + "reward": 1.6500000357627869, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1205 + }, + { + "completion_length": 192.2999973297119, + "epoch": 1.2498703991705546, + "grad_norm": 0.5509751439094543, + "kl": 3.65380859375, + "learning_rate": 2.172419414416162e-06, + "loss": 0.3186, + "reward": 1.475000023841858, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1206 + }, + { + "completion_length": 156.47500228881836, + "epoch": 1.2509072058061173, + "grad_norm": 15.85297966003418, + "kl": 1.678466796875, + "learning_rate": 2.1708000264046784e-06, + "loss": 0.1949, + "reward": 1.6500000059604645, + "reward_std": 0.32818929851055145, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1207 + }, + { + "completion_length": 170.90000343322754, + "epoch": 1.2519440124416796, + "grad_norm": 0.9234507083892822, + "kl": 1.376220703125, + "learning_rate": 2.1691796605298223e-06, + "loss": 0.1696, + "reward": 1.5000000298023224, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 0.9750000089406967, + "step": 1208 + }, + { + "completion_length": 107.37500190734863, + "epoch": 1.252980819077242, + "grad_norm": 0.49394702911376953, + "kl": 0.27685546875, + "learning_rate": 2.1675583191536936e-06, + "loss": 0.0603, + "reward": 1.425000011920929, + "reward_std": 0.18026871606707573, + "rewards/accuracy_reward": 0.4250000063329935, + "rewards/format_reward": 1.0, + "step": 1209 + }, + { + "completion_length": 118.07500076293945, + "epoch": 1.2540176257128046, + "grad_norm": 0.31334245204925537, + "kl": 0.250244140625, + "learning_rate": 2.1659360046398127e-06, + "loss": 0.0077, + "reward": 1.600000023841858, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.6000000033527613, + "rewards/format_reward": 1.0, + "step": 1210 + }, + { + "completion_length": 263.7000045776367, + "epoch": 1.255054432348367, + "grad_norm": 2.3956122398376465, + "kl": 2.690673828125, + "learning_rate": 2.164312719353121e-06, + "loss": 0.3025, + "reward": 1.6250000298023224, + "reward_std": 0.4290027767419815, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9000000059604645, + "step": 1211 + }, + { + "completion_length": 104.75000190734863, + "epoch": 1.2560912389839296, + "grad_norm": 1.3687725067138672, + "kl": 0.489501953125, + "learning_rate": 2.1626884656599734e-06, + "loss": 0.0095, + "reward": 1.675000011920929, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 1212 + }, + { + "completion_length": 207.15000534057617, + "epoch": 1.257128045619492, + "grad_norm": 0.5240602493286133, + "kl": 2.193359375, + "learning_rate": 2.1610632459281375e-06, + "loss": 0.2094, + "reward": 1.375, + "reward_std": 0.3354102037847042, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 0.925000011920929, + "step": 1213 + }, + { + "completion_length": 101.80000305175781, + "epoch": 1.2581648522550544, + "grad_norm": 2.601689577102661, + "kl": 0.537109375, + "learning_rate": 2.159437062526788e-06, + "loss": 0.0006, + "reward": 1.7750000059604645, + "reward_std": 0.10458250343799591, + "rewards/accuracy_reward": 0.7999999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 1214 + }, + { + "completion_length": 133.20000076293945, + "epoch": 1.259201658890617, + "grad_norm": 0.0533587783575058, + "kl": 0.259521484375, + "learning_rate": 2.1578099178265045e-06, + "loss": 0.0064, + "reward": 1.4749999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 1.0, + "step": 1215 + }, + { + "completion_length": 160.20000076293945, + "epoch": 1.2602384655261794, + "grad_norm": 0.5499846935272217, + "kl": 1.1591796875, + "learning_rate": 2.1561818141992697e-06, + "loss": 0.1508, + "reward": 1.3250000327825546, + "reward_std": 0.31719937175512314, + "rewards/accuracy_reward": 0.35000000335276127, + "rewards/format_reward": 0.9750000089406967, + "step": 1216 + }, + { + "completion_length": 104.64999961853027, + "epoch": 1.261275272161742, + "grad_norm": 0.9835723042488098, + "kl": 0.248291015625, + "learning_rate": 2.1545527540184615e-06, + "loss": 0.0182, + "reward": 1.7249999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 1.0, + "step": 1217 + }, + { + "completion_length": 165.10000610351562, + "epoch": 1.2623120787973043, + "grad_norm": 0.298230916261673, + "kl": 0.4833984375, + "learning_rate": 2.1529227396588546e-06, + "loss": 0.0823, + "reward": 1.7000000178813934, + "reward_std": 0.22360678017139435, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1218 + }, + { + "completion_length": 167.60000038146973, + "epoch": 1.2633488854328667, + "grad_norm": 3.0810084342956543, + "kl": 1.11328125, + "learning_rate": 2.151291773496614e-06, + "loss": 0.1332, + "reward": 1.4000000059604645, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9750000089406967, + "step": 1219 + }, + { + "completion_length": 127.27500343322754, + "epoch": 1.264385692068429, + "grad_norm": 0.4667617976665497, + "kl": 0.2464599609375, + "learning_rate": 2.149659857909291e-06, + "loss": 0.0454, + "reward": 1.350000023841858, + "reward_std": 0.36053744703531265, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/format_reward": 0.9750000089406967, + "step": 1220 + }, + { + "completion_length": 100.97500038146973, + "epoch": 1.2654224987039917, + "grad_norm": 0.531950831413269, + "kl": 0.354248046875, + "learning_rate": 2.148026995275822e-06, + "loss": 0.0324, + "reward": 1.6250000298023224, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.6500000096857548, + "rewards/format_reward": 0.9750000089406967, + "step": 1221 + }, + { + "completion_length": 110.25000190734863, + "epoch": 1.2664593053395543, + "grad_norm": 0.46680694818496704, + "kl": 0.443359375, + "learning_rate": 2.1463931879765247e-06, + "loss": 0.0523, + "reward": 1.675000011920929, + "reward_std": 0.22894952073693275, + "rewards/accuracy_reward": 0.6999999955296516, + "rewards/format_reward": 0.9750000089406967, + "step": 1222 + }, + { + "completion_length": 115.62500190734863, + "epoch": 1.2674961119751167, + "grad_norm": 0.4408380687236786, + "kl": 0.2998046875, + "learning_rate": 2.1447584383930915e-06, + "loss": -0.0039, + "reward": 1.5999999940395355, + "reward_std": 0.30463574826717377, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9750000089406967, + "step": 1223 + }, + { + "completion_length": 116.375, + "epoch": 1.268532918610679, + "grad_norm": 0.40410295128822327, + "kl": 0.280517578125, + "learning_rate": 2.143122748908591e-06, + "loss": 0.0256, + "reward": 1.475000023841858, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1224 + }, + { + "completion_length": 94.60000228881836, + "epoch": 1.2695697252462415, + "grad_norm": 0.5488646030426025, + "kl": 0.27587890625, + "learning_rate": 2.14148612190746e-06, + "loss": -0.0308, + "reward": 1.75, + "reward_std": 0.31719936802983284, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 1.0, + "step": 1225 + }, + { + "completion_length": 204.25000190734863, + "epoch": 1.270606531881804, + "grad_norm": 1.7064461708068848, + "kl": 1.0234375, + "learning_rate": 2.1398485597755035e-06, + "loss": 0.1092, + "reward": 1.4749999940395355, + "reward_std": 0.27950847521424294, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.925000011920929, + "step": 1226 + }, + { + "completion_length": 171.100004196167, + "epoch": 1.2716433385173664, + "grad_norm": 3.355642080307007, + "kl": 0.697509765625, + "learning_rate": 2.138210064899889e-06, + "loss": 0.068, + "reward": 1.800000011920929, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 1227 + }, + { + "completion_length": 140.57500076293945, + "epoch": 1.272680145152929, + "grad_norm": 0.8206340670585632, + "kl": 0.77099609375, + "learning_rate": 2.1365706396691432e-06, + "loss": 0.0585, + "reward": 1.699999988079071, + "reward_std": 0.20019177719950676, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1228 + }, + { + "completion_length": 211.6500015258789, + "epoch": 1.2737169517884914, + "grad_norm": 0.5457781553268433, + "kl": 0.89453125, + "learning_rate": 2.13493028647315e-06, + "loss": 0.0865, + "reward": 1.675000011920929, + "reward_std": 0.25609346106648445, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 1229 + }, + { + "completion_length": 258.8000030517578, + "epoch": 1.2747537584240538, + "grad_norm": 0.5949947237968445, + "kl": 4.61669921875, + "learning_rate": 2.1332890077031454e-06, + "loss": 0.4629, + "reward": 1.5, + "reward_std": 0.32818930223584175, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 1230 + }, + { + "completion_length": 118.4000015258789, + "epoch": 1.2757905650596164, + "grad_norm": 12.690463066101074, + "kl": 0.2841796875, + "learning_rate": 2.1316468057517154e-06, + "loss": 0.0061, + "reward": 1.75, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 1.0, + "step": 1231 + }, + { + "completion_length": 165.52500343322754, + "epoch": 1.2768273716951788, + "grad_norm": 1.1037925481796265, + "kl": 1.39501953125, + "learning_rate": 2.130003683012791e-06, + "loss": 0.1823, + "reward": 1.574999988079071, + "reward_std": 0.27228759229183197, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9500000178813934, + "step": 1232 + }, + { + "completion_length": 150.10000038146973, + "epoch": 1.2778641783307414, + "grad_norm": 0.7952080965042114, + "kl": 0.2578125, + "learning_rate": 2.1283596418816447e-06, + "loss": 0.0616, + "reward": 1.600000023841858, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9750000089406967, + "step": 1233 + }, + { + "completion_length": 225.67499923706055, + "epoch": 1.2789009849663038, + "grad_norm": 2.4153053760528564, + "kl": 2.948974609375, + "learning_rate": 2.1267146847548895e-06, + "loss": 0.3239, + "reward": 1.2500000298023224, + "reward_std": 0.3407529219985008, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.9500000178813934, + "step": 1234 + }, + { + "completion_length": 307.8000087738037, + "epoch": 1.2799377916018662, + "grad_norm": 4.414966583251953, + "kl": 3.880859375, + "learning_rate": 2.1250688140304736e-06, + "loss": 0.4783, + "reward": 1.3000000417232513, + "reward_std": 0.3046357333660126, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8750000149011612, + "step": 1235 + }, + { + "completion_length": 210.35000610351562, + "epoch": 1.2809745982374288, + "grad_norm": 2.01659893989563, + "kl": 1.3486328125, + "learning_rate": 2.1234220321076763e-06, + "loss": 0.2044, + "reward": 1.5250000357627869, + "reward_std": 0.49746809899806976, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.9500000178813934, + "step": 1236 + }, + { + "completion_length": 358.0000057220459, + "epoch": 1.2820114048729911, + "grad_norm": 4.004034042358398, + "kl": 3.3154296875, + "learning_rate": 2.1217743413871044e-06, + "loss": 0.3045, + "reward": 1.2750000357627869, + "reward_std": 0.4850429967045784, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.8500000238418579, + "step": 1237 + }, + { + "completion_length": 276.2250003814697, + "epoch": 1.2830482115085537, + "grad_norm": 0.434862345457077, + "kl": 6.9912109375, + "learning_rate": 2.1201257442706914e-06, + "loss": 0.5656, + "reward": 1.4750000536441803, + "reward_std": 0.20539596676826477, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9000000059604645, + "step": 1238 + }, + { + "completion_length": 161.89999961853027, + "epoch": 1.2840850181441161, + "grad_norm": 16.989086151123047, + "kl": 0.656005859375, + "learning_rate": 2.1184762431616906e-06, + "loss": 0.0833, + "reward": 1.6250000298023224, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9750000089406967, + "step": 1239 + }, + { + "completion_length": 247.2500057220459, + "epoch": 1.2851218247796785, + "grad_norm": 1.4616671800613403, + "kl": 5.37646484375, + "learning_rate": 2.1168258404646737e-06, + "loss": 0.493, + "reward": 1.800000011920929, + "reward_std": 0.36053743958473206, + "rewards/accuracy_reward": 0.9000000208616257, + "rewards/format_reward": 0.9000000208616257, + "step": 1240 + }, + { + "completion_length": 257.8250026702881, + "epoch": 1.2861586314152411, + "grad_norm": 12.225122451782227, + "kl": 0.751220703125, + "learning_rate": 2.1151745385855266e-06, + "loss": 0.1749, + "reward": 1.5500000417232513, + "reward_std": 0.36789683997631073, + "rewards/accuracy_reward": 0.6250000111758709, + "rewards/format_reward": 0.9250000268220901, + "step": 1241 + }, + { + "completion_length": 136.85000038146973, + "epoch": 1.2871954380508035, + "grad_norm": 2.7704896926879883, + "kl": 1.5244140625, + "learning_rate": 2.113522339931447e-06, + "loss": 0.1441, + "reward": 1.3750000149011612, + "reward_std": 0.43999267742037773, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.8750000298023224, + "step": 1242 + }, + { + "completion_length": 151.17500114440918, + "epoch": 1.288232244686366, + "grad_norm": 0.6818279027938843, + "kl": 1.033935546875, + "learning_rate": 2.111869246910937e-06, + "loss": 0.0704, + "reward": 1.5250000059604645, + "reward_std": 0.3047742433845997, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1243 + }, + { + "completion_length": 309.1500053405762, + "epoch": 1.2892690513219285, + "grad_norm": 1.6926321983337402, + "kl": 2.467529296875, + "learning_rate": 2.110215261933806e-06, + "loss": 0.2825, + "reward": 1.5500000417232513, + "reward_std": 0.41643911972641945, + "rewards/accuracy_reward": 0.6500000152736902, + "rewards/format_reward": 0.9000000059604645, + "step": 1244 + }, + { + "completion_length": 262.52500915527344, + "epoch": 1.2903058579574909, + "grad_norm": 4.2472052574157715, + "kl": 1.349609375, + "learning_rate": 2.1085603874111612e-06, + "loss": 0.1518, + "reward": 1.5000000298023224, + "reward_std": 0.3354101926088333, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.875, + "step": 1245 + }, + { + "completion_length": 214.975004196167, + "epoch": 1.2913426645930535, + "grad_norm": 2.5802834033966064, + "kl": 0.95361328125, + "learning_rate": 2.106904625755408e-06, + "loss": 0.1651, + "reward": 1.300000011920929, + "reward_std": 0.22894951701164246, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.9500000029802322, + "step": 1246 + }, + { + "completion_length": 234.22500610351562, + "epoch": 1.2923794712286159, + "grad_norm": 1.8437985181808472, + "kl": 1.1201171875, + "learning_rate": 2.1052479793802435e-06, + "loss": 0.2049, + "reward": 1.350000023841858, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 1247 + }, + { + "completion_length": 357.10001373291016, + "epoch": 1.2934162778641785, + "grad_norm": 0.7537873983383179, + "kl": 3.578857421875, + "learning_rate": 2.103590450700656e-06, + "loss": 0.4093, + "reward": 1.0750000327825546, + "reward_std": 0.5049660094082355, + "rewards/accuracy_reward": 0.22500000707805157, + "rewards/format_reward": 0.8500000089406967, + "step": 1248 + }, + { + "completion_length": 180.5500030517578, + "epoch": 1.2944530844997408, + "grad_norm": 1.090117335319519, + "kl": 2.805419921875, + "learning_rate": 2.1019320421329195e-06, + "loss": 0.197, + "reward": 1.6000000536441803, + "reward_std": 0.3354101851582527, + "rewards/accuracy_reward": 0.6750000193715096, + "rewards/format_reward": 0.9250000268220901, + "step": 1249 + }, + { + "completion_length": 309.7250061035156, + "epoch": 1.2954898911353032, + "grad_norm": 1.707935094833374, + "kl": 9.330078125, + "learning_rate": 2.1002727560945897e-06, + "loss": 0.7876, + "reward": 1.2000000029802322, + "reward_std": 0.3479738011956215, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.875, + "step": 1250 + }, + { + "completion_length": 259.02501106262207, + "epoch": 1.2965266977708656, + "grad_norm": 26.302583694458008, + "kl": 1.01171875, + "learning_rate": 2.098612595004504e-06, + "loss": 0.1072, + "reward": 1.4500000327825546, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.8750000149011612, + "step": 1251 + }, + { + "completion_length": 339.9750061035156, + "epoch": 1.2975635044064282, + "grad_norm": 2.3541882038116455, + "kl": 4.7470703125, + "learning_rate": 2.0969515612827717e-06, + "loss": 0.5587, + "reward": 1.5750000327825546, + "reward_std": 0.5163118913769722, + "rewards/accuracy_reward": 0.725000012665987, + "rewards/format_reward": 0.8500000089406967, + "step": 1252 + }, + { + "completion_length": 308.075008392334, + "epoch": 1.2986003110419908, + "grad_norm": 5.078311443328857, + "kl": 1.603515625, + "learning_rate": 2.0952896573507772e-06, + "loss": 0.2998, + "reward": 1.5250000208616257, + "reward_std": 0.3297629877924919, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8750000149011612, + "step": 1253 + }, + { + "completion_length": 181.25000762939453, + "epoch": 1.2996371176775532, + "grad_norm": 1.3796796798706055, + "kl": 1.64599609375, + "learning_rate": 2.0936268856311727e-06, + "loss": 0.1874, + "reward": 1.5, + "reward_std": 0.3479738049209118, + "rewards/accuracy_reward": 0.5999999940395355, + "rewards/format_reward": 0.9000000059604645, + "step": 1254 + }, + { + "completion_length": 295.6750087738037, + "epoch": 1.3006739243131156, + "grad_norm": 2.7654173374176025, + "kl": 9.52099609375, + "learning_rate": 2.0919632485478746e-06, + "loss": 0.8503, + "reward": 1.5000000298023224, + "reward_std": 0.32818928360939026, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 1255 + }, + { + "completion_length": 267.07500648498535, + "epoch": 1.301710730948678, + "grad_norm": 37.04022216796875, + "kl": 1.9921875, + "learning_rate": 2.090298748526061e-06, + "loss": 0.1794, + "reward": 1.300000011920929, + "reward_std": 0.45977720245718956, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.8250000327825546, + "step": 1256 + }, + { + "completion_length": 344.2500057220459, + "epoch": 1.3027475375842406, + "grad_norm": 9.935186386108398, + "kl": 13.21484375, + "learning_rate": 2.08863338799217e-06, + "loss": 1.1489, + "reward": 1.225000038743019, + "reward_std": 0.4472135864198208, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.8250000178813934, + "step": 1257 + }, + { + "completion_length": 280.99999713897705, + "epoch": 1.303784344219803, + "grad_norm": 26.916645050048828, + "kl": 6.21875, + "learning_rate": 2.0869671693738904e-06, + "loss": 0.3927, + "reward": 1.5250000059604645, + "reward_std": 0.40387552976608276, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9000000059604645, + "step": 1258 + }, + { + "completion_length": 246.9000015258789, + "epoch": 1.3048211508553655, + "grad_norm": 8.15865421295166, + "kl": 4.12744140625, + "learning_rate": 2.0853000951001652e-06, + "loss": 0.348, + "reward": 1.1000000089406967, + "reward_std": 0.3768701143562794, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/format_reward": 0.8500000089406967, + "step": 1259 + }, + { + "completion_length": 472.5750045776367, + "epoch": 1.305857957490928, + "grad_norm": 6.539979934692383, + "kl": 7.273193359375, + "learning_rate": 2.083632167601183e-06, + "loss": 0.5544, + "reward": 1.3500000536441803, + "reward_std": 0.4723408482968807, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.7500000149011612, + "step": 1260 + }, + { + "completion_length": 333.7750129699707, + "epoch": 1.3068947641264903, + "grad_norm": 12.980135917663574, + "kl": 3.122314453125, + "learning_rate": 2.081963389308376e-06, + "loss": 0.3177, + "reward": 1.475000023841858, + "reward_std": 0.5210216455161572, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8500000089406967, + "step": 1261 + }, + { + "completion_length": 358.12501525878906, + "epoch": 1.307931570762053, + "grad_norm": 5.874717712402344, + "kl": 4.38330078125, + "learning_rate": 2.080293762654419e-06, + "loss": 0.4428, + "reward": 1.4250000268220901, + "reward_std": 0.4290027692914009, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.8500000089406967, + "step": 1262 + }, + { + "completion_length": 479.4750061035156, + "epoch": 1.3089683773976153, + "grad_norm": 48.87544250488281, + "kl": 14.296875, + "learning_rate": 2.07862329007322e-06, + "loss": 1.1627, + "reward": 1.100000038743019, + "reward_std": 0.4415663853287697, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.75, + "step": 1263 + }, + { + "completion_length": 372.8250045776367, + "epoch": 1.310005184033178, + "grad_norm": 28.217016220092773, + "kl": 19.96484375, + "learning_rate": 2.076951973999923e-06, + "loss": 0.7583, + "reward": 1.2000000476837158, + "reward_std": 0.5535083115100861, + "rewards/accuracy_reward": 0.4000000115483999, + "rewards/format_reward": 0.800000011920929, + "step": 1264 + }, + { + "completion_length": 453.80001640319824, + "epoch": 1.3110419906687403, + "grad_norm": 5.039831638336182, + "kl": 2.38525390625, + "learning_rate": 2.0752798168709e-06, + "loss": 0.3843, + "reward": 1.2000000476837158, + "reward_std": 0.40387551859021187, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/format_reward": 0.8250000178813934, + "step": 1265 + }, + { + "completion_length": 349.4000053405762, + "epoch": 1.3120787973043027, + "grad_norm": 9.782047271728516, + "kl": 4.740234375, + "learning_rate": 2.07360682112375e-06, + "loss": 0.5457, + "reward": 1.3250000178813934, + "reward_std": 0.6726711243391037, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.8500000238418579, + "step": 1266 + }, + { + "completion_length": 303.5500087738037, + "epoch": 1.3131156039398653, + "grad_norm": 3.975912570953369, + "kl": 1.66748046875, + "learning_rate": 2.071932989197294e-06, + "loss": 0.2409, + "reward": 1.4500000178813934, + "reward_std": 0.42963578924536705, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.8500000089406967, + "step": 1267 + }, + { + "completion_length": 201.92500686645508, + "epoch": 1.3141524105754276, + "grad_norm": 2.8953194618225098, + "kl": 0.730224609375, + "learning_rate": 2.0702583235315716e-06, + "loss": 0.1217, + "reward": 1.4749999940395355, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.9500000178813934, + "step": 1268 + }, + { + "completion_length": 375.975004196167, + "epoch": 1.3151892172109902, + "grad_norm": 1.7686223983764648, + "kl": 1.0029296875, + "learning_rate": 2.0685828265678388e-06, + "loss": 0.2452, + "reward": 1.2750000208616257, + "reward_std": 0.5288755223155022, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.8000000193715096, + "step": 1269 + }, + { + "completion_length": 157.02499771118164, + "epoch": 1.3162260238465526, + "grad_norm": 1.5042879581451416, + "kl": 1.02978515625, + "learning_rate": 2.066906500748563e-06, + "loss": 0.1275, + "reward": 1.800000011920929, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.8500000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 1270 + }, + { + "completion_length": 257.100004196167, + "epoch": 1.317262830482115, + "grad_norm": 2.236104965209961, + "kl": 0.393798828125, + "learning_rate": 2.065229348517419e-06, + "loss": 0.0451, + "reward": 1.5250000059604645, + "reward_std": 0.21638590842485428, + "rewards/accuracy_reward": 0.6000000033527613, + "rewards/format_reward": 0.925000011920929, + "step": 1271 + }, + { + "completion_length": 295.95000648498535, + "epoch": 1.3182996371176776, + "grad_norm": 0.46052783727645874, + "kl": 2.16015625, + "learning_rate": 2.0635513723192876e-06, + "loss": 0.1901, + "reward": 1.475000023841858, + "reward_std": 0.41643913462758064, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.8750000149011612, + "step": 1272 + }, + { + "completion_length": 356.8250045776367, + "epoch": 1.31933644375324, + "grad_norm": 6.156704425811768, + "kl": 2.18310546875, + "learning_rate": 2.06187257460025e-06, + "loss": 0.4086, + "reward": 1.2750000357627869, + "reward_std": 0.6291945055127144, + "rewards/accuracy_reward": 0.42500000819563866, + "rewards/format_reward": 0.8500000238418579, + "step": 1273 + }, + { + "completion_length": 139.95000457763672, + "epoch": 1.3203732503888026, + "grad_norm": 6.010787487030029, + "kl": 2.75927734375, + "learning_rate": 2.0601929578075857e-06, + "loss": 0.2277, + "reward": 1.600000023841858, + "reward_std": 0.30477424710989, + "rewards/accuracy_reward": 0.6750000156462193, + "rewards/format_reward": 0.9250000268220901, + "step": 1274 + }, + { + "completion_length": 162.87500190734863, + "epoch": 1.321410057024365, + "grad_norm": 1.2043706178665161, + "kl": 0.456298828125, + "learning_rate": 2.0585125243897673e-06, + "loss": 0.0403, + "reward": 1.5250000357627869, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.5500000137835741, + "rewards/format_reward": 0.9750000089406967, + "step": 1275 + }, + { + "completion_length": 306.22500228881836, + "epoch": 1.3224468636599274, + "grad_norm": 4.1956706047058105, + "kl": 0.492919921875, + "learning_rate": 2.0568312767964573e-06, + "loss": 0.1374, + "reward": 1.125, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.9000000059604645, + "step": 1276 + }, + { + "completion_length": 276.97501373291016, + "epoch": 1.32348367029549, + "grad_norm": 0.5680062770843506, + "kl": 1.7470703125, + "learning_rate": 2.0551492174785077e-06, + "loss": 0.1708, + "reward": 1.475000023841858, + "reward_std": 0.5769233480095863, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9000000208616257, + "step": 1277 + }, + { + "completion_length": 200.02500343322754, + "epoch": 1.3245204769310523, + "grad_norm": 6.632391929626465, + "kl": 0.780029296875, + "learning_rate": 2.053466348887951e-06, + "loss": 0.1845, + "reward": 1.800000011920929, + "reward_std": 0.40387552604079247, + "rewards/accuracy_reward": 0.8750000149011612, + "rewards/format_reward": 0.925000011920929, + "step": 1278 + }, + { + "completion_length": 244.70000457763672, + "epoch": 1.325557283566615, + "grad_norm": 3.6601107120513916, + "kl": 0.83251953125, + "learning_rate": 2.0517826734780003e-06, + "loss": 0.1642, + "reward": 1.4500000178813934, + "reward_std": 0.3046357333660126, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9000000059604645, + "step": 1279 + }, + { + "completion_length": 99.9000015258789, + "epoch": 1.3265940902021773, + "grad_norm": 0.35116100311279297, + "kl": 0.29541015625, + "learning_rate": 2.050098193703046e-06, + "loss": 0.0298, + "reward": 1.800000011920929, + "reward_std": 0.16770510375499725, + "rewards/accuracy_reward": 0.8250000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 1280 + }, + { + "completion_length": 151.14999961853027, + "epoch": 1.3276308968377397, + "grad_norm": 2.063858985900879, + "kl": 1.90966796875, + "learning_rate": 2.04841291201865e-06, + "loss": 0.2199, + "reward": 1.4249999821186066, + "reward_std": 0.21638590842485428, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 1281 + }, + { + "completion_length": 202.80000686645508, + "epoch": 1.328667703473302, + "grad_norm": 1.1473495960235596, + "kl": 0.9140625, + "learning_rate": 2.0467268308815423e-06, + "loss": 0.1356, + "reward": 1.550000011920929, + "reward_std": 0.4723408445715904, + "rewards/accuracy_reward": 0.6250000037252903, + "rewards/format_reward": 0.9250000268220901, + "step": 1282 + }, + { + "completion_length": 108.72500419616699, + "epoch": 1.3297045101088647, + "grad_norm": 0.49558553099632263, + "kl": 0.28662109375, + "learning_rate": 2.0450399527496203e-06, + "loss": 0.0247, + "reward": 1.5750000178813934, + "reward_std": 0.20539596676826477, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 1.0, + "step": 1283 + }, + { + "completion_length": 221.8250026702881, + "epoch": 1.330741316744427, + "grad_norm": 0.9170733094215393, + "kl": 1.209716796875, + "learning_rate": 2.043352280081942e-06, + "loss": 0.2026, + "reward": 1.5500000417232513, + "reward_std": 0.5659333989024162, + "rewards/accuracy_reward": 0.6249999925494194, + "rewards/format_reward": 0.925000011920929, + "step": 1284 + }, + { + "completion_length": 179.37500381469727, + "epoch": 1.3317781233799897, + "grad_norm": 0.7182348966598511, + "kl": 0.850341796875, + "learning_rate": 2.041663815338724e-06, + "loss": 0.1016, + "reward": 1.5000000149011612, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1285 + }, + { + "completion_length": 164.39999961853027, + "epoch": 1.332814930015552, + "grad_norm": 2.3556573390960693, + "kl": 2.061279296875, + "learning_rate": 2.039974560981337e-06, + "loss": 0.1464, + "reward": 1.6500000059604645, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1286 + }, + { + "completion_length": 309.7999954223633, + "epoch": 1.3338517366511145, + "grad_norm": 15.727317810058594, + "kl": 16.7392578125, + "learning_rate": 2.0382845194723033e-06, + "loss": 1.4665, + "reward": 1.1250000298023224, + "reward_std": 0.6401844471693039, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.800000011920929, + "step": 1287 + }, + { + "completion_length": 123.27500343322754, + "epoch": 1.334888543286677, + "grad_norm": 30.45476722717285, + "kl": 0.635986328125, + "learning_rate": 2.036593693275293e-06, + "loss": 0.019, + "reward": 1.5250000059604645, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 1288 + }, + { + "completion_length": 248.42500686645508, + "epoch": 1.3359253499222394, + "grad_norm": 0.7280150055885315, + "kl": 1.158203125, + "learning_rate": 2.0349020848551197e-06, + "loss": 0.1204, + "reward": 1.275000050663948, + "reward_std": 0.4849044606089592, + "rewards/accuracy_reward": 0.42500001564621925, + "rewards/format_reward": 0.8500000238418579, + "step": 1289 + }, + { + "completion_length": 212.7750072479248, + "epoch": 1.336962156557802, + "grad_norm": 1.562294840812683, + "kl": 0.937255859375, + "learning_rate": 2.0332096966777368e-06, + "loss": 0.131, + "reward": 1.2750000357627869, + "reward_std": 0.3804604709148407, + "rewards/accuracy_reward": 0.32500000298023224, + "rewards/format_reward": 0.9500000178813934, + "step": 1290 + }, + { + "completion_length": 317.0250053405762, + "epoch": 1.3379989631933644, + "grad_norm": 6.326425075531006, + "kl": 5.015625, + "learning_rate": 2.0315165312102347e-06, + "loss": 0.4916, + "reward": 1.449999988079071, + "reward_std": 0.4723408445715904, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.8750000149011612, + "step": 1291 + }, + { + "completion_length": 169.32500076293945, + "epoch": 1.3390357698289268, + "grad_norm": 39.2647819519043, + "kl": 11.615234375, + "learning_rate": 2.0298225909208375e-06, + "loss": 0.8409, + "reward": 1.4500000476837158, + "reward_std": 0.20539595931768417, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9500000029802322, + "step": 1292 + }, + { + "completion_length": 164.32500457763672, + "epoch": 1.3400725764644894, + "grad_norm": 1.0471571683883667, + "kl": 0.7060546875, + "learning_rate": 2.0281278782788985e-06, + "loss": 0.0951, + "reward": 1.4749999940395355, + "reward_std": 0.17304782569408417, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9750000089406967, + "step": 1293 + }, + { + "completion_length": 199.62500381469727, + "epoch": 1.3411093831000518, + "grad_norm": 0.6253998875617981, + "kl": 1.9755859375, + "learning_rate": 2.0264323957548966e-06, + "loss": 0.2257, + "reward": 1.475000023841858, + "reward_std": 0.3047742620110512, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.925000011920929, + "step": 1294 + }, + { + "completion_length": 267.0750026702881, + "epoch": 1.3421461897356144, + "grad_norm": 1.7353161573410034, + "kl": 2.8291015625, + "learning_rate": 2.0247361458204336e-06, + "loss": 0.3355, + "reward": 1.1250000298023224, + "reward_std": 0.36053743213415146, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/format_reward": 0.8750000298023224, + "step": 1295 + }, + { + "completion_length": 255.30000495910645, + "epoch": 1.3431829963711768, + "grad_norm": 1.32989501953125, + "kl": 1.211181640625, + "learning_rate": 2.0230391309482278e-06, + "loss": 0.2027, + "reward": 1.425000011920929, + "reward_std": 0.47247935086488724, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 1296 + }, + { + "completion_length": 353.1750030517578, + "epoch": 1.3442198030067392, + "grad_norm": 0.5341919660568237, + "kl": 4.642578125, + "learning_rate": 2.0213413536121164e-06, + "loss": 0.5698, + "reward": 1.3750000298023224, + "reward_std": 0.7769765928387642, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8250000178813934, + "step": 1297 + }, + { + "completion_length": 345.5750141143799, + "epoch": 1.3452566096423018, + "grad_norm": 3.6098403930664062, + "kl": 2.868896484375, + "learning_rate": 2.0196428162870455e-06, + "loss": 0.4044, + "reward": 1.350000023841858, + "reward_std": 0.5732928328216076, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.8500000089406967, + "step": 1298 + }, + { + "completion_length": 258.3500061035156, + "epoch": 1.3462934162778641, + "grad_norm": 3.2862420082092285, + "kl": 0.93603515625, + "learning_rate": 2.017943521449069e-06, + "loss": 0.1827, + "reward": 1.225000023841858, + "reward_std": 0.42178186774253845, + "rewards/accuracy_reward": 0.3250000011175871, + "rewards/format_reward": 0.9000000208616257, + "step": 1299 + }, + { + "completion_length": 159.47500038146973, + "epoch": 1.3473302229134267, + "grad_norm": 2.2226035594940186, + "kl": 0.3154296875, + "learning_rate": 2.016243471575346e-06, + "loss": 0.0745, + "reward": 1.600000023841858, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.6250000111758709, + "rewards/format_reward": 0.9750000089406967, + "step": 1300 + }, + { + "completion_length": 241.12500190734863, + "epoch": 1.3483670295489891, + "grad_norm": 4.94175386428833, + "kl": 3.16162109375, + "learning_rate": 2.014542669144136e-06, + "loss": 0.3438, + "reward": 1.5750000178813934, + "reward_std": 0.4290027543902397, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 1301 + }, + { + "completion_length": 285.5750045776367, + "epoch": 1.3494038361845515, + "grad_norm": 0.8761221170425415, + "kl": 1.416259765625, + "learning_rate": 2.012841116634796e-06, + "loss": 0.2923, + "reward": 1.4250000417232513, + "reward_std": 0.5894870012998581, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.8750000298023224, + "step": 1302 + }, + { + "completion_length": 299.7750129699707, + "epoch": 1.3504406428201141, + "grad_norm": 1.5156244039535522, + "kl": 2.20556640625, + "learning_rate": 2.011138816527775e-06, + "loss": 0.3612, + "reward": 1.6250000596046448, + "reward_std": 0.40387552231550217, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.8750000298023224, + "step": 1303 + }, + { + "completion_length": 199.1250057220459, + "epoch": 1.3514774494556765, + "grad_norm": 0.821329653263092, + "kl": 0.312255859375, + "learning_rate": 2.009435771304613e-06, + "loss": 0.0109, + "reward": 1.5250000059604645, + "reward_std": 0.1369306445121765, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000029802322, + "step": 1304 + }, + { + "completion_length": 206.35000228881836, + "epoch": 1.352514256091239, + "grad_norm": 0.8979102373123169, + "kl": 1.183837890625, + "learning_rate": 2.0077319834479364e-06, + "loss": 0.0987, + "reward": 1.6000000089406967, + "reward_std": 0.17304782569408417, + "rewards/accuracy_reward": 0.6750000007450581, + "rewards/format_reward": 0.9249999970197678, + "step": 1305 + }, + { + "completion_length": 161.97500228881836, + "epoch": 1.3535510627268015, + "grad_norm": 0.4250115752220154, + "kl": 0.3291015625, + "learning_rate": 2.0060274554414534e-06, + "loss": 0.0632, + "reward": 1.4500000178813934, + "reward_std": 0.30463573709130287, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1306 + }, + { + "completion_length": 255.27500343322754, + "epoch": 1.3545878693623639, + "grad_norm": 2.771432876586914, + "kl": 1.2548828125, + "learning_rate": 2.0043221897699516e-06, + "loss": 0.1758, + "reward": 1.5999999940395355, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 1307 + }, + { + "completion_length": 153.32500076293945, + "epoch": 1.3556246759979265, + "grad_norm": 1.119118332862854, + "kl": 1.18994140625, + "learning_rate": 2.0026161889192937e-06, + "loss": 0.1228, + "reward": 1.5250000059604645, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.5500000063329935, + "rewards/format_reward": 0.9750000089406967, + "step": 1308 + }, + { + "completion_length": 196.87500953674316, + "epoch": 1.3566614826334888, + "grad_norm": 0.8415226340293884, + "kl": 1.863037109375, + "learning_rate": 2.000909455376414e-06, + "loss": 0.2324, + "reward": 1.675000011920929, + "reward_std": 0.3407529294490814, + "rewards/accuracy_reward": 0.7249999940395355, + "rewards/format_reward": 0.9500000178813934, + "step": 1309 + }, + { + "completion_length": 159.50000381469727, + "epoch": 1.3576982892690515, + "grad_norm": 1.2588375806808472, + "kl": 1.39892578125, + "learning_rate": 1.999201991629316e-06, + "loss": 0.1529, + "reward": 1.6500000357627869, + "reward_std": 0.335410188883543, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9500000029802322, + "step": 1310 + }, + { + "completion_length": 238.82500839233398, + "epoch": 1.3587350959046138, + "grad_norm": 2.7150652408599854, + "kl": 1.916748046875, + "learning_rate": 1.9974938001670653e-06, + "loss": 0.2961, + "reward": 1.7000000476837158, + "reward_std": 0.3913118839263916, + "rewards/accuracy_reward": 0.7750000134110451, + "rewards/format_reward": 0.9250000268220901, + "step": 1311 + }, + { + "completion_length": 343.2000045776367, + "epoch": 1.3597719025401762, + "grad_norm": 1.2668253183364868, + "kl": 4.1181640625, + "learning_rate": 1.9957848834797907e-06, + "loss": 0.3664, + "reward": 1.1250000149011612, + "reward_std": 0.34860678389668465, + "rewards/accuracy_reward": 0.25, + "rewards/format_reward": 0.8750000149011612, + "step": 1312 + }, + { + "completion_length": 220.475004196167, + "epoch": 1.3608087091757386, + "grad_norm": 4.202035427093506, + "kl": 1.3388671875, + "learning_rate": 1.994075244058678e-06, + "loss": 0.1765, + "reward": 1.5499999821186066, + "reward_std": 0.36053744703531265, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1313 + }, + { + "completion_length": 244.44999885559082, + "epoch": 1.3618455158113012, + "grad_norm": 0.495758980512619, + "kl": 1.9638671875, + "learning_rate": 1.992364884395964e-06, + "loss": 0.2726, + "reward": 1.3750000149011612, + "reward_std": 0.3678968548774719, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.9000000208616257, + "step": 1314 + }, + { + "completion_length": 220.75000190734863, + "epoch": 1.3628823224468636, + "grad_norm": 7.6908745765686035, + "kl": 3.709228515625, + "learning_rate": 1.9906538069849394e-06, + "loss": 0.4358, + "reward": 1.600000023841858, + "reward_std": 0.3407529145479202, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9000000208616257, + "step": 1315 + }, + { + "completion_length": 149.04999923706055, + "epoch": 1.3639191290824262, + "grad_norm": 0.523462176322937, + "kl": 1.835693359375, + "learning_rate": 1.9889420143199378e-06, + "loss": 0.1679, + "reward": 1.5000000298023224, + "reward_std": 0.3354101814329624, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 1316 + }, + { + "completion_length": 117.12499809265137, + "epoch": 1.3649559357179886, + "grad_norm": 1.952399492263794, + "kl": 4.02685546875, + "learning_rate": 1.987229508896337e-06, + "loss": 0.1088, + "reward": 1.5250000059604645, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1317 + }, + { + "completion_length": 201.97500228881836, + "epoch": 1.365992742353551, + "grad_norm": 5.1983842849731445, + "kl": 0.33984375, + "learning_rate": 1.9855162932105538e-06, + "loss": 0.1112, + "reward": 1.5000000149011612, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.9500000178813934, + "step": 1318 + }, + { + "completion_length": 192.45000648498535, + "epoch": 1.3670295489891136, + "grad_norm": 0.7994645833969116, + "kl": 2.2470703125, + "learning_rate": 1.98380236976004e-06, + "loss": 0.2463, + "reward": 1.725000023841858, + "reward_std": 0.3354102075099945, + "rewards/accuracy_reward": 0.8000000268220901, + "rewards/format_reward": 0.9250000268220901, + "step": 1319 + }, + { + "completion_length": 297.1500015258789, + "epoch": 1.368066355624676, + "grad_norm": 3.1475718021392822, + "kl": 2.51220703125, + "learning_rate": 1.9820877410432804e-06, + "loss": 0.3617, + "reward": 1.4249999970197678, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1320 + }, + { + "completion_length": 104.80000114440918, + "epoch": 1.3691031622602385, + "grad_norm": 0.5522794723510742, + "kl": 0.20751953125, + "learning_rate": 1.9803724095597845e-06, + "loss": 0.0092, + "reward": 1.7000000178813934, + "reward_std": 0.12436700984835625, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 1.0, + "step": 1321 + }, + { + "completion_length": 345.65000915527344, + "epoch": 1.370139968895801, + "grad_norm": 2.9535956382751465, + "kl": 4.50244140625, + "learning_rate": 1.978656377810091e-06, + "loss": 0.3934, + "reward": 1.425000011920929, + "reward_std": 0.497468076646328, + "rewards/accuracy_reward": 0.5749999992549419, + "rewards/format_reward": 0.8500000089406967, + "step": 1322 + }, + { + "completion_length": 343.65000343322754, + "epoch": 1.3711767755313633, + "grad_norm": 1.0855612754821777, + "kl": 1.342041015625, + "learning_rate": 1.976939648295756e-06, + "loss": 0.3052, + "reward": 1.1750000268220901, + "reward_std": 0.44892581552267075, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.8750000149011612, + "step": 1323 + }, + { + "completion_length": 297.0500030517578, + "epoch": 1.372213582166926, + "grad_norm": 1.9501731395721436, + "kl": 4.21484375, + "learning_rate": 1.9752222235193536e-06, + "loss": 0.3417, + "reward": 1.275000050663948, + "reward_std": 0.5048274956643581, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.8500000238418579, + "step": 1324 + }, + { + "completion_length": 353.6750030517578, + "epoch": 1.3732503888024883, + "grad_norm": 3.1614885330200195, + "kl": 2.147705078125, + "learning_rate": 1.9735041059844727e-06, + "loss": 0.4237, + "reward": 1.0750000178813934, + "reward_std": 0.45977721363306046, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.8500000089406967, + "step": 1325 + }, + { + "completion_length": 254.95001029968262, + "epoch": 1.374287195438051, + "grad_norm": 1.7063990831375122, + "kl": 3.00390625, + "learning_rate": 1.9717852981957092e-06, + "loss": 0.3488, + "reward": 1.25, + "reward_std": 0.5643596909940243, + "rewards/accuracy_reward": 0.3999999985098839, + "rewards/format_reward": 0.8500000238418579, + "step": 1326 + }, + { + "completion_length": 395.6250057220459, + "epoch": 1.3753240020736133, + "grad_norm": 3.218597650527954, + "kl": 4.737060546875, + "learning_rate": 1.9700658026586687e-06, + "loss": 0.4662, + "reward": 1.0250000208616257, + "reward_std": 0.36588016897439957, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.7999999970197678, + "step": 1327 + }, + { + "completion_length": 307.10000228881836, + "epoch": 1.3763608087091757, + "grad_norm": 1.8125945329666138, + "kl": 3.138671875, + "learning_rate": 1.968345621879957e-06, + "loss": 0.4781, + "reward": 1.5250000059604645, + "reward_std": 0.5715806037187576, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9000000208616257, + "step": 1328 + }, + { + "completion_length": 217.62500381469727, + "epoch": 1.3773976153447383, + "grad_norm": 1.349875807762146, + "kl": 1.9482421875, + "learning_rate": 1.9666247583671796e-06, + "loss": 0.2255, + "reward": 1.574999988079071, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9500000178813934, + "step": 1329 + }, + { + "completion_length": 163.37500381469727, + "epoch": 1.3784344219803006, + "grad_norm": 2.4353957176208496, + "kl": 0.28515625, + "learning_rate": 1.9649032146289377e-06, + "loss": 0.0256, + "reward": 1.3499999940395355, + "reward_std": 0.16770510375499725, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9750000089406967, + "step": 1330 + }, + { + "completion_length": 176.20000648498535, + "epoch": 1.3794712286158632, + "grad_norm": 0.4093508720397949, + "kl": 3.2236328125, + "learning_rate": 1.9631809931748218e-06, + "loss": 0.3603, + "reward": 1.4749999940395355, + "reward_std": 0.4525563195347786, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.925000011920929, + "step": 1331 + }, + { + "completion_length": 354.90000343322754, + "epoch": 1.3805080352514256, + "grad_norm": 120.4134750366211, + "kl": 50.94775390625, + "learning_rate": 1.9614580965154143e-06, + "loss": 2.1455, + "reward": 1.5000000298023224, + "reward_std": 0.5461488850414753, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.8500000238418579, + "step": 1332 + }, + { + "completion_length": 248.62500381469727, + "epoch": 1.381544841886988, + "grad_norm": 1.1852550506591797, + "kl": 2.482666015625, + "learning_rate": 1.9597345271622784e-06, + "loss": 0.2442, + "reward": 1.5, + "reward_std": 0.2854842245578766, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.925000011920929, + "step": 1333 + }, + { + "completion_length": 407.375, + "epoch": 1.3825816485225506, + "grad_norm": 5.533868312835693, + "kl": 4.03466796875, + "learning_rate": 1.9580102876279596e-06, + "loss": 0.5496, + "reward": 1.350000023841858, + "reward_std": 0.5210216753184795, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8000000268220901, + "step": 1334 + }, + { + "completion_length": 191.85000038146973, + "epoch": 1.383618455158113, + "grad_norm": 3.50870418548584, + "kl": 0.885498046875, + "learning_rate": 1.95628538042598e-06, + "loss": 0.1257, + "reward": 1.5500000417232513, + "reward_std": 0.40387553349137306, + "rewards/accuracy_reward": 0.6250000223517418, + "rewards/format_reward": 0.925000011920929, + "step": 1335 + }, + { + "completion_length": 148.4249973297119, + "epoch": 1.3846552617936756, + "grad_norm": 5.719729423522949, + "kl": 0.66552734375, + "learning_rate": 1.9545598080708355e-06, + "loss": 0.093, + "reward": 1.4249999970197678, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9250000268220901, + "step": 1336 + }, + { + "completion_length": 190.05000495910645, + "epoch": 1.385692068429238, + "grad_norm": 1.0899783372879028, + "kl": 2.8780517578125, + "learning_rate": 1.95283357307799e-06, + "loss": 0.2897, + "reward": 1.7000000178813934, + "reward_std": 0.20019173622131348, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1337 + }, + { + "completion_length": 390.47501373291016, + "epoch": 1.3867288750648004, + "grad_norm": 14.337153434753418, + "kl": 6.26953125, + "learning_rate": 1.9511066779638754e-06, + "loss": 0.7641, + "reward": 1.2000000476837158, + "reward_std": 0.4399926923215389, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/format_reward": 0.8250000178813934, + "step": 1338 + }, + { + "completion_length": 139.92499923706055, + "epoch": 1.387765681700363, + "grad_norm": 3.601635694503784, + "kl": 0.927734375, + "learning_rate": 1.9493791252458842e-06, + "loss": 0.109, + "reward": 1.5750000178813934, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.6250000037252903, + "rewards/format_reward": 0.9500000178813934, + "step": 1339 + }, + { + "completion_length": 147.82500076293945, + "epoch": 1.3888024883359253, + "grad_norm": 2.475231647491455, + "kl": 1.29345703125, + "learning_rate": 1.9476509174423695e-06, + "loss": 0.1011, + "reward": 1.2750000059604645, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 0.9750000089406967, + "step": 1340 + }, + { + "completion_length": 272.7000026702881, + "epoch": 1.389839294971488, + "grad_norm": 3.056259870529175, + "kl": 2.72265625, + "learning_rate": 1.9459220570726375e-06, + "loss": 0.3923, + "reward": 1.6500000357627869, + "reward_std": 0.47768355160951614, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9000000208616257, + "step": 1341 + }, + { + "completion_length": 379.625, + "epoch": 1.3908761016070503, + "grad_norm": 2.5830676555633545, + "kl": 2.778564453125, + "learning_rate": 1.9441925466569454e-06, + "loss": 0.4417, + "reward": 1.350000023841858, + "reward_std": 0.5461488962173462, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.800000011920929, + "step": 1342 + }, + { + "completion_length": 274.6750030517578, + "epoch": 1.3919129082426127, + "grad_norm": 2.2792797088623047, + "kl": 1.99365234375, + "learning_rate": 1.9424623887165005e-06, + "loss": 0.3201, + "reward": 1.475000023841858, + "reward_std": 0.3804604522883892, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9000000357627869, + "step": 1343 + }, + { + "completion_length": 327.22500228881836, + "epoch": 1.392949714878175, + "grad_norm": 2.4801011085510254, + "kl": 1.701171875, + "learning_rate": 1.9407315857734516e-06, + "loss": 0.1142, + "reward": 1.5500000417232513, + "reward_std": 0.3804604634642601, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.8499999940395355, + "step": 1344 + }, + { + "completion_length": 249.52500343322754, + "epoch": 1.3939865215137377, + "grad_norm": 3.4298818111419678, + "kl": 1.82177734375, + "learning_rate": 1.939000140350889e-06, + "loss": 0.2832, + "reward": 1.7000000178813934, + "reward_std": 0.3913118876516819, + "rewards/accuracy_reward": 0.7750000134110451, + "rewards/format_reward": 0.925000011920929, + "step": 1345 + }, + { + "completion_length": 142.95000648498535, + "epoch": 1.3950233281493, + "grad_norm": 3.945064067840576, + "kl": 2.00634765625, + "learning_rate": 1.9372680549728394e-06, + "loss": 0.1959, + "reward": 1.7249999940395355, + "reward_std": 0.21638591215014458, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1346 + }, + { + "completion_length": 241.475004196167, + "epoch": 1.3960601347848627, + "grad_norm": 2.005232095718384, + "kl": 1.3388671875, + "learning_rate": 1.9355353321642615e-06, + "loss": 0.1454, + "reward": 1.475000023841858, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9000000208616257, + "step": 1347 + }, + { + "completion_length": 243.9500026702881, + "epoch": 1.397096941420425, + "grad_norm": 2.505420684814453, + "kl": 3.240966796875, + "learning_rate": 1.9338019744510444e-06, + "loss": 0.378, + "reward": 1.4000000357627869, + "reward_std": 0.4237985163927078, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.925000011920929, + "step": 1348 + }, + { + "completion_length": 260.20000076293945, + "epoch": 1.3981337480559874, + "grad_norm": 5.2407546043396, + "kl": 1.7978515625, + "learning_rate": 1.932067984360002e-06, + "loss": 0.2908, + "reward": 1.7000000476837158, + "reward_std": 0.447213601320982, + "rewards/accuracy_reward": 0.8000000193715096, + "rewards/format_reward": 0.9000000357627869, + "step": 1349 + }, + { + "completion_length": 243.350004196167, + "epoch": 1.39917055469155, + "grad_norm": 1.1847293376922607, + "kl": 1.4765625, + "learning_rate": 1.9303333644188712e-06, + "loss": 0.2516, + "reward": 1.6500000059604645, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1350 + }, + { + "completion_length": 183.25000190734863, + "epoch": 1.4002073613271124, + "grad_norm": 2.2103288173675537, + "kl": 2.0478515625, + "learning_rate": 1.9285981171563054e-06, + "loss": 0.267, + "reward": 1.7000000476837158, + "reward_std": 0.3804604858160019, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9250000268220901, + "step": 1351 + }, + { + "completion_length": 312.20000648498535, + "epoch": 1.401244167962675, + "grad_norm": 4.538423538208008, + "kl": 3.09814453125, + "learning_rate": 1.9268622451018734e-06, + "loss": 0.3925, + "reward": 1.350000023841858, + "reward_std": 0.42963574081659317, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.875, + "step": 1352 + }, + { + "completion_length": 240.82500648498535, + "epoch": 1.4022809745982374, + "grad_norm": 0.9791562557220459, + "kl": 3.373046875, + "learning_rate": 1.925125750786055e-06, + "loss": 0.3425, + "reward": 1.4000000357627869, + "reward_std": 0.5715806111693382, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.8750000298023224, + "step": 1353 + }, + { + "completion_length": 279.325008392334, + "epoch": 1.4033177812337998, + "grad_norm": 2.6334242820739746, + "kl": 2.3408203125, + "learning_rate": 1.9233886367402364e-06, + "loss": 0.3473, + "reward": 1.1500000208616257, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 0.8750000149011612, + "step": 1354 + }, + { + "completion_length": 195.25, + "epoch": 1.4043545878693624, + "grad_norm": 2.7357373237609863, + "kl": 0.626708984375, + "learning_rate": 1.9216509054967076e-06, + "loss": 0.0761, + "reward": 1.550000011920929, + "reward_std": 0.28485121205449104, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9250000268220901, + "step": 1355 + }, + { + "completion_length": 150.5750026702881, + "epoch": 1.4053913945049248, + "grad_norm": 2.6245734691619873, + "kl": 1.7177734375, + "learning_rate": 1.919912559588659e-06, + "loss": 0.1806, + "reward": 1.5250000059604645, + "reward_std": 0.279508501291275, + "rewards/accuracy_reward": 0.5500000063329935, + "rewards/format_reward": 0.9750000089406967, + "step": 1356 + }, + { + "completion_length": 197.17501068115234, + "epoch": 1.4064282011404874, + "grad_norm": 2.298128604888916, + "kl": 2.6669921875, + "learning_rate": 1.9181736015501754e-06, + "loss": 0.2947, + "reward": 1.4250000417232513, + "reward_std": 0.28485123068094254, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 1357 + }, + { + "completion_length": 193.05000495910645, + "epoch": 1.4074650077760498, + "grad_norm": 1.1060773134231567, + "kl": 2.9697265625, + "learning_rate": 1.916434033916235e-06, + "loss": 0.3492, + "reward": 1.350000023841858, + "reward_std": 0.45977720618247986, + "rewards/accuracy_reward": 0.44999999925494194, + "rewards/format_reward": 0.9000000059604645, + "step": 1358 + }, + { + "completion_length": 333.15001106262207, + "epoch": 1.4085018144116122, + "grad_norm": 1.7391694784164429, + "kl": 2.19970703125, + "learning_rate": 1.914693859222705e-06, + "loss": 0.3666, + "reward": 1.3750000298023224, + "reward_std": 0.3856646940112114, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.8750000149011612, + "step": 1359 + }, + { + "completion_length": 310.1750030517578, + "epoch": 1.4095386210471748, + "grad_norm": 5.543538570404053, + "kl": 25.02783203125, + "learning_rate": 1.9129530800063377e-06, + "loss": 2.0473, + "reward": 1.5750000178813934, + "reward_std": 0.3533165454864502, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.8750000149011612, + "step": 1360 + }, + { + "completion_length": 303.875, + "epoch": 1.4105754276827371, + "grad_norm": 72.21014404296875, + "kl": 1.54541015625, + "learning_rate": 1.9112116988047646e-06, + "loss": 0.2335, + "reward": 1.3750000298023224, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.9000000059604645, + "step": 1361 + }, + { + "completion_length": 286.0500011444092, + "epoch": 1.4116122343182997, + "grad_norm": 65.4577407836914, + "kl": 22.9541015625, + "learning_rate": 1.9094697181564967e-06, + "loss": 1.9458, + "reward": 1.4500000178813934, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.8750000149011612, + "step": 1362 + }, + { + "completion_length": 199.5499973297119, + "epoch": 1.4126490409538621, + "grad_norm": 3.5749762058258057, + "kl": 2.787109375, + "learning_rate": 1.907727140600919e-06, + "loss": 0.1877, + "reward": 1.5250000357627869, + "reward_std": 0.4290027320384979, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 1363 + }, + { + "completion_length": 184.100004196167, + "epoch": 1.4136858475894245, + "grad_norm": 4.195121765136719, + "kl": 3.4029541015625, + "learning_rate": 1.9059839686782853e-06, + "loss": 0.1832, + "reward": 1.5750000178813934, + "reward_std": 0.4415663816034794, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9000000208616257, + "step": 1364 + }, + { + "completion_length": 201.15000915527344, + "epoch": 1.414722654224987, + "grad_norm": 2.0089199542999268, + "kl": 1.5, + "learning_rate": 1.9042402049297163e-06, + "loss": 0.1528, + "reward": 1.475000023841858, + "reward_std": 0.45977721735835075, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9000000357627869, + "step": 1365 + }, + { + "completion_length": 242.22500610351562, + "epoch": 1.4157594608605495, + "grad_norm": 4.145914077758789, + "kl": 3.361572265625, + "learning_rate": 1.9024958518971964e-06, + "loss": 0.3238, + "reward": 1.475000038743019, + "reward_std": 0.33712238073349, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9000000059604645, + "step": 1366 + }, + { + "completion_length": 273.37500762939453, + "epoch": 1.416796267496112, + "grad_norm": 1.178970217704773, + "kl": 1.828125, + "learning_rate": 1.9007509121235676e-06, + "loss": 0.2617, + "reward": 1.475000023841858, + "reward_std": 0.3479738049209118, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9000000208616257, + "step": 1367 + }, + { + "completion_length": 146.25000190734863, + "epoch": 1.4178330741316745, + "grad_norm": 2.0879671573638916, + "kl": 1.00146484375, + "learning_rate": 1.8990053881525282e-06, + "loss": 0.1353, + "reward": 1.5250000357627869, + "reward_std": 0.31719937175512314, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1368 + }, + { + "completion_length": 292.45000648498535, + "epoch": 1.4188698807672369, + "grad_norm": 1.4969731569290161, + "kl": 2.66796875, + "learning_rate": 1.897259282528628e-06, + "loss": 0.324, + "reward": 1.225000038743019, + "reward_std": 0.31719936802983284, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.8750000149011612, + "step": 1369 + }, + { + "completion_length": 196.87500381469727, + "epoch": 1.4199066874027995, + "grad_norm": 0.8825180530548096, + "kl": 1.77880859375, + "learning_rate": 1.8955125977972644e-06, + "loss": 0.1783, + "reward": 1.6500000357627869, + "reward_std": 0.38409100100398064, + "rewards/accuracy_reward": 0.7250000163912773, + "rewards/format_reward": 0.9250000268220901, + "step": 1370 + }, + { + "completion_length": 238.02501106262207, + "epoch": 1.4209434940383618, + "grad_norm": 1.4576854705810547, + "kl": 1.9921875, + "learning_rate": 1.8937653365046792e-06, + "loss": 0.2668, + "reward": 1.3000000417232513, + "reward_std": 0.4723408482968807, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.8750000149011612, + "step": 1371 + }, + { + "completion_length": 196.47500133514404, + "epoch": 1.4219803006739244, + "grad_norm": 1.6968486309051514, + "kl": 0.517578125, + "learning_rate": 1.892017501197954e-06, + "loss": 0.1419, + "reward": 1.5000000149011612, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 1372 + }, + { + "completion_length": 389.4500045776367, + "epoch": 1.4230171073094868, + "grad_norm": 3.5373055934906006, + "kl": 2.83740234375, + "learning_rate": 1.8902690944250086e-06, + "loss": 0.5057, + "reward": 1.3750000298023224, + "reward_std": 0.6149186939001083, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.8500000238418579, + "step": 1373 + }, + { + "completion_length": 185.05000495910645, + "epoch": 1.4240539139450492, + "grad_norm": 3.059485673904419, + "kl": 1.74072265625, + "learning_rate": 1.8885201187345948e-06, + "loss": 0.2048, + "reward": 1.625, + "reward_std": 0.22360679134726524, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 1374 + }, + { + "completion_length": 190.62500381469727, + "epoch": 1.4250907205806116, + "grad_norm": 1.2744885683059692, + "kl": 2.24658203125, + "learning_rate": 1.8867705766762936e-06, + "loss": 0.1086, + "reward": 1.1500000208616257, + "reward_std": 0.3407529145479202, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 1375 + }, + { + "completion_length": 94.32500267028809, + "epoch": 1.4261275272161742, + "grad_norm": 0.6009107232093811, + "kl": 0.4296875, + "learning_rate": 1.8850204708005127e-06, + "loss": 0.1247, + "reward": 1.6500000357627869, + "reward_std": 0.31719936430454254, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1376 + }, + { + "completion_length": 202.850004196167, + "epoch": 1.4271643338517366, + "grad_norm": 0.49962109327316284, + "kl": 0.6357421875, + "learning_rate": 1.8832698036584805e-06, + "loss": 0.1524, + "reward": 1.600000023841858, + "reward_std": 0.41643913835287094, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 1377 + }, + { + "completion_length": 136.77499961853027, + "epoch": 1.4282011404872992, + "grad_norm": 2.90053129196167, + "kl": 0.54052734375, + "learning_rate": 1.8815185778022447e-06, + "loss": 0.079, + "reward": 1.3250000327825546, + "reward_std": 0.24873404577374458, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.9750000089406967, + "step": 1378 + }, + { + "completion_length": 106.85000038146973, + "epoch": 1.4292379471228616, + "grad_norm": 0.4404795467853546, + "kl": 1.04052734375, + "learning_rate": 1.8797667957846663e-06, + "loss": 0.0969, + "reward": 1.550000011920929, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 1379 + }, + { + "completion_length": 192.80000686645508, + "epoch": 1.430274753758424, + "grad_norm": 1.2581452131271362, + "kl": 1.458984375, + "learning_rate": 1.878014460159418e-06, + "loss": 0.1953, + "reward": 1.675000011920929, + "reward_std": 0.16770508512854576, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1380 + }, + { + "completion_length": 428.6000118255615, + "epoch": 1.4313115603939865, + "grad_norm": 14.338582038879395, + "kl": 6.367919921875, + "learning_rate": 1.8762615734809787e-06, + "loss": 0.6138, + "reward": 1.3500000089406967, + "reward_std": 0.576923344284296, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.8250000029802322, + "step": 1381 + }, + { + "completion_length": 345.4000015258789, + "epoch": 1.432348367029549, + "grad_norm": 13.323136329650879, + "kl": 4.8623046875, + "learning_rate": 1.8745081383046317e-06, + "loss": 0.583, + "reward": 1.5000000298023224, + "reward_std": 0.5445752069354057, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8750000149011612, + "step": 1382 + }, + { + "completion_length": 221.0500087738037, + "epoch": 1.4333851736651115, + "grad_norm": 0.6289466023445129, + "kl": 2.7431640625, + "learning_rate": 1.8727541571864588e-06, + "loss": 0.3706, + "reward": 1.6250000596046448, + "reward_std": 0.3913118876516819, + "rewards/accuracy_reward": 0.700000025331974, + "rewards/format_reward": 0.9250000268220901, + "step": 1383 + }, + { + "completion_length": 168.15000343322754, + "epoch": 1.434421980300674, + "grad_norm": 2.4519495964050293, + "kl": 2.47216796875, + "learning_rate": 1.870999632683338e-06, + "loss": 0.2879, + "reward": 1.675000011920929, + "reward_std": 0.16770510375499725, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 1384 + }, + { + "completion_length": 101.60000228881836, + "epoch": 1.4354587869362363, + "grad_norm": 3.0664381980895996, + "kl": 0.244873046875, + "learning_rate": 1.8692445673529398e-06, + "loss": 0.004, + "reward": 1.7000000178813934, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 1.0, + "step": 1385 + }, + { + "completion_length": 288.49999809265137, + "epoch": 1.436495593571799, + "grad_norm": 0.5069672465324402, + "kl": 8.235595703125, + "learning_rate": 1.867488963753723e-06, + "loss": 0.8141, + "reward": 1.3750000298023224, + "reward_std": 0.39728759974241257, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.9000000208616257, + "step": 1386 + }, + { + "completion_length": 181.87500381469727, + "epoch": 1.4375324002073613, + "grad_norm": 29.51852798461914, + "kl": 4.054443359375, + "learning_rate": 1.8657328244449301e-06, + "loss": 0.3738, + "reward": 1.75, + "reward_std": 0.1767766997218132, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1387 + }, + { + "completion_length": 245.850004196167, + "epoch": 1.4385692068429239, + "grad_norm": 19.067703247070312, + "kl": 2.6376953125, + "learning_rate": 1.8639761519865866e-06, + "loss": 0.2866, + "reward": 1.6500000208616257, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.7250000182539225, + "rewards/format_reward": 0.925000011920929, + "step": 1388 + }, + { + "completion_length": 167.39999961853027, + "epoch": 1.4396060134784863, + "grad_norm": 0.3625662326812744, + "kl": 1.27001953125, + "learning_rate": 1.8622189489394926e-06, + "loss": 0.1855, + "reward": 1.550000011920929, + "reward_std": 0.16770510375499725, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9500000029802322, + "step": 1389 + }, + { + "completion_length": 330.87500762939453, + "epoch": 1.4406428201140487, + "grad_norm": 5.789668560028076, + "kl": 5.293212890625, + "learning_rate": 1.8604612178652234e-06, + "loss": 0.4377, + "reward": 1.2500000149011612, + "reward_std": 0.40387551859021187, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/format_reward": 0.875, + "step": 1390 + }, + { + "completion_length": 145.75000381469727, + "epoch": 1.4416796267496113, + "grad_norm": 9.631103515625, + "kl": 2.2275390625, + "learning_rate": 1.8587029613261241e-06, + "loss": 0.0912, + "reward": 1.5500000417232513, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.600000012665987, + "rewards/format_reward": 0.9500000178813934, + "step": 1391 + }, + { + "completion_length": 237.0750026702881, + "epoch": 1.4427164333851736, + "grad_norm": 1.7411144971847534, + "kl": 1.08447265625, + "learning_rate": 1.8569441818853047e-06, + "loss": 0.1139, + "reward": 1.5750000178813934, + "reward_std": 0.30463575199246407, + "rewards/accuracy_reward": 0.6500000208616257, + "rewards/format_reward": 0.925000011920929, + "step": 1392 + }, + { + "completion_length": 177.82500648498535, + "epoch": 1.4437532400207362, + "grad_norm": 1.8259129524230957, + "kl": 2.00634765625, + "learning_rate": 1.8551848821066383e-06, + "loss": 0.2332, + "reward": 1.4500000178813934, + "reward_std": 0.41643914580345154, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1393 + }, + { + "completion_length": 297.4250011444092, + "epoch": 1.4447900466562986, + "grad_norm": 0.878925621509552, + "kl": 2.45947265625, + "learning_rate": 1.8534250645547568e-06, + "loss": 0.2842, + "reward": 1.425000011920929, + "reward_std": 0.36789683997631073, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1394 + }, + { + "completion_length": 291.425012588501, + "epoch": 1.445826853291861, + "grad_norm": 2.780809164047241, + "kl": 4.52490234375, + "learning_rate": 1.8516647317950455e-06, + "loss": 0.3475, + "reward": 1.2750000208616257, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9000000059604645, + "step": 1395 + }, + { + "completion_length": 232.52500534057617, + "epoch": 1.4468636599274236, + "grad_norm": 11.077188491821289, + "kl": 1.074951171875, + "learning_rate": 1.849903886393642e-06, + "loss": 0.2272, + "reward": 1.600000023841858, + "reward_std": 0.3354102112352848, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 1396 + }, + { + "completion_length": 134.60000228881836, + "epoch": 1.447900466562986, + "grad_norm": 0.9025780558586121, + "kl": 0.34912109375, + "learning_rate": 1.848142530917431e-06, + "loss": 0.0357, + "reward": 1.3250000178813934, + "reward_std": 0.15685366094112396, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1397 + }, + { + "completion_length": 238.72500228881836, + "epoch": 1.4489372731985486, + "grad_norm": 3.455339193344116, + "kl": 1.527587890625, + "learning_rate": 1.8463806679340409e-06, + "loss": 0.2651, + "reward": 1.7000000476837158, + "reward_std": 0.3479738011956215, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9250000268220901, + "step": 1398 + }, + { + "completion_length": 89.25000190734863, + "epoch": 1.449974079834111, + "grad_norm": 0.4030129313468933, + "kl": 0.3681640625, + "learning_rate": 1.844618300011839e-06, + "loss": 0.0142, + "reward": 1.5, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 1.0, + "step": 1399 + }, + { + "completion_length": 258.55001068115234, + "epoch": 1.4510108864696734, + "grad_norm": 2.5429656505584717, + "kl": 1.571533203125, + "learning_rate": 1.8428554297199293e-06, + "loss": 0.2594, + "reward": 1.4750000536441803, + "reward_std": 0.3966546058654785, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9000000357627869, + "step": 1400 + }, + { + "completion_length": 346.0, + "epoch": 1.4520476931052357, + "grad_norm": 2.216012954711914, + "kl": 1.3095703125, + "learning_rate": 1.8410920596281483e-06, + "loss": 0.3248, + "reward": 1.3250000327825546, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.8750000298023224, + "step": 1401 + }, + { + "completion_length": 239.6750030517578, + "epoch": 1.4530844997407983, + "grad_norm": 3.4484033584594727, + "kl": 2.7900390625, + "learning_rate": 1.8393281923070614e-06, + "loss": 0.3214, + "reward": 1.4000000357627869, + "reward_std": 0.4472135752439499, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.9250000268220901, + "step": 1402 + }, + { + "completion_length": 190.600004196167, + "epoch": 1.454121306376361, + "grad_norm": 0.9628894925117493, + "kl": 0.29833984375, + "learning_rate": 1.837563830327958e-06, + "loss": 0.1015, + "reward": 1.475000023841858, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9250000268220901, + "step": 1403 + }, + { + "completion_length": 299.62500953674316, + "epoch": 1.4551581130119233, + "grad_norm": 16.452903747558594, + "kl": 14.7822265625, + "learning_rate": 1.8357989762628493e-06, + "loss": 1.2684, + "reward": 1.6250000596046448, + "reward_std": 0.4290027618408203, + "rewards/accuracy_reward": 0.7250000238418579, + "rewards/format_reward": 0.9000000208616257, + "step": 1404 + }, + { + "completion_length": 147.2500057220459, + "epoch": 1.4561949196474857, + "grad_norm": 48.58760452270508, + "kl": 3.30419921875, + "learning_rate": 1.8340336326844627e-06, + "loss": 0.2987, + "reward": 1.4500000178813934, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.9750000089406967, + "step": 1405 + }, + { + "completion_length": 139.2500057220459, + "epoch": 1.457231726283048, + "grad_norm": 12.247923851013184, + "kl": 1.459716796875, + "learning_rate": 1.832267802166241e-06, + "loss": 0.1251, + "reward": 1.625, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1406 + }, + { + "completion_length": 126.57500267028809, + "epoch": 1.4582685329186107, + "grad_norm": 0.12291527539491653, + "kl": 1.54150390625, + "learning_rate": 1.8305014872823346e-06, + "loss": 0.1752, + "reward": 1.5750000178813934, + "reward_std": 0.17304781079292297, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1407 + }, + { + "completion_length": 135.95000076293945, + "epoch": 1.459305339554173, + "grad_norm": 3.804716110229492, + "kl": 1.363525390625, + "learning_rate": 1.8287346906076024e-06, + "loss": 0.1411, + "reward": 1.5750000178813934, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1408 + }, + { + "completion_length": 150.75000381469727, + "epoch": 1.4603421461897357, + "grad_norm": 2.0299649238586426, + "kl": 0.79345703125, + "learning_rate": 1.8269674147176044e-06, + "loss": 0.0995, + "reward": 1.5250000357627869, + "reward_std": 0.403875507414341, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1409 + }, + { + "completion_length": 245.3250026702881, + "epoch": 1.461378952825298, + "grad_norm": 2.6532015800476074, + "kl": 0.55029296875, + "learning_rate": 1.825199662188598e-06, + "loss": 0.0766, + "reward": 1.2750000059604645, + "reward_std": 0.37844379246234894, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.925000011920929, + "step": 1410 + }, + { + "completion_length": 151.9000015258789, + "epoch": 1.4624157594608604, + "grad_norm": 0.45854178071022034, + "kl": 0.419921875, + "learning_rate": 1.823431435597538e-06, + "loss": 0.0667, + "reward": 1.6500000059604645, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 1411 + }, + { + "completion_length": 393.9000053405762, + "epoch": 1.463452566096423, + "grad_norm": 3.40095591545105, + "kl": 4.75830078125, + "learning_rate": 1.8216627375220684e-06, + "loss": 0.4276, + "reward": 1.4500000178813934, + "reward_std": 0.31719936430454254, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.8500000089406967, + "step": 1412 + }, + { + "completion_length": 210.37500190734863, + "epoch": 1.4644893727319854, + "grad_norm": 11.09891128540039, + "kl": 0.6650390625, + "learning_rate": 1.819893570540521e-06, + "loss": 0.1252, + "reward": 1.550000011920929, + "reward_std": 0.25609345361590385, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9500000178813934, + "step": 1413 + }, + { + "completion_length": 99.67500114440918, + "epoch": 1.465526179367548, + "grad_norm": 0.5185099244117737, + "kl": 0.38720703125, + "learning_rate": 1.818123937231911e-06, + "loss": 0.0038, + "reward": 1.625, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.650000000372529, + "rewards/format_reward": 0.9750000089406967, + "step": 1414 + }, + { + "completion_length": 251.52500534057617, + "epoch": 1.4665629860031104, + "grad_norm": 3.0295069217681885, + "kl": 1.4248046875, + "learning_rate": 1.8163538401759342e-06, + "loss": 0.2459, + "reward": 1.5250000059604645, + "reward_std": 0.3354102075099945, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9250000268220901, + "step": 1415 + }, + { + "completion_length": 244.975004196167, + "epoch": 1.4675997926386728, + "grad_norm": 5.051865577697754, + "kl": 3.203125, + "learning_rate": 1.8145832819529614e-06, + "loss": 0.3631, + "reward": 1.5500000417232513, + "reward_std": 0.45977726206183434, + "rewards/accuracy_reward": 0.6250000223517418, + "rewards/format_reward": 0.9250000268220901, + "step": 1416 + }, + { + "completion_length": 122.9000015258789, + "epoch": 1.4686365992742354, + "grad_norm": 4.377063274383545, + "kl": 2.48486328125, + "learning_rate": 1.8128122651440362e-06, + "loss": 0.1759, + "reward": 1.3500000089406967, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1417 + }, + { + "completion_length": 96.32500076293945, + "epoch": 1.4696734059097978, + "grad_norm": 6.121978759765625, + "kl": 0.453857421875, + "learning_rate": 1.8110407923308702e-06, + "loss": 0.0328, + "reward": 1.7000000178813934, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1418 + }, + { + "completion_length": 189.47500610351562, + "epoch": 1.4707102125453604, + "grad_norm": 7.747506141662598, + "kl": 3.40234375, + "learning_rate": 1.8092688660958408e-06, + "loss": 0.3534, + "reward": 1.225000023841858, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1419 + }, + { + "completion_length": 232.7000026702881, + "epoch": 1.4717470191809228, + "grad_norm": 6.417865753173828, + "kl": 3.60107421875, + "learning_rate": 1.807496489021985e-06, + "loss": 0.3762, + "reward": 1.5500000417232513, + "reward_std": 0.3245587833225727, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.925000011920929, + "step": 1420 + }, + { + "completion_length": 244.12500190734863, + "epoch": 1.4727838258164851, + "grad_norm": 5.064550399780273, + "kl": 2.221435546875, + "learning_rate": 1.8057236636929988e-06, + "loss": 0.3437, + "reward": 1.5750000178813934, + "reward_std": 0.3354101926088333, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.8750000298023224, + "step": 1421 + }, + { + "completion_length": 234.55000686645508, + "epoch": 1.4738206324520478, + "grad_norm": 3.3823163509368896, + "kl": 2.0166015625, + "learning_rate": 1.8039503926932294e-06, + "loss": 0.2434, + "reward": 1.5000000298023224, + "reward_std": 0.3354102149605751, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 0.9000000208616257, + "step": 1422 + }, + { + "completion_length": 145.2500057220459, + "epoch": 1.4748574390876101, + "grad_norm": 4.185642719268799, + "kl": 0.616943359375, + "learning_rate": 1.8021766786076754e-06, + "loss": 0.0692, + "reward": 1.25, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1423 + }, + { + "completion_length": 341.6500129699707, + "epoch": 1.4758942457231727, + "grad_norm": 6.28878116607666, + "kl": 2.884765625, + "learning_rate": 1.8004025240219807e-06, + "loss": 0.4261, + "reward": 1.5500000417232513, + "reward_std": 0.40387551859021187, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.8500000238418579, + "step": 1424 + }, + { + "completion_length": 166.67500114440918, + "epoch": 1.4769310523587351, + "grad_norm": 3.579341411590576, + "kl": 1.2705078125, + "learning_rate": 1.7986279315224315e-06, + "loss": 0.1264, + "reward": 1.550000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 1425 + }, + { + "completion_length": 151.92500114440918, + "epoch": 1.4779678589942975, + "grad_norm": 0.24669739603996277, + "kl": 0.78076171875, + "learning_rate": 1.7968529036959518e-06, + "loss": 0.0722, + "reward": 1.5250000059604645, + "reward_std": 0.10458250343799591, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 1426 + }, + { + "completion_length": 211.15000343322754, + "epoch": 1.47900466562986, + "grad_norm": 0.9443327784538269, + "kl": 0.842529296875, + "learning_rate": 1.7950774431301009e-06, + "loss": 0.1043, + "reward": 1.7000000178813934, + "reward_std": 0.3407529294490814, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1427 + }, + { + "completion_length": 150.10000038146973, + "epoch": 1.4800414722654225, + "grad_norm": 1.4577233791351318, + "kl": 3.143310546875, + "learning_rate": 1.7933015524130691e-06, + "loss": 0.2531, + "reward": 1.475000023841858, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1428 + }, + { + "completion_length": 88.25000190734863, + "epoch": 1.481078278900985, + "grad_norm": 9.675968170166016, + "kl": 0.2255859375, + "learning_rate": 1.7915252341336732e-06, + "loss": -0.0211, + "reward": 1.5000000298023224, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 1.0, + "step": 1429 + }, + { + "completion_length": 197.47500228881836, + "epoch": 1.4821150855365475, + "grad_norm": 0.5715732574462891, + "kl": 0.4140625, + "learning_rate": 1.7897484908813534e-06, + "loss": 0.0736, + "reward": 1.5750000476837158, + "reward_std": 0.41643913462758064, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1430 + }, + { + "completion_length": 144.27500343322754, + "epoch": 1.4831518921721099, + "grad_norm": 5.203686714172363, + "kl": 2.2109375, + "learning_rate": 1.7879713252461689e-06, + "loss": 0.1812, + "reward": 1.2750000059604645, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.3000000063329935, + "rewards/format_reward": 0.9750000089406967, + "step": 1431 + }, + { + "completion_length": 202.17500495910645, + "epoch": 1.4841886988076722, + "grad_norm": 3.741713285446167, + "kl": 2.13818359375, + "learning_rate": 1.786193739818796e-06, + "loss": 0.2348, + "reward": 1.550000011920929, + "reward_std": 0.16770508512854576, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 1432 + }, + { + "completion_length": 376.2000045776367, + "epoch": 1.4852255054432348, + "grad_norm": 4.261992931365967, + "kl": 4.01025390625, + "learning_rate": 1.7844157371905217e-06, + "loss": 0.5322, + "reward": 1.2000000327825546, + "reward_std": 0.484904445707798, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.8250000029802322, + "step": 1433 + }, + { + "completion_length": 185.77500343322754, + "epoch": 1.4862623120787974, + "grad_norm": 1.0868406295776367, + "kl": 3.3740234375, + "learning_rate": 1.782637319953242e-06, + "loss": 0.3241, + "reward": 1.5000000298023224, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9500000029802322, + "step": 1434 + }, + { + "completion_length": 294.6500015258789, + "epoch": 1.4872991187143598, + "grad_norm": 7.851423263549805, + "kl": 1.19677734375, + "learning_rate": 1.7808584906994563e-06, + "loss": 0.2256, + "reward": 1.2000000327825546, + "reward_std": 0.403875507414341, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.9000000357627869, + "step": 1435 + }, + { + "completion_length": 359.1250057220459, + "epoch": 1.4883359253499222, + "grad_norm": 1.853022575378418, + "kl": 1.81689453125, + "learning_rate": 1.7790792520222657e-06, + "loss": 0.1214, + "reward": 1.4250000268220901, + "reward_std": 0.41643913090229034, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.800000011920929, + "step": 1436 + }, + { + "completion_length": 228.52500534057617, + "epoch": 1.4893727319854846, + "grad_norm": 0.7809069752693176, + "kl": 0.87060546875, + "learning_rate": 1.7772996065153676e-06, + "loss": 0.081, + "reward": 1.2750000059604645, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9000000059604645, + "step": 1437 + }, + { + "completion_length": 208.5000057220459, + "epoch": 1.4904095386210472, + "grad_norm": 3.053826093673706, + "kl": 1.2216796875, + "learning_rate": 1.775519556773052e-06, + "loss": 0.1771, + "reward": 1.425000011920929, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.9500000178813934, + "step": 1438 + }, + { + "completion_length": 215.40000343322754, + "epoch": 1.4914463452566096, + "grad_norm": 4.729836463928223, + "kl": 1.341552734375, + "learning_rate": 1.7737391053901998e-06, + "loss": 0.1857, + "reward": 1.350000023841858, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1439 + }, + { + "completion_length": 202.62500381469727, + "epoch": 1.4924831518921722, + "grad_norm": 0.5005960464477539, + "kl": 1.277099609375, + "learning_rate": 1.7719582549622755e-06, + "loss": 0.1389, + "reward": 1.4000000208616257, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.925000011920929, + "step": 1440 + }, + { + "completion_length": 259.65000343322754, + "epoch": 1.4935199585277346, + "grad_norm": 1.7078673839569092, + "kl": 1.71240234375, + "learning_rate": 1.7701770080853261e-06, + "loss": 0.2559, + "reward": 1.4000000208616257, + "reward_std": 0.27228762954473495, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 1441 + }, + { + "completion_length": 164.92500495910645, + "epoch": 1.494556765163297, + "grad_norm": 0.4092227518558502, + "kl": 1.8466796875, + "learning_rate": 1.7683953673559771e-06, + "loss": 0.1498, + "reward": 1.7500000298023224, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1442 + }, + { + "completion_length": 158.20000457763672, + "epoch": 1.4955935717988595, + "grad_norm": 3.3610963821411133, + "kl": 1.3515625, + "learning_rate": 1.7666133353714268e-06, + "loss": 0.1636, + "reward": 1.4000000059604645, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.42499999701976776, + "rewards/format_reward": 0.9750000089406967, + "step": 1443 + }, + { + "completion_length": 200.0750026702881, + "epoch": 1.496630378434422, + "grad_norm": 0.3641982972621918, + "kl": 1.6630859375, + "learning_rate": 1.7648309147294452e-06, + "loss": 0.2213, + "reward": 1.6500000059604645, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1444 + }, + { + "completion_length": 221.5250072479248, + "epoch": 1.4976671850699845, + "grad_norm": 2.0081844329833984, + "kl": 2.14794921875, + "learning_rate": 1.7630481080283683e-06, + "loss": 0.2462, + "reward": 1.2250000089406967, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.2749999985098839, + "rewards/format_reward": 0.9500000178813934, + "step": 1445 + }, + { + "completion_length": 218.32500457763672, + "epoch": 1.498703991705547, + "grad_norm": 1.0023314952850342, + "kl": 1.4482421875, + "learning_rate": 1.7612649178670946e-06, + "loss": 0.1987, + "reward": 1.5500000417232513, + "reward_std": 0.40558771044015884, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.925000011920929, + "step": 1446 + }, + { + "completion_length": 217.5500030517578, + "epoch": 1.4997407983411093, + "grad_norm": 1.1659680604934692, + "kl": 1.01171875, + "learning_rate": 1.7594813468450822e-06, + "loss": 0.1317, + "reward": 1.75, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1447 + }, + { + "completion_length": 263.3000011444092, + "epoch": 1.500777604976672, + "grad_norm": 0.3303806781768799, + "kl": 2.60009765625, + "learning_rate": 1.7576973975623436e-06, + "loss": 0.2062, + "reward": 1.4000000357627869, + "reward_std": 0.4237985461950302, + "rewards/accuracy_reward": 0.47500001080334187, + "rewards/format_reward": 0.925000011920929, + "step": 1448 + }, + { + "completion_length": 217.8250026702881, + "epoch": 1.5018144116122343, + "grad_norm": 1.0276261568069458, + "kl": 1.326416015625, + "learning_rate": 1.7559130726194434e-06, + "loss": 0.1334, + "reward": 1.4749999940395355, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000029802322, + "step": 1449 + }, + { + "completion_length": 437.4250068664551, + "epoch": 1.5028512182477969, + "grad_norm": 1.156264066696167, + "kl": 1.967529296875, + "learning_rate": 1.7541283746174943e-06, + "loss": 0.4732, + "reward": 1.475000023841858, + "reward_std": 0.5156789049506187, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.8250000029802322, + "step": 1450 + }, + { + "completion_length": 170.72500038146973, + "epoch": 1.5038880248833593, + "grad_norm": 0.7211430668830872, + "kl": 0.8359375, + "learning_rate": 1.7523433061581511e-06, + "loss": 0.0813, + "reward": 1.6500000059604645, + "reward_std": 0.3245587684214115, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1451 + }, + { + "completion_length": 233.29999923706055, + "epoch": 1.5049248315189216, + "grad_norm": 1.8370705842971802, + "kl": 1.745849609375, + "learning_rate": 1.7505578698436103e-06, + "loss": 0.21, + "reward": 1.3750000447034836, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9500000178813934, + "step": 1452 + }, + { + "completion_length": 317.9750003814697, + "epoch": 1.505961638154484, + "grad_norm": 9.33315658569336, + "kl": 6.892333984375, + "learning_rate": 1.7487720682766035e-06, + "loss": 0.6925, + "reward": 1.5250000357627869, + "reward_std": 0.41643912345170975, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8750000149011612, + "step": 1453 + }, + { + "completion_length": 237.50000381469727, + "epoch": 1.5069984447900466, + "grad_norm": 7.423142910003662, + "kl": 3.14892578125, + "learning_rate": 1.7469859040603956e-06, + "loss": 0.2776, + "reward": 1.600000023841858, + "reward_std": 0.3046357296407223, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 1454 + }, + { + "completion_length": 331.25001335144043, + "epoch": 1.5080352514256092, + "grad_norm": 2.906614065170288, + "kl": 3.95947265625, + "learning_rate": 1.74519937979878e-06, + "loss": 0.4799, + "reward": 1.5750000476837158, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.8750000149011612, + "step": 1455 + }, + { + "completion_length": 140.75000190734863, + "epoch": 1.5090720580611716, + "grad_norm": 3.1353986263275146, + "kl": 1.02490234375, + "learning_rate": 1.7434124980960746e-06, + "loss": 0.0841, + "reward": 1.425000011920929, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 1456 + }, + { + "completion_length": 327.9750061035156, + "epoch": 1.510108864696734, + "grad_norm": 2.66587495803833, + "kl": 2.343505859375, + "learning_rate": 1.7416252615571188e-06, + "loss": 0.3487, + "reward": 1.1750000417232513, + "reward_std": 0.48867350444197655, + "rewards/accuracy_reward": 0.27500000409781933, + "rewards/format_reward": 0.9000000208616257, + "step": 1457 + }, + { + "completion_length": 328.7750129699707, + "epoch": 1.5111456713322964, + "grad_norm": 2.1067070960998535, + "kl": 2.60693359375, + "learning_rate": 1.7398376727872688e-06, + "loss": 0.3707, + "reward": 1.4000000059604645, + "reward_std": 0.5210216231644154, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 1458 + }, + { + "completion_length": 226.82500076293945, + "epoch": 1.512182477967859, + "grad_norm": 1.5981792211532593, + "kl": 2.7060546875, + "learning_rate": 1.7380497343923939e-06, + "loss": 0.2833, + "reward": 1.475000023841858, + "reward_std": 0.3371223956346512, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9000000208616257, + "step": 1459 + }, + { + "completion_length": 310.27499771118164, + "epoch": 1.5132192846034216, + "grad_norm": 1.6134463548660278, + "kl": 1.82861328125, + "learning_rate": 1.7362614489788746e-06, + "loss": 0.2917, + "reward": 1.3750000298023224, + "reward_std": 0.40387551486492157, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.9000000208616257, + "step": 1460 + }, + { + "completion_length": 259.0750026702881, + "epoch": 1.514256091238984, + "grad_norm": 2.272364616394043, + "kl": 2.445068359375, + "learning_rate": 1.734472819153596e-06, + "loss": 0.3094, + "reward": 1.4000000059604645, + "reward_std": 0.3407529182732105, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1461 + }, + { + "completion_length": 174.95000457763672, + "epoch": 1.5152928978745464, + "grad_norm": 0.6720190048217773, + "kl": 0.226318359375, + "learning_rate": 1.7326838475239455e-06, + "loss": 0.0733, + "reward": 1.75, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1462 + }, + { + "completion_length": 485.5500030517578, + "epoch": 1.5163297045101087, + "grad_norm": 3.0803070068359375, + "kl": 4.446044921875, + "learning_rate": 1.7308945366978087e-06, + "loss": 0.6859, + "reward": 1.1000000089406967, + "reward_std": 0.6020505987107754, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.7750000059604645, + "step": 1463 + }, + { + "completion_length": 275.40000343322754, + "epoch": 1.5173665111456713, + "grad_norm": 0.49723660945892334, + "kl": 2.191650390625, + "learning_rate": 1.7291048892835658e-06, + "loss": 0.32, + "reward": 1.475000038743019, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 1464 + }, + { + "completion_length": 190.39999961853027, + "epoch": 1.518403317781234, + "grad_norm": 0.2684459984302521, + "kl": 0.29150390625, + "learning_rate": 1.727314907890088e-06, + "loss": 0.0613, + "reward": 1.4749999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9750000089406967, + "step": 1465 + }, + { + "completion_length": 127.55000114440918, + "epoch": 1.5194401244167963, + "grad_norm": 0.3302256464958191, + "kl": 0.25146484375, + "learning_rate": 1.725524595126733e-06, + "loss": 0.0094, + "reward": 1.4500000476837158, + "reward_std": 0.2487340271472931, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 1.0, + "step": 1466 + }, + { + "completion_length": 195.600004196167, + "epoch": 1.5204769310523587, + "grad_norm": 0.4501000642776489, + "kl": 1.028076171875, + "learning_rate": 1.7237339536033417e-06, + "loss": 0.1607, + "reward": 1.4750000089406967, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1467 + }, + { + "completion_length": 174.77500343322754, + "epoch": 1.521513737687921, + "grad_norm": 0.3627508580684662, + "kl": 0.228271484375, + "learning_rate": 1.7219429859302336e-06, + "loss": 0.0539, + "reward": 1.725000023841858, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.7500000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1468 + }, + { + "completion_length": 184.475004196167, + "epoch": 1.5225505443234837, + "grad_norm": 0.448893666267395, + "kl": 0.669921875, + "learning_rate": 1.7201516947182045e-06, + "loss": 0.1209, + "reward": 1.6750000417232513, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.7250000238418579, + "rewards/format_reward": 0.9500000178813934, + "step": 1469 + }, + { + "completion_length": 267.6750030517578, + "epoch": 1.5235873509590463, + "grad_norm": 1.081580638885498, + "kl": 2.113525390625, + "learning_rate": 1.7183600825785218e-06, + "loss": 0.2725, + "reward": 1.7000000178813934, + "reward_std": 0.27228760346770287, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 1470 + }, + { + "completion_length": 362.82501220703125, + "epoch": 1.5246241575946087, + "grad_norm": 3.3258347511291504, + "kl": 2.7412109375, + "learning_rate": 1.71656815212292e-06, + "loss": 0.4794, + "reward": 1.4500000178813934, + "reward_std": 0.6149186752736568, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.8500000238418579, + "step": 1471 + }, + { + "completion_length": 126.37500190734863, + "epoch": 1.525660964230171, + "grad_norm": 0.5189715027809143, + "kl": 0.267333984375, + "learning_rate": 1.7147759059635987e-06, + "loss": 0.0503, + "reward": 1.5500000417232513, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9750000089406967, + "step": 1472 + }, + { + "completion_length": 112.82500076293945, + "epoch": 1.5266977708657334, + "grad_norm": 0.3307163715362549, + "kl": 0.283447265625, + "learning_rate": 1.7129833467132167e-06, + "loss": -0.003, + "reward": 1.4000000059604645, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1473 + }, + { + "completion_length": 179.07500076293945, + "epoch": 1.527734577501296, + "grad_norm": 0.42351534962654114, + "kl": 1.09619140625, + "learning_rate": 1.7111904769848892e-06, + "loss": 0.0423, + "reward": 1.5, + "reward_std": 0.1369306445121765, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.9500000029802322, + "step": 1474 + }, + { + "completion_length": 222.62500381469727, + "epoch": 1.5287713841368584, + "grad_norm": 0.4376266598701477, + "kl": 0.935791015625, + "learning_rate": 1.7093972993921855e-06, + "loss": 0.1267, + "reward": 1.625, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1475 + }, + { + "completion_length": 411.6250114440918, + "epoch": 1.529808190772421, + "grad_norm": 3.8951869010925293, + "kl": 3.93310546875, + "learning_rate": 1.7076038165491215e-06, + "loss": 0.5593, + "reward": 1.4500000178813934, + "reward_std": 0.5590169690549374, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8250000029802322, + "step": 1476 + }, + { + "completion_length": 133.95000076293945, + "epoch": 1.5308449974079834, + "grad_norm": 4.108523845672607, + "kl": 0.268310546875, + "learning_rate": 1.7058100310701605e-06, + "loss": -0.03, + "reward": 1.425000011920929, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 1.0, + "step": 1477 + }, + { + "completion_length": 160.47500228881836, + "epoch": 1.5318818040435458, + "grad_norm": 0.4052504897117615, + "kl": 1.052978515625, + "learning_rate": 1.7040159455702044e-06, + "loss": 0.1561, + "reward": 1.4500000178813934, + "reward_std": 0.3046357296407223, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.9750000089406967, + "step": 1478 + }, + { + "completion_length": 221.30000114440918, + "epoch": 1.5329186106791084, + "grad_norm": 0.8664781451225281, + "kl": 2.45751953125, + "learning_rate": 1.7022215626645942e-06, + "loss": 0.2669, + "reward": 1.7500000298023224, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1479 + }, + { + "completion_length": 115.20000267028809, + "epoch": 1.5339554173146708, + "grad_norm": 2.1453802585601807, + "kl": 0.21435546875, + "learning_rate": 1.7004268849691045e-06, + "loss": 0.0208, + "reward": 1.7249999940395355, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.7250000238418579, + "rewards/format_reward": 1.0, + "step": 1480 + }, + { + "completion_length": 97.4000015258789, + "epoch": 1.5349922239502334, + "grad_norm": 0.1985270082950592, + "kl": 0.3603515625, + "learning_rate": 1.6986319150999382e-06, + "loss": 0.0396, + "reward": 1.7249999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 1.0, + "step": 1481 + }, + { + "completion_length": 162.35000038146973, + "epoch": 1.5360290305857958, + "grad_norm": 0.38188207149505615, + "kl": 1.603271484375, + "learning_rate": 1.696836655673726e-06, + "loss": 0.1492, + "reward": 1.600000023841858, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1482 + }, + { + "completion_length": 313.3750057220459, + "epoch": 1.5370658372213581, + "grad_norm": 3.4883224964141846, + "kl": 7.850341796875, + "learning_rate": 1.695041109307519e-06, + "loss": 0.7717, + "reward": 1.0750000178813934, + "reward_std": 0.5533697828650475, + "rewards/accuracy_reward": 0.25000000931322575, + "rewards/format_reward": 0.8250000178813934, + "step": 1483 + }, + { + "completion_length": 365.2750053405762, + "epoch": 1.5381026438569205, + "grad_norm": 12.899331092834473, + "kl": 2.89892578125, + "learning_rate": 1.6932452786187875e-06, + "loss": 0.4664, + "reward": 1.225000023841858, + "reward_std": 0.3354101814329624, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.8750000149011612, + "step": 1484 + }, + { + "completion_length": 254.17500114440918, + "epoch": 1.5391394504924831, + "grad_norm": 1.319108247756958, + "kl": 3.1611328125, + "learning_rate": 1.6914491662254159e-06, + "loss": 0.3845, + "reward": 1.6500000059604645, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 0.9250000268220901, + "step": 1485 + }, + { + "completion_length": 169.95000457763672, + "epoch": 1.5401762571280457, + "grad_norm": 1.1633234024047852, + "kl": 0.43603515625, + "learning_rate": 1.6896527747456997e-06, + "loss": 0.0378, + "reward": 1.5250000208616257, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1486 + }, + { + "completion_length": 106.10000228881836, + "epoch": 1.5412130637636081, + "grad_norm": 0.48456689715385437, + "kl": 0.24267578125, + "learning_rate": 1.6878561067983412e-06, + "loss": -0.0568, + "reward": 1.8250000178813934, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.8250000029802322, + "rewards/format_reward": 1.0, + "step": 1487 + }, + { + "completion_length": 240.45000457763672, + "epoch": 1.5422498703991705, + "grad_norm": 0.9362939596176147, + "kl": 2.4794921875, + "learning_rate": 1.6860591650024454e-06, + "loss": 0.3434, + "reward": 1.475000038743019, + "reward_std": 0.2920721061527729, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 1488 + }, + { + "completion_length": 284.25000762939453, + "epoch": 1.5432866770347329, + "grad_norm": 2.9841935634613037, + "kl": 1.6005859375, + "learning_rate": 1.684261951977516e-06, + "loss": 0.2635, + "reward": 1.3250000178813934, + "reward_std": 0.3354101814329624, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9000000208616257, + "step": 1489 + }, + { + "completion_length": 156.02500534057617, + "epoch": 1.5443234836702955, + "grad_norm": 0.9497570395469666, + "kl": 1.114501953125, + "learning_rate": 1.6824644703434537e-06, + "loss": 0.1514, + "reward": 1.5750000178813934, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 0.9750000089406967, + "step": 1490 + }, + { + "completion_length": 209.2750072479248, + "epoch": 1.545360290305858, + "grad_norm": 0.2899838387966156, + "kl": 2.5693359375, + "learning_rate": 1.6806667227205487e-06, + "loss": 0.2662, + "reward": 1.4500000178813934, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 1491 + }, + { + "completion_length": 160.82500457763672, + "epoch": 1.5463970969414205, + "grad_norm": 2.7640109062194824, + "kl": 1.458984375, + "learning_rate": 1.6788687117294811e-06, + "loss": 0.1608, + "reward": 1.4000000357627869, + "reward_std": 0.37310105934739113, + "rewards/accuracy_reward": 0.45000001415610313, + "rewards/format_reward": 0.9500000178813934, + "step": 1492 + }, + { + "completion_length": 157.67500114440918, + "epoch": 1.5474339035769828, + "grad_norm": 0.7922787070274353, + "kl": 1.201416015625, + "learning_rate": 1.677070439991313e-06, + "loss": 0.1548, + "reward": 1.6500000059604645, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1493 + }, + { + "completion_length": 162.29999923706055, + "epoch": 1.5484707102125452, + "grad_norm": 4.747604846954346, + "kl": 2.978515625, + "learning_rate": 1.6752719101274876e-06, + "loss": 0.2523, + "reward": 1.5750000476837158, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.6250000223517418, + "rewards/format_reward": 0.9500000178813934, + "step": 1494 + }, + { + "completion_length": 156.52500534057617, + "epoch": 1.5495075168481078, + "grad_norm": 1.032403826713562, + "kl": 0.344970703125, + "learning_rate": 1.673473124759825e-06, + "loss": 0.0548, + "reward": 1.800000011920929, + "reward_std": 0.16770508512854576, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1495 + }, + { + "completion_length": 162.87500190734863, + "epoch": 1.5505443234836704, + "grad_norm": 0.3170575499534607, + "kl": 1.254638671875, + "learning_rate": 1.671674086510516e-06, + "loss": 0.1184, + "reward": 1.6500000059604645, + "reward_std": 0.32818929478526115, + "rewards/accuracy_reward": 0.6750000268220901, + "rewards/format_reward": 0.9750000089406967, + "step": 1496 + }, + { + "completion_length": 209.50000190734863, + "epoch": 1.5515811301192328, + "grad_norm": 1.0937821865081787, + "kl": 1.545654296875, + "learning_rate": 1.6698747980021213e-06, + "loss": 0.133, + "reward": 1.5750000178813934, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000178813934, + "step": 1497 + }, + { + "completion_length": 104.4000015258789, + "epoch": 1.5526179367547952, + "grad_norm": 1.5526025295257568, + "kl": 0.243896484375, + "learning_rate": 1.6680752618575665e-06, + "loss": -0.0018, + "reward": 1.9000000059604645, + "reward_std": 0.18026872351765633, + "rewards/accuracy_reward": 0.9000000059604645, + "rewards/format_reward": 1.0, + "step": 1498 + }, + { + "completion_length": 203.17499923706055, + "epoch": 1.5536547433903576, + "grad_norm": 0.3623444736003876, + "kl": 2.51123046875, + "learning_rate": 1.6662754807001376e-06, + "loss": 0.1818, + "reward": 1.550000011920929, + "reward_std": 0.4290027692914009, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9249999970197678, + "step": 1499 + }, + { + "completion_length": 203.7500057220459, + "epoch": 1.5546915500259202, + "grad_norm": 1.1907838582992554, + "kl": 1.669921875, + "learning_rate": 1.664475457153478e-06, + "loss": 0.2063, + "reward": 1.6500000059604645, + "reward_std": 0.24873405322432518, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9249999970197678, + "step": 1500 + }, + { + "completion_length": 168.37500381469727, + "epoch": 1.5557283566614828, + "grad_norm": 1.3985354900360107, + "kl": 0.7236328125, + "learning_rate": 1.6626751938415847e-06, + "loss": 0.1602, + "reward": 1.5500000268220901, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 1501 + }, + { + "completion_length": 253.4250087738037, + "epoch": 1.5567651632970452, + "grad_norm": 2.0021469593048096, + "kl": 1.75341796875, + "learning_rate": 1.6608746933888036e-06, + "loss": 0.1339, + "reward": 1.7750000059604645, + "reward_std": 0.2974148504436016, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 1502 + }, + { + "completion_length": 243.32500457763672, + "epoch": 1.5578019699326076, + "grad_norm": 0.421642541885376, + "kl": 1.3193359375, + "learning_rate": 1.6590739584198273e-06, + "loss": 0.2241, + "reward": 1.4000000208616257, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9000000208616257, + "step": 1503 + }, + { + "completion_length": 194.5750026702881, + "epoch": 1.55883877656817, + "grad_norm": 0.661420464515686, + "kl": 1.241455078125, + "learning_rate": 1.6572729915596884e-06, + "loss": 0.0457, + "reward": 1.425000011920929, + "reward_std": 0.3479738049209118, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.9500000029802322, + "step": 1504 + }, + { + "completion_length": 349.5250053405762, + "epoch": 1.5598755832037325, + "grad_norm": 8.147127151489258, + "kl": 6.46875, + "learning_rate": 1.6554717954337604e-06, + "loss": 0.4454, + "reward": 1.2000000178813934, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.8500000089406967, + "step": 1505 + }, + { + "completion_length": 212.5500087738037, + "epoch": 1.560912389839295, + "grad_norm": 0.6585958003997803, + "kl": 3.0361328125, + "learning_rate": 1.6536703726677483e-06, + "loss": 0.3097, + "reward": 1.4000000059604645, + "reward_std": 0.4237985350191593, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.9000000357627869, + "step": 1506 + }, + { + "completion_length": 208.1250057220459, + "epoch": 1.5619491964748575, + "grad_norm": 1.088865876197815, + "kl": 0.254638671875, + "learning_rate": 1.6518687258876884e-06, + "loss": 0.0666, + "reward": 1.1250000149011612, + "reward_std": 0.30463573709130287, + "rewards/accuracy_reward": 0.17500000074505806, + "rewards/format_reward": 0.9500000178813934, + "step": 1507 + }, + { + "completion_length": 252.65000534057617, + "epoch": 1.56298600311042, + "grad_norm": 0.4406292736530304, + "kl": 0.35107421875, + "learning_rate": 1.6500668577199448e-06, + "loss": 0.18, + "reward": 1.600000038743019, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 1508 + }, + { + "completion_length": 252.37499809265137, + "epoch": 1.5640228097459823, + "grad_norm": 0.7803511023521423, + "kl": 2.29052734375, + "learning_rate": 1.648264770791202e-06, + "loss": 0.3081, + "reward": 1.475000023841858, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 1509 + }, + { + "completion_length": 150.12500190734863, + "epoch": 1.565059616381545, + "grad_norm": 0.6692258715629578, + "kl": 0.607666015625, + "learning_rate": 1.6464624677284663e-06, + "loss": 0.0313, + "reward": 1.5500000417232513, + "reward_std": 0.41643911972641945, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 1510 + }, + { + "completion_length": 272.35000228881836, + "epoch": 1.5660964230171073, + "grad_norm": 0.6051040291786194, + "kl": 2.58056640625, + "learning_rate": 1.6446599511590558e-06, + "loss": 0.3326, + "reward": 1.2750000208616257, + "reward_std": 0.4092182256281376, + "rewards/accuracy_reward": 0.42499999701976776, + "rewards/format_reward": 0.8500000089406967, + "step": 1511 + }, + { + "completion_length": 297.75000190734863, + "epoch": 1.5671332296526699, + "grad_norm": 2.3226115703582764, + "kl": 0.70849609375, + "learning_rate": 1.6428572237106023e-06, + "loss": 0.1106, + "reward": 1.550000011920929, + "reward_std": 0.284851249307394, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9000000059604645, + "step": 1512 + }, + { + "completion_length": 287.9000053405762, + "epoch": 1.5681700362882323, + "grad_norm": 3.223996162414551, + "kl": 2.470458984375, + "learning_rate": 1.641054288011044e-06, + "loss": 0.3918, + "reward": 1.4750000089406967, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.8750000149011612, + "step": 1513 + }, + { + "completion_length": 201.95000076293945, + "epoch": 1.5692068429237946, + "grad_norm": 6.340351581573486, + "kl": 0.7060546875, + "learning_rate": 1.6392511466886231e-06, + "loss": 0.134, + "reward": 1.2000000327825546, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 0.925000011920929, + "step": 1514 + }, + { + "completion_length": 175.0250072479248, + "epoch": 1.570243649559357, + "grad_norm": 2.9794259071350098, + "kl": 0.84814453125, + "learning_rate": 1.637447802371881e-06, + "loss": 0.1037, + "reward": 1.425000011920929, + "reward_std": 0.12436702102422714, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 1515 + }, + { + "completion_length": 256.1250057220459, + "epoch": 1.5712804561949196, + "grad_norm": 1.7978872060775757, + "kl": 1.8125, + "learning_rate": 1.635644257689656e-06, + "loss": 0.2891, + "reward": 1.550000011920929, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 1516 + }, + { + "completion_length": 358.0750026702881, + "epoch": 1.5723172628304822, + "grad_norm": 1.119584560394287, + "kl": 3.794189453125, + "learning_rate": 1.6338405152710764e-06, + "loss": 0.5007, + "reward": 1.600000023841858, + "reward_std": 0.4723408445715904, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.875, + "step": 1517 + }, + { + "completion_length": 263.29999923706055, + "epoch": 1.5733540694660446, + "grad_norm": 0.628728985786438, + "kl": 1.5361328125, + "learning_rate": 1.6320365777455616e-06, + "loss": 0.2288, + "reward": 1.3500000089406967, + "reward_std": 0.36053743958473206, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9000000208616257, + "step": 1518 + }, + { + "completion_length": 221.97500228881836, + "epoch": 1.574390876101607, + "grad_norm": 4.136834621429443, + "kl": 5.1005859375, + "learning_rate": 1.6302324477428133e-06, + "loss": 0.4654, + "reward": 1.5250000357627869, + "reward_std": 0.5533697605133057, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.8750000149011612, + "step": 1519 + }, + { + "completion_length": 345.6000099182129, + "epoch": 1.5754276827371694, + "grad_norm": 17.222375869750977, + "kl": 6.73828125, + "learning_rate": 1.6284281278928149e-06, + "loss": 0.7314, + "reward": 1.6500000059604645, + "reward_std": 0.6635995022952557, + "rewards/accuracy_reward": 0.8000000268220901, + "rewards/format_reward": 0.8500000089406967, + "step": 1520 + }, + { + "completion_length": 163.07500648498535, + "epoch": 1.576464489372732, + "grad_norm": 4.385284900665283, + "kl": 1.533935546875, + "learning_rate": 1.6266236208258257e-06, + "loss": 0.1571, + "reward": 1.7249999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.75, + "rewards/format_reward": 0.9750000089406967, + "step": 1521 + }, + { + "completion_length": 264.12500190734863, + "epoch": 1.5775012960082946, + "grad_norm": 1.2917828559875488, + "kl": 2.667236328125, + "learning_rate": 1.6248189291723782e-06, + "loss": 0.3353, + "reward": 1.4000000059604645, + "reward_std": 0.3354101777076721, + "rewards/accuracy_reward": 0.47500000335276127, + "rewards/format_reward": 0.925000011920929, + "step": 1522 + }, + { + "completion_length": 322.75000953674316, + "epoch": 1.578538102643857, + "grad_norm": 1.4020047187805176, + "kl": 0.292236328125, + "learning_rate": 1.6230140555632748e-06, + "loss": 0.0598, + "reward": 1.4000000059604645, + "reward_std": 0.34860679507255554, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9000000059604645, + "step": 1523 + }, + { + "completion_length": 355.3250102996826, + "epoch": 1.5795749092794193, + "grad_norm": 1.2283453941345215, + "kl": 3.8046875, + "learning_rate": 1.6212090026295819e-06, + "loss": 0.4933, + "reward": 1.5500000268220901, + "reward_std": 0.403875507414341, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 1524 + }, + { + "completion_length": 208.0250072479248, + "epoch": 1.5806117159149817, + "grad_norm": 4.229063987731934, + "kl": 2.224609375, + "learning_rate": 1.6194037730026283e-06, + "loss": 0.2554, + "reward": 1.5750000178813934, + "reward_std": 0.36053745821118355, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000178813934, + "step": 1525 + }, + { + "completion_length": 317.55000495910645, + "epoch": 1.5816485225505443, + "grad_norm": 1.123126745223999, + "kl": 1.52490234375, + "learning_rate": 1.6175983693139994e-06, + "loss": 0.2079, + "reward": 1.475000023841858, + "reward_std": 0.4092182330787182, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9000000059604645, + "step": 1526 + }, + { + "completion_length": 107.67500114440918, + "epoch": 1.582685329186107, + "grad_norm": 1.740889310836792, + "kl": 0.271484375, + "learning_rate": 1.6157927941955354e-06, + "loss": 0.0049, + "reward": 1.425000011920929, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 1.0, + "step": 1527 + }, + { + "completion_length": 151.25000190734863, + "epoch": 1.5837221358216693, + "grad_norm": 1.7476391792297363, + "kl": 1.57470703125, + "learning_rate": 1.613987050279326e-06, + "loss": 0.1968, + "reward": 1.4000000208616257, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.45000001788139343, + "rewards/format_reward": 0.9500000178813934, + "step": 1528 + }, + { + "completion_length": 170.39999771118164, + "epoch": 1.5847589424572317, + "grad_norm": 0.23554155230522156, + "kl": 1.2509765625, + "learning_rate": 1.6121811401977062e-06, + "loss": 0.1608, + "reward": 1.550000011920929, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.925000011920929, + "step": 1529 + }, + { + "completion_length": 241.22500610351562, + "epoch": 1.585795749092794, + "grad_norm": 2.0842747688293457, + "kl": 0.82373046875, + "learning_rate": 1.6103750665832547e-06, + "loss": 0.1447, + "reward": 1.5999999940395355, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.650000000372529, + "rewards/format_reward": 0.9500000178813934, + "step": 1530 + }, + { + "completion_length": 215.600004196167, + "epoch": 1.5868325557283567, + "grad_norm": 0.22583654522895813, + "kl": 1.046630859375, + "learning_rate": 1.608568832068787e-06, + "loss": 0.1359, + "reward": 1.5250000208616257, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000029802322, + "step": 1531 + }, + { + "completion_length": 217.25000381469727, + "epoch": 1.5878693623639193, + "grad_norm": 7.7239089012146, + "kl": 1.028076171875, + "learning_rate": 1.6067624392873545e-06, + "loss": 0.1174, + "reward": 1.600000023841858, + "reward_std": 0.45413000881671906, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9500000029802322, + "step": 1532 + }, + { + "completion_length": 201.600004196167, + "epoch": 1.5889061689994817, + "grad_norm": 2.654834032058716, + "kl": 1.376953125, + "learning_rate": 1.6049558908722385e-06, + "loss": 0.1816, + "reward": 1.6750000417232513, + "reward_std": 0.4849044531583786, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1533 + }, + { + "completion_length": 245.22500228881836, + "epoch": 1.589942975635044, + "grad_norm": 1.0713359117507935, + "kl": 2.8271484375, + "learning_rate": 1.603149189456948e-06, + "loss": 0.3116, + "reward": 1.725000023841858, + "reward_std": 0.3354102075099945, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1534 + }, + { + "completion_length": 148.20000457763672, + "epoch": 1.5909797822706064, + "grad_norm": 0.886328399181366, + "kl": 1.534423828125, + "learning_rate": 1.6013423376752131e-06, + "loss": 0.098, + "reward": 1.5250000357627869, + "reward_std": 0.13693063706159592, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000029802322, + "step": 1535 + }, + { + "completion_length": 347.8999996185303, + "epoch": 1.592016588906169, + "grad_norm": 2.5690722465515137, + "kl": 2.233642578125, + "learning_rate": 1.5995353381609864e-06, + "loss": 0.2542, + "reward": 1.350000038743019, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.8750000149011612, + "step": 1536 + }, + { + "completion_length": 158.79999923706055, + "epoch": 1.5930533955417314, + "grad_norm": 0.9137997627258301, + "kl": 0.298828125, + "learning_rate": 1.597728193548432e-06, + "loss": 0.0629, + "reward": 1.6500000059604645, + "reward_std": 0.18026873469352722, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 1537 + }, + { + "completion_length": 174.25000190734863, + "epoch": 1.594090202177294, + "grad_norm": 0.7593628764152527, + "kl": 1.142333984375, + "learning_rate": 1.5959209064719291e-06, + "loss": 0.1649, + "reward": 1.6250000298023224, + "reward_std": 0.3479738011956215, + "rewards/accuracy_reward": 0.6750000063329935, + "rewards/format_reward": 0.9500000178813934, + "step": 1538 + }, + { + "completion_length": 327.800012588501, + "epoch": 1.5951270088128564, + "grad_norm": 2.3554060459136963, + "kl": 2.83251953125, + "learning_rate": 1.594113479566062e-06, + "loss": 0.2768, + "reward": 1.475000023841858, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.8500000089406967, + "step": 1539 + }, + { + "completion_length": 216.55000686645508, + "epoch": 1.5961638154484188, + "grad_norm": 1.6953859329223633, + "kl": 0.8837890625, + "learning_rate": 1.5923059154656195e-06, + "loss": 0.1239, + "reward": 1.675000011920929, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.7250000033527613, + "rewards/format_reward": 0.9500000178813934, + "step": 1540 + }, + { + "completion_length": 132.6500015258789, + "epoch": 1.5972006220839814, + "grad_norm": 0.8555593490600586, + "kl": 0.274658203125, + "learning_rate": 1.5904982168055912e-06, + "loss": 0.0857, + "reward": 1.4749999940395355, + "reward_std": 0.3046357296407223, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 1.0, + "step": 1541 + }, + { + "completion_length": 159.29999923706055, + "epoch": 1.5982374287195438, + "grad_norm": 0.48500317335128784, + "kl": 0.344970703125, + "learning_rate": 1.5886903862211622e-06, + "loss": 0.0902, + "reward": 1.5500000417232513, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 1542 + }, + { + "completion_length": 303.6000003814697, + "epoch": 1.5992742353551064, + "grad_norm": 1.3442599773406982, + "kl": 3.2978515625, + "learning_rate": 1.58688242634771e-06, + "loss": 0.3328, + "reward": 1.4000000059604645, + "reward_std": 0.31199514865875244, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 0.8750000298023224, + "step": 1543 + }, + { + "completion_length": 387.8250160217285, + "epoch": 1.6003110419906688, + "grad_norm": 1.801302433013916, + "kl": 5.2314453125, + "learning_rate": 1.5850743398208008e-06, + "loss": 0.513, + "reward": 1.425000011920929, + "reward_std": 0.5841442123055458, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.8250000029802322, + "step": 1544 + }, + { + "completion_length": 244.67499923706055, + "epoch": 1.6013478486262311, + "grad_norm": 2.613616943359375, + "kl": 1.364501953125, + "learning_rate": 1.5832661292761848e-06, + "loss": 0.1866, + "reward": 1.425000011920929, + "reward_std": 0.40387551486492157, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.9500000178813934, + "step": 1545 + }, + { + "completion_length": 295.6000003814697, + "epoch": 1.6023846552617935, + "grad_norm": 4.084285736083984, + "kl": 3.672119140625, + "learning_rate": 1.5814577973497938e-06, + "loss": 0.4096, + "reward": 1.7249999940395355, + "reward_std": 0.2650667019188404, + "rewards/accuracy_reward": 0.8250000029802322, + "rewards/format_reward": 0.9000000208616257, + "step": 1546 + }, + { + "completion_length": 166.1500015258789, + "epoch": 1.6034214618973561, + "grad_norm": 2.872854232788086, + "kl": 0.380859375, + "learning_rate": 1.5796493466777356e-06, + "loss": 0.0488, + "reward": 1.6000000089406967, + "reward_std": 0.1442900411784649, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1547 + }, + { + "completion_length": 196.4250030517578, + "epoch": 1.6044582685329187, + "grad_norm": 2.0663416385650635, + "kl": 1.898193359375, + "learning_rate": 1.5778407798962925e-06, + "loss": 0.1581, + "reward": 1.5750000178813934, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000029802322, + "step": 1548 + }, + { + "completion_length": 305.82501220703125, + "epoch": 1.6054950751684811, + "grad_norm": 1.5803462266921997, + "kl": 0.78271484375, + "learning_rate": 1.576032099641915e-06, + "loss": 0.2732, + "reward": 1.4000000059604645, + "reward_std": 0.6040672399103642, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.8750000298023224, + "step": 1549 + }, + { + "completion_length": 174.47500038146973, + "epoch": 1.6065318818040435, + "grad_norm": 1.148290753364563, + "kl": 1.66748046875, + "learning_rate": 1.5742233085512183e-06, + "loss": 0.0676, + "reward": 1.5750000178813934, + "reward_std": 0.40387552976608276, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000178813934, + "step": 1550 + }, + { + "completion_length": 211.50000953674316, + "epoch": 1.6075686884396059, + "grad_norm": 1.313335657119751, + "kl": 1.8564453125, + "learning_rate": 1.572414409260981e-06, + "loss": 0.2061, + "reward": 1.600000023841858, + "reward_std": 0.38409099727869034, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9000000208616257, + "step": 1551 + }, + { + "completion_length": 98.52500343322754, + "epoch": 1.6086054950751685, + "grad_norm": 1.1569651365280151, + "kl": 0.256103515625, + "learning_rate": 1.5706054044081385e-06, + "loss": 0.0219, + "reward": 1.7249999940395355, + "reward_std": 0.1442900449037552, + "rewards/accuracy_reward": 0.75, + "rewards/format_reward": 0.9750000089406967, + "step": 1552 + }, + { + "completion_length": 393.100004196167, + "epoch": 1.609642301710731, + "grad_norm": 0.44840702414512634, + "kl": 1.95068359375, + "learning_rate": 1.56879629662978e-06, + "loss": 0.4238, + "reward": 1.475000023841858, + "reward_std": 0.5282425284385681, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.8500000089406967, + "step": 1553 + }, + { + "completion_length": 203.4250030517578, + "epoch": 1.6106791083462935, + "grad_norm": 9.99445915222168, + "kl": 1.92578125, + "learning_rate": 1.566987088563145e-06, + "loss": 0.2378, + "reward": 1.600000023841858, + "reward_std": 0.21638590842485428, + "rewards/accuracy_reward": 0.6500000022351742, + "rewards/format_reward": 0.9500000178813934, + "step": 1554 + }, + { + "completion_length": 396.0250015258789, + "epoch": 1.6117159149818558, + "grad_norm": 12.977739334106445, + "kl": 6.28515625, + "learning_rate": 1.5651777828456183e-06, + "loss": 0.6928, + "reward": 1.6500000357627869, + "reward_std": 0.6959476321935654, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.8250000178813934, + "step": 1555 + }, + { + "completion_length": 399.100004196167, + "epoch": 1.6127527216174182, + "grad_norm": 5.44423770904541, + "kl": 3.6025390625, + "learning_rate": 1.5633683821147291e-06, + "loss": 0.534, + "reward": 1.4000000208616257, + "reward_std": 0.5517960898578167, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.8250000178813934, + "step": 1556 + }, + { + "completion_length": 339.4250011444092, + "epoch": 1.6137895282529808, + "grad_norm": 2.6319031715393066, + "kl": 4.25048828125, + "learning_rate": 1.5615588890081424e-06, + "loss": 0.5455, + "reward": 1.2500000298023224, + "reward_std": 0.540806170552969, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.8250000029802322, + "step": 1557 + }, + { + "completion_length": 691.7500228881836, + "epoch": 1.6148263348885434, + "grad_norm": 2.091921329498291, + "kl": 4.77490234375, + "learning_rate": 1.5597493061636607e-06, + "loss": 0.6976, + "reward": 1.0750000178813934, + "reward_std": 0.7314124554395676, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.6749999970197678, + "step": 1558 + }, + { + "completion_length": 453.77501106262207, + "epoch": 1.6158631415241058, + "grad_norm": 2.37636399269104, + "kl": 3.71484375, + "learning_rate": 1.5579396362192148e-06, + "loss": 0.3806, + "reward": 1.0750000029802322, + "reward_std": 0.5100317224860191, + "rewards/accuracy_reward": 0.29999999701976776, + "rewards/format_reward": 0.7750000059604645, + "step": 1559 + }, + { + "completion_length": 495.950008392334, + "epoch": 1.6168999481596682, + "grad_norm": 4.216372489929199, + "kl": 4.93701171875, + "learning_rate": 1.556129881812864e-06, + "loss": 0.6983, + "reward": 1.225000038743019, + "reward_std": 0.6092714965343475, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.7500000149011612, + "step": 1560 + }, + { + "completion_length": 203.35000228881836, + "epoch": 1.6179367547952306, + "grad_norm": 5.362270355224609, + "kl": 1.5576171875, + "learning_rate": 1.55432004558279e-06, + "loss": 0.2287, + "reward": 1.5250000059604645, + "reward_std": 0.2920721098780632, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1561 + }, + { + "completion_length": 442.7000141143799, + "epoch": 1.6189735614307932, + "grad_norm": 3.0055434703826904, + "kl": 1.2265625, + "learning_rate": 1.5525101301672946e-06, + "loss": 0.3656, + "reward": 1.3750000149011612, + "reward_std": 0.6281152926385403, + "rewards/accuracy_reward": 0.5500000081956387, + "rewards/format_reward": 0.8250000029802322, + "step": 1562 + }, + { + "completion_length": 301.3250045776367, + "epoch": 1.6200103680663558, + "grad_norm": 2.8841614723205566, + "kl": 1.67919921875, + "learning_rate": 1.550700138204794e-06, + "loss": 0.2526, + "reward": 1.550000011920929, + "reward_std": 0.4045085087418556, + "rewards/accuracy_reward": 0.6500000096857548, + "rewards/format_reward": 0.9000000208616257, + "step": 1563 + }, + { + "completion_length": 256.25000190734863, + "epoch": 1.6210471747019182, + "grad_norm": 1.5231977701187134, + "kl": 3.0517578125, + "learning_rate": 1.5488900723338173e-06, + "loss": 0.284, + "reward": 1.25, + "reward_std": 0.4291412979364395, + "rewards/accuracy_reward": 0.350000012665987, + "rewards/format_reward": 0.9000000208616257, + "step": 1564 + }, + { + "completion_length": 308.4500045776367, + "epoch": 1.6220839813374806, + "grad_norm": 0.8757073879241943, + "kl": 4.7578125, + "learning_rate": 1.547079935192999e-06, + "loss": 0.568, + "reward": 1.449999988079071, + "reward_std": 0.50311528891325, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.9000000208616257, + "step": 1565 + }, + { + "completion_length": 253.64999961853027, + "epoch": 1.623120787973043, + "grad_norm": 5.401318073272705, + "kl": 4.2685546875, + "learning_rate": 1.5452697294210807e-06, + "loss": 0.4422, + "reward": 1.2750000059604645, + "reward_std": 0.2974148392677307, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9000000059604645, + "step": 1566 + }, + { + "completion_length": 255.12500190734863, + "epoch": 1.6241575946086055, + "grad_norm": 10.566208839416504, + "kl": 3.754638671875, + "learning_rate": 1.5434594576569009e-06, + "loss": 0.421, + "reward": 1.2750000357627869, + "reward_std": 0.4958943948149681, + "rewards/accuracy_reward": 0.4000000059604645, + "rewards/format_reward": 0.8750000149011612, + "step": 1567 + }, + { + "completion_length": 104.60000228881836, + "epoch": 1.625194401244168, + "grad_norm": 0.4463522434234619, + "kl": 0.328125, + "learning_rate": 1.541649122539397e-06, + "loss": 0.0442, + "reward": 1.300000011920929, + "reward_std": 0.30463575199246407, + "rewards/accuracy_reward": 0.30000000819563866, + "rewards/format_reward": 1.0, + "step": 1568 + }, + { + "completion_length": 107.47500228881836, + "epoch": 1.6262312078797305, + "grad_norm": 0.45555442571640015, + "kl": 0.220947265625, + "learning_rate": 1.5398387267075966e-06, + "loss": -0.0156, + "reward": 1.5250000059604645, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 1.0, + "step": 1569 + }, + { + "completion_length": 153.92500114440918, + "epoch": 1.627268014515293, + "grad_norm": 0.24410176277160645, + "kl": 0.6796875, + "learning_rate": 1.5380282728006174e-06, + "loss": 0.0791, + "reward": 1.600000023841858, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1570 + }, + { + "completion_length": 181.37500190734863, + "epoch": 1.6283048211508553, + "grad_norm": 4.3899312019348145, + "kl": 1.115966796875, + "learning_rate": 1.5362177634576608e-06, + "loss": 0.1201, + "reward": 1.5750000178813934, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1571 + }, + { + "completion_length": 298.2000026702881, + "epoch": 1.6293416277864177, + "grad_norm": 3.581695079803467, + "kl": 3.4921875, + "learning_rate": 1.5344072013180093e-06, + "loss": 0.3671, + "reward": 1.3250000327825546, + "reward_std": 0.6047002524137497, + "rewards/accuracy_reward": 0.47500000707805157, + "rewards/format_reward": 0.8500000238418579, + "step": 1572 + }, + { + "completion_length": 156.0, + "epoch": 1.6303784344219803, + "grad_norm": 1.390693187713623, + "kl": 0.886474609375, + "learning_rate": 1.5325965890210223e-06, + "loss": 0.0974, + "reward": 1.6500000059604645, + "reward_std": 0.18026871606707573, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 1573 + }, + { + "completion_length": 359.3500099182129, + "epoch": 1.6314152410575429, + "grad_norm": 2.581305742263794, + "kl": 3.68212890625, + "learning_rate": 1.5307859292061326e-06, + "loss": 0.4285, + "reward": 1.375, + "reward_std": 0.46511995792388916, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.8500000089406967, + "step": 1574 + }, + { + "completion_length": 210.4500026702881, + "epoch": 1.6324520476931053, + "grad_norm": 6.038915157318115, + "kl": 0.876220703125, + "learning_rate": 1.5289752245128416e-06, + "loss": 0.1413, + "reward": 1.475000023841858, + "reward_std": 0.28485120832920074, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 0.9500000178813934, + "step": 1575 + }, + { + "completion_length": 207.3249969482422, + "epoch": 1.6334888543286676, + "grad_norm": 1.6099038124084473, + "kl": 2.033447265625, + "learning_rate": 1.5271644775807173e-06, + "loss": 0.231, + "reward": 1.5, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1576 + }, + { + "completion_length": 122.95000267028809, + "epoch": 1.63452566096423, + "grad_norm": 0.775686502456665, + "kl": 0.7314453125, + "learning_rate": 1.525353691049388e-06, + "loss": 0.0345, + "reward": 1.8250000178813934, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1577 + }, + { + "completion_length": 154.67500495910645, + "epoch": 1.6355624675997926, + "grad_norm": 0.4884706139564514, + "kl": 0.86083984375, + "learning_rate": 1.5235428675585404e-06, + "loss": 0.1491, + "reward": 1.6500000357627869, + "reward_std": 0.41643914207816124, + "rewards/accuracy_reward": 0.700000025331974, + "rewards/format_reward": 0.9500000178813934, + "step": 1578 + }, + { + "completion_length": 149.8750057220459, + "epoch": 1.6365992742353552, + "grad_norm": 1.7934447526931763, + "kl": 0.4140625, + "learning_rate": 1.5217320097479152e-06, + "loss": 0.0524, + "reward": 1.5000000298023224, + "reward_std": 0.41643914952874184, + "rewards/accuracy_reward": 0.5250000022351742, + "rewards/format_reward": 0.9750000089406967, + "step": 1579 + }, + { + "completion_length": 199.40000534057617, + "epoch": 1.6376360808709176, + "grad_norm": 0.4854935109615326, + "kl": 1.685791015625, + "learning_rate": 1.5199211202573021e-06, + "loss": 0.1541, + "reward": 1.600000023841858, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.925000011920929, + "step": 1580 + }, + { + "completion_length": 110.95000267028809, + "epoch": 1.63867288750648, + "grad_norm": 2.2596967220306396, + "kl": 0.27294921875, + "learning_rate": 1.5181102017265389e-06, + "loss": -0.0106, + "reward": 1.7750000059604645, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.7749999985098839, + "rewards/format_reward": 1.0, + "step": 1581 + }, + { + "completion_length": 110.32500076293945, + "epoch": 1.6397096941420424, + "grad_norm": 0.32057473063468933, + "kl": 0.295654296875, + "learning_rate": 1.5162992567955044e-06, + "loss": -0.0126, + "reward": 1.375, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 1.0, + "step": 1582 + }, + { + "completion_length": 410.02501678466797, + "epoch": 1.640746500777605, + "grad_norm": 0.5861947536468506, + "kl": 4.6337890625, + "learning_rate": 1.5144882881041147e-06, + "loss": 0.5881, + "reward": 1.4000000059604645, + "reward_std": 0.6274823173880577, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8500000238418579, + "step": 1583 + }, + { + "completion_length": 155.55000495910645, + "epoch": 1.6417833074131676, + "grad_norm": 6.338595867156982, + "kl": 1.751953125, + "learning_rate": 1.5126772982923235e-06, + "loss": 0.1598, + "reward": 1.3750000298023224, + "reward_std": 0.38409099727869034, + "rewards/accuracy_reward": 0.4500000160187483, + "rewards/format_reward": 0.925000011920929, + "step": 1584 + }, + { + "completion_length": 159.07500457763672, + "epoch": 1.64282011404873, + "grad_norm": 1.1443921327590942, + "kl": 2.912841796875, + "learning_rate": 1.510866290000113e-06, + "loss": 0.2813, + "reward": 1.6250000298023224, + "reward_std": 0.23617041110992432, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1585 + }, + { + "completion_length": 250.20001029968262, + "epoch": 1.6438569206842923, + "grad_norm": 5.0934953689575195, + "kl": 1.823486328125, + "learning_rate": 1.5090552658674932e-06, + "loss": 0.256, + "reward": 1.5500000417232513, + "reward_std": 0.32818929851055145, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 1586 + }, + { + "completion_length": 281.1500015258789, + "epoch": 1.6448937273198547, + "grad_norm": 1.359196424484253, + "kl": 6.293212890625, + "learning_rate": 1.5072442285344971e-06, + "loss": 0.5661, + "reward": 1.2500000149011612, + "reward_std": 0.46511995047330856, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 1587 + }, + { + "completion_length": 154.77499961853027, + "epoch": 1.6459305339554173, + "grad_norm": 11.460147857666016, + "kl": 1.093994140625, + "learning_rate": 1.5054331806411767e-06, + "loss": 0.0877, + "reward": 1.4500000178813934, + "reward_std": 0.16770508512854576, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1588 + }, + { + "completion_length": 200.77500343322754, + "epoch": 1.64696734059098, + "grad_norm": 0.5465800762176514, + "kl": 2.06982421875, + "learning_rate": 1.5036221248275996e-06, + "loss": 0.2584, + "reward": 1.6000000536441803, + "reward_std": 0.3479738123714924, + "rewards/accuracy_reward": 0.650000024586916, + "rewards/format_reward": 0.9500000178813934, + "step": 1589 + }, + { + "completion_length": 221.95000457763672, + "epoch": 1.6480041472265423, + "grad_norm": 0.8402000665664673, + "kl": 2.9384765625, + "learning_rate": 1.5018110637338452e-06, + "loss": 0.1917, + "reward": 1.3500000089406967, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9000000059604645, + "step": 1590 + }, + { + "completion_length": 241.94999885559082, + "epoch": 1.6490409538621047, + "grad_norm": 1.211803674697876, + "kl": 2.736083984375, + "learning_rate": 1.5e-06, + "loss": 0.2163, + "reward": 1.5250000208616257, + "reward_std": 0.22894951701164246, + "rewards/accuracy_reward": 0.6000000033527613, + "rewards/format_reward": 0.9249999970197678, + "step": 1591 + }, + { + "completion_length": 159.375, + "epoch": 1.650077760497667, + "grad_norm": 2.434783935546875, + "kl": 2.064208984375, + "learning_rate": 1.498188936266155e-06, + "loss": 0.1954, + "reward": 1.7750000357627869, + "reward_std": 0.36053744331002235, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1592 + }, + { + "completion_length": 262.2250118255615, + "epoch": 1.6511145671332297, + "grad_norm": 0.5691776871681213, + "kl": 1.02587890625, + "learning_rate": 1.4963778751724005e-06, + "loss": 0.2409, + "reward": 1.725000023841858, + "reward_std": 0.3354102075099945, + "rewards/accuracy_reward": 0.8000000193715096, + "rewards/format_reward": 0.9250000268220901, + "step": 1593 + }, + { + "completion_length": 107.75000190734863, + "epoch": 1.6521513737687923, + "grad_norm": 0.8404366970062256, + "kl": 0.28369140625, + "learning_rate": 1.4945668193588233e-06, + "loss": 0.0043, + "reward": 1.550000011920929, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 1.0, + "step": 1594 + }, + { + "completion_length": 216.37500381469727, + "epoch": 1.6531881804043547, + "grad_norm": 0.4120844006538391, + "kl": 0.34814453125, + "learning_rate": 1.492755771465503e-06, + "loss": 0.1214, + "reward": 1.5250000059604645, + "reward_std": 0.3354102112352848, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 1595 + }, + { + "completion_length": 199.2000026702881, + "epoch": 1.654224987039917, + "grad_norm": 0.5536773800849915, + "kl": 1.704833984375, + "learning_rate": 1.4909447341325067e-06, + "loss": 0.1642, + "reward": 1.550000011920929, + "reward_std": 0.36053743213415146, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1596 + }, + { + "completion_length": 163.5750026702881, + "epoch": 1.6552617936754794, + "grad_norm": 5.34987211227417, + "kl": 0.679443359375, + "learning_rate": 1.4891337099998875e-06, + "loss": 0.0953, + "reward": 1.4500000476837158, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.4750000163912773, + "rewards/format_reward": 0.9750000089406967, + "step": 1597 + }, + { + "completion_length": 123.05000305175781, + "epoch": 1.656298600311042, + "grad_norm": 0.5703464150428772, + "kl": 0.23193359375, + "learning_rate": 1.4873227017076763e-06, + "loss": 0.0431, + "reward": 1.8250000178813934, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.8250000029802322, + "rewards/format_reward": 1.0, + "step": 1598 + }, + { + "completion_length": 304.200008392334, + "epoch": 1.6573354069466044, + "grad_norm": 0.6134169697761536, + "kl": 2.039794921875, + "learning_rate": 1.4855117118958856e-06, + "loss": 0.2122, + "reward": 1.3500000089406967, + "reward_std": 0.2361704260110855, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9000000059604645, + "step": 1599 + }, + { + "completion_length": 170.00000190734863, + "epoch": 1.658372213582167, + "grad_norm": 0.38233059644699097, + "kl": 0.826904296875, + "learning_rate": 1.4837007432044961e-06, + "loss": 0.0966, + "reward": 1.375, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.4000000078231096, + "rewards/format_reward": 0.9750000089406967, + "step": 1600 + }, + { + "completion_length": 318.7750015258789, + "epoch": 1.6594090202177294, + "grad_norm": 4.728154182434082, + "kl": 3.4892578125, + "learning_rate": 1.4818897982734612e-06, + "loss": 0.2733, + "reward": 1.6500000357627869, + "reward_std": 0.5084579959511757, + "rewards/accuracy_reward": 0.7750000134110451, + "rewards/format_reward": 0.8750000149011612, + "step": 1601 + }, + { + "completion_length": 281.7500114440918, + "epoch": 1.6604458268532918, + "grad_norm": 2.4841439723968506, + "kl": 1.1044921875, + "learning_rate": 1.4800788797426977e-06, + "loss": 0.2515, + "reward": 1.175000011920929, + "reward_std": 0.25609343871474266, + "rewards/accuracy_reward": 0.27500000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 1602 + }, + { + "completion_length": 349.22501373291016, + "epoch": 1.6614826334888542, + "grad_norm": 1.3508840799331665, + "kl": 3.63525390625, + "learning_rate": 1.478267990252085e-06, + "loss": 0.3127, + "reward": 1.1750000268220901, + "reward_std": 0.44892580807209015, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.8750000149011612, + "step": 1603 + }, + { + "completion_length": 203.3750057220459, + "epoch": 1.6625194401244168, + "grad_norm": 2.026071310043335, + "kl": 1.2587890625, + "learning_rate": 1.4764571324414595e-06, + "loss": 0.1681, + "reward": 1.5250000357627869, + "reward_std": 0.292072132229805, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9500000178813934, + "step": 1604 + }, + { + "completion_length": 163.80000114440918, + "epoch": 1.6635562467599794, + "grad_norm": 0.2313051074743271, + "kl": 0.25537109375, + "learning_rate": 1.474646308950612e-06, + "loss": 0.0659, + "reward": 1.600000023841858, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9750000089406967, + "step": 1605 + }, + { + "completion_length": 122.25, + "epoch": 1.6645930533955418, + "grad_norm": 0.35142838954925537, + "kl": 0.445556640625, + "learning_rate": 1.4728355224192826e-06, + "loss": 0.0286, + "reward": 1.5999999940395355, + "reward_std": 0.21638590842485428, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1606 + }, + { + "completion_length": 227.82500457763672, + "epoch": 1.6656298600311041, + "grad_norm": 0.5352864861488342, + "kl": 1.51806640625, + "learning_rate": 1.4710247754871587e-06, + "loss": 0.205, + "reward": 1.5000000298023224, + "reward_std": 0.4415663778781891, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9249999970197678, + "step": 1607 + }, + { + "completion_length": 116.69999885559082, + "epoch": 1.6666666666666665, + "grad_norm": 0.6908564567565918, + "kl": 0.255859375, + "learning_rate": 1.4692140707938675e-06, + "loss": 0.0067, + "reward": 1.9500000178813934, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.9500000029802322, + "rewards/format_reward": 1.0, + "step": 1608 + }, + { + "completion_length": 117.20000076293945, + "epoch": 1.6677034733022291, + "grad_norm": 0.4036959111690521, + "kl": 0.218505859375, + "learning_rate": 1.467403410978978e-06, + "loss": 0.0104, + "reward": 1.425000011920929, + "reward_std": 0.26129765436053276, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 1.0, + "step": 1609 + }, + { + "completion_length": 216.85000228881836, + "epoch": 1.6687402799377917, + "grad_norm": 0.5966922640800476, + "kl": 0.321533203125, + "learning_rate": 1.465592798681991e-06, + "loss": 0.0942, + "reward": 1.5250000357627869, + "reward_std": 0.3354101851582527, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1610 + }, + { + "completion_length": 110.35000610351562, + "epoch": 1.669777086573354, + "grad_norm": 0.4677380323410034, + "kl": 0.333984375, + "learning_rate": 1.4637822365423397e-06, + "loss": 0.0127, + "reward": 1.550000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 1611 + }, + { + "completion_length": 100.60000038146973, + "epoch": 1.6708138932089165, + "grad_norm": 0.29280760884284973, + "kl": 0.275146484375, + "learning_rate": 1.4619717271993828e-06, + "loss": 0.0177, + "reward": 1.7750000059604645, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 1.0, + "step": 1612 + }, + { + "completion_length": 207.77500534057617, + "epoch": 1.6718506998444789, + "grad_norm": 1.227375864982605, + "kl": 2.556640625, + "learning_rate": 1.4601612732924036e-06, + "loss": 0.2885, + "reward": 1.550000011920929, + "reward_std": 0.4363621845841408, + "rewards/accuracy_reward": 0.6250000223517418, + "rewards/format_reward": 0.9250000268220901, + "step": 1613 + }, + { + "completion_length": 159.8000030517578, + "epoch": 1.6728875064800415, + "grad_norm": 0.9971756339073181, + "kl": 1.3330078125, + "learning_rate": 1.4583508774606033e-06, + "loss": 0.0911, + "reward": 1.6500000059604645, + "reward_std": 0.24873404577374458, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9500000029802322, + "step": 1614 + }, + { + "completion_length": 268.62500953674316, + "epoch": 1.673924313115604, + "grad_norm": 2.016587495803833, + "kl": 1.754150390625, + "learning_rate": 1.4565405423430994e-06, + "loss": 0.1122, + "reward": 1.6500000357627869, + "reward_std": 0.3052687272429466, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9249999970197678, + "step": 1615 + }, + { + "completion_length": 108.97500228881836, + "epoch": 1.6749611197511665, + "grad_norm": 0.03610905632376671, + "kl": 0.216796875, + "learning_rate": 1.4547302705789198e-06, + "loss": 0.0083, + "reward": 1.75, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.75, + "rewards/format_reward": 1.0, + "step": 1616 + }, + { + "completion_length": 127.0, + "epoch": 1.6759979263867288, + "grad_norm": 0.3895142078399658, + "kl": 0.29296875, + "learning_rate": 1.4529200648070016e-06, + "loss": 0.0191, + "reward": 1.800000011920929, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 1.0, + "step": 1617 + }, + { + "completion_length": 167.25, + "epoch": 1.6770347330222912, + "grad_norm": 0.15201283991336823, + "kl": 0.28759765625, + "learning_rate": 1.4511099276661832e-06, + "loss": 0.0544, + "reward": 1.3499999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9750000089406967, + "step": 1618 + }, + { + "completion_length": 207.55000686645508, + "epoch": 1.6780715396578538, + "grad_norm": 0.21813489496707916, + "kl": 0.4111328125, + "learning_rate": 1.4492998617952062e-06, + "loss": 0.1047, + "reward": 1.5250000357627869, + "reward_std": 0.22360678017139435, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 1619 + }, + { + "completion_length": 171.8000030517578, + "epoch": 1.6791083462934164, + "grad_norm": 0.6411747336387634, + "kl": 1.269287109375, + "learning_rate": 1.447489869832706e-06, + "loss": 0.0934, + "reward": 1.3250000178813934, + "reward_std": 0.31719937548041344, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9500000029802322, + "step": 1620 + }, + { + "completion_length": 116.4749984741211, + "epoch": 1.6801451529289788, + "grad_norm": 1.831243634223938, + "kl": 0.17822265625, + "learning_rate": 1.4456799544172104e-06, + "loss": -0.0055, + "reward": 1.8499999940395355, + "reward_std": 0.23617041110992432, + "rewards/accuracy_reward": 0.8500000238418579, + "rewards/format_reward": 1.0, + "step": 1621 + }, + { + "completion_length": 184.05000686645508, + "epoch": 1.6811819595645412, + "grad_norm": 0.413693904876709, + "kl": 0.224853515625, + "learning_rate": 1.4438701181871363e-06, + "loss": 0.0633, + "reward": 1.5750000178813934, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.6000000201165676, + "rewards/format_reward": 0.9750000089406967, + "step": 1622 + }, + { + "completion_length": 113.0250015258789, + "epoch": 1.6822187662001036, + "grad_norm": 0.29346320033073425, + "kl": 0.248046875, + "learning_rate": 1.4420603637807857e-06, + "loss": -0.0001, + "reward": 1.5250000357627869, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 1.0, + "step": 1623 + }, + { + "completion_length": 262.75000381469727, + "epoch": 1.6832555728356662, + "grad_norm": 1.1153827905654907, + "kl": 1.1875, + "learning_rate": 1.4402506938363398e-06, + "loss": 0.1261, + "reward": 1.5, + "reward_std": 0.30463573709130287, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.925000011920929, + "step": 1624 + }, + { + "completion_length": 229.87500762939453, + "epoch": 1.6842923794712286, + "grad_norm": 1.9630428552627563, + "kl": 0.88818359375, + "learning_rate": 1.4384411109918577e-06, + "loss": 0.1332, + "reward": 1.7000000178813934, + "reward_std": 0.3479737974703312, + "rewards/accuracy_reward": 0.75, + "rewards/format_reward": 0.9500000029802322, + "step": 1625 + }, + { + "completion_length": 214.90000534057617, + "epoch": 1.6853291861067912, + "grad_norm": 0.46512189507484436, + "kl": 0.61181640625, + "learning_rate": 1.4366316178852714e-06, + "loss": 0.1527, + "reward": 1.6250000298023224, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9500000178813934, + "step": 1626 + }, + { + "completion_length": 116.64999961853027, + "epoch": 1.6863659927423535, + "grad_norm": 2.697237253189087, + "kl": 0.9498291015625, + "learning_rate": 1.4348222171543822e-06, + "loss": 0.095, + "reward": 1.550000011920929, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 1627 + }, + { + "completion_length": 184.7500057220459, + "epoch": 1.687402799377916, + "grad_norm": 0.5999835133552551, + "kl": 0.671630859375, + "learning_rate": 1.4330129114368554e-06, + "loss": 0.0842, + "reward": 1.675000011920929, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 1628 + }, + { + "completion_length": 204.97500228881836, + "epoch": 1.6884396060134785, + "grad_norm": 2.4206836223602295, + "kl": 1.625, + "learning_rate": 1.4312037033702205e-06, + "loss": 0.1712, + "reward": 1.75, + "reward_std": 0.32818931341171265, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.9000000059604645, + "step": 1629 + }, + { + "completion_length": 107.70000267028809, + "epoch": 1.689476412649041, + "grad_norm": 0.4894731044769287, + "kl": 0.219970703125, + "learning_rate": 1.4293945955918617e-06, + "loss": -0.0092, + "reward": 1.4500000178813934, + "reward_std": 0.06846531480550766, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 1.0, + "step": 1630 + }, + { + "completion_length": 176.52500343322754, + "epoch": 1.6905132192846035, + "grad_norm": 0.2627868354320526, + "kl": 1.17529296875, + "learning_rate": 1.4275855907390186e-06, + "loss": 0.1381, + "reward": 1.600000023841858, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1631 + }, + { + "completion_length": 219.15000343322754, + "epoch": 1.691550025920166, + "grad_norm": 4.7461466789245605, + "kl": 3.079345703125, + "learning_rate": 1.425776691448782e-06, + "loss": 0.3135, + "reward": 1.5999999940395355, + "reward_std": 0.2795085050165653, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000029802322, + "step": 1632 + }, + { + "completion_length": 201.17500495910645, + "epoch": 1.6925868325557283, + "grad_norm": 0.37242236733436584, + "kl": 3.163330078125, + "learning_rate": 1.4239679003580854e-06, + "loss": 0.3266, + "reward": 1.3000000268220901, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1633 + }, + { + "completion_length": 115.22500228881836, + "epoch": 1.6936236391912907, + "grad_norm": 4.702274799346924, + "kl": 0.3017578125, + "learning_rate": 1.4221592201037075e-06, + "loss": 0.0156, + "reward": 1.550000011920929, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.5749999973922968, + "rewards/format_reward": 0.9750000089406967, + "step": 1634 + }, + { + "completion_length": 270.3250045776367, + "epoch": 1.6946604458268533, + "grad_norm": 8.115036964416504, + "kl": 5.443359375, + "learning_rate": 1.420350653322264e-06, + "loss": 0.4798, + "reward": 1.475000023841858, + "reward_std": 0.32818933203816414, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9250000268220901, + "step": 1635 + }, + { + "completion_length": 218.47500610351562, + "epoch": 1.6956972524624159, + "grad_norm": 5.428323268890381, + "kl": 3.03564453125, + "learning_rate": 1.4185422026502062e-06, + "loss": 0.3195, + "reward": 1.1250000149011612, + "reward_std": 0.2812207005918026, + "rewards/accuracy_reward": 0.22500000335276127, + "rewards/format_reward": 0.9000000208616257, + "step": 1636 + }, + { + "completion_length": 201.3250026702881, + "epoch": 1.6967340590979783, + "grad_norm": 0.41541728377342224, + "kl": 2.009033203125, + "learning_rate": 1.416733870723815e-06, + "loss": 0.2323, + "reward": 1.6500000059604645, + "reward_std": 0.3354102112352848, + "rewards/accuracy_reward": 0.7250000238418579, + "rewards/format_reward": 0.925000011920929, + "step": 1637 + }, + { + "completion_length": 260.975004196167, + "epoch": 1.6977708657335406, + "grad_norm": 0.4679148197174072, + "kl": 2.0595703125, + "learning_rate": 1.4149256601791993e-06, + "loss": 0.3123, + "reward": 1.4500000178813934, + "reward_std": 0.3930240832269192, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.9250000268220901, + "step": 1638 + }, + { + "completion_length": 106.625, + "epoch": 1.698807672369103, + "grad_norm": 3.114511489868164, + "kl": 0.36279296875, + "learning_rate": 1.4131175736522897e-06, + "loss": 0.0242, + "reward": 1.9749999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.9750000089406967, + "rewards/format_reward": 1.0, + "step": 1639 + }, + { + "completion_length": 157.3499984741211, + "epoch": 1.6998444790046656, + "grad_norm": 0.44841909408569336, + "kl": 1.156982421875, + "learning_rate": 1.411309613778838e-06, + "loss": 0.1365, + "reward": 1.699999988079071, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1640 + }, + { + "completion_length": 271.9000072479248, + "epoch": 1.7008812856402282, + "grad_norm": 0.4403691291809082, + "kl": 0.539306640625, + "learning_rate": 1.4095017831944087e-06, + "loss": 0.1344, + "reward": 1.3500000089406967, + "reward_std": 0.2795085050165653, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9250000268220901, + "step": 1641 + }, + { + "completion_length": 118.9000015258789, + "epoch": 1.7019180922757906, + "grad_norm": 0.30528485774993896, + "kl": 0.2333984375, + "learning_rate": 1.4076940845343808e-06, + "loss": 0.0026, + "reward": 1.6500000059604645, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.6500000152736902, + "rewards/format_reward": 1.0, + "step": 1642 + }, + { + "completion_length": 167.05000114440918, + "epoch": 1.702954898911353, + "grad_norm": 1.617598295211792, + "kl": 0.653076171875, + "learning_rate": 1.4058865204339383e-06, + "loss": 0.0373, + "reward": 1.7000000178813934, + "reward_std": 0.15685367584228516, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1643 + }, + { + "completion_length": 242.39999771118164, + "epoch": 1.7039917055469154, + "grad_norm": 0.044586073607206345, + "kl": 2.680908203125, + "learning_rate": 1.4040790935280714e-06, + "loss": 0.3243, + "reward": 1.600000023841858, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9000000059604645, + "step": 1644 + }, + { + "completion_length": 118.62500190734863, + "epoch": 1.705028512182478, + "grad_norm": 1.295025110244751, + "kl": 0.28564453125, + "learning_rate": 1.4022718064515678e-06, + "loss": 0.0111, + "reward": 1.625, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 1.0, + "step": 1645 + }, + { + "completion_length": 202.25000190734863, + "epoch": 1.7060653188180406, + "grad_norm": 1.3782089948654175, + "kl": 0.6884765625, + "learning_rate": 1.4004646618390139e-06, + "loss": 0.1291, + "reward": 1.5, + "reward_std": 0.27228761836886406, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9500000029802322, + "step": 1646 + }, + { + "completion_length": 260.6250114440918, + "epoch": 1.707102125453603, + "grad_norm": 2.5833706855773926, + "kl": 2.291015625, + "learning_rate": 1.3986576623247865e-06, + "loss": 0.2426, + "reward": 1.4749999940395355, + "reward_std": 0.38109345734119415, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9000000208616257, + "step": 1647 + }, + { + "completion_length": 113.37500190734863, + "epoch": 1.7081389320891653, + "grad_norm": 0.9900299906730652, + "kl": 0.28271484375, + "learning_rate": 1.3968508105430525e-06, + "loss": 0.0018, + "reward": 1.625, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 1.0, + "step": 1648 + }, + { + "completion_length": 206.67500114440918, + "epoch": 1.7091757387247277, + "grad_norm": 0.37716108560562134, + "kl": 1.8212890625, + "learning_rate": 1.3950441091277612e-06, + "loss": 0.2549, + "reward": 1.4000000059604645, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.4500000011175871, + "rewards/format_reward": 0.9500000178813934, + "step": 1649 + }, + { + "completion_length": 188.55000114440918, + "epoch": 1.7102125453602903, + "grad_norm": 1.695388913154602, + "kl": 1.5244140625, + "learning_rate": 1.393237560712646e-06, + "loss": 0.1435, + "reward": 1.475000023841858, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9750000089406967, + "step": 1650 + }, + { + "completion_length": 160.92500495910645, + "epoch": 1.711249351995853, + "grad_norm": 1.9501590728759766, + "kl": 0.2236328125, + "learning_rate": 1.391431167931213e-06, + "loss": 0.0411, + "reward": 1.75, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1651 + }, + { + "completion_length": 202.375, + "epoch": 1.7122861586314153, + "grad_norm": 0.16232004761695862, + "kl": 1.083251953125, + "learning_rate": 1.389624933416746e-06, + "loss": 0.1748, + "reward": 1.6250000298023224, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9500000178813934, + "step": 1652 + }, + { + "completion_length": 176.90000343322754, + "epoch": 1.7133229652669777, + "grad_norm": 0.8067182898521423, + "kl": 1.25048828125, + "learning_rate": 1.3878188598022941e-06, + "loss": 0.1323, + "reward": 1.449999988079071, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1653 + }, + { + "completion_length": 179.80000114440918, + "epoch": 1.71435977190254, + "grad_norm": 0.28877392411231995, + "kl": 0.2696533203125, + "learning_rate": 1.3860129497206747e-06, + "loss": 0.0804, + "reward": 1.5999999940395355, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9750000089406967, + "step": 1654 + }, + { + "completion_length": 260.5000057220459, + "epoch": 1.7153965785381027, + "grad_norm": 0.5322703123092651, + "kl": 2.345947265625, + "learning_rate": 1.3842072058044649e-06, + "loss": 0.2565, + "reward": 1.5250000357627869, + "reward_std": 0.40387552976608276, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1655 + }, + { + "completion_length": 130.4000015258789, + "epoch": 1.716433385173665, + "grad_norm": 0.6268528699874878, + "kl": 0.223876953125, + "learning_rate": 1.382401630686001e-06, + "loss": 0.0517, + "reward": 1.7750000059604645, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 1.0, + "step": 1656 + }, + { + "completion_length": 116.57499885559082, + "epoch": 1.7174701918092277, + "grad_norm": 0.3194897174835205, + "kl": 0.26318359375, + "learning_rate": 1.3805962269973722e-06, + "loss": -0.0135, + "reward": 1.600000023841858, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 1.0, + "step": 1657 + }, + { + "completion_length": 319.850004196167, + "epoch": 1.71850699844479, + "grad_norm": 1.0248870849609375, + "kl": 4.117431640625, + "learning_rate": 1.3787909973704186e-06, + "loss": 0.4725, + "reward": 1.5000000447034836, + "reward_std": 0.40387551113963127, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 0.9000000208616257, + "step": 1658 + }, + { + "completion_length": 108.22500228881836, + "epoch": 1.7195438050803524, + "grad_norm": 3.075923204421997, + "kl": 0.233642578125, + "learning_rate": 1.3769859444367255e-06, + "loss": 0.0176, + "reward": 1.675000011920929, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 1.0, + "step": 1659 + }, + { + "completion_length": 210.6250057220459, + "epoch": 1.720580611715915, + "grad_norm": 0.2640899419784546, + "kl": 1.01953125, + "learning_rate": 1.3751810708276225e-06, + "loss": 0.1408, + "reward": 1.475000023841858, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.9500000029802322, + "step": 1660 + }, + { + "completion_length": 117.40000343322754, + "epoch": 1.7216174183514774, + "grad_norm": 1.2959752082824707, + "kl": 0.2353515625, + "learning_rate": 1.3733763791741748e-06, + "loss": 0.0243, + "reward": 1.550000011920929, + "reward_std": 0.06846532225608826, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 1.0, + "step": 1661 + }, + { + "completion_length": 197.52500343322754, + "epoch": 1.72265422498704, + "grad_norm": 0.5643317699432373, + "kl": 2.107177734375, + "learning_rate": 1.3715718721071858e-06, + "loss": 0.2302, + "reward": 1.725000023841858, + "reward_std": 0.40387552976608276, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 1662 + }, + { + "completion_length": 437.15000915527344, + "epoch": 1.7236910316226024, + "grad_norm": 1.0964477062225342, + "kl": 4.185546875, + "learning_rate": 1.3697675522571872e-06, + "loss": 0.5379, + "reward": 1.3000000417232513, + "reward_std": 0.6094099879264832, + "rewards/accuracy_reward": 0.475000012665987, + "rewards/format_reward": 0.8250000178813934, + "step": 1663 + }, + { + "completion_length": 219.9500026702881, + "epoch": 1.7247278382581648, + "grad_norm": 1.2283393144607544, + "kl": 1.347900390625, + "learning_rate": 1.367963422254439e-06, + "loss": 0.1927, + "reward": 1.425000011920929, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1664 + }, + { + "completion_length": 113.72500419616699, + "epoch": 1.7257646448937272, + "grad_norm": 0.6352521181106567, + "kl": 0.247802734375, + "learning_rate": 1.3661594847289239e-06, + "loss": 0.0097, + "reward": 1.625, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 1.0, + "step": 1665 + }, + { + "completion_length": 161.4500026702881, + "epoch": 1.7268014515292898, + "grad_norm": 0.9560681581497192, + "kl": 0.270263671875, + "learning_rate": 1.3643557423103448e-06, + "loss": 0.0937, + "reward": 1.800000011920929, + "reward_std": 0.22360679879784584, + "rewards/accuracy_reward": 0.8250000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 1666 + }, + { + "completion_length": 165.62500190734863, + "epoch": 1.7278382581648524, + "grad_norm": 0.7034723162651062, + "kl": 1.19287109375, + "learning_rate": 1.3625521976281192e-06, + "loss": 0.1538, + "reward": 1.5750000327825546, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9500000178813934, + "step": 1667 + }, + { + "completion_length": 258.9499988555908, + "epoch": 1.7288750648004148, + "grad_norm": 0.8078364133834839, + "kl": 1.2313232421875, + "learning_rate": 1.3607488533113774e-06, + "loss": 0.2207, + "reward": 1.2750000208616257, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9000000059604645, + "step": 1668 + }, + { + "completion_length": 167.100004196167, + "epoch": 1.7299118714359771, + "grad_norm": 0.26684144139289856, + "kl": 0.213623046875, + "learning_rate": 1.358945711988956e-06, + "loss": 0.0518, + "reward": 1.699999988079071, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1669 + }, + { + "completion_length": 162.95000076293945, + "epoch": 1.7309486780715395, + "grad_norm": 0.2954098880290985, + "kl": 1.203125, + "learning_rate": 1.3571427762893975e-06, + "loss": 0.107, + "reward": 1.5750000178813934, + "reward_std": 0.18026871606707573, + "rewards/accuracy_reward": 0.6000000033527613, + "rewards/format_reward": 0.9750000089406967, + "step": 1670 + }, + { + "completion_length": 266.37500953674316, + "epoch": 1.7319854847071021, + "grad_norm": 0.40566912293434143, + "kl": 1.091796875, + "learning_rate": 1.3553400488409442e-06, + "loss": 0.2136, + "reward": 1.574999988079071, + "reward_std": 0.27228759974241257, + "rewards/accuracy_reward": 0.6500000096857548, + "rewards/format_reward": 0.925000011920929, + "step": 1671 + }, + { + "completion_length": 275.875, + "epoch": 1.7330222913426647, + "grad_norm": 4.127599716186523, + "kl": 3.32080078125, + "learning_rate": 1.3535375322715338e-06, + "loss": 0.3764, + "reward": 1.449999988079071, + "reward_std": 0.5048274919390678, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.8750000149011612, + "step": 1672 + }, + { + "completion_length": 145.82500648498535, + "epoch": 1.734059097978227, + "grad_norm": 1.3361109495162964, + "kl": 1.259033203125, + "learning_rate": 1.351735229208798e-06, + "loss": 0.1036, + "reward": 1.425000011920929, + "reward_std": 0.20019175112247467, + "rewards/accuracy_reward": 0.4500000011175871, + "rewards/format_reward": 0.9750000089406967, + "step": 1673 + }, + { + "completion_length": 206.55000495910645, + "epoch": 1.7350959046137895, + "grad_norm": 1.2055398225784302, + "kl": 1.6787109375, + "learning_rate": 1.349933142280055e-06, + "loss": 0.1976, + "reward": 1.4750000536441803, + "reward_std": 0.4723408371210098, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.9250000268220901, + "step": 1674 + }, + { + "completion_length": 395.5500068664551, + "epoch": 1.7361327112493519, + "grad_norm": 0.9115234613418579, + "kl": 3.5263671875, + "learning_rate": 1.3481312741123117e-06, + "loss": 0.2621, + "reward": 1.375, + "reward_std": 0.4981010966002941, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.8500000089406967, + "step": 1675 + }, + { + "completion_length": 305.97501373291016, + "epoch": 1.7371695178849145, + "grad_norm": 2.5328431129455566, + "kl": 2.07373046875, + "learning_rate": 1.346329627332252e-06, + "loss": 0.3261, + "reward": 1.4500000178813934, + "reward_std": 0.3731010779738426, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.9000000059604645, + "step": 1676 + }, + { + "completion_length": 309.8000087738037, + "epoch": 1.738206324520477, + "grad_norm": 0.9459800720214844, + "kl": 1.197998046875, + "learning_rate": 1.34452820456624e-06, + "loss": 0.2595, + "reward": 1.1500000208616257, + "reward_std": 0.41643916070461273, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.8750000149011612, + "step": 1677 + }, + { + "completion_length": 153.8750057220459, + "epoch": 1.7392431311560395, + "grad_norm": 0.6364434957504272, + "kl": 0.249267578125, + "learning_rate": 1.3427270084403114e-06, + "loss": 0.0669, + "reward": 1.675000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9750000089406967, + "step": 1678 + }, + { + "completion_length": 255.24999809265137, + "epoch": 1.7402799377916018, + "grad_norm": 0.2327195107936859, + "kl": 2.372802734375, + "learning_rate": 1.3409260415801732e-06, + "loss": 0.3188, + "reward": 1.4250000417232513, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.925000011920929, + "step": 1679 + }, + { + "completion_length": 314.7250061035156, + "epoch": 1.7413167444271642, + "grad_norm": 4.738442897796631, + "kl": 1.777099609375, + "learning_rate": 1.3391253066111962e-06, + "loss": 0.2694, + "reward": 1.375, + "reward_std": 0.3606759384274483, + "rewards/accuracy_reward": 0.47500000707805157, + "rewards/format_reward": 0.9000000208616257, + "step": 1680 + }, + { + "completion_length": 243.1250057220459, + "epoch": 1.7423535510627268, + "grad_norm": 0.2917191982269287, + "kl": 2.29541015625, + "learning_rate": 1.3373248061584154e-06, + "loss": 0.3098, + "reward": 1.7750000059604645, + "reward_std": 0.25609345361590385, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1681 + }, + { + "completion_length": 298.89999771118164, + "epoch": 1.7433903576982894, + "grad_norm": 1.8764444589614868, + "kl": 2.10302734375, + "learning_rate": 1.3355245428465218e-06, + "loss": 0.2798, + "reward": 1.675000011920929, + "reward_std": 0.4415663853287697, + "rewards/accuracy_reward": 0.7750000059604645, + "rewards/format_reward": 0.9000000059604645, + "step": 1682 + }, + { + "completion_length": 393.37499809265137, + "epoch": 1.7444271643338518, + "grad_norm": 8.258137702941895, + "kl": 3.956787109375, + "learning_rate": 1.3337245192998627e-06, + "loss": 0.5742, + "reward": 1.3000000268220901, + "reward_std": 0.45318930596113205, + "rewards/accuracy_reward": 0.450000012293458, + "rewards/format_reward": 0.8500000089406967, + "step": 1683 + }, + { + "completion_length": 227.02500915527344, + "epoch": 1.7454639709694142, + "grad_norm": 1.1636691093444824, + "kl": 1.012939453125, + "learning_rate": 1.3319247381424333e-06, + "loss": 0.1633, + "reward": 1.5000000298023224, + "reward_std": 0.3479738235473633, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1684 + }, + { + "completion_length": 544.9750061035156, + "epoch": 1.7465007776049766, + "grad_norm": 2.0953562259674072, + "kl": 5.66015625, + "learning_rate": 1.330125201997879e-06, + "loss": 0.5279, + "reward": 1.1250000298023224, + "reward_std": 0.5607291907072067, + "rewards/accuracy_reward": 0.3500000052154064, + "rewards/format_reward": 0.7750000059604645, + "step": 1685 + }, + { + "completion_length": 348.375, + "epoch": 1.7475375842405392, + "grad_norm": 5.163078784942627, + "kl": 3.19091796875, + "learning_rate": 1.3283259134894844e-06, + "loss": 0.4759, + "reward": 1.4500000476837158, + "reward_std": 0.4604101926088333, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.8750000149011612, + "step": 1686 + }, + { + "completion_length": 400.35001373291016, + "epoch": 1.7485743908761016, + "grad_norm": 3.5182042121887207, + "kl": 4.67578125, + "learning_rate": 1.3265268752401755e-06, + "loss": 0.3921, + "reward": 1.1750000268220901, + "reward_std": 0.47234081104397774, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.800000011920929, + "step": 1687 + }, + { + "completion_length": 489.5500183105469, + "epoch": 1.7496111975116642, + "grad_norm": 0.4392010271549225, + "kl": 2.59814453125, + "learning_rate": 1.3247280898725125e-06, + "loss": 0.3865, + "reward": 1.3250000476837158, + "reward_std": 0.5649927109479904, + "rewards/accuracy_reward": 0.5500000156462193, + "rewards/format_reward": 0.7750000059604645, + "step": 1688 + }, + { + "completion_length": 481.375, + "epoch": 1.7506480041472265, + "grad_norm": 2.8733747005462646, + "kl": 4.671875, + "learning_rate": 1.3229295600086873e-06, + "loss": 0.5259, + "reward": 1.2250000089406967, + "reward_std": 0.6130405366420746, + "rewards/accuracy_reward": 0.4250000100582838, + "rewards/format_reward": 0.800000011920929, + "step": 1689 + }, + { + "completion_length": 292.2749996185303, + "epoch": 1.751684810782789, + "grad_norm": 1.616410732269287, + "kl": 1.1044921875, + "learning_rate": 1.3211312882705192e-06, + "loss": 0.2193, + "reward": 1.3250000327825546, + "reward_std": 0.3046357296407223, + "rewards/accuracy_reward": 0.4250000026077032, + "rewards/format_reward": 0.9000000059604645, + "step": 1690 + }, + { + "completion_length": 357.9250144958496, + "epoch": 1.7527216174183515, + "grad_norm": 1.5785585641860962, + "kl": 4.5654296875, + "learning_rate": 1.3193332772794515e-06, + "loss": 0.2922, + "reward": 1.2500000149011612, + "reward_std": 0.4415663853287697, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8250000178813934, + "step": 1691 + }, + { + "completion_length": 393.9750061035156, + "epoch": 1.753758424053914, + "grad_norm": 2.245445489883423, + "kl": 4.9765625, + "learning_rate": 1.3175355296565466e-06, + "loss": 0.6097, + "reward": 1.2000000327825546, + "reward_std": 0.41643913462758064, + "rewards/accuracy_reward": 0.37500000186264515, + "rewards/format_reward": 0.8250000178813934, + "step": 1692 + }, + { + "completion_length": 450.95001220703125, + "epoch": 1.7547952306894765, + "grad_norm": 1.7147431373596191, + "kl": 1.007568359375, + "learning_rate": 1.3157380480224844e-06, + "loss": 0.3057, + "reward": 1.350000023841858, + "reward_std": 0.6166309043765068, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.8250000178813934, + "step": 1693 + }, + { + "completion_length": 200.52500534057617, + "epoch": 1.755832037325039, + "grad_norm": 0.47567418217658997, + "kl": 0.35205078125, + "learning_rate": 1.3139408349975549e-06, + "loss": 0.1028, + "reward": 1.4250000417232513, + "reward_std": 0.3046357184648514, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 1694 + }, + { + "completion_length": 199.29999923706055, + "epoch": 1.7568688439606013, + "grad_norm": 2.4895272254943848, + "kl": 1.810546875, + "learning_rate": 1.3121438932016595e-06, + "loss": 0.2271, + "reward": 1.6250000298023224, + "reward_std": 0.27950847521424294, + "rewards/accuracy_reward": 0.6750000193715096, + "rewards/format_reward": 0.9500000178813934, + "step": 1695 + }, + { + "completion_length": 93.2750015258789, + "epoch": 1.7579056505961637, + "grad_norm": 0.4038166105747223, + "kl": 0.654296875, + "learning_rate": 1.3103472252543003e-06, + "loss": 0.0834, + "reward": 1.5250000208616257, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1696 + }, + { + "completion_length": 195.37500953674316, + "epoch": 1.7589424572317263, + "grad_norm": 2.357900381088257, + "kl": 2.333984375, + "learning_rate": 1.3085508337745844e-06, + "loss": 0.2501, + "reward": 1.725000023841858, + "reward_std": 0.3354101926088333, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 1697 + }, + { + "completion_length": 299.55000495910645, + "epoch": 1.7599792638672889, + "grad_norm": 4.066470146179199, + "kl": 3.29541015625, + "learning_rate": 1.306754721381213e-06, + "loss": 0.3886, + "reward": 1.5, + "reward_std": 0.3479738272726536, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9000000208616257, + "step": 1698 + }, + { + "completion_length": 195.05000114440918, + "epoch": 1.7610160705028512, + "grad_norm": 2.4610085487365723, + "kl": 2.865234375, + "learning_rate": 1.3049588906924814e-06, + "loss": 0.2184, + "reward": 1.4000000208616257, + "reward_std": 0.45977721735835075, + "rewards/accuracy_reward": 0.5249999985098839, + "rewards/format_reward": 0.8750000298023224, + "step": 1699 + }, + { + "completion_length": 143.95000457763672, + "epoch": 1.7620528771384136, + "grad_norm": 1.7119483947753906, + "kl": 1.673828125, + "learning_rate": 1.3031633443262742e-06, + "loss": 0.1461, + "reward": 1.425000011920929, + "reward_std": 0.2795084938406944, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 1700 + }, + { + "completion_length": 207.39999771118164, + "epoch": 1.763089683773976, + "grad_norm": 1.165863037109375, + "kl": 1.869873046875, + "learning_rate": 1.3013680849000619e-06, + "loss": 0.2443, + "reward": 1.625, + "reward_std": 0.2361704222857952, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.9500000178813934, + "step": 1701 + }, + { + "completion_length": 139.10000228881836, + "epoch": 1.7641264904095386, + "grad_norm": 1.0776835680007935, + "kl": 1.6533203125, + "learning_rate": 1.2995731150308957e-06, + "loss": 0.1518, + "reward": 1.875, + "reward_std": 0.22360678017139435, + "rewards/accuracy_reward": 0.9249999970197678, + "rewards/format_reward": 0.9500000029802322, + "step": 1702 + }, + { + "completion_length": 206.87500190734863, + "epoch": 1.7651632970451012, + "grad_norm": 2.873706340789795, + "kl": 2.02880859375, + "learning_rate": 1.2977784373354061e-06, + "loss": 0.2131, + "reward": 1.375, + "reward_std": 0.3479738235473633, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.9250000268220901, + "step": 1703 + }, + { + "completion_length": 253.75000381469727, + "epoch": 1.7662001036806636, + "grad_norm": 0.7443107962608337, + "kl": 1.505859375, + "learning_rate": 1.2959840544297957e-06, + "loss": 0.2283, + "reward": 1.4250000566244125, + "reward_std": 0.4038754962384701, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9250000268220901, + "step": 1704 + }, + { + "completion_length": 237.8499984741211, + "epoch": 1.767236910316226, + "grad_norm": 2.906121253967285, + "kl": 3.813720703125, + "learning_rate": 1.2941899689298402e-06, + "loss": 0.4106, + "reward": 1.7250000536441803, + "reward_std": 0.3913118876516819, + "rewards/accuracy_reward": 0.8000000268220901, + "rewards/format_reward": 0.9250000268220901, + "step": 1705 + }, + { + "completion_length": 139.92500114440918, + "epoch": 1.7682737169517884, + "grad_norm": 2.2459774017333984, + "kl": 1.25634765625, + "learning_rate": 1.2923961834508785e-06, + "loss": 0.0538, + "reward": 1.350000023841858, + "reward_std": 0.27950847521424294, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9250000268220901, + "step": 1706 + }, + { + "completion_length": 160.55000686645508, + "epoch": 1.769310523587351, + "grad_norm": 0.6946339011192322, + "kl": 1.557861328125, + "learning_rate": 1.2906027006078152e-06, + "loss": 0.1377, + "reward": 1.7750000059604645, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.8250000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 1707 + }, + { + "completion_length": 193.47500610351562, + "epoch": 1.7703473302229136, + "grad_norm": 2.409801483154297, + "kl": 2.78955078125, + "learning_rate": 1.288809523015111e-06, + "loss": 0.3181, + "reward": 1.3250000178813934, + "reward_std": 0.48867350444197655, + "rewards/accuracy_reward": 0.4249999951571226, + "rewards/format_reward": 0.9000000208616257, + "step": 1708 + }, + { + "completion_length": 150.92500114440918, + "epoch": 1.771384136858476, + "grad_norm": 0.6411232948303223, + "kl": 0.862548828125, + "learning_rate": 1.287016653286784e-06, + "loss": 0.1047, + "reward": 1.425000011920929, + "reward_std": 0.11180340498685837, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9750000089406967, + "step": 1709 + }, + { + "completion_length": 294.50000190734863, + "epoch": 1.7724209434940383, + "grad_norm": 3.621253490447998, + "kl": 3.5322265625, + "learning_rate": 1.2852240940364018e-06, + "loss": 0.4518, + "reward": 1.4000000059604645, + "reward_std": 0.38943372294306755, + "rewards/accuracy_reward": 0.5000000037252903, + "rewards/format_reward": 0.9000000208616257, + "step": 1710 + }, + { + "completion_length": 248.05000686645508, + "epoch": 1.7734577501296007, + "grad_norm": 1.0850621461868286, + "kl": 1.03369140625, + "learning_rate": 1.28343184787708e-06, + "loss": 0.204, + "reward": 1.6250000298023224, + "reward_std": 0.4525563307106495, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9250000268220901, + "step": 1711 + }, + { + "completion_length": 188.02500343322754, + "epoch": 1.7744945567651633, + "grad_norm": 2.1935651302337646, + "kl": 1.64306640625, + "learning_rate": 1.2816399174214785e-06, + "loss": 0.1811, + "reward": 1.725000023841858, + "reward_std": 0.2795085124671459, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 1712 + }, + { + "completion_length": 195.0000057220459, + "epoch": 1.775531363400726, + "grad_norm": 1.4455136060714722, + "kl": 1.5380859375, + "learning_rate": 1.2798483052817956e-06, + "loss": 0.183, + "reward": 1.5250000059604645, + "reward_std": 0.3354101963341236, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1713 + }, + { + "completion_length": 151.30000686645508, + "epoch": 1.7765681700362883, + "grad_norm": 1.1415375471115112, + "kl": 0.890869140625, + "learning_rate": 1.2780570140697667e-06, + "loss": 0.0601, + "reward": 1.675000011920929, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9750000089406967, + "step": 1714 + }, + { + "completion_length": 289.45000076293945, + "epoch": 1.7776049766718507, + "grad_norm": 1.0119621753692627, + "kl": 1.9130859375, + "learning_rate": 1.2762660463966584e-06, + "loss": 0.1743, + "reward": 1.5250000059604645, + "reward_std": 0.373101070523262, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9000000059604645, + "step": 1715 + }, + { + "completion_length": 152.47500228881836, + "epoch": 1.778641783307413, + "grad_norm": 0.6372405886650085, + "kl": 2.10205078125, + "learning_rate": 1.2744754048732672e-06, + "loss": 0.2332, + "reward": 1.8000000417232513, + "reward_std": 0.3479737974703312, + "rewards/accuracy_reward": 0.8500000238418579, + "rewards/format_reward": 0.9500000178813934, + "step": 1716 + }, + { + "completion_length": 294.5750045776367, + "epoch": 1.7796785899429757, + "grad_norm": 0.4199138283729553, + "kl": 3.05908203125, + "learning_rate": 1.2726850921099117e-06, + "loss": 0.4174, + "reward": 1.4250000566244125, + "reward_std": 0.3913118727505207, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.9000000357627869, + "step": 1717 + }, + { + "completion_length": 103.77499961853027, + "epoch": 1.780715396578538, + "grad_norm": 0.9671004414558411, + "kl": 0.2900390625, + "learning_rate": 1.2708951107164345e-06, + "loss": -0.0021, + "reward": 1.5000000298023224, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 1.0, + "step": 1718 + }, + { + "completion_length": 206.975004196167, + "epoch": 1.7817522032141007, + "grad_norm": 2.2271177768707275, + "kl": 2.0693359375, + "learning_rate": 1.2691054633021914e-06, + "loss": 0.2187, + "reward": 1.2750000208616257, + "reward_std": 0.3371223844587803, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.8750000298023224, + "step": 1719 + }, + { + "completion_length": 155.97500610351562, + "epoch": 1.782789009849663, + "grad_norm": 0.91446453332901, + "kl": 0.313720703125, + "learning_rate": 1.2673161524760548e-06, + "loss": 0.0516, + "reward": 1.6500000357627869, + "reward_std": 0.23617041110992432, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9500000178813934, + "step": 1720 + }, + { + "completion_length": 160.30000495910645, + "epoch": 1.7838258164852254, + "grad_norm": 0.9384238719940186, + "kl": 1.2392578125, + "learning_rate": 1.2655271808464041e-06, + "loss": 0.1296, + "reward": 1.600000023841858, + "reward_std": 0.24873404204845428, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9750000089406967, + "step": 1721 + }, + { + "completion_length": 204.350004196167, + "epoch": 1.784862623120788, + "grad_norm": 0.4190017879009247, + "kl": 2.197021484375, + "learning_rate": 1.2637385510211255e-06, + "loss": 0.2493, + "reward": 1.300000011920929, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.3500000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 1722 + }, + { + "completion_length": 209.25000190734863, + "epoch": 1.7858994297563504, + "grad_norm": 0.058677058666944504, + "kl": 1.091796875, + "learning_rate": 1.261950265607606e-06, + "loss": 0.1672, + "reward": 1.6250000298023224, + "reward_std": 0.23617040365934372, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9500000029802322, + "step": 1723 + }, + { + "completion_length": 241.8750057220459, + "epoch": 1.786936236391913, + "grad_norm": 2.870100259780884, + "kl": 2.03955078125, + "learning_rate": 1.2601623272127315e-06, + "loss": 0.2512, + "reward": 1.3000000268220901, + "reward_std": 0.39365706965327263, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 0.9000000208616257, + "step": 1724 + }, + { + "completion_length": 148.17499923706055, + "epoch": 1.7879730430274754, + "grad_norm": 0.45264932513237, + "kl": 0.904296875, + "learning_rate": 1.2583747384428813e-06, + "loss": 0.1311, + "reward": 1.3750000298023224, + "reward_std": 0.18026870489120483, + "rewards/accuracy_reward": 0.4000000022351742, + "rewards/format_reward": 0.9750000089406967, + "step": 1725 + }, + { + "completion_length": 153.14999771118164, + "epoch": 1.7890098496630378, + "grad_norm": 1.5778464078903198, + "kl": 1.006591796875, + "learning_rate": 1.2565875019039259e-06, + "loss": 0.1198, + "reward": 1.4250000417232513, + "reward_std": 0.24873404949903488, + "rewards/accuracy_reward": 0.44999999925494194, + "rewards/format_reward": 0.9750000089406967, + "step": 1726 + }, + { + "completion_length": 303.72500801086426, + "epoch": 1.7900466562986002, + "grad_norm": 1.394773006439209, + "kl": 3.194580078125, + "learning_rate": 1.2548006202012198e-06, + "loss": 0.4276, + "reward": 1.5500000417232513, + "reward_std": 0.40387551486492157, + "rewards/accuracy_reward": 0.6499999985098839, + "rewards/format_reward": 0.9000000208616257, + "step": 1727 + }, + { + "completion_length": 395.25000762939453, + "epoch": 1.7910834629341628, + "grad_norm": 2.0842819213867188, + "kl": 3.142578125, + "learning_rate": 1.2530140959396049e-06, + "loss": 0.4972, + "reward": 1.350000023841858, + "reward_std": 0.7085113078355789, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.8500000238418579, + "step": 1728 + }, + { + "completion_length": 93.37500190734863, + "epoch": 1.7921202695697254, + "grad_norm": 1.1371591091156006, + "kl": 0.28076171875, + "learning_rate": 1.2512279317233968e-06, + "loss": 0.0116, + "reward": 1.5000000298023224, + "reward_std": 0.13693062961101532, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 1.0, + "step": 1729 + }, + { + "completion_length": 240.55000114440918, + "epoch": 1.7931570762052877, + "grad_norm": 0.3758859634399414, + "kl": 3.274658203125, + "learning_rate": 1.2494421301563904e-06, + "loss": 0.3887, + "reward": 1.5750000476837158, + "reward_std": 0.37844377011060715, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9249999970197678, + "step": 1730 + }, + { + "completion_length": 247.3250026702881, + "epoch": 1.7941938828408501, + "grad_norm": 13.461213111877441, + "kl": 4.13427734375, + "learning_rate": 1.2476566938418494e-06, + "loss": 0.4357, + "reward": 1.2500000447034836, + "reward_std": 0.30463574454188347, + "rewards/accuracy_reward": 0.3250000048428774, + "rewards/format_reward": 0.925000011920929, + "step": 1731 + }, + { + "completion_length": 164.7750072479248, + "epoch": 1.7952306894764125, + "grad_norm": 0.4518619775772095, + "kl": 1.48828125, + "learning_rate": 1.2458716253825062e-06, + "loss": 0.1191, + "reward": 1.2000000327825546, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.22500000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1732 + }, + { + "completion_length": 177.95000457763672, + "epoch": 1.7962674961119751, + "grad_norm": 1.2171616554260254, + "kl": 0.61328125, + "learning_rate": 1.2440869273805566e-06, + "loss": 0.1541, + "reward": 1.5750000178813934, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9500000178813934, + "step": 1733 + }, + { + "completion_length": 156.32500076293945, + "epoch": 1.7973043027475377, + "grad_norm": 2.8522934913635254, + "kl": 1.1611328125, + "learning_rate": 1.242302602437657e-06, + "loss": 0.114, + "reward": 1.725000023841858, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.75, + "rewards/format_reward": 0.9750000089406967, + "step": 1734 + }, + { + "completion_length": 140.27499961853027, + "epoch": 1.7983411093831, + "grad_norm": 0.3764083683490753, + "kl": 1.037841796875, + "learning_rate": 1.2405186531549181e-06, + "loss": 0.0853, + "reward": 1.699999988079071, + "reward_std": 0.20019175112247467, + "rewards/accuracy_reward": 0.7250000163912773, + "rewards/format_reward": 0.9750000089406967, + "step": 1735 + }, + { + "completion_length": 194.6750087738037, + "epoch": 1.7993779160186625, + "grad_norm": 1.3961740732192993, + "kl": 0.92138671875, + "learning_rate": 1.2387350821329061e-06, + "loss": 0.1744, + "reward": 1.725000023841858, + "reward_std": 0.2920720987021923, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 1736 + }, + { + "completion_length": 257.575008392334, + "epoch": 1.8004147226542249, + "grad_norm": 1.2615939378738403, + "kl": 1.832275390625, + "learning_rate": 1.2369518919716318e-06, + "loss": 0.2549, + "reward": 1.5000000447034836, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.5750000178813934, + "rewards/format_reward": 0.9250000268220901, + "step": 1737 + }, + { + "completion_length": 250.00000762939453, + "epoch": 1.8014515292897875, + "grad_norm": 0.34040528535842896, + "kl": 1.0751953125, + "learning_rate": 1.2351690852705553e-06, + "loss": 0.2489, + "reward": 1.375, + "reward_std": 0.3354102037847042, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9250000268220901, + "step": 1738 + }, + { + "completion_length": 208.95000648498535, + "epoch": 1.80248833592535, + "grad_norm": 0.7414734959602356, + "kl": 2.16748046875, + "learning_rate": 1.2333866646285734e-06, + "loss": 0.2397, + "reward": 1.4500000178813934, + "reward_std": 0.22360680624842644, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.9500000029802322, + "step": 1739 + }, + { + "completion_length": 147.5250015258789, + "epoch": 1.8035251425609125, + "grad_norm": 0.5828838348388672, + "kl": 1.392822265625, + "learning_rate": 1.2316046326440236e-06, + "loss": 0.1437, + "reward": 1.6000000536441803, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1740 + }, + { + "completion_length": 190.62500190734863, + "epoch": 1.8045619491964748, + "grad_norm": 1.4413193464279175, + "kl": 2.00341796875, + "learning_rate": 1.2298229919146742e-06, + "loss": 0.2509, + "reward": 1.4000000059604645, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.45000000298023224, + "rewards/format_reward": 0.9500000178813934, + "step": 1741 + }, + { + "completion_length": 146.20000648498535, + "epoch": 1.8055987558320372, + "grad_norm": 0.4844883382320404, + "kl": 0.25341796875, + "learning_rate": 1.228041745037725e-06, + "loss": 0.0718, + "reward": 1.4000000059604645, + "reward_std": 0.1442900486290455, + "rewards/accuracy_reward": 0.4250000063329935, + "rewards/format_reward": 0.9750000089406967, + "step": 1742 + }, + { + "completion_length": 207.55000114440918, + "epoch": 1.8066355624675998, + "grad_norm": 2.3084030151367188, + "kl": 0.81298828125, + "learning_rate": 1.2262608946098004e-06, + "loss": 0.1324, + "reward": 1.5750000178813934, + "reward_std": 0.26129766181111336, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9500000029802322, + "step": 1743 + }, + { + "completion_length": 158.89999961853027, + "epoch": 1.8076723691031624, + "grad_norm": 0.41830238699913025, + "kl": 2.94482421875, + "learning_rate": 1.224480443226948e-06, + "loss": 0.3097, + "reward": 1.675000011920929, + "reward_std": 0.3913118727505207, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9500000029802322, + "step": 1744 + }, + { + "completion_length": 293.1750030517578, + "epoch": 1.8087091757387248, + "grad_norm": 3.3972856998443604, + "kl": 4.56640625, + "learning_rate": 1.2227003934846327e-06, + "loss": 0.4965, + "reward": 1.625, + "reward_std": 0.4614894241094589, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 1745 + }, + { + "completion_length": 394.25000381469727, + "epoch": 1.8097459823742872, + "grad_norm": 2.418309450149536, + "kl": 1.626220703125, + "learning_rate": 1.220920747977735e-06, + "loss": 0.271, + "reward": 0.925000011920929, + "reward_std": 0.3966546207666397, + "rewards/accuracy_reward": 0.10000000335276127, + "rewards/format_reward": 0.8250000029802322, + "step": 1746 + }, + { + "completion_length": 243.85000801086426, + "epoch": 1.8107827890098496, + "grad_norm": 0.8050068020820618, + "kl": 2.1611328125, + "learning_rate": 1.2191415093005438e-06, + "loss": 0.2769, + "reward": 1.5, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.9250000268220901, + "step": 1747 + }, + { + "completion_length": 212.2000026702881, + "epoch": 1.8118195956454122, + "grad_norm": 0.7463563680648804, + "kl": 0.81689453125, + "learning_rate": 1.2173626800467587e-06, + "loss": 0.141, + "reward": 1.3750000149011612, + "reward_std": 0.3913118802011013, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9000000208616257, + "step": 1748 + }, + { + "completion_length": 89.70000267028809, + "epoch": 1.8128564022809746, + "grad_norm": 0.7790433168411255, + "kl": 0.46875, + "learning_rate": 1.2155842628094786e-06, + "loss": 0.0045, + "reward": 1.5999999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 1.0, + "step": 1749 + }, + { + "completion_length": 209.25000190734863, + "epoch": 1.8138932089165372, + "grad_norm": 1.4553327560424805, + "kl": 1.29931640625, + "learning_rate": 1.2138062601812042e-06, + "loss": 0.1713, + "reward": 1.5, + "reward_std": 0.27950848639011383, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 1750 + }, + { + "completion_length": 99.82500267028809, + "epoch": 1.8149300155520995, + "grad_norm": 0.4023449122905731, + "kl": 0.25341796875, + "learning_rate": 1.2120286747538314e-06, + "loss": -0.0057, + "reward": 1.675000011920929, + "reward_std": 0.06846531480550766, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 1.0, + "step": 1751 + }, + { + "completion_length": 215.1750030517578, + "epoch": 1.815966822187662, + "grad_norm": 5.118110656738281, + "kl": 2.81103515625, + "learning_rate": 1.210251509118647e-06, + "loss": 0.293, + "reward": 1.5250000059604645, + "reward_std": 0.1369306445121765, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000029802322, + "step": 1752 + }, + { + "completion_length": 104.70000267028809, + "epoch": 1.8170036288232243, + "grad_norm": 0.2352413386106491, + "kl": 0.262451171875, + "learning_rate": 1.208474765866327e-06, + "loss": 0.0062, + "reward": 1.7249999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 1.0, + "step": 1753 + }, + { + "completion_length": 170.12500190734863, + "epoch": 1.818040435458787, + "grad_norm": 0.32878950238227844, + "kl": 0.37255859375, + "learning_rate": 1.206698447586931e-06, + "loss": 0.0954, + "reward": 1.5999999940395355, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1754 + }, + { + "completion_length": 154.725004196167, + "epoch": 1.8190772420943495, + "grad_norm": 1.2951363325119019, + "kl": 1.0927734375, + "learning_rate": 1.2049225568698992e-06, + "loss": 0.1117, + "reward": 1.4000000059604645, + "reward_std": 0.12436701729893684, + "rewards/accuracy_reward": 0.42499999701976776, + "rewards/format_reward": 0.9750000089406967, + "step": 1755 + }, + { + "completion_length": 155.22500038146973, + "epoch": 1.820114048729912, + "grad_norm": 0.4612466096878052, + "kl": 1.195068359375, + "learning_rate": 1.2031470963040483e-06, + "loss": 0.1253, + "reward": 1.5250000357627869, + "reward_std": 0.4290027692914009, + "rewards/accuracy_reward": 0.5500000007450581, + "rewards/format_reward": 0.9750000089406967, + "step": 1756 + }, + { + "completion_length": 220.72500038146973, + "epoch": 1.8211508553654743, + "grad_norm": 1.5667455196380615, + "kl": 2.27685546875, + "learning_rate": 1.201372068477569e-06, + "loss": 0.2602, + "reward": 1.6250000298023224, + "reward_std": 0.2812206968665123, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1757 + }, + { + "completion_length": 201.92500495910645, + "epoch": 1.8221876620010367, + "grad_norm": 1.2501370906829834, + "kl": 2.5498046875, + "learning_rate": 1.1995974759780194e-06, + "loss": 0.2853, + "reward": 1.425000011920929, + "reward_std": 0.16770508512854576, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9500000178813934, + "step": 1758 + }, + { + "completion_length": 114.87500190734863, + "epoch": 1.8232244686365993, + "grad_norm": 0.5402330756187439, + "kl": 0.31103515625, + "learning_rate": 1.197823321392325e-06, + "loss": 0.0147, + "reward": 1.800000011920929, + "reward_std": 0.12436702102422714, + "rewards/accuracy_reward": 0.7999999970197678, + "rewards/format_reward": 1.0, + "step": 1759 + }, + { + "completion_length": 205.850004196167, + "epoch": 1.8242612752721619, + "grad_norm": 0.4346194267272949, + "kl": 1.8212890625, + "learning_rate": 1.1960496073067707e-06, + "loss": 0.2358, + "reward": 1.425000011920929, + "reward_std": 0.16770510375499725, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9500000029802322, + "step": 1760 + }, + { + "completion_length": 156.875, + "epoch": 1.8252980819077242, + "grad_norm": 1.3507357835769653, + "kl": 1.203369140625, + "learning_rate": 1.1942763363070014e-06, + "loss": 0.0688, + "reward": 1.600000023841858, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1761 + }, + { + "completion_length": 296.3500118255615, + "epoch": 1.8263348885432866, + "grad_norm": 4.305059432983398, + "kl": 3.052001953125, + "learning_rate": 1.1925035109780146e-06, + "loss": 0.3334, + "reward": 1.5, + "reward_std": 0.38409097492694855, + "rewards/accuracy_reward": 0.5999999977648258, + "rewards/format_reward": 0.9000000059604645, + "step": 1762 + }, + { + "completion_length": 155.44999885559082, + "epoch": 1.827371695178849, + "grad_norm": 0.37885773181915283, + "kl": 1.04296875, + "learning_rate": 1.1907311339041592e-06, + "loss": 0.0823, + "reward": 1.8250000178813934, + "reward_std": 0.27228759601712227, + "rewards/accuracy_reward": 0.8500000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1763 + }, + { + "completion_length": 210.77499961853027, + "epoch": 1.8284085018144116, + "grad_norm": 0.6505246758460999, + "kl": 0.936767578125, + "learning_rate": 1.1889592076691297e-06, + "loss": 0.1182, + "reward": 1.4500000178813934, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.5, + "rewards/format_reward": 0.9500000178813934, + "step": 1764 + }, + { + "completion_length": 102.07500267028809, + "epoch": 1.8294453084499742, + "grad_norm": 0.9405878782272339, + "kl": 0.290283203125, + "learning_rate": 1.1871877348559639e-06, + "loss": 0.005, + "reward": 1.5, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1765 + }, + { + "completion_length": 155.0250015258789, + "epoch": 1.8304821150855366, + "grad_norm": 0.451928049325943, + "kl": 1.29931640625, + "learning_rate": 1.1854167180470385e-06, + "loss": 0.1072, + "reward": 1.6250000298023224, + "reward_std": 0.22531898319721222, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.9750000089406967, + "step": 1766 + }, + { + "completion_length": 147.97500228881836, + "epoch": 1.831518921721099, + "grad_norm": 1.0946840047836304, + "kl": 0.22412109375, + "learning_rate": 1.183646159824066e-06, + "loss": 0.0549, + "reward": 1.5750000029802322, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.6000000014901161, + "rewards/format_reward": 0.9750000089406967, + "step": 1767 + }, + { + "completion_length": 156.4250030517578, + "epoch": 1.8325557283566614, + "grad_norm": 0.3518161475658417, + "kl": 0.310546875, + "learning_rate": 1.181876062768089e-06, + "loss": 0.0465, + "reward": 1.6500000059604645, + "reward_std": 0.36053745448589325, + "rewards/accuracy_reward": 0.6750000044703484, + "rewards/format_reward": 0.9750000089406967, + "step": 1768 + }, + { + "completion_length": 206.225004196167, + "epoch": 1.833592534992224, + "grad_norm": 0.8643704056739807, + "kl": 1.40771484375, + "learning_rate": 1.1801064294594797e-06, + "loss": 0.2228, + "reward": 1.6500000059604645, + "reward_std": 0.3354101963341236, + "rewards/accuracy_reward": 0.7000000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1769 + }, + { + "completion_length": 223.64999961853027, + "epoch": 1.8346293416277866, + "grad_norm": 0.5106009840965271, + "kl": 1.833251953125, + "learning_rate": 1.178337262477932e-06, + "loss": 0.1917, + "reward": 1.5250000059604645, + "reward_std": 0.25609344616532326, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9500000178813934, + "step": 1770 + }, + { + "completion_length": 171.9250087738037, + "epoch": 1.835666148263349, + "grad_norm": 1.0903598070144653, + "kl": 1.017822265625, + "learning_rate": 1.1765685644024624e-06, + "loss": 0.0995, + "reward": 1.7500000298023224, + "reward_std": 0.18026871234178543, + "rewards/accuracy_reward": 0.7749999985098839, + "rewards/format_reward": 0.9750000089406967, + "step": 1771 + }, + { + "completion_length": 269.050012588501, + "epoch": 1.8367029548989113, + "grad_norm": 1.58475661277771, + "kl": 1.67724609375, + "learning_rate": 1.1748003378114023e-06, + "loss": 0.1046, + "reward": 1.5000000298023224, + "reward_std": 0.42963574454188347, + "rewards/accuracy_reward": 0.5750000067055225, + "rewards/format_reward": 0.9249999970197678, + "step": 1772 + }, + { + "completion_length": 124.97500228881836, + "epoch": 1.8377397615344737, + "grad_norm": 0.42571085691452026, + "kl": 0.212646484375, + "learning_rate": 1.1730325852823965e-06, + "loss": 0.0375, + "reward": 1.4749999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 1.0, + "step": 1773 + }, + { + "completion_length": 190.22500228881836, + "epoch": 1.8387765681700363, + "grad_norm": 2.391396999359131, + "kl": 0.664306640625, + "learning_rate": 1.1712653093923977e-06, + "loss": 0.1056, + "reward": 1.6250000298023224, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9750000089406967, + "step": 1774 + }, + { + "completion_length": 178.65000534057617, + "epoch": 1.839813374805599, + "grad_norm": 0.3581797480583191, + "kl": 1.29443359375, + "learning_rate": 1.1694985127176657e-06, + "loss": 0.1552, + "reward": 1.6000000536441803, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1775 + }, + { + "completion_length": 182.50000381469727, + "epoch": 1.8408501814411613, + "grad_norm": 1.3145884275436401, + "kl": 0.640625, + "learning_rate": 1.1677321978337596e-06, + "loss": 0.003, + "reward": 1.475000023841858, + "reward_std": 0.21275537088513374, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1776 + }, + { + "completion_length": 226.77501106262207, + "epoch": 1.8418869880767237, + "grad_norm": 0.6741175055503845, + "kl": 1.28173828125, + "learning_rate": 1.165966367315538e-06, + "loss": 0.1894, + "reward": 1.8499999940395355, + "reward_std": 0.2795085124671459, + "rewards/accuracy_reward": 0.9000000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 1777 + }, + { + "completion_length": 273.4250068664551, + "epoch": 1.842923794712286, + "grad_norm": 0.37939608097076416, + "kl": 1.74267578125, + "learning_rate": 1.164201023737151e-06, + "loss": 0.2582, + "reward": 1.425000011920929, + "reward_std": 0.39302409440279007, + "rewards/accuracy_reward": 0.5000000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 1778 + }, + { + "completion_length": 263.87500381469727, + "epoch": 1.8439606013478487, + "grad_norm": 2.4952118396759033, + "kl": 1.37109375, + "learning_rate": 1.1624361696720424e-06, + "loss": 0.1401, + "reward": 1.4500000476837158, + "reward_std": 0.45977719873189926, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.925000011920929, + "step": 1779 + }, + { + "completion_length": 228.975004196167, + "epoch": 1.844997407983411, + "grad_norm": 7.276212692260742, + "kl": 1.14599609375, + "learning_rate": 1.1606718076929387e-06, + "loss": 0.1986, + "reward": 1.550000011920929, + "reward_std": 0.2920721210539341, + "rewards/accuracy_reward": 0.6000000052154064, + "rewards/format_reward": 0.9500000178813934, + "step": 1780 + }, + { + "completion_length": 143.85000228881836, + "epoch": 1.8460342146189737, + "grad_norm": 2.0451505184173584, + "kl": 0.3984375, + "learning_rate": 1.158907940371852e-06, + "loss": 0.0495, + "reward": 1.4500000178813934, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9750000089406967, + "step": 1781 + }, + { + "completion_length": 127.5250015258789, + "epoch": 1.847071021254536, + "grad_norm": 0.33295977115631104, + "kl": 0.20556640625, + "learning_rate": 1.1571445702800712e-06, + "loss": 0.018, + "reward": 1.5999999940395355, + "reward_std": 0.16770509257912636, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 1.0, + "step": 1782 + }, + { + "completion_length": 121.67500114440918, + "epoch": 1.8481078278900984, + "grad_norm": 0.32990562915802, + "kl": 0.212890625, + "learning_rate": 1.1553816999881617e-06, + "loss": -0.0039, + "reward": 1.7249999940395355, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.7250000014901161, + "rewards/format_reward": 1.0, + "step": 1783 + }, + { + "completion_length": 253.75000381469727, + "epoch": 1.8491446345256608, + "grad_norm": 0.3000059127807617, + "kl": 1.16552734375, + "learning_rate": 1.1536193320659596e-06, + "loss": 0.189, + "reward": 1.5250000059604645, + "reward_std": 0.21638590097427368, + "rewards/accuracy_reward": 0.5749999973922968, + "rewards/format_reward": 0.9500000178813934, + "step": 1784 + }, + { + "completion_length": 271.82500076293945, + "epoch": 1.8501814411612234, + "grad_norm": 1.5088744163513184, + "kl": 3.0732421875, + "learning_rate": 1.1518574690825691e-06, + "loss": 0.3893, + "reward": 1.6750000417232513, + "reward_std": 0.3913118988275528, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 1785 + }, + { + "completion_length": 172.65000343322754, + "epoch": 1.851218247796786, + "grad_norm": 5.147366523742676, + "kl": 1.6201171875, + "learning_rate": 1.150096113606358e-06, + "loss": 0.2063, + "reward": 1.675000011920929, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.7000000029802322, + "rewards/format_reward": 0.9750000089406967, + "step": 1786 + }, + { + "completion_length": 314.1750030517578, + "epoch": 1.8522550544323484, + "grad_norm": 2.1646902561187744, + "kl": 4.66015625, + "learning_rate": 1.1483352682049552e-06, + "loss": 0.5139, + "reward": 1.300000011920929, + "reward_std": 0.5409446619451046, + "rewards/accuracy_reward": 0.4250000063329935, + "rewards/format_reward": 0.8750000298023224, + "step": 1787 + }, + { + "completion_length": 228.42501068115234, + "epoch": 1.8532918610679108, + "grad_norm": 2.4354076385498047, + "kl": 0.26904296875, + "learning_rate": 1.1465749354452435e-06, + "loss": 0.0876, + "reward": 1.625, + "reward_std": 0.20019174739718437, + "rewards/accuracy_reward": 0.6750000007450581, + "rewards/format_reward": 0.9500000178813934, + "step": 1788 + }, + { + "completion_length": 123.35000228881836, + "epoch": 1.8543286677034732, + "grad_norm": 0.36107137799263, + "kl": 0.206298828125, + "learning_rate": 1.1448151178933613e-06, + "loss": 0.0454, + "reward": 1.5750000178813934, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.5749999992549419, + "rewards/format_reward": 1.0, + "step": 1789 + }, + { + "completion_length": 175.1500072479248, + "epoch": 1.8553654743390358, + "grad_norm": 1.548614501953125, + "kl": 0.971923828125, + "learning_rate": 1.1430558181146956e-06, + "loss": 0.1015, + "reward": 1.2000000178813934, + "reward_std": 0.2795084975659847, + "rewards/accuracy_reward": 0.22500000707805157, + "rewards/format_reward": 0.9750000089406967, + "step": 1790 + }, + { + "completion_length": 266.82501220703125, + "epoch": 1.8564022809745984, + "grad_norm": 1.0396159887313843, + "kl": 1.69189453125, + "learning_rate": 1.1412970386738757e-06, + "loss": 0.2626, + "reward": 1.3000000268220901, + "reward_std": 0.35704541578888893, + "rewards/accuracy_reward": 0.37500001303851604, + "rewards/format_reward": 0.925000011920929, + "step": 1791 + }, + { + "completion_length": 272.77500915527344, + "epoch": 1.8574390876101607, + "grad_norm": 2.518148183822632, + "kl": 2.255615234375, + "learning_rate": 1.1395387821347766e-06, + "loss": 0.2858, + "reward": 1.5000000298023224, + "reward_std": 0.4525563009083271, + "rewards/accuracy_reward": 0.5750000216066837, + "rewards/format_reward": 0.925000011920929, + "step": 1792 + }, + { + "completion_length": 152.60000228881836, + "epoch": 1.8584758942457231, + "grad_norm": 5.471208095550537, + "kl": 0.904052734375, + "learning_rate": 1.1377810510605075e-06, + "loss": 0.1344, + "reward": 1.725000023841858, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.7500000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1793 + }, + { + "completion_length": 215.92500114440918, + "epoch": 1.8595127008812855, + "grad_norm": 0.036844827234745026, + "kl": 1.365234375, + "learning_rate": 1.136023848013414e-06, + "loss": 0.2076, + "reward": 1.3250000178813934, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9500000178813934, + "step": 1794 + }, + { + "completion_length": 186.94999885559082, + "epoch": 1.8605495075168481, + "grad_norm": 1.577744722366333, + "kl": 0.36279296875, + "learning_rate": 1.1342671755550695e-06, + "loss": 0.0738, + "reward": 1.5, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 0.9750000089406967, + "step": 1795 + }, + { + "completion_length": 236.3750114440918, + "epoch": 1.8615863141524107, + "grad_norm": 0.6909179091453552, + "kl": 0.349609375, + "learning_rate": 1.1325110362462771e-06, + "loss": 0.1053, + "reward": 1.5250000357627869, + "reward_std": 0.2920721173286438, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9000000059604645, + "step": 1796 + }, + { + "completion_length": 249.350004196167, + "epoch": 1.862623120787973, + "grad_norm": 0.4353656470775604, + "kl": 3.93359375, + "learning_rate": 1.13075543264706e-06, + "loss": 0.4378, + "reward": 1.5, + "reward_std": 0.27950848266482353, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 1797 + }, + { + "completion_length": 203.45001029968262, + "epoch": 1.8636599274235355, + "grad_norm": 7.002437591552734, + "kl": 2.75146484375, + "learning_rate": 1.129000367316662e-06, + "loss": 0.2961, + "reward": 1.7750000059604645, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1798 + }, + { + "completion_length": 122.72500038146973, + "epoch": 1.8646967340590979, + "grad_norm": 0.2963770031929016, + "kl": 0.27001953125, + "learning_rate": 1.1272458428135413e-06, + "loss": 0.0223, + "reward": 1.6500000059604645, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.650000000372529, + "rewards/format_reward": 1.0, + "step": 1799 + }, + { + "completion_length": 270.20001220703125, + "epoch": 1.8657335406946605, + "grad_norm": 0.3787246346473694, + "kl": 1.204833984375, + "learning_rate": 1.1254918616953688e-06, + "loss": 0.2406, + "reward": 1.6500000059604645, + "reward_std": 0.28485123440623283, + "rewards/accuracy_reward": 0.7249999940395355, + "rewards/format_reward": 0.925000011920929, + "step": 1800 + }, + { + "completion_length": 323.62500381469727, + "epoch": 1.866770347330223, + "grad_norm": 4.747866630554199, + "kl": 4.41650390625, + "learning_rate": 1.1237384265190213e-06, + "loss": 0.4899, + "reward": 1.4500000178813934, + "reward_std": 0.4415663927793503, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.875, + "step": 1801 + }, + { + "completion_length": 162.25000190734863, + "epoch": 1.8678071539657854, + "grad_norm": 3.6878278255462646, + "kl": 1.429443359375, + "learning_rate": 1.1219855398405825e-06, + "loss": 0.1522, + "reward": 1.3000000268220901, + "reward_std": 0.2487340308725834, + "rewards/accuracy_reward": 0.32500000670552254, + "rewards/format_reward": 0.9750000089406967, + "step": 1802 + }, + { + "completion_length": 214.67500495910645, + "epoch": 1.8688439606013478, + "grad_norm": 0.9373306632041931, + "kl": 1.298095703125, + "learning_rate": 1.1202332042153338e-06, + "loss": 0.1917, + "reward": 1.2750000357627869, + "reward_std": 0.28485123068094254, + "rewards/accuracy_reward": 0.32499999925494194, + "rewards/format_reward": 0.9500000178813934, + "step": 1803 + }, + { + "completion_length": 221.52499961853027, + "epoch": 1.8698807672369102, + "grad_norm": 0.8081947565078735, + "kl": 0.2734375, + "learning_rate": 1.1184814221977554e-06, + "loss": 0.1022, + "reward": 1.3750000298023224, + "reward_std": 0.31733788549900055, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9500000178813934, + "step": 1804 + }, + { + "completion_length": 181.975004196167, + "epoch": 1.8709175738724728, + "grad_norm": 0.3742946982383728, + "kl": 0.85986328125, + "learning_rate": 1.1167301963415193e-06, + "loss": 0.0933, + "reward": 1.350000023841858, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9750000089406967, + "step": 1805 + }, + { + "completion_length": 266.27499771118164, + "epoch": 1.8719543805080352, + "grad_norm": 5.045395851135254, + "kl": 2.68798828125, + "learning_rate": 1.1149795291994874e-06, + "loss": 0.3369, + "reward": 1.550000011920929, + "reward_std": 0.31719936430454254, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9249999970197678, + "step": 1806 + }, + { + "completion_length": 116.7750015258789, + "epoch": 1.8729911871435978, + "grad_norm": 0.8432880640029907, + "kl": 0.2197265625, + "learning_rate": 1.1132294233237064e-06, + "loss": 0.0056, + "reward": 1.6000000089406967, + "reward_std": 0.05590169504284859, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9750000089406967, + "step": 1807 + }, + { + "completion_length": 321.4000053405762, + "epoch": 1.8740279937791602, + "grad_norm": 2.597256660461426, + "kl": 2.2613525390625, + "learning_rate": 1.1114798812654055e-06, + "loss": 0.3055, + "reward": 1.550000011920929, + "reward_std": 0.49226386845111847, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9000000208616257, + "step": 1808 + }, + { + "completion_length": 207.87500190734863, + "epoch": 1.8750648004147226, + "grad_norm": 0.5160976648330688, + "kl": 1.82470703125, + "learning_rate": 1.1097309055749915e-06, + "loss": 0.2081, + "reward": 1.4000000059604645, + "reward_std": 0.21275535970926285, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9500000178813934, + "step": 1809 + }, + { + "completion_length": 113.17500305175781, + "epoch": 1.8761016070502852, + "grad_norm": 0.7997815012931824, + "kl": 0.281982421875, + "learning_rate": 1.1079824988020467e-06, + "loss": 0.0544, + "reward": 1.675000011920929, + "reward_std": 0.12436702474951744, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 1.0, + "step": 1810 + }, + { + "completion_length": 203.90000343322754, + "epoch": 1.8771384136858476, + "grad_norm": 1.020841121673584, + "kl": 2.4033203125, + "learning_rate": 1.1062346634953213e-06, + "loss": 0.2653, + "reward": 1.7750000357627869, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1811 + }, + { + "completion_length": 355.0250053405762, + "epoch": 1.8781752203214102, + "grad_norm": 5.2479705810546875, + "kl": 4.98486328125, + "learning_rate": 1.1044874022027363e-06, + "loss": 0.5485, + "reward": 1.4250000417232513, + "reward_std": 0.5533697754144669, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.8500000089406967, + "step": 1812 + }, + { + "completion_length": 355.5750026702881, + "epoch": 1.8792120269569725, + "grad_norm": 9.089150428771973, + "kl": 4.91796875, + "learning_rate": 1.1027407174713724e-06, + "loss": 0.5858, + "reward": 1.0500000417232513, + "reward_std": 0.49226390197873116, + "rewards/accuracy_reward": 0.2000000048428774, + "rewards/format_reward": 0.8500000238418579, + "step": 1813 + }, + { + "completion_length": 298.5750102996826, + "epoch": 1.880248833592535, + "grad_norm": 5.795817852020264, + "kl": 4.29833984375, + "learning_rate": 1.100994611847472e-06, + "loss": 0.4714, + "reward": 1.6000000536441803, + "reward_std": 0.5031153038144112, + "rewards/accuracy_reward": 0.7000000216066837, + "rewards/format_reward": 0.9000000208616257, + "step": 1814 + }, + { + "completion_length": 179.1750030517578, + "epoch": 1.8812856402280973, + "grad_norm": 0.4812382459640503, + "kl": 1.103271484375, + "learning_rate": 1.0992490878764327e-06, + "loss": 0.1006, + "reward": 1.75, + "reward_std": 0.1604842059314251, + "rewards/accuracy_reward": 0.7999999970197678, + "rewards/format_reward": 0.9500000178813934, + "step": 1815 + }, + { + "completion_length": 413.0250015258789, + "epoch": 1.88232244686366, + "grad_norm": 2.979985237121582, + "kl": 4.40380859375, + "learning_rate": 1.097504148102804e-06, + "loss": 0.6087, + "reward": 1.2500000298023224, + "reward_std": 0.45977722853422165, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8250000178813934, + "step": 1816 + }, + { + "completion_length": 248.55000495910645, + "epoch": 1.8833592534992225, + "grad_norm": 1.9136056900024414, + "kl": 2.117431640625, + "learning_rate": 1.0957597950702837e-06, + "loss": 0.2905, + "reward": 1.4500000029802322, + "reward_std": 0.27228761091828346, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 1817 + }, + { + "completion_length": 149.9000015258789, + "epoch": 1.884396060134785, + "grad_norm": 3.6969919204711914, + "kl": 1.101806640625, + "learning_rate": 1.0940160313217152e-06, + "loss": 0.1208, + "reward": 1.475000038743019, + "reward_std": 0.1928323395550251, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.9500000178813934, + "step": 1818 + }, + { + "completion_length": 262.2250118255615, + "epoch": 1.8854328667703473, + "grad_norm": 0.5404413938522339, + "kl": 1.449462890625, + "learning_rate": 1.0922728593990813e-06, + "loss": 0.2044, + "reward": 1.6250000298023224, + "reward_std": 0.5031153149902821, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.9250000268220901, + "step": 1819 + }, + { + "completion_length": 303.17499923706055, + "epoch": 1.8864696734059097, + "grad_norm": 2.9051778316497803, + "kl": 2.6064453125, + "learning_rate": 1.0905302818435038e-06, + "loss": 0.2468, + "reward": 1.375, + "reward_std": 0.2686570808291435, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.8750000149011612, + "step": 1820 + }, + { + "completion_length": 217.55000495910645, + "epoch": 1.8875064800414723, + "grad_norm": 2.8954715728759766, + "kl": 1.001953125, + "learning_rate": 1.0887883011952357e-06, + "loss": 0.1922, + "reward": 1.574999988079071, + "reward_std": 0.22360680997371674, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.9500000178813934, + "step": 1821 + }, + { + "completion_length": 306.35000228881836, + "epoch": 1.8885432866770349, + "grad_norm": 1.8013978004455566, + "kl": 2.07275390625, + "learning_rate": 1.087046919993663e-06, + "loss": 0.3474, + "reward": 1.7000000178813934, + "reward_std": 0.5031152777373791, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9000000208616257, + "step": 1822 + }, + { + "completion_length": 303.6500015258789, + "epoch": 1.8895800933125972, + "grad_norm": 0.41030386090278625, + "kl": 1.762939453125, + "learning_rate": 1.085306140777295e-06, + "loss": 0.2851, + "reward": 1.2000000029802322, + "reward_std": 0.4092182330787182, + "rewards/accuracy_reward": 0.325000012293458, + "rewards/format_reward": 0.8750000298023224, + "step": 1823 + }, + { + "completion_length": 271.50000762939453, + "epoch": 1.8906168999481596, + "grad_norm": 3.335235118865967, + "kl": 2.220947265625, + "learning_rate": 1.0835659660837653e-06, + "loss": 0.2506, + "reward": 1.300000011920929, + "reward_std": 0.25609346479177475, + "rewards/accuracy_reward": 0.40000000037252903, + "rewards/format_reward": 0.9000000059604645, + "step": 1824 + }, + { + "completion_length": 262.85000228881836, + "epoch": 1.891653706583722, + "grad_norm": 0.33565187454223633, + "kl": 2.37890625, + "learning_rate": 1.0818263984498253e-06, + "loss": 0.3014, + "reward": 1.5750000476837158, + "reward_std": 0.3046357333660126, + "rewards/accuracy_reward": 0.6500000059604645, + "rewards/format_reward": 0.925000011920929, + "step": 1825 + }, + { + "completion_length": 358.3000068664551, + "epoch": 1.8926905132192846, + "grad_norm": 0.7493380904197693, + "kl": 5.809326171875, + "learning_rate": 1.0800874404113417e-06, + "loss": 0.6782, + "reward": 1.475000023841858, + "reward_std": 0.4525563269853592, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.8750000149011612, + "step": 1826 + }, + { + "completion_length": 642.1250190734863, + "epoch": 1.8937273198548472, + "grad_norm": 21.93817138671875, + "kl": 8.14306640625, + "learning_rate": 1.0783490945032925e-06, + "loss": 0.8109, + "reward": 1.0750000029802322, + "reward_std": 0.5769233554601669, + "rewards/accuracy_reward": 0.3499999977648258, + "rewards/format_reward": 0.7249999940395355, + "step": 1827 + }, + { + "completion_length": 187.100004196167, + "epoch": 1.8947641264904096, + "grad_norm": 0.654755711555481, + "kl": 1.386962890625, + "learning_rate": 1.0766113632597636e-06, + "loss": 0.1964, + "reward": 1.550000011920929, + "reward_std": 0.16770510002970695, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1828 + }, + { + "completion_length": 152.35000610351562, + "epoch": 1.895800933125972, + "grad_norm": 1.1775734424591064, + "kl": 0.19580078125, + "learning_rate": 1.074874249213945e-06, + "loss": 0.0854, + "reward": 1.850000023841858, + "reward_std": 0.2361704297363758, + "rewards/accuracy_reward": 0.8750000149011612, + "rewards/format_reward": 0.9750000089406967, + "step": 1829 + }, + { + "completion_length": 398.40000343322754, + "epoch": 1.8968377397615344, + "grad_norm": 2.1487603187561035, + "kl": 4.818359375, + "learning_rate": 1.0731377548981265e-06, + "loss": 0.4655, + "reward": 1.350000038743019, + "reward_std": 0.5967078916728497, + "rewards/accuracy_reward": 0.5250000208616257, + "rewards/format_reward": 0.8250000327825546, + "step": 1830 + }, + { + "completion_length": 494.6250057220459, + "epoch": 1.897874546397097, + "grad_norm": 3.8342268466949463, + "kl": 6.6142578125, + "learning_rate": 1.0714018828436947e-06, + "loss": 0.6902, + "reward": 1.4250000268220901, + "reward_std": 0.40921827033162117, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8000000193715096, + "step": 1831 + }, + { + "completion_length": 362.75000762939453, + "epoch": 1.8989113530326596, + "grad_norm": 6.237865447998047, + "kl": 2.207275390625, + "learning_rate": 1.0696666355811287e-06, + "loss": 0.3944, + "reward": 1.350000038743019, + "reward_std": 0.5715806297957897, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.8750000298023224, + "step": 1832 + }, + { + "completion_length": 351.225004196167, + "epoch": 1.899948159668222, + "grad_norm": 3.259511947631836, + "kl": 2.474853515625, + "learning_rate": 1.0679320156399981e-06, + "loss": 0.2497, + "reward": 1.4500000178813934, + "reward_std": 0.19346532225608826, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.875, + "step": 1833 + }, + { + "completion_length": 355.1499996185303, + "epoch": 1.9009849663037843, + "grad_norm": 3.741173028945923, + "kl": 3.993408203125, + "learning_rate": 1.0661980255489555e-06, + "loss": 0.5295, + "reward": 1.4250000417232513, + "reward_std": 0.31719934940338135, + "rewards/accuracy_reward": 0.550000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 1834 + }, + { + "completion_length": 151.57500457763672, + "epoch": 1.9020217729393467, + "grad_norm": 3.732590436935425, + "kl": 2.19287109375, + "learning_rate": 1.064464667835739e-06, + "loss": 0.1009, + "reward": 1.4500000178813934, + "reward_std": 0.5156789049506187, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 0.9250000268220901, + "step": 1835 + }, + { + "completion_length": 451.87500762939453, + "epoch": 1.9030585795749093, + "grad_norm": 2.423400402069092, + "kl": 2.49462890625, + "learning_rate": 1.0627319450271611e-06, + "loss": 0.4216, + "reward": 1.2000000327825546, + "reward_std": 0.690300427377224, + "rewards/accuracy_reward": 0.4000000134110451, + "rewards/format_reward": 0.800000011920929, + "step": 1836 + }, + { + "completion_length": 342.95000648498535, + "epoch": 1.9040953862104717, + "grad_norm": 4.658113956451416, + "kl": 3.33251953125, + "learning_rate": 1.060999859649111e-06, + "loss": 0.4544, + "reward": 1.600000023841858, + "reward_std": 0.36053744331002235, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.8750000149011612, + "step": 1837 + }, + { + "completion_length": 433.6000061035156, + "epoch": 1.9051321928460343, + "grad_norm": 1.5304324626922607, + "kl": 3.76513671875, + "learning_rate": 1.0592684142265484e-06, + "loss": 0.5858, + "reward": 1.3500000536441803, + "reward_std": 0.45977720618247986, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.8250000327825546, + "step": 1838 + }, + { + "completion_length": 491.9250030517578, + "epoch": 1.9061689994815967, + "grad_norm": 2.982764959335327, + "kl": 1.926513671875, + "learning_rate": 1.0575376112834998e-06, + "loss": 0.3757, + "reward": 1.3499999940395355, + "reward_std": 0.4045084938406944, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.8000000268220901, + "step": 1839 + }, + { + "completion_length": 252.82501220703125, + "epoch": 1.907205806117159, + "grad_norm": 0.9904256463050842, + "kl": 1.57568359375, + "learning_rate": 1.0558074533430546e-06, + "loss": 0.1511, + "reward": 1.1250000149011612, + "reward_std": 0.1928323432803154, + "rewards/accuracy_reward": 0.20000000298023224, + "rewards/format_reward": 0.925000011920929, + "step": 1840 + }, + { + "completion_length": 99.25000381469727, + "epoch": 1.9082426127527217, + "grad_norm": 1.0968436002731323, + "kl": 0.47216796875, + "learning_rate": 1.0540779429273632e-06, + "loss": 0.0489, + "reward": 1.5250000208616257, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.9500000178813934, + "step": 1841 + }, + { + "completion_length": 241.75000381469727, + "epoch": 1.909279419388284, + "grad_norm": 1.6434630155563354, + "kl": 1.84716796875, + "learning_rate": 1.0523490825576304e-06, + "loss": 0.2819, + "reward": 1.7750000059604645, + "reward_std": 0.38409100845456123, + "rewards/accuracy_reward": 0.8500000238418579, + "rewards/format_reward": 0.925000011920929, + "step": 1842 + }, + { + "completion_length": 196.2000026702881, + "epoch": 1.9103162260238467, + "grad_norm": 4.200102806091309, + "kl": 1.955322265625, + "learning_rate": 1.0506208747541158e-06, + "loss": 0.2434, + "reward": 1.3250000029802322, + "reward_std": 0.11180339008569717, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.9500000029802322, + "step": 1843 + }, + { + "completion_length": 210.4000015258789, + "epoch": 1.911353032659409, + "grad_norm": 0.38087666034698486, + "kl": 0.367919921875, + "learning_rate": 1.0488933220361246e-06, + "loss": 0.0898, + "reward": 1.6250000596046448, + "reward_std": 0.41643914580345154, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9500000178813934, + "step": 1844 + }, + { + "completion_length": 151.8250026702881, + "epoch": 1.9123898392949714, + "grad_norm": 0.7956162095069885, + "kl": 1.002685546875, + "learning_rate": 1.0471664269220103e-06, + "loss": 0.1219, + "reward": 1.475000023841858, + "reward_std": 0.3297629877924919, + "rewards/accuracy_reward": 0.5000000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1845 + }, + { + "completion_length": 274.3000087738037, + "epoch": 1.9134266459305338, + "grad_norm": 0.9696609973907471, + "kl": 5.397705078125, + "learning_rate": 1.0454401919291648e-06, + "loss": 0.6238, + "reward": 1.5000000149011612, + "reward_std": 0.45977720618247986, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.8750000149011612, + "step": 1846 + }, + { + "completion_length": 331.57501220703125, + "epoch": 1.9144634525660964, + "grad_norm": 11.981886863708496, + "kl": 3.90234375, + "learning_rate": 1.04371461957402e-06, + "loss": 0.5325, + "reward": 1.4500000476837158, + "reward_std": 0.436362162232399, + "rewards/accuracy_reward": 0.6000000182539225, + "rewards/format_reward": 0.8500000238418579, + "step": 1847 + }, + { + "completion_length": 352.42501068115234, + "epoch": 1.915500259201659, + "grad_norm": 8.505236625671387, + "kl": 4.437255859375, + "learning_rate": 1.0419897123720403e-06, + "loss": 0.5587, + "reward": 1.6250000596046448, + "reward_std": 0.4723408594727516, + "rewards/accuracy_reward": 0.7500000223517418, + "rewards/format_reward": 0.8750000298023224, + "step": 1848 + }, + { + "completion_length": 158.5750026702881, + "epoch": 1.9165370658372214, + "grad_norm": 1.6408089399337769, + "kl": 1.48681640625, + "learning_rate": 1.0402654728377219e-06, + "loss": 0.1275, + "reward": 1.6500000059604645, + "reward_std": 0.2920720987021923, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1849 + }, + { + "completion_length": 100.72500228881836, + "epoch": 1.9175738724727838, + "grad_norm": 0.6208429336547852, + "kl": 0.25, + "learning_rate": 1.0385419034845858e-06, + "loss": -0.0043, + "reward": 1.425000011920929, + "reward_std": 0.27228759601712227, + "rewards/accuracy_reward": 0.45000000670552254, + "rewards/format_reward": 0.9750000089406967, + "step": 1850 + }, + { + "completion_length": 154.95000648498535, + "epoch": 1.9186106791083462, + "grad_norm": 2.2388131618499756, + "kl": 1.45458984375, + "learning_rate": 1.036819006825178e-06, + "loss": 0.1409, + "reward": 1.5, + "reward_std": 0.30463575199246407, + "rewards/accuracy_reward": 0.575000012293458, + "rewards/format_reward": 0.9249999970197678, + "step": 1851 + }, + { + "completion_length": 275.7750072479248, + "epoch": 1.9196474857439088, + "grad_norm": 1.517973780632019, + "kl": 0.9404296875, + "learning_rate": 1.0350967853710628e-06, + "loss": 0.1928, + "reward": 1.5250000059604645, + "reward_std": 0.5715805999934673, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9000000208616257, + "step": 1852 + }, + { + "completion_length": 155.72500610351562, + "epoch": 1.9206842923794714, + "grad_norm": 0.5169199109077454, + "kl": 1.428466796875, + "learning_rate": 1.0333752416328206e-06, + "loss": 0.0918, + "reward": 1.3500000089406967, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/format_reward": 0.9750000089406967, + "step": 1853 + }, + { + "completion_length": 210.600004196167, + "epoch": 1.9217210990150337, + "grad_norm": 0.7431930303573608, + "kl": 1.59765625, + "learning_rate": 1.0316543781200429e-06, + "loss": 0.1882, + "reward": 1.5750000476837158, + "reward_std": 0.5408061742782593, + "rewards/accuracy_reward": 0.6500000134110451, + "rewards/format_reward": 0.9250000268220901, + "step": 1854 + }, + { + "completion_length": 312.65000343322754, + "epoch": 1.9227579056505961, + "grad_norm": 2.7001216411590576, + "kl": 1.943359375, + "learning_rate": 1.0299341973413316e-06, + "loss": 0.2359, + "reward": 1.1750000268220901, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9000000208616257, + "step": 1855 + }, + { + "completion_length": 210.70000648498535, + "epoch": 1.9237947122861585, + "grad_norm": 4.911709785461426, + "kl": 0.89599609375, + "learning_rate": 1.0282147018042908e-06, + "loss": 0.1635, + "reward": 1.7750000059604645, + "reward_std": 0.22360678762197495, + "rewards/accuracy_reward": 0.8250000178813934, + "rewards/format_reward": 0.9500000178813934, + "step": 1856 + }, + { + "completion_length": 302.22500228881836, + "epoch": 1.924831518921721, + "grad_norm": 8.21536922454834, + "kl": 1.489990234375, + "learning_rate": 1.0264958940155282e-06, + "loss": 0.273, + "reward": 1.5250000357627869, + "reward_std": 0.4922638610005379, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.9000000208616257, + "step": 1857 + }, + { + "completion_length": 204.2249984741211, + "epoch": 1.9258683255572837, + "grad_norm": 5.6338725090026855, + "kl": 1.3759765625, + "learning_rate": 1.0247777764806463e-06, + "loss": 0.1064, + "reward": 1.6749999821186066, + "reward_std": 0.25609347969293594, + "rewards/accuracy_reward": 0.7250000238418579, + "rewards/format_reward": 0.9500000029802322, + "step": 1858 + }, + { + "completion_length": 208.62500762939453, + "epoch": 1.926905132192846, + "grad_norm": 0.6463760733604431, + "kl": 0.350830078125, + "learning_rate": 1.0230603517042444e-06, + "loss": 0.0992, + "reward": 1.5499999970197678, + "reward_std": 0.2361704185605049, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9249999970197678, + "step": 1859 + }, + { + "completion_length": 149.17500495910645, + "epoch": 1.9279419388284085, + "grad_norm": 0.46324238181114197, + "kl": 1.07275390625, + "learning_rate": 1.0213436221899091e-06, + "loss": 0.0835, + "reward": 1.6500000059604645, + "reward_std": 0.10458250343799591, + "rewards/accuracy_reward": 0.6749999970197678, + "rewards/format_reward": 0.9750000089406967, + "step": 1860 + }, + { + "completion_length": 295.1750068664551, + "epoch": 1.9289787454639709, + "grad_norm": 0.3605247735977173, + "kl": 4.64794921875, + "learning_rate": 1.0196275904402154e-06, + "loss": 0.5267, + "reward": 1.2000000178813934, + "reward_std": 0.3354101702570915, + "rewards/accuracy_reward": 0.29999999701976776, + "rewards/format_reward": 0.9000000208616257, + "step": 1861 + }, + { + "completion_length": 154.10000038146973, + "epoch": 1.9300155520995335, + "grad_norm": 2.2866482734680176, + "kl": 0.276611328125, + "learning_rate": 1.0179122589567201e-06, + "loss": 0.0411, + "reward": 1.4000000059604645, + "reward_std": 0.14429005607962608, + "rewards/accuracy_reward": 0.42500000074505806, + "rewards/format_reward": 0.9750000089406967, + "step": 1862 + }, + { + "completion_length": 294.5999984741211, + "epoch": 1.931052358735096, + "grad_norm": 0.3986778259277344, + "kl": 1.63330078125, + "learning_rate": 1.0161976302399604e-06, + "loss": 0.3185, + "reward": 1.5750000327825546, + "reward_std": 0.34075290337204933, + "rewards/accuracy_reward": 0.6750000156462193, + "rewards/format_reward": 0.9000000059604645, + "step": 1863 + }, + { + "completion_length": 393.9000015258789, + "epoch": 1.9320891653706584, + "grad_norm": 2.3842616081237793, + "kl": 2.63671875, + "learning_rate": 1.014483706789446e-06, + "loss": 0.4191, + "reward": 1.6250000298023224, + "reward_std": 0.5973408445715904, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.8250000178813934, + "step": 1864 + }, + { + "completion_length": 448.62500762939453, + "epoch": 1.9331259720062208, + "grad_norm": 8.79536247253418, + "kl": 3.38671875, + "learning_rate": 1.0127704911036636e-06, + "loss": 0.4878, + "reward": 1.3250000178813934, + "reward_std": 0.6183430999517441, + "rewards/accuracy_reward": 0.5250000059604645, + "rewards/format_reward": 0.8000000268220901, + "step": 1865 + }, + { + "completion_length": 406.0999984741211, + "epoch": 1.9341627786417832, + "grad_norm": 2.2143993377685547, + "kl": 5.48046875, + "learning_rate": 1.0110579856800627e-06, + "loss": 0.7072, + "reward": 1.225000023841858, + "reward_std": 0.5590169690549374, + "rewards/accuracy_reward": 0.375, + "rewards/format_reward": 0.8500000238418579, + "step": 1866 + }, + { + "completion_length": 331.2250061035156, + "epoch": 1.9351995852773458, + "grad_norm": 1.509391188621521, + "kl": 2.050537109375, + "learning_rate": 1.0093461930150604e-06, + "loss": 0.3489, + "reward": 1.5000000298023224, + "reward_std": 0.41657764464616776, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8750000149011612, + "step": 1867 + }, + { + "completion_length": 151.22500038146973, + "epoch": 1.9362363919129082, + "grad_norm": 1.790948748588562, + "kl": 0.880615234375, + "learning_rate": 1.007635115604036e-06, + "loss": 0.0995, + "reward": 1.5000000298023224, + "reward_std": 0.2920721285045147, + "rewards/accuracy_reward": 0.525000000372529, + "rewards/format_reward": 0.9750000089406967, + "step": 1868 + }, + { + "completion_length": 389.9000072479248, + "epoch": 1.9372731985484708, + "grad_norm": 0.3396458923816681, + "kl": 2.37158203125, + "learning_rate": 1.0059247559413224e-06, + "loss": 0.2399, + "reward": 1.3750000447034836, + "reward_std": 0.4849044792354107, + "rewards/accuracy_reward": 0.5500000193715096, + "rewards/format_reward": 0.8250000178813934, + "step": 1869 + }, + { + "completion_length": 339.0500030517578, + "epoch": 1.9383100051840332, + "grad_norm": 1.0203490257263184, + "kl": 4.2451171875, + "learning_rate": 1.0042151165202094e-06, + "loss": 0.5323, + "reward": 1.2750000357627869, + "reward_std": 0.41643913835287094, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.8500000089406967, + "step": 1870 + }, + { + "completion_length": 316.4250068664551, + "epoch": 1.9393468118195956, + "grad_norm": 3.4523062705993652, + "kl": 5.239501953125, + "learning_rate": 1.0025061998329348e-06, + "loss": 0.5682, + "reward": 1.3750000596046448, + "reward_std": 0.565933421254158, + "rewards/accuracy_reward": 0.5250000208616257, + "rewards/format_reward": 0.8500000238418579, + "step": 1871 + }, + { + "completion_length": 697.7250137329102, + "epoch": 1.9403836184551582, + "grad_norm": 1.787365436553955, + "kl": 6.781494140625, + "learning_rate": 1.0007980083706845e-06, + "loss": 0.7802, + "reward": 1.0500000417232513, + "reward_std": 0.6658061668276787, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.7000000029802322, + "step": 1872 + }, + { + "completion_length": 150.75000381469727, + "epoch": 1.9414204250907205, + "grad_norm": 3.339010715484619, + "kl": 1.56787109375, + "learning_rate": 9.99090544623586e-07, + "loss": 0.1362, + "reward": 1.7249999940395355, + "reward_std": 0.3479738309979439, + "rewards/accuracy_reward": 0.7750000208616257, + "rewards/format_reward": 0.9500000178813934, + "step": 1873 + }, + { + "completion_length": 443.3750057220459, + "epoch": 1.9424572317262832, + "grad_norm": 2.877967119216919, + "kl": 3.479736328125, + "learning_rate": 9.973838110807066e-07, + "loss": 0.4387, + "reward": 1.1249999850988388, + "reward_std": 0.4290027692914009, + "rewards/accuracy_reward": 0.30000000447034836, + "rewards/format_reward": 0.8250000029802322, + "step": 1874 + }, + { + "completion_length": 303.7250099182129, + "epoch": 1.9434940383618455, + "grad_norm": 0.6481685638427734, + "kl": 3.72802734375, + "learning_rate": 9.956778102300485e-07, + "loss": 0.4148, + "reward": 1.4000000059604645, + "reward_std": 0.49226387962698936, + "rewards/accuracy_reward": 0.5250000078231096, + "rewards/format_reward": 0.8750000149011612, + "step": 1875 + }, + { + "completion_length": 266.450008392334, + "epoch": 1.944530844997408, + "grad_norm": 1.3731906414031982, + "kl": 3.2451171875, + "learning_rate": 9.939725445585467e-07, + "loss": 0.3688, + "reward": 1.5750000327825546, + "reward_std": 0.3354102000594139, + "rewards/accuracy_reward": 0.6750000268220901, + "rewards/format_reward": 0.9000000208616257, + "step": 1876 + }, + { + "completion_length": 262.5500011444092, + "epoch": 1.9455676516329703, + "grad_norm": 1.7140556573867798, + "kl": 3.1298828125, + "learning_rate": 9.922680165520633e-07, + "loss": 0.3096, + "reward": 1.350000023841858, + "reward_std": 0.3966546207666397, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.8750000149011612, + "step": 1877 + }, + { + "completion_length": 102.92500305175781, + "epoch": 1.946604458268533, + "grad_norm": 3.247180223464966, + "kl": 0.350830078125, + "learning_rate": 9.90564228695387e-07, + "loss": -0.0103, + "reward": 1.425000011920929, + "reward_std": 0.18026871979236603, + "rewards/accuracy_reward": 0.4249999988824129, + "rewards/format_reward": 1.0, + "step": 1878 + }, + { + "completion_length": 410.7500057220459, + "epoch": 1.9476412649040955, + "grad_norm": 0.5788431167602539, + "kl": 1.53564453125, + "learning_rate": 9.888611834722254e-07, + "loss": 0.176, + "reward": 1.3750000149011612, + "reward_std": 0.3731010667979717, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.8499999940395355, + "step": 1879 + }, + { + "completion_length": 203.5500030517578, + "epoch": 1.9486780715396579, + "grad_norm": 0.4296620488166809, + "kl": 2.08447265625, + "learning_rate": 9.871588833652043e-07, + "loss": 0.2596, + "reward": 1.5750000029802322, + "reward_std": 0.11180339753627777, + "rewards/accuracy_reward": 0.625, + "rewards/format_reward": 0.9500000178813934, + "step": 1880 + }, + { + "completion_length": 302.9750003814697, + "epoch": 1.9497148781752203, + "grad_norm": 1.4566224813461304, + "kl": 0.8955078125, + "learning_rate": 9.854573308558638e-07, + "loss": 0.1655, + "reward": 1.3499999940395355, + "reward_std": 0.2920721136033535, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/format_reward": 0.8499999940395355, + "step": 1881 + }, + { + "completion_length": 197.2750072479248, + "epoch": 1.9507516848107826, + "grad_norm": 1.4151586294174194, + "kl": 2.52099609375, + "learning_rate": 9.837565284246541e-07, + "loss": 0.3082, + "reward": 1.6500000059604645, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.7250000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1882 + }, + { + "completion_length": 199.32500076293945, + "epoch": 1.9517884914463453, + "grad_norm": 2.0754356384277344, + "kl": 1.36572265625, + "learning_rate": 9.820564785509312e-07, + "loss": 0.1703, + "reward": 1.4250000566244125, + "reward_std": 0.45977723225951195, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1883 + }, + { + "completion_length": 241.00000381469727, + "epoch": 1.9528252980819079, + "grad_norm": 0.5099223256111145, + "kl": 1.987548828125, + "learning_rate": 9.803571837129548e-07, + "loss": 0.2949, + "reward": 1.5500000268220901, + "reward_std": 0.25609343498945236, + "rewards/accuracy_reward": 0.6250000093132257, + "rewards/format_reward": 0.925000011920929, + "step": 1884 + }, + { + "completion_length": 307.07500648498535, + "epoch": 1.9538621047174702, + "grad_norm": 0.44971713423728943, + "kl": 3.94189453125, + "learning_rate": 9.786586463878835e-07, + "loss": 0.5022, + "reward": 1.2750000059604645, + "reward_std": 0.3407529219985008, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.9000000059604645, + "step": 1885 + }, + { + "completion_length": 312.40000343322754, + "epoch": 1.9548989113530326, + "grad_norm": 6.418180465698242, + "kl": 3.01806640625, + "learning_rate": 9.769608690517723e-07, + "loss": 0.3368, + "reward": 1.3250000178813934, + "reward_std": 0.47234082594513893, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.7750000059604645, + "step": 1886 + }, + { + "completion_length": 197.40000534057617, + "epoch": 1.955935717988595, + "grad_norm": 1.2701430320739746, + "kl": 1.67822265625, + "learning_rate": 9.752638541795667e-07, + "loss": 0.1824, + "reward": 1.4500000178813934, + "reward_std": 0.2920721247792244, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.925000011920929, + "step": 1887 + }, + { + "completion_length": 455.57501220703125, + "epoch": 1.9569725246241576, + "grad_norm": 3.930828094482422, + "kl": 3.11572265625, + "learning_rate": 9.735676042451037e-07, + "loss": 0.4811, + "reward": 1.3250000476837158, + "reward_std": 0.5356019139289856, + "rewards/accuracy_reward": 0.5000000111758709, + "rewards/format_reward": 0.8250000178813934, + "step": 1888 + }, + { + "completion_length": 197.5000057220459, + "epoch": 1.9580093312597202, + "grad_norm": 4.1649346351623535, + "kl": 0.983642578125, + "learning_rate": 9.718721217211015e-07, + "loss": 0.1585, + "reward": 1.5500000417232513, + "reward_std": 0.23617041483521461, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 1889 + }, + { + "completion_length": 250.600004196167, + "epoch": 1.9590461378952826, + "grad_norm": 0.3494587242603302, + "kl": 1.55078125, + "learning_rate": 9.701774090791628e-07, + "loss": 0.217, + "reward": 1.5, + "reward_std": 0.1928323470056057, + "rewards/accuracy_reward": 0.5750000029802322, + "rewards/format_reward": 0.925000011920929, + "step": 1890 + }, + { + "completion_length": 153.9749984741211, + "epoch": 1.960082944530845, + "grad_norm": 0.8926106691360474, + "kl": 0.245361328125, + "learning_rate": 9.684834687897656e-07, + "loss": 0.0825, + "reward": 1.7750000059604645, + "reward_std": 0.22360679134726524, + "rewards/accuracy_reward": 0.800000011920929, + "rewards/format_reward": 0.9750000089406967, + "step": 1891 + }, + { + "completion_length": 458.95001220703125, + "epoch": 1.9611197511664074, + "grad_norm": 1.8985650539398193, + "kl": 3.068115234375, + "learning_rate": 9.66790303322264e-07, + "loss": 0.5416, + "reward": 1.425000011920929, + "reward_std": 0.540806170552969, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.800000011920929, + "step": 1892 + }, + { + "completion_length": 395.15001106262207, + "epoch": 1.96215655780197, + "grad_norm": 2.027270793914795, + "kl": 2.281005859375, + "learning_rate": 9.650979151448808e-07, + "loss": 0.3557, + "reward": 1.3250000327825546, + "reward_std": 0.39728761464357376, + "rewards/accuracy_reward": 0.4750000052154064, + "rewards/format_reward": 0.8500000238418579, + "step": 1893 + }, + { + "completion_length": 156.17500495910645, + "epoch": 1.9631933644375326, + "grad_norm": 3.82222580909729, + "kl": 0.25, + "learning_rate": 9.634063067247074e-07, + "loss": 0.0566, + "reward": 1.5750000476837158, + "reward_std": 0.2487340345978737, + "rewards/accuracy_reward": 0.6000000052154064, + "rewards/format_reward": 0.9750000089406967, + "step": 1894 + }, + { + "completion_length": 385.0000057220459, + "epoch": 1.964230171073095, + "grad_norm": 0.30109697580337524, + "kl": 2.16796875, + "learning_rate": 9.61715480527697e-07, + "loss": 0.2793, + "reward": 1.4500000178813934, + "reward_std": 0.5533697828650475, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.8499999940395355, + "step": 1895 + }, + { + "completion_length": 404.350004196167, + "epoch": 1.9652669777086573, + "grad_norm": 2.2040598392486572, + "kl": 4.5947265625, + "learning_rate": 9.600254390186637e-07, + "loss": 0.566, + "reward": 1.2750000208616257, + "reward_std": 0.5535082593560219, + "rewards/accuracy_reward": 0.45000001043081284, + "rewards/format_reward": 0.8250000178813934, + "step": 1896 + }, + { + "completion_length": 497.7000141143799, + "epoch": 1.9663037843442197, + "grad_norm": 5.503700256347656, + "kl": 3.828857421875, + "learning_rate": 9.583361846612765e-07, + "loss": 0.5718, + "reward": 1.475000023841858, + "reward_std": 0.7416308633983135, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.800000011920929, + "step": 1897 + }, + { + "completion_length": 211.2500057220459, + "epoch": 1.9673405909797823, + "grad_norm": 3.9459316730499268, + "kl": 1.50146484375, + "learning_rate": 9.566477199180586e-07, + "loss": 0.1866, + "reward": 1.5500000417232513, + "reward_std": 0.27950849011540413, + "rewards/accuracy_reward": 0.6250000074505806, + "rewards/format_reward": 0.925000011920929, + "step": 1898 + }, + { + "completion_length": 304.97500228881836, + "epoch": 1.9683773976153447, + "grad_norm": 1.4846832752227783, + "kl": 3.548828125, + "learning_rate": 9.549600472503802e-07, + "loss": 0.36, + "reward": 1.2500000149011612, + "reward_std": 0.4363621734082699, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/format_reward": 0.8750000149011612, + "step": 1899 + }, + { + "completion_length": 494.24999809265137, + "epoch": 1.9694142042509073, + "grad_norm": 5.710992336273193, + "kl": 6.60107421875, + "learning_rate": 9.532731691184581e-07, + "loss": 0.8096, + "reward": 1.1500000208616257, + "reward_std": 0.4415663927793503, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/format_reward": 0.7750000059604645, + "step": 1900 + }, + { + "completion_length": 345.05000495910645, + "epoch": 1.9704510108864697, + "grad_norm": 1.5863420963287354, + "kl": 2.3662109375, + "learning_rate": 9.515870879813505e-07, + "loss": 0.3272, + "reward": 1.5250000357627869, + "reward_std": 0.5847772285342216, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.8250000178813934, + "step": 1901 + }, + { + "completion_length": 181.57500839233398, + "epoch": 1.971487817522032, + "grad_norm": 2.184415340423584, + "kl": 2.1875, + "learning_rate": 9.499018062969543e-07, + "loss": 0.2611, + "reward": 1.4500000476837158, + "reward_std": 0.3479738086462021, + "rewards/accuracy_reward": 0.5250000134110451, + "rewards/format_reward": 0.9250000268220901, + "step": 1902 + }, + { + "completion_length": 252.7500057220459, + "epoch": 1.9725246241575947, + "grad_norm": 1.3678945302963257, + "kl": 2.06884765625, + "learning_rate": 9.482173265219997e-07, + "loss": 0.2767, + "reward": 1.5750000476837158, + "reward_std": 0.4363621771335602, + "rewards/accuracy_reward": 0.6750000268220901, + "rewards/format_reward": 0.9000000208616257, + "step": 1903 + }, + { + "completion_length": 151.92500495910645, + "epoch": 1.973561430793157, + "grad_norm": 1.299222707748413, + "kl": 1.2138671875, + "learning_rate": 9.465336511120496e-07, + "loss": 0.1129, + "reward": 1.475000038743019, + "reward_std": 0.23617040738463402, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.9500000178813934, + "step": 1904 + }, + { + "completion_length": 382.5499973297119, + "epoch": 1.9745982374287196, + "grad_norm": 6.568785190582275, + "kl": 4.384521484375, + "learning_rate": 9.448507825214927e-07, + "loss": 0.5428, + "reward": 1.350000023841858, + "reward_std": 0.49746809154748917, + "rewards/accuracy_reward": 0.5250000096857548, + "rewards/format_reward": 0.8250000178813934, + "step": 1905 + }, + { + "completion_length": 249.35000038146973, + "epoch": 1.975635044064282, + "grad_norm": 0.8650876879692078, + "kl": 3.6494140625, + "learning_rate": 9.431687232035433e-07, + "loss": 0.4195, + "reward": 1.6500000357627869, + "reward_std": 0.4604102075099945, + "rewards/accuracy_reward": 0.7500000298023224, + "rewards/format_reward": 0.9000000059604645, + "step": 1906 + }, + { + "completion_length": 245.6500072479248, + "epoch": 1.9766718506998444, + "grad_norm": 4.518108367919922, + "kl": 3.12109375, + "learning_rate": 9.414874756102333e-07, + "loss": 0.331, + "reward": 1.7000000476837158, + "reward_std": 0.4237985536456108, + "rewards/accuracy_reward": 0.8250000327825546, + "rewards/format_reward": 0.8750000298023224, + "step": 1907 + }, + { + "completion_length": 301.8499984741211, + "epoch": 1.9777086573354068, + "grad_norm": 0.5200906991958618, + "kl": 3.833984375, + "learning_rate": 9.398070421924143e-07, + "loss": 0.4564, + "reward": 1.4750000536441803, + "reward_std": 0.6149186789989471, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.8500000238418579, + "step": 1908 + }, + { + "completion_length": 346.9000015258789, + "epoch": 1.9787454639709694, + "grad_norm": 1.6915509700775146, + "kl": 5.0751953125, + "learning_rate": 9.381274253997501e-07, + "loss": 0.2743, + "reward": 1.425000011920929, + "reward_std": 0.22360679507255554, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.8750000149011612, + "step": 1909 + }, + { + "completion_length": 305.325008392334, + "epoch": 1.979782270606532, + "grad_norm": 5.309218883514404, + "kl": 3.3740234375, + "learning_rate": 9.364486276807123e-07, + "loss": 0.4393, + "reward": 1.300000011920929, + "reward_std": 0.45413000881671906, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.8750000149011612, + "step": 1910 + }, + { + "completion_length": 154.75000381469727, + "epoch": 1.9808190772420944, + "grad_norm": 2.066056728363037, + "kl": 0.671142578125, + "learning_rate": 9.347706514825813e-07, + "loss": 0.0532, + "reward": 1.3750000447034836, + "reward_std": 0.353316530585289, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.9000000208616257, + "step": 1911 + }, + { + "completion_length": 387.37500762939453, + "epoch": 1.9818558838776568, + "grad_norm": 1.2347073554992676, + "kl": 4.736572265625, + "learning_rate": 9.330934992514374e-07, + "loss": 0.6312, + "reward": 1.6250000298023224, + "reward_std": 0.4472135975956917, + "rewards/accuracy_reward": 0.7750000134110451, + "rewards/format_reward": 0.8500000089406967, + "step": 1912 + }, + { + "completion_length": 406.7500114440918, + "epoch": 1.9828926905132191, + "grad_norm": 0.9930393099784851, + "kl": 3.92578125, + "learning_rate": 9.314171734321614e-07, + "loss": 0.4137, + "reward": 1.4750000089406967, + "reward_std": 0.38409099355340004, + "rewards/accuracy_reward": 0.6250000037252903, + "rewards/format_reward": 0.8500000238418579, + "step": 1913 + }, + { + "completion_length": 217.45000648498535, + "epoch": 1.9839294971487818, + "grad_norm": 3.541429281234741, + "kl": 1.580810546875, + "learning_rate": 9.297416764684285e-07, + "loss": 0.2431, + "reward": 1.2000000327825546, + "reward_std": 0.32818930223584175, + "rewards/accuracy_reward": 0.2750000059604645, + "rewards/format_reward": 0.9250000268220901, + "step": 1914 + }, + { + "completion_length": 195.02500534057617, + "epoch": 1.9849663037843444, + "grad_norm": 1.2164496183395386, + "kl": 2.34326171875, + "learning_rate": 9.280670108027065e-07, + "loss": 0.2629, + "reward": 1.3000000268220901, + "reward_std": 0.16770510748028755, + "rewards/accuracy_reward": 0.3500000089406967, + "rewards/format_reward": 0.9500000178813934, + "step": 1915 + }, + { + "completion_length": 152.05000495910645, + "epoch": 1.9860031104199067, + "grad_norm": 1.7377318143844604, + "kl": 1.78955078125, + "learning_rate": 9.2639317887625e-07, + "loss": 0.1915, + "reward": 1.5500000417232513, + "reward_std": 0.447213601320982, + "rewards/accuracy_reward": 0.6250000149011612, + "rewards/format_reward": 0.9250000268220901, + "step": 1916 + }, + { + "completion_length": 388.225004196167, + "epoch": 1.9870399170554691, + "grad_norm": 0.8515089154243469, + "kl": 3.8017578125, + "learning_rate": 9.247201831291001e-07, + "loss": 0.4497, + "reward": 1.4000000059604645, + "reward_std": 0.4958944320678711, + "rewards/accuracy_reward": 0.5500000044703484, + "rewards/format_reward": 0.8500000089406967, + "step": 1917 + }, + { + "completion_length": 235.10000610351562, + "epoch": 1.9880767236910315, + "grad_norm": 1.671351671218872, + "kl": 0.8701171875, + "learning_rate": 9.230480260000769e-07, + "loss": 0.1938, + "reward": 1.5250000059604645, + "reward_std": 0.22360680252313614, + "rewards/accuracy_reward": 0.6000000089406967, + "rewards/format_reward": 0.9250000268220901, + "step": 1918 + }, + { + "completion_length": 310.70000648498535, + "epoch": 1.989113530326594, + "grad_norm": 1.697324514389038, + "kl": 2.175048828125, + "learning_rate": 9.213767099267801e-07, + "loss": 0.34, + "reward": 1.5000000298023224, + "reward_std": 0.5533697865903378, + "rewards/accuracy_reward": 0.6000000163912773, + "rewards/format_reward": 0.9000000059604645, + "step": 1919 + }, + { + "completion_length": 456.2000102996826, + "epoch": 1.9901503369621567, + "grad_norm": 9.253554344177246, + "kl": 5.18310546875, + "learning_rate": 9.19706237345581e-07, + "loss": 0.6159, + "reward": 1.2750000357627869, + "reward_std": 0.31719935685396194, + "rewards/accuracy_reward": 0.4750000014901161, + "rewards/format_reward": 0.800000011920929, + "step": 1920 + }, + { + "completion_length": 253.95000457763672, + "epoch": 1.991187143597719, + "grad_norm": 0.7404049634933472, + "kl": 0.40478515625, + "learning_rate": 9.180366106916239e-07, + "loss": 0.1588, + "reward": 1.6250000298023224, + "reward_std": 0.2795084938406944, + "rewards/accuracy_reward": 0.7000000104308128, + "rewards/format_reward": 0.925000011920929, + "step": 1921 + }, + { + "completion_length": 138.0000057220459, + "epoch": 1.9922239502332815, + "grad_norm": 0.33539578318595886, + "kl": 0.913330078125, + "learning_rate": 9.163678323988172e-07, + "loss": 0.0743, + "reward": 1.4749999940395355, + "reward_std": 0.055901702493429184, + "rewards/accuracy_reward": 0.5000000018626451, + "rewards/format_reward": 0.9750000089406967, + "step": 1922 + }, + { + "completion_length": 251.1750030517578, + "epoch": 1.9932607568688439, + "grad_norm": 2.1586689949035645, + "kl": 2.477783203125, + "learning_rate": 9.146999048998349e-07, + "loss": 0.2985, + "reward": 1.4000000208616257, + "reward_std": 0.18026872724294662, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.925000011920929, + "step": 1923 + }, + { + "completion_length": 294.5500030517578, + "epoch": 1.9942975635044065, + "grad_norm": 0.4859706163406372, + "kl": 2.2275390625, + "learning_rate": 9.130328306261095e-07, + "loss": 0.3606, + "reward": 1.3250000327825546, + "reward_std": 0.3804604671895504, + "rewards/accuracy_reward": 0.42500000447034836, + "rewards/format_reward": 0.9000000208616257, + "step": 1924 + }, + { + "completion_length": 399.6999988555908, + "epoch": 1.995334370139969, + "grad_norm": 4.5429158210754395, + "kl": 2.865966796875, + "learning_rate": 9.113666120078303e-07, + "loss": 0.4637, + "reward": 1.3250000178813934, + "reward_std": 0.5156789049506187, + "rewards/accuracy_reward": 0.4750000089406967, + "rewards/format_reward": 0.8500000089406967, + "step": 1925 + }, + { + "completion_length": 296.475004196167, + "epoch": 1.9963711767755314, + "grad_norm": 1.8262897729873657, + "kl": 2.650146484375, + "learning_rate": 9.097012514739386e-07, + "loss": 0.3707, + "reward": 1.5500000417232513, + "reward_std": 0.4797002449631691, + "rewards/accuracy_reward": 0.675000011920929, + "rewards/format_reward": 0.8750000149011612, + "step": 1926 + }, + { + "completion_length": 298.425012588501, + "epoch": 1.9974079834110938, + "grad_norm": 3.1941144466400146, + "kl": 2.65869140625, + "learning_rate": 9.080367514521257e-07, + "loss": 0.3707, + "reward": 1.3250000327825546, + "reward_std": 0.4922638610005379, + "rewards/accuracy_reward": 0.42500001192092896, + "rewards/format_reward": 0.9000000208616257, + "step": 1927 + }, + { + "completion_length": 332.325008392334, + "epoch": 1.9984447900466562, + "grad_norm": 0.6302455067634583, + "kl": 1.169677734375, + "learning_rate": 9.063731143688274e-07, + "loss": 0.1151, + "reward": 1.550000011920929, + "reward_std": 0.3479738160967827, + "rewards/accuracy_reward": 0.6750000212341547, + "rewards/format_reward": 0.875, + "step": 1928 + }, + { + "completion_length": 448.6000061035156, + "epoch": 1.9994815966822188, + "grad_norm": 1.1343616247177124, + "kl": 5.01953125, + "learning_rate": 9.047103426492231e-07, + "loss": 0.6224, + "reward": 1.3750000447034836, + "reward_std": 0.6526094973087311, + "rewards/accuracy_reward": 0.5750000104308128, + "rewards/format_reward": 0.8000000268220901, + "step": 1929 + }, + { + "completion_length": 101.1500015258789, + "epoch": 2.0, + "grad_norm": 1.2842036485671997, + "kl": 0.703125, + "learning_rate": 9.030484387172285e-07, + "loss": -0.0309, + "reward": 1.5, + "reward_std": 0.3209684118628502, + "rewards/accuracy_reward": 0.5499999970197678, + "rewards/format_reward": 0.9500000178813934, + "step": 1930 + } + ], + "logging_steps": 1, + "max_steps": 2892, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}