{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.1365374112506827,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0625,
      "epoch": 0.00013653741125068268,
      "grad_norm": 0.08739373087882996,
      "kl": 0.0006317789470813295,
      "learning_rate": 1.0000000000000001e-07,
      "loss": -0.0002,
      "num_tokens": 11046.0,
      "reward": 0.7437499761581421,
      "reward_std": 0.19240379333496094,
      "rewards/reward_fn": 0.7437499761581421,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00027307482250136535,
      "grad_norm": 0.08738827705383301,
      "kl": 0.0006317789470813295,
      "learning_rate": 2.0000000000000002e-07,
      "loss": -0.0002,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00040961223375204805,
      "grad_norm": 0.08647612482309341,
      "kl": 0.0006050676793165621,
      "learning_rate": 3.0000000000000004e-07,
      "loss": -0.0001,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005461496450027307,
      "grad_norm": 0.08993478119373322,
      "kl": 0.0005967747410977609,
      "learning_rate": 4.0000000000000003e-07,
      "loss": -0.0,
      "step": 4
    },
    {
      "clip_ratio": 0.0009092581894947216,
      "completion_length": 123.28125,
      "epoch": 0.0006826870562534135,
      "grad_norm": 0.09589574486017227,
      "kl": 0.0005814487990392081,
      "learning_rate": 5.000000000000001e-07,
      "loss": 0.0,
      "num_tokens": 22015.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 5
    },
    {
      "clip_ratio": 0.00021701389050576836,
      "epoch": 0.0008192244675040961,
      "grad_norm": 0.09353803843259811,
      "kl": 0.0005364549010664632,
      "learning_rate": 6.000000000000001e-07,
      "loss": 0.0001,
      "step": 6
    },
    {
      "clip_ratio": 0.001004464298603125,
      "epoch": 0.0009557618787547789,
      "grad_norm": 0.10129779577255249,
      "kl": 0.000562284074476338,
      "learning_rate": 7.000000000000001e-07,
      "loss": 0.0003,
      "step": 7
    },
    {
      "clip_ratio": 0.00021701389050576836,
      "epoch": 0.0010922992900054614,
      "grad_norm": 0.09212316572666168,
      "kl": 0.0005370138956095616,
      "learning_rate": 8.000000000000001e-07,
      "loss": 0.0001,
      "step": 8
    },
    {
      "clip_ratio": 0.0009476926934439689,
      "completion_length": 152.6875,
      "epoch": 0.0012288367012561442,
      "grad_norm": 0.10085771977901459,
      "kl": 0.0005878389974895981,
      "learning_rate": 9.000000000000001e-07,
      "loss": -0.0,
      "num_tokens": 34213.0,
      "reward": 0.746874988079071,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.746874988079071,
      "step": 9
    },
    {
      "clip_ratio": 0.00126093348080758,
      "epoch": 0.001365374112506827,
      "grad_norm": 0.1008201465010643,
      "kl": 0.0005425554691100842,
      "learning_rate": 1.0000000000000002e-06,
      "loss": 0.0001,
      "step": 10
    },
    {
      "clip_ratio": 0.0007717385306023061,
      "epoch": 0.0015019115237575095,
      "grad_norm": 0.10120692104101181,
      "kl": 0.0005051454013482726,
      "learning_rate": 1.1e-06,
      "loss": 0.0,
      "step": 11
    },
    {
      "clip_ratio": 0.0012008554913336411,
      "epoch": 0.0016384489350081922,
      "grad_norm": 0.10010115802288055,
      "kl": 0.0005464185369419283,
      "learning_rate": 1.2000000000000002e-06,
      "loss": -0.0001,
      "step": 12
    },
    {
      "clip_ratio": 0.0009030445798998699,
      "completion_length": 145.59375,
      "epoch": 0.001774986346258875,
      "grad_norm": 0.09043823927640915,
      "kl": 0.0005211641523601429,
      "learning_rate": 1.3e-06,
      "loss": 0.0002,
      "num_tokens": 45996.0,
      "reward": 0.6312500238418579,
      "reward_std": 0.2836301922798157,
      "rewards/reward_fn": 0.6312500238418579,
      "step": 13
    },
    {
      "clip_ratio": 0.0006189533451106399,
      "epoch": 0.0019115237575095577,
      "grad_norm": 0.09039849042892456,
      "kl": 0.0005784431677966495,
      "learning_rate": 1.4000000000000001e-06,
      "loss": 0.0002,
      "step": 14
    },
    {
      "clip_ratio": 0.00031675209174863994,
      "epoch": 0.0020480611687602405,
      "grad_norm": 0.09077497571706772,
      "kl": 0.0005650495991176285,
      "learning_rate": 1.5e-06,
      "loss": 0.0002,
      "step": 15
    },
    {
      "clip_ratio": 0.000734518383978866,
      "epoch": 0.002184598580010923,
      "grad_norm": 0.09000656008720398,
      "kl": 0.0005918515398661839,
      "learning_rate": 1.6000000000000001e-06,
      "loss": 0.0001,
      "step": 16
    },
    {
      "clip_ratio": 0.0001270325155928731,
      "completion_length": 151.8125,
      "epoch": 0.0023211359912616056,
      "grad_norm": 0.08065712451934814,
      "kl": 0.0005675913225786644,
      "learning_rate": 1.7000000000000002e-06,
      "loss": 0.0001,
      "num_tokens": 58366.0,
      "reward": 0.578125,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.578125,
      "step": 17
    },
    {
      "clip_ratio": 0.00040856405394151807,
      "epoch": 0.0024576734025122883,
      "grad_norm": 0.08062373846769333,
      "kl": 0.0006601297482120572,
      "learning_rate": 1.8000000000000001e-06,
      "loss": 0.0001,
      "step": 18
    },
    {
      "clip_ratio": 0.0001270325155928731,
      "epoch": 0.002594210813762971,
      "grad_norm": 0.08101943135261536,
      "kl": 0.0006057311593394843,
      "learning_rate": 1.9000000000000002e-06,
      "loss": 0.0002,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002730748225013654,
      "grad_norm": 0.08095166832208633,
      "kl": 0.0007306283941943548,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0001,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 175.4375,
      "epoch": 0.0028672856362643366,
      "grad_norm": 0.06835271418094635,
      "kl": 0.0006434723027268774,
      "learning_rate": 2.1000000000000002e-06,
      "loss": 0.0,
      "num_tokens": 71024.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.606249988079071,
      "step": 21
    },
    {
      "clip_ratio": 0.00040108305984176695,
      "epoch": 0.003003823047515019,
      "grad_norm": 0.06890249252319336,
      "kl": 0.0007574095243398915,
      "learning_rate": 2.2e-06,
      "loss": -0.0001,
      "step": 22
    },
    {
      "clip_ratio": 0.0003356586676090956,
      "epoch": 0.0031403604587657017,
      "grad_norm": 0.07008787244558334,
      "kl": 0.0008008444001461612,
      "learning_rate": 2.3000000000000004e-06,
      "loss": -0.0001,
      "step": 23
    },
    {
      "clip_ratio": 0.000147405662573874,
      "epoch": 0.0032768978700163844,
      "grad_norm": 0.06931491941213608,
      "kl": 0.0008262821156677091,
      "learning_rate": 2.4000000000000003e-06,
      "loss": -0.0003,
      "step": 24
    },
    {
      "clip_ratio": 0.0011949757172260433,
      "completion_length": 154.78125,
      "epoch": 0.003413435281267067,
      "grad_norm": 0.09464246779680252,
      "kl": 0.0008019438510018517,
      "learning_rate": 2.5e-06,
      "loss": -0.0002,
      "num_tokens": 82917.0,
      "reward": 0.550000011920929,
      "reward_std": 0.30735570192337036,
      "rewards/reward_fn": 0.550000011920929,
      "step": 25
    },
    {
      "clip_ratio": 0.0008628188807051629,
      "epoch": 0.00354997269251775,
      "grad_norm": 0.09641154855489731,
      "kl": 0.0008660993607918499,
      "learning_rate": 2.6e-06,
      "loss": 0.0,
      "step": 26
    },
    {
      "clip_ratio": 0.000868194954819046,
      "epoch": 0.0036865101037684327,
      "grad_norm": 0.09419838339090347,
      "kl": 0.0010259089904138818,
      "learning_rate": 2.7000000000000004e-06,
      "loss": -0.0001,
      "step": 27
    },
    {
      "clip_ratio": 0.0008564656018279493,
      "epoch": 0.0038230475150191155,
      "grad_norm": 0.08919408172369003,
      "kl": 0.0012204149879835313,
      "learning_rate": 2.8000000000000003e-06,
      "loss": -0.0004,
      "step": 28
    },
    {
      "clip_ratio": 0.0008514263026881963,
      "completion_length": 140.875,
      "epoch": 0.003959584926269798,
      "grad_norm": 0.08971544355154037,
      "kl": 0.001275990204703703,
      "learning_rate": 2.9e-06,
      "loss": -0.0001,
      "num_tokens": 94653.0,
      "reward": 0.606249988079071,
      "reward_std": 0.1948557198047638,
      "rewards/reward_fn": 0.606249988079071,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004096122337520481,
      "grad_norm": 0.08967753499746323,
      "kl": 0.0016255443697446026,
      "learning_rate": 3e-06,
      "loss": -0.0,
      "step": 30
    },
    {
      "clip_ratio": 0.000954879040364176,
      "epoch": 0.004232659748771164,
      "grad_norm": 0.0948873832821846,
      "kl": 0.0017126279599324334,
      "learning_rate": 3.1000000000000004e-06,
      "loss": -0.0001,
      "step": 31
    },
    {
      "clip_ratio": 0.0009331955807283521,
      "epoch": 0.004369197160021846,
      "grad_norm": 0.0915728509426117,
      "kl": 0.0019823211387119954,
      "learning_rate": 3.2000000000000003e-06,
      "loss": -0.0001,
      "step": 32
    },
    {
      "clip_ratio": 0.00015624999650754035,
      "completion_length": 156.3125,
      "epoch": 0.004505734571272528,
      "grad_norm": 0.07780904322862625,
      "kl": 0.0020532281778287143,
      "learning_rate": 3.3000000000000006e-06,
      "loss": -0.0001,
      "num_tokens": 107451.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 33
    },
    {
      "clip_ratio": 0.0005133637605467811,
      "epoch": 0.004642271982523211,
      "grad_norm": 0.07766616344451904,
      "kl": 0.00238418992375955,
      "learning_rate": 3.4000000000000005e-06,
      "loss": 0.0,
      "step": 34
    },
    {
      "clip_ratio": 0.0011453818005975336,
      "epoch": 0.004778809393773894,
      "grad_norm": 0.0797438770532608,
      "kl": 0.002831721540133003,
      "learning_rate": 3.5e-06,
      "loss": -0.0001,
      "step": 35
    },
    {
      "clip_ratio": 0.0010551480372669175,
      "epoch": 0.004915346805024577,
      "grad_norm": 0.07885287702083588,
      "kl": 0.003094232171861222,
      "learning_rate": 3.6000000000000003e-06,
      "loss": -0.0001,
      "step": 36
    },
    {
      "clip_ratio": 0.0001382743357680738,
      "completion_length": 159.625,
      "epoch": 0.005051884216275259,
      "grad_norm": 0.06207679212093353,
      "kl": 0.0033535315305925906,
      "learning_rate": 3.7e-06,
      "loss": -0.0001,
      "num_tokens": 119355.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 37
    },
    {
      "clip_ratio": 0.00040536836604587734,
      "epoch": 0.005188421627525942,
      "grad_norm": 0.062118254601955414,
      "kl": 0.0036885815097775776,
      "learning_rate": 3.8000000000000005e-06,
      "loss": -0.0001,
      "step": 38
    },
    {
      "clip_ratio": 0.000545094400877133,
      "epoch": 0.005324959038776625,
      "grad_norm": 0.06358339637517929,
      "kl": 0.0039351375453406945,
      "learning_rate": 3.900000000000001e-06,
      "loss": 0.0,
      "step": 39
    },
    {
      "clip_ratio": 0.0005789794813608751,
      "epoch": 0.005461496450027308,
      "grad_norm": 0.06142238900065422,
      "kl": 0.004162700774031691,
      "learning_rate": 4.000000000000001e-06,
      "loss": -0.0001,
      "step": 40
    },
    {
      "clip_ratio": 0.0004968734428985044,
      "completion_length": 135.09375,
      "epoch": 0.00559803386127799,
      "grad_norm": 0.10866480320692062,
      "kl": 0.00454872860427713,
      "learning_rate": 4.1e-06,
      "loss": 0.0002,
      "num_tokens": 130746.0,
      "reward": 0.746874988079071,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.746874988079071,
      "step": 41
    },
    {
      "clip_ratio": 0.0002297794126207009,
      "epoch": 0.005734571272528673,
      "grad_norm": 0.09940455108880997,
      "kl": 0.004684018753323471,
      "learning_rate": 4.2000000000000004e-06,
      "loss": -0.0002,
      "step": 42
    },
    {
      "clip_ratio": 0.0002297794126207009,
      "epoch": 0.005871108683779356,
      "grad_norm": 0.09981972724199295,
      "kl": 0.004767102662299294,
      "learning_rate": 4.3e-06,
      "loss": -0.0003,
      "step": 43
    },
    {
      "clip_ratio": 0.00021551724057644606,
      "epoch": 0.006007646095030038,
      "grad_norm": 0.0989937111735344,
      "kl": 0.00463390517688822,
      "learning_rate": 4.4e-06,
      "loss": -0.0006,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.40625,
      "epoch": 0.006144183506280721,
      "grad_norm": 0.06329631805419922,
      "kl": 0.004799115195055492,
      "learning_rate": 4.5e-06,
      "loss": -0.0,
      "num_tokens": 142491.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 45
    },
    {
      "clip_ratio": 0.00017655367264524102,
      "epoch": 0.006280720917531403,
      "grad_norm": 0.06211785599589348,
      "kl": 0.0047047465850482695,
      "learning_rate": 4.600000000000001e-06,
      "loss": -0.0002,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006417258328782086,
      "grad_norm": 0.060480426996946335,
      "kl": 0.004680782469222322,
      "learning_rate": 4.7e-06,
      "loss": -0.0003,
      "step": 47
    },
    {
      "clip_ratio": 0.0008001700480235741,
      "epoch": 0.006553795740032769,
      "grad_norm": 0.06124988943338394,
      "kl": 0.004546212963759899,
      "learning_rate": 4.800000000000001e-06,
      "loss": -0.0004,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.84375,
      "epoch": 0.006690333151283452,
      "grad_norm": 0.09592146426439285,
      "kl": 0.004827649225262576,
      "learning_rate": 4.9000000000000005e-06,
      "loss": -0.0001,
      "num_tokens": 153834.0,
      "reward": 0.550000011920929,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.550000011920929,
      "step": 49
    },
    {
      "clip_ratio": 0.0011473706836113706,
      "epoch": 0.006826870562534134,
      "grad_norm": 0.09477215260267258,
      "kl": 0.004913321821732097,
      "learning_rate": 5e-06,
      "loss": -0.0002,
      "step": 50
    },
    {
      "clip_ratio": 0.00017655367264524102,
      "epoch": 0.006963407973784817,
      "grad_norm": 0.09420502185821533,
      "kl": 0.005133655802637804,
      "learning_rate": 4.9947368421052636e-06,
      "loss": -0.0006,
      "step": 51
    },
    {
      "clip_ratio": 0.0006438815908040851,
      "epoch": 0.0070999453850355,
      "grad_norm": 0.08837559819221497,
      "kl": 0.005440949782496318,
      "learning_rate": 4.989473684210527e-06,
      "loss": -0.0011,
      "step": 52
    },
    {
      "clip_ratio": 0.0003052657120861113,
      "completion_length": 140.1875,
      "epoch": 0.007236482796286183,
      "grad_norm": 0.07635519653558731,
      "kl": 0.004967198314261623,
      "learning_rate": 4.98421052631579e-06,
      "loss": 0.0001,
      "num_tokens": 165068.0,
      "reward": 0.26875001192092896,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.26875001192092896,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.007373020207536865,
      "grad_norm": 0.07609064877033234,
      "kl": 0.005105830910906661,
      "learning_rate": 4.978947368421053e-06,
      "loss": 0.0001,
      "step": 54
    },
    {
      "clip_ratio": 0.0004236875247443095,
      "epoch": 0.007509557618787548,
      "grad_norm": 0.07511620223522186,
      "kl": 0.0052619903508457355,
      "learning_rate": 4.973684210526316e-06,
      "loss": -0.0003,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.007646095030038231,
      "grad_norm": 0.0717560425400734,
      "kl": 0.005385495860537048,
      "learning_rate": 4.968421052631579e-06,
      "loss": -0.0006,
      "step": 56
    },
    {
      "clip_ratio": 0.0006011680379742756,
      "completion_length": 143.28125,
      "epoch": 0.007782632441288913,
      "grad_norm": 0.10058116912841797,
      "kl": 0.007372127176495269,
      "learning_rate": 4.9631578947368426e-06,
      "loss": 0.0004,
      "num_tokens": 177057.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.30735570192337036,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 57
    },
    {
      "clip_ratio": 0.0006176497845444828,
      "epoch": 0.007919169852539596,
      "grad_norm": 0.10059979557991028,
      "kl": 0.007655073924979661,
      "learning_rate": 4.957894736842106e-06,
      "loss": -0.0,
      "step": 58
    },
    {
      "clip_ratio": 0.0002929344482254237,
      "epoch": 0.008055707263790278,
      "grad_norm": 0.0992376059293747,
      "kl": 0.008138025958032813,
      "learning_rate": 4.952631578947369e-06,
      "loss": -0.0006,
      "step": 59
    },
    {
      "clip_ratio": 0.0005786725378129631,
      "epoch": 0.008192244675040962,
      "grad_norm": 0.09717098623514175,
      "kl": 0.008423170140304137,
      "learning_rate": 4.947368421052632e-06,
      "loss": -0.0012,
      "step": 60
    },
    {
      "clip_ratio": 0.0005044226418249309,
      "completion_length": 145.75,
      "epoch": 0.008328782086291644,
      "grad_norm": 0.07611903548240662,
      "kl": 0.008245052813435905,
      "learning_rate": 4.942105263157895e-06,
      "loss": 0.0003,
      "num_tokens": 189173.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 61
    },
    {
      "clip_ratio": 0.0007626870938111097,
      "epoch": 0.008465319497542327,
      "grad_norm": 0.0758456438779831,
      "kl": 0.009095820518268738,
      "learning_rate": 4.936842105263158e-06,
      "loss": -0.0001,
      "step": 62
    },
    {
      "clip_ratio": 0.0004581221583066508,
      "epoch": 0.00860185690879301,
      "grad_norm": 0.07229050993919373,
      "kl": 0.009441651978704613,
      "learning_rate": 4.9315789473684215e-06,
      "loss": -0.0003,
      "step": 63
    },
    {
      "clip_ratio": 0.00047310019726864994,
      "epoch": 0.008738394320043691,
      "grad_norm": 0.07003813982009888,
      "kl": 0.010120850245584734,
      "learning_rate": 4.926315789473685e-06,
      "loss": -0.0007,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.375,
      "epoch": 0.008874931731294375,
      "grad_norm": 0.03567022085189819,
      "kl": 0.011777133724535815,
      "learning_rate": 4.921052631578948e-06,
      "loss": 0.0001,
      "num_tokens": 200529.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.009011469142545057,
      "grad_norm": 0.03496097773313522,
      "kl": 0.012535931979073212,
      "learning_rate": 4.915789473684211e-06,
      "loss": 0.0,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00914800655379574,
      "grad_norm": 0.03556264564394951,
      "kl": 0.012659623011131771,
      "learning_rate": 4.910526315789474e-06,
      "loss": -0.0,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.009284543965046422,
      "grad_norm": 0.03532101958990097,
      "kl": 0.013175853382563218,
      "learning_rate": 4.905263157894737e-06,
      "loss": -0.0,
      "step": 68
    },
    {
      "clip_ratio": 0.0006605616363231093,
      "completion_length": 124.90625,
      "epoch": 0.009421081376297106,
      "grad_norm": 0.05857838690280914,
      "kl": 0.012000912392977625,
      "learning_rate": 4.9000000000000005e-06,
      "loss": 0.0003,
      "num_tokens": 211414.0,
      "reward": 0.8875000476837158,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.8875000476837158,
      "step": 69
    },
    {
      "clip_ratio": 0.0002765486715361476,
      "epoch": 0.009557618787547788,
      "grad_norm": 0.060380011796951294,
      "kl": 0.012359667627606541,
      "learning_rate": 4.894736842105264e-06,
      "loss": 0.0001,
      "step": 70
    },
    {
      "clip_ratio": 0.0006605616363231093,
      "epoch": 0.009694156198798471,
      "grad_norm": 0.05945640802383423,
      "kl": 0.012303199764573947,
      "learning_rate": 4.889473684210527e-06,
      "loss": -0.0,
      "step": 71
    },
    {
      "clip_ratio": 0.0006619231717195362,
      "epoch": 0.009830693610049153,
      "grad_norm": 0.052826281636953354,
      "kl": 0.012528339851996861,
      "learning_rate": 4.88421052631579e-06,
      "loss": -0.0004,
      "step": 72
    },
    {
      "clip_ratio": 0.00036407963489182293,
      "completion_length": 142.125,
      "epoch": 0.009967231021299837,
      "grad_norm": 0.08659490197896957,
      "kl": 0.010383772038039751,
      "learning_rate": 4.878947368421053e-06,
      "loss": 0.0001,
      "num_tokens": 223626.0,
      "reward": 0.6031250357627869,
      "reward_std": 0.23124998807907104,
      "rewards/reward_fn": 0.6031250357627869,
      "step": 73
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.010103768432550519,
      "grad_norm": 0.08382167667150497,
      "kl": 0.010327450014301576,
      "learning_rate": 4.873684210526316e-06,
      "loss": -0.0002,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0102403058438012,
      "grad_norm": 0.0809021145105362,
      "kl": 0.010177173695410602,
      "learning_rate": 4.8684210526315795e-06,
      "loss": -0.0006,
      "step": 75
    },
    {
      "clip_ratio": 0.0005547723558265716,
      "epoch": 0.010376843255051884,
      "grad_norm": 0.07807288318872452,
      "kl": 0.010375534344348125,
      "learning_rate": 4.863157894736843e-06,
      "loss": -0.0011,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.15625,
      "epoch": 0.010513380666302566,
      "grad_norm": 0.09454550594091415,
      "kl": 0.01368577299581375,
      "learning_rate": 4.857894736842106e-06,
      "loss": 0.0002,
      "num_tokens": 235095.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01064991807755325,
      "grad_norm": 0.09300191700458527,
      "kl": 0.013703606295166537,
      "learning_rate": 4.852631578947369e-06,
      "loss": -0.0002,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.010786455488803932,
      "grad_norm": 0.08737168461084366,
      "kl": 0.01426290805102326,
      "learning_rate": 4.847368421052631e-06,
      "loss": -0.0006,
      "step": 79
    },
    {
      "clip_ratio": 0.00027173911803402007,
      "epoch": 0.010922992900054615,
      "grad_norm": 0.0839756429195404,
      "kl": 0.014389160307473503,
      "learning_rate": 4.842105263157895e-06,
      "loss": -0.0011,
      "step": 80
    },
    {
      "clip_ratio": 0.00023148147738538682,
      "completion_length": 115.40625,
      "epoch": 0.011059530311305297,
      "grad_norm": 0.0841306746006012,
      "kl": 0.019477701251162216,
      "learning_rate": 4.8368421052631585e-06,
      "loss": 0.0003,
      "num_tokens": 245552.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 81
    },
    {
      "clip_ratio": 0.00021551724057644606,
      "epoch": 0.01119606772255598,
      "grad_norm": 0.08322307467460632,
      "kl": 0.019782173665589653,
      "learning_rate": 4.831578947368422e-06,
      "loss": -0.0001,
      "step": 82
    },
    {
      "clip_ratio": 0.00021551724057644606,
      "epoch": 0.011332605133806663,
      "grad_norm": 0.08051823079586029,
      "kl": 0.019718343231943436,
      "learning_rate": 4.826315789473685e-06,
      "loss": -0.0006,
      "step": 83
    },
    {
      "clip_ratio": 0.0007434187136823311,
      "epoch": 0.011469142545057346,
      "grad_norm": 0.07694952934980392,
      "kl": 0.01945641718339175,
      "learning_rate": 4.821052631578948e-06,
      "loss": -0.0009,
      "step": 84
    },
    {
      "clip_ratio": 0.000730298925191164,
      "completion_length": 113.375,
      "epoch": 0.011605679956308028,
      "grad_norm": 0.0934474989771843,
      "kl": 0.01406104041961953,
      "learning_rate": 4.815789473684211e-06,
      "loss": 0.0001,
      "num_tokens": 256096.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 85
    },
    {
      "clip_ratio": 0.0006386408931575716,
      "epoch": 0.011742217367558712,
      "grad_norm": 0.09243673086166382,
      "kl": 0.013716014829697087,
      "learning_rate": 4.8105263157894735e-06,
      "loss": -0.0002,
      "step": 86
    },
    {
      "clip_ratio": 0.00016983695968519896,
      "epoch": 0.011878754778809394,
      "grad_norm": 0.08725881576538086,
      "kl": 0.013890811271267012,
      "learning_rate": 4.8052631578947375e-06,
      "loss": -0.0007,
      "step": 87
    },
    {
      "clip_ratio": 0.00024801588733680546,
      "epoch": 0.012015292190060076,
      "grad_norm": 0.07987057417631149,
      "kl": 0.013736142864217982,
      "learning_rate": 4.800000000000001e-06,
      "loss": -0.0012,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.03125,
      "epoch": 0.01215182960131076,
      "grad_norm": 0.035524703562259674,
      "kl": 0.014395015765330754,
      "learning_rate": 4.794736842105264e-06,
      "loss": 0.0001,
      "num_tokens": 267685.0,
      "reward": 0.746874988079071,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.746874988079071,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.012288367012561441,
      "grad_norm": 0.03528278321027756,
      "kl": 0.014380123306182213,
      "learning_rate": 4.789473684210527e-06,
      "loss": 0.0001,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.012424904423812125,
      "grad_norm": 0.03564177826046944,
      "kl": 0.014681547458167188,
      "learning_rate": 4.78421052631579e-06,
      "loss": -0.0,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.012561441835062807,
      "grad_norm": 0.03350580483675003,
      "kl": 0.01473659290059004,
      "learning_rate": 4.778947368421053e-06,
      "loss": -0.0001,
      "step": 92
    },
    {
      "clip_ratio": 0.0010092409647768363,
      "completion_length": 116.34375,
      "epoch": 0.01269797924631349,
      "grad_norm": 0.11783453822135925,
      "kl": 0.016927717690123245,
      "learning_rate": 4.773684210526316e-06,
      "loss": 0.0002,
      "num_tokens": 278480.0,
      "reward": 0.46562501788139343,
      "reward_std": 0.36360570788383484,
      "rewards/reward_fn": 0.46562501788139343,
      "step": 93
    },
    {
      "clip_ratio": 0.0005676986766047776,
      "epoch": 0.012834516657564172,
      "grad_norm": 0.1174498200416565,
      "kl": 0.016682636385667138,
      "learning_rate": 4.76842105263158e-06,
      "loss": -0.0001,
      "step": 94
    },
    {
      "clip_ratio": 0.00021258502965793014,
      "epoch": 0.012971054068814856,
      "grad_norm": 0.1170513778924942,
      "kl": 0.016236918556387536,
      "learning_rate": 4.763157894736842e-06,
      "loss": -0.001,
      "step": 95
    },
    {
      "clip_ratio": 0.0008842054230626673,
      "epoch": 0.013107591480065538,
      "grad_norm": 0.11030783504247665,
      "kl": 0.016805329025373794,
      "learning_rate": 4.757894736842106e-06,
      "loss": -0.0018,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.34375,
      "epoch": 0.013244128891316221,
      "grad_norm": 0.05533390864729881,
      "kl": 0.012452395327272825,
      "learning_rate": 4.752631578947369e-06,
      "loss": 0.0,
      "num_tokens": 289523.0,
      "reward": 0.859375,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.859375,
      "step": 97
    },
    {
      "clip_ratio": 0.00028935185400769114,
      "epoch": 0.013380666302566903,
      "grad_norm": 0.05383645370602608,
      "kl": 0.012898564287752379,
      "learning_rate": 4.747368421052632e-06,
      "loss": -0.0,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.013517203713817587,
      "grad_norm": 0.045518334954977036,
      "kl": 0.012721069382678252,
      "learning_rate": 4.7421052631578954e-06,
      "loss": -0.0002,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.013653741125068269,
      "grad_norm": 0.043640851974487305,
      "kl": 0.013114535111526493,
      "learning_rate": 4.736842105263158e-06,
      "loss": -0.0004,
      "step": 100
    },
    {
      "clip_ratio": 0.0003751235635718331,
      "completion_length": 133.21875,
      "epoch": 0.01379027853631895,
      "grad_norm": 0.08343350887298584,
      "kl": 0.011969710292760283,
      "learning_rate": 4.731578947368422e-06,
      "loss": 0.0003,
      "num_tokens": 301274.0,
      "reward": 0.690625011920929,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.690625011920929,
      "step": 101
    },
    {
      "clip_ratio": 0.0003751235635718331,
      "epoch": 0.013926815947569634,
      "grad_norm": 0.08278653025627136,
      "kl": 0.011722918432496954,
      "learning_rate": 4.726315789473684e-06,
      "loss": -0.0001,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.014063353358820316,
      "grad_norm": 0.08366868644952774,
      "kl": 0.011790596836362965,
      "learning_rate": 4.721052631578948e-06,
      "loss": -0.0006,
      "step": 103
    },
    {
      "clip_ratio": 0.00020161290012765676,
      "epoch": 0.014199890770071,
      "grad_norm": 0.07245514541864395,
      "kl": 0.011914224931388162,
      "learning_rate": 4.71578947368421e-06,
      "loss": -0.001,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 105.34375,
      "epoch": 0.014336428181321682,
      "grad_norm": 0.06334606558084488,
      "kl": 0.014175733827869408,
      "learning_rate": 4.710526315789474e-06,
      "loss": 0.0002,
      "num_tokens": 312005.0,
      "reward": 0.9156249761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.9156249761581421,
      "step": 105
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.014472965592572365,
      "grad_norm": 0.06191639602184296,
      "kl": 0.014437062345677987,
      "learning_rate": 4.705263157894738e-06,
      "loss": 0.0,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.014609503003823047,
      "grad_norm": 0.06282703578472137,
      "kl": 0.014868097176076844,
      "learning_rate": 4.7e-06,
      "loss": -0.0002,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01474604041507373,
      "grad_norm": 0.06213715299963951,
      "kl": 0.015245917427819222,
      "learning_rate": 4.694736842105264e-06,
      "loss": -0.0006,
      "step": 108
    },
    {
      "clip_ratio": 0.00048828125,
      "completion_length": 114.46875,
      "epoch": 0.014882577826324413,
      "grad_norm": 0.06244153529405594,
      "kl": 0.014229173626517877,
      "learning_rate": 4.689473684210526e-06,
      "loss": 0.0002,
      "num_tokens": 323008.0,
      "reward": 0.856249988079071,
      "reward_std": 0.12358209490776062,
      "rewards/reward_fn": 0.856249988079071,
      "step": 109
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.015019115237575096,
      "grad_norm": 0.06384127587080002,
      "kl": 0.014452488074311987,
      "learning_rate": 4.68421052631579e-06,
      "loss": -0.0001,
      "step": 110
    },
    {
      "clip_ratio": 0.000732421875,
      "epoch": 0.015155652648825778,
      "grad_norm": 0.05795735865831375,
      "kl": 0.014755003750906326,
      "learning_rate": 4.6789473684210525e-06,
      "loss": -0.0003,
      "step": 111
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.015292190060076462,
      "grad_norm": 0.06043582037091255,
      "kl": 0.014582981850253418,
      "learning_rate": 4.6736842105263166e-06,
      "loss": -0.0004,
      "step": 112
    },
    {
      "clip_ratio": 0.0005965908931102604,
      "completion_length": 109.96875,
      "epoch": 0.015428727471327144,
      "grad_norm": 0.06778660416603088,
      "kl": 0.015787007549079135,
      "learning_rate": 4.668421052631579e-06,
      "loss": 0.0003,
      "num_tokens": 333667.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.015565264882577826,
      "grad_norm": 0.07079855352640152,
      "kl": 0.015863822191022336,
      "learning_rate": 4.663157894736842e-06,
      "loss": -0.0,
      "step": 114
    },
    {
      "clip_ratio": 0.0005342270596884191,
      "epoch": 0.015701802293828507,
      "grad_norm": 0.07030012458562851,
      "kl": 0.015859683539019898,
      "learning_rate": 4.657894736842106e-06,
      "loss": -0.0003,
      "step": 115
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.015838339705079193,
      "grad_norm": 0.05977308005094528,
      "kl": 0.015181809227215126,
      "learning_rate": 4.652631578947368e-06,
      "loss": -0.0007,
      "step": 116
    },
    {
      "clip_ratio": 0.000294811325147748,
      "completion_length": 102.75,
      "epoch": 0.015974877116329875,
      "grad_norm": 0.09741310030221939,
      "kl": 0.016162807471118867,
      "learning_rate": 4.647368421052632e-06,
      "loss": 0.0004,
      "num_tokens": 343995.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 117
    },
    {
      "clip_ratio": 0.00020973154460079968,
      "epoch": 0.016111414527580557,
      "grad_norm": 0.09549403935670853,
      "kl": 0.015964071644702926,
      "learning_rate": 4.642105263157895e-06,
      "loss": 0.0001,
      "step": 118
    },
    {
      "clip_ratio": 0.0010666992020560429,
      "epoch": 0.01624795193883124,
      "grad_norm": 0.09260634332895279,
      "kl": 0.016316924724378623,
      "learning_rate": 4.636842105263159e-06,
      "loss": -0.0007,
      "step": 119
    },
    {
      "clip_ratio": 0.0005621563323074952,
      "epoch": 0.016384489350081924,
      "grad_norm": 0.07998410612344742,
      "kl": 0.016282340366160497,
      "learning_rate": 4.631578947368421e-06,
      "loss": -0.0016,
      "step": 120
    },
    {
      "clip_ratio": 0.0005777268670499325,
      "completion_length": 105.71875,
      "epoch": 0.016521026761332606,
      "grad_norm": 0.11327945441007614,
      "kl": 0.014576794725144282,
      "learning_rate": 4.626315789473684e-06,
      "loss": -0.0,
      "num_tokens": 354366.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.289951890707016,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 121
    },
    {
      "clip_ratio": 0.0010138368816114962,
      "epoch": 0.016657564172583288,
      "grad_norm": 0.11017230898141861,
      "kl": 0.015154679873376153,
      "learning_rate": 4.621052631578948e-06,
      "loss": -0.0003,
      "step": 122
    },
    {
      "clip_ratio": 0.0029109671886544675,
      "epoch": 0.01679410158383397,
      "grad_norm": 0.09320779144763947,
      "kl": 0.01626152906101197,
      "learning_rate": 4.6157894736842105e-06,
      "loss": -0.001,
      "step": 123
    },
    {
      "clip_ratio": 0.003067635203478858,
      "epoch": 0.016930638995084655,
      "grad_norm": 0.08714654296636581,
      "kl": 0.01717385504161939,
      "learning_rate": 4.6105263157894745e-06,
      "loss": -0.0019,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 104.625,
      "epoch": 0.017067176406335337,
      "grad_norm": 0.06164494529366493,
      "kl": 0.022742171786376275,
      "learning_rate": 4.605263157894737e-06,
      "loss": 0.0001,
      "num_tokens": 365118.0,
      "reward": 0.4937500059604645,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.4937500059604645,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01720371381758602,
      "grad_norm": 0.05765366554260254,
      "kl": 0.023318831241340376,
      "learning_rate": 4.600000000000001e-06,
      "loss": 0.0001,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0173402512288367,
      "grad_norm": 0.057430919259786606,
      "kl": 0.024255198208265938,
      "learning_rate": 4.594736842105263e-06,
      "loss": -0.0001,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.017476788640087382,
      "grad_norm": 0.0521625392138958,
      "kl": 0.024568809458287433,
      "learning_rate": 4.589473684210526e-06,
      "loss": -0.0003,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 97.4375,
      "epoch": 0.017613326051338068,
      "grad_norm": 0.06752870976924896,
      "kl": 0.02813084190711379,
      "learning_rate": 4.5842105263157895e-06,
      "loss": 0.0003,
      "num_tokens": 375444.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01774986346258875,
      "grad_norm": 0.06426940858364105,
      "kl": 0.02811072752228938,
      "learning_rate": 4.578947368421053e-06,
      "loss": -0.0001,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01788640087383943,
      "grad_norm": 0.052824705839157104,
      "kl": 0.02794748015003279,
      "learning_rate": 4.573684210526317e-06,
      "loss": -0.0003,
      "step": 131
    },
    {
      "clip_ratio": 0.0005663055635523051,
      "epoch": 0.018022938285090113,
      "grad_norm": 0.05740916728973389,
      "kl": 0.027399677026551217,
      "learning_rate": 4.568421052631579e-06,
      "loss": -0.0006,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.0625,
      "epoch": 0.0181594756963408,
      "grad_norm": 0.08138228207826614,
      "kl": 0.021528464392758906,
      "learning_rate": 4.563157894736843e-06,
      "loss": 0.0004,
      "num_tokens": 385342.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01829601310759148,
      "grad_norm": 0.07547755539417267,
      "kl": 0.021013552715885453,
      "learning_rate": 4.557894736842105e-06,
      "loss": 0.0001,
      "step": 134
    },
    {
      "clip_ratio": 0.0004340277810115367,
      "epoch": 0.018432550518842163,
      "grad_norm": 0.06484001874923706,
      "kl": 0.020921625837218016,
      "learning_rate": 4.552631578947369e-06,
      "loss": -0.0003,
      "step": 135
    },
    {
      "clip_ratio": 0.0004340277810115367,
      "epoch": 0.018569087930092845,
      "grad_norm": 0.051754601299762726,
      "kl": 0.021819816451170482,
      "learning_rate": 4.547368421052632e-06,
      "loss": -0.0008,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 102.28125,
      "epoch": 0.01870562534134353,
      "grad_norm": 0.04303466156125069,
      "kl": 0.02289676680811681,
      "learning_rate": 4.542105263157895e-06,
      "loss": 0.0002,
      "num_tokens": 396003.0,
      "reward": 0.971875011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.971875011920929,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.018842162752594212,
      "grad_norm": 0.04287838935852051,
      "kl": 0.02256237791152671,
      "learning_rate": 4.536842105263158e-06,
      "loss": 0.0001,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.018978700163844894,
      "grad_norm": 0.04330320656299591,
      "kl": 0.022393779450794682,
      "learning_rate": 4.531578947368421e-06,
      "loss": 0.0001,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.019115237575095576,
      "grad_norm": 0.039670608937740326,
      "kl": 0.022108391451183707,
      "learning_rate": 4.526315789473685e-06,
      "loss": -0.0002,
      "step": 140
    },
    {
      "clip_ratio": 0.0011110177874797955,
      "completion_length": 124.75,
      "epoch": 0.019251774986346257,
      "grad_norm": 0.08559431880712509,
      "kl": 0.02040508067875635,
      "learning_rate": 4.5210526315789475e-06,
      "loss": 0.0004,
      "num_tokens": 407619.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.690625011920929,
      "step": 141
    },
    {
      "clip_ratio": 0.0009295805648434907,
      "epoch": 0.019388312397596943,
      "grad_norm": 0.08730953186750412,
      "kl": 0.0202211801224621,
      "learning_rate": 4.5157894736842115e-06,
      "loss": 0.0001,
      "step": 142
    },
    {
      "clip_ratio": 0.0001990445889532566,
      "epoch": 0.019524849808847625,
      "grad_norm": 0.08218631148338318,
      "kl": 0.020368822690215893,
      "learning_rate": 4.510526315789474e-06,
      "loss": -0.0005,
      "step": 143
    },
    {
      "clip_ratio": 0.0001990445889532566,
      "epoch": 0.019661387220098307,
      "grad_norm": 0.07692748308181763,
      "kl": 0.02033531089546159,
      "learning_rate": 4.505263157894737e-06,
      "loss": -0.0011,
      "step": 144
    },
    {
      "clip_ratio": 0.00027173911803402007,
      "completion_length": 93.125,
      "epoch": 0.01979792463134899,
      "grad_norm": 0.06828766316175461,
      "kl": 0.026619675249094144,
      "learning_rate": 4.5e-06,
      "loss": 0.0003,
      "num_tokens": 417195.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.019934462042599674,
      "grad_norm": 0.07163087278604507,
      "kl": 0.027274078922346234,
      "learning_rate": 4.494736842105263e-06,
      "loss": 0.0001,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.020070999453850356,
      "grad_norm": 0.0677010715007782,
      "kl": 0.02618727515800856,
      "learning_rate": 4.489473684210527e-06,
      "loss": -0.0003,
      "step": 147
    },
    {
      "clip_ratio": 0.0005972599610686302,
      "epoch": 0.020207536865101038,
      "grad_norm": 0.05918688699603081,
      "kl": 0.0264243567653466,
      "learning_rate": 4.48421052631579e-06,
      "loss": -0.0005,
      "step": 148
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "completion_length": 114.15625,
      "epoch": 0.02034407427635172,
      "grad_norm": 0.10522865504026413,
      "kl": 0.025243330223020166,
      "learning_rate": 4.478947368421054e-06,
      "loss": 0.0004,
      "num_tokens": 427584.0,
      "reward": 0.6343750357627869,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.6343750357627869,
      "step": 149
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "epoch": 0.0204806116876024,
      "grad_norm": 0.10407311469316483,
      "kl": 0.02505102547002025,
      "learning_rate": 4.473684210526316e-06,
      "loss": -0.0003,
      "step": 150
    },
    {
      "clip_ratio": 0.0009035185066750273,
      "epoch": 0.020617149098853087,
      "grad_norm": 0.10190074890851974,
      "kl": 0.025200023461366072,
      "learning_rate": 4.468421052631579e-06,
      "loss": -0.001,
      "step": 151
    },
    {
      "clip_ratio": 0.0014054116763873026,
      "epoch": 0.02075368651010377,
      "grad_norm": 0.0961143970489502,
      "kl": 0.025511470768833533,
      "learning_rate": 4.463157894736842e-06,
      "loss": -0.0019,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 107.71875,
      "epoch": 0.02089022392135445,
      "grad_norm": 0.059831857681274414,
      "kl": 0.023090639937436208,
      "learning_rate": 4.4578947368421054e-06,
      "loss": 0.0003,
      "num_tokens": 438211.0,
      "reward": 0.3812500238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.3812500238418579,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.021026761332605132,
      "grad_norm": 0.059171441942453384,
      "kl": 0.02331032755319029,
      "learning_rate": 4.452631578947369e-06,
      "loss": 0.0001,
      "step": 154
    },
    {
      "clip_ratio": 0.0006432863010559231,
      "epoch": 0.021163298743855818,
      "grad_norm": 0.05819160118699074,
      "kl": 0.023676532495301217,
      "learning_rate": 4.447368421052632e-06,
      "loss": -0.0,
      "step": 155
    },
    {
      "clip_ratio": 0.0005681818001903594,
      "epoch": 0.0212998361551065,
      "grad_norm": 0.05214538797736168,
      "kl": 0.02407641662284732,
      "learning_rate": 4.442105263157896e-06,
      "loss": -0.0004,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 111.25,
      "epoch": 0.02143637356635718,
      "grad_norm": 0.06498148292303085,
      "kl": 0.028692521911580116,
      "learning_rate": 4.436842105263158e-06,
      "loss": 0.0002,
      "num_tokens": 448495.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.021572910977607863,
      "grad_norm": 0.06544307619333267,
      "kl": 0.029219213291071355,
      "learning_rate": 4.431578947368421e-06,
      "loss": 0.0003,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02170944838885855,
      "grad_norm": 0.06307835131883621,
      "kl": 0.0294132886629086,
      "learning_rate": 4.426315789473684e-06,
      "loss": -0.0002,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02184598580010923,
      "grad_norm": 0.06254979223012924,
      "kl": 0.03018665249692276,
      "learning_rate": 4.4210526315789476e-06,
      "loss": -0.0003,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.75,
      "epoch": 0.021982523211359913,
      "grad_norm": 0.09002089500427246,
      "kl": 0.025298897293396294,
      "learning_rate": 4.415789473684211e-06,
      "loss": 0.0001,
      "num_tokens": 459767.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.2337018847465515,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.022119060622610594,
      "grad_norm": 0.0899246335029602,
      "kl": 0.025724691950017586,
      "learning_rate": 4.410526315789474e-06,
      "loss": -0.0,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.022255598033861276,
      "grad_norm": 0.08925420790910721,
      "kl": 0.02707095700316131,
      "learning_rate": 4.405263157894737e-06,
      "loss": -0.0007,
      "step": 163
    },
    {
      "clip_ratio": 0.00017361111531499773,
      "epoch": 0.02239213544511196,
      "grad_norm": 0.0802987664937973,
      "kl": 0.027386941597796977,
      "learning_rate": 4.4e-06,
      "loss": -0.0015,
      "step": 164
    },
    {
      "clip_ratio": 0.0005446214054245502,
      "completion_length": 112.78125,
      "epoch": 0.022528672856362644,
      "grad_norm": 0.07420904189348221,
      "kl": 0.028630498243728653,
      "learning_rate": 4.394736842105263e-06,
      "loss": 0.0003,
      "num_tokens": 470856.0,
      "reward": 0.859375,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.859375,
      "step": 165
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.022665210267613325,
      "grad_norm": 0.07424714416265488,
      "kl": 0.02978915546555072,
      "learning_rate": 4.3894736842105266e-06,
      "loss": 0.0001,
      "step": 166
    },
    {
      "clip_ratio": 0.0005446214054245502,
      "epoch": 0.022801747678864007,
      "grad_norm": 0.0685018002986908,
      "kl": 0.03043460487970151,
      "learning_rate": 4.38421052631579e-06,
      "loss": -0.0003,
      "step": 167
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.022938285090114693,
      "grad_norm": 0.06347281485795975,
      "kl": 0.03080894984304905,
      "learning_rate": 4.378947368421053e-06,
      "loss": -0.0009,
      "step": 168
    },
    {
      "clip_ratio": 0.0003324467979837209,
      "completion_length": 115.125,
      "epoch": 0.023074822501365375,
      "grad_norm": 0.09582728147506714,
      "kl": 0.029706816072575748,
      "learning_rate": 4.373684210526316e-06,
      "loss": 0.0005,
      "num_tokens": 481480.0,
      "reward": 0.4937500059604645,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.4937500059604645,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.023211359912616056,
      "grad_norm": 0.08268621563911438,
      "kl": 0.029470782610587776,
      "learning_rate": 4.368421052631579e-06,
      "loss": -0.0,
      "step": 170
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "epoch": 0.02334789732386674,
      "grad_norm": 0.07390839606523514,
      "kl": 0.029934398946352303,
      "learning_rate": 4.363157894736842e-06,
      "loss": -0.0005,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.023484434735117424,
      "grad_norm": 0.07234860211610794,
      "kl": 0.02987176994793117,
      "learning_rate": 4.3578947368421055e-06,
      "loss": -0.0009,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.84375,
      "epoch": 0.023620972146368106,
      "grad_norm": 0.07804840058088303,
      "kl": 0.030131216015433893,
      "learning_rate": 4.352631578947369e-06,
      "loss": 0.0002,
      "num_tokens": 491971.0,
      "reward": 0.887499988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.887499988079071,
      "step": 173
    },
    {
      "clip_ratio": 0.0005341880605556071,
      "epoch": 0.023757509557618788,
      "grad_norm": 0.07273226976394653,
      "kl": 0.030887642147717997,
      "learning_rate": 4.347368421052632e-06,
      "loss": 0.0,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02389404696886947,
      "grad_norm": 0.06586578488349915,
      "kl": 0.031295952590880916,
      "learning_rate": 4.342105263157895e-06,
      "loss": -0.0005,
      "step": 175
    },
    {
      "clip_ratio": 0.0008401820377912372,
      "epoch": 0.02403058438012015,
      "grad_norm": 0.059170957654714584,
      "kl": 0.03198954090476036,
      "learning_rate": 4.336842105263158e-06,
      "loss": -0.001,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 106.65625,
      "epoch": 0.024167121791370837,
      "grad_norm": 0.08889607340097427,
      "kl": 0.04427822964498773,
      "learning_rate": 4.331578947368421e-06,
      "loss": 0.0006,
      "num_tokens": 502488.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02430365920262152,
      "grad_norm": 0.07279235869646072,
      "kl": 0.0449052246985957,
      "learning_rate": 4.3263157894736845e-06,
      "loss": 0.0001,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0244401966138722,
      "grad_norm": 0.072224460542202,
      "kl": 0.04581551585579291,
      "learning_rate": 4.321052631578948e-06,
      "loss": -0.0002,
      "step": 179
    },
    {
      "clip_ratio": 0.00018825300503522158,
      "epoch": 0.024576734025122882,
      "grad_norm": 0.06858509033918381,
      "kl": 0.04448665684321895,
      "learning_rate": 4.315789473684211e-06,
      "loss": -0.0008,
      "step": 180
    },
    {
      "clip_ratio": 0.00035919540096074343,
      "completion_length": 99.4375,
      "epoch": 0.024713271436373568,
      "grad_norm": 0.06964289397001266,
      "kl": 0.03402404289226979,
      "learning_rate": 4.310526315789474e-06,
      "loss": 0.0004,
      "num_tokens": 513246.0,
      "reward": 0.746874988079071,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.746874988079071,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02484980884762425,
      "grad_norm": 0.06922803074121475,
      "kl": 0.03361908288206905,
      "learning_rate": 4.305263157894737e-06,
      "loss": 0.0003,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02498634625887493,
      "grad_norm": 0.0668816789984703,
      "kl": 0.033550983702298254,
      "learning_rate": 4.3e-06,
      "loss": -0.0,
      "step": 183
    },
    {
      "clip_ratio": 0.0003511235991027206,
      "epoch": 0.025122883670125613,
      "grad_norm": 0.05455351620912552,
      "kl": 0.03327295114286244,
      "learning_rate": 4.2947368421052635e-06,
      "loss": -0.0005,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 106.65625,
      "epoch": 0.0252594210813763,
      "grad_norm": 0.11938021332025528,
      "kl": 0.044678817328531295,
      "learning_rate": 4.289473684210527e-06,
      "loss": 0.0005,
      "num_tokens": 524007.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.2511056959629059,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 185
    },
    {
      "clip_ratio": 0.00034722223062999547,
      "epoch": 0.02539595849262698,
      "grad_norm": 0.11370915919542313,
      "kl": 0.04234666272532195,
      "learning_rate": 4.28421052631579e-06,
      "loss": -0.0003,
      "step": 186
    },
    {
      "clip_ratio": 0.000662878795992583,
      "epoch": 0.025532495903877662,
      "grad_norm": 0.10087863355875015,
      "kl": 0.04062945069745183,
      "learning_rate": 4.278947368421053e-06,
      "loss": -0.001,
      "step": 187
    },
    {
      "clip_ratio": 0.0005541758728213608,
      "epoch": 0.025669033315128344,
      "grad_norm": 0.09143723547458649,
      "kl": 0.0406409723800607,
      "learning_rate": 4.273684210526316e-06,
      "loss": -0.0017,
      "step": 188
    },
    {
      "clip_ratio": 0.0005189153016544878,
      "completion_length": 103.1875,
      "epoch": 0.025805570726379026,
      "grad_norm": 0.11307642608880997,
      "kl": 0.03528324858052656,
      "learning_rate": 4.268421052631579e-06,
      "loss": 0.0005,
      "num_tokens": 533865.0,
      "reward": 0.7750000357627869,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.7750000357627869,
      "step": 189
    },
    {
      "clip_ratio": 0.0003124999930150807,
      "epoch": 0.02594210813762971,
      "grad_norm": 0.1095878928899765,
      "kl": 0.03600154822925106,
      "learning_rate": 4.2631578947368425e-06,
      "loss": -0.0002,
      "step": 190
    },
    {
      "clip_ratio": 0.0006060116575099528,
      "epoch": 0.026078645548880394,
      "grad_norm": 0.0969381183385849,
      "kl": 0.035806038504233584,
      "learning_rate": 4.257894736842106e-06,
      "loss": -0.0011,
      "step": 191
    },
    {
      "clip_ratio": 0.0005024050769861788,
      "epoch": 0.026215182960131075,
      "grad_norm": 0.08637399971485138,
      "kl": 0.03535072586964816,
      "learning_rate": 4.252631578947369e-06,
      "loss": -0.0018,
      "step": 192
    },
    {
      "completion_length": 106.0,
      "epoch": 0.026351720371381757,
      "grad_norm": 0.0,
      "learning_rate": 4.247368421052632e-06,
      "loss": 0.0,
      "num_tokens": 543893.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.0,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 193
    },
    {
      "epoch": 0.026488257782632443,
      "grad_norm": 0.0,
      "learning_rate": 4.242105263157895e-06,
      "loss": 0.0,
      "step": 194
    },
    {
      "epoch": 0.026624795193883125,
      "grad_norm": 0.0,
      "learning_rate": 4.236842105263158e-06,
      "loss": 0.0,
      "step": 195
    },
    {
      "epoch": 0.026761332605133806,
      "grad_norm": 0.0,
      "learning_rate": 4.2315789473684215e-06,
      "loss": 0.0,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 104.1875,
      "epoch": 0.02689787001638449,
      "grad_norm": 0.10188470780849457,
      "kl": 0.03902641427703202,
      "learning_rate": 4.226315789473685e-06,
      "loss": 0.0002,
      "num_tokens": 554083.0,
      "reward": 0.859375,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.859375,
      "step": 197
    },
    {
      "clip_ratio": 0.00021404109429568052,
      "epoch": 0.027034407427635174,
      "grad_norm": 0.09636596590280533,
      "kl": 0.03859919391106814,
      "learning_rate": 4.221052631578948e-06,
      "loss": 0.0,
      "step": 198
    },
    {
      "clip_ratio": 0.00045628915540874004,
      "epoch": 0.027170944838885856,
      "grad_norm": 0.07624981552362442,
      "kl": 0.03810006860294379,
      "learning_rate": 4.215789473684211e-06,
      "loss": -0.0007,
      "step": 199
    },
    {
      "clip_ratio": 0.0005362060328479856,
      "epoch": 0.027307482250136537,
      "grad_norm": 0.07269936800003052,
      "kl": 0.038183362572453916,
      "learning_rate": 4.210526315789474e-06,
      "loss": -0.0012,
      "step": 200
    },
    {
      "clip_ratio": 0.0004032258002553135,
      "completion_length": 105.375,
      "epoch": 0.02744401966138722,
      "grad_norm": 0.06504663079977036,
      "kl": 0.03330190840642899,
      "learning_rate": 4.205263157894737e-06,
      "loss": 0.0003,
      "num_tokens": 564759.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 201
    },
    {
      "clip_ratio": 0.0004032258002553135,
      "epoch": 0.0275805570726379,
      "grad_norm": 0.0650041475892067,
      "kl": 0.03405800275504589,
      "learning_rate": 4.2000000000000004e-06,
      "loss": 0.0001,
      "step": 202
    },
    {
      "clip_ratio": 0.0006048387149348855,
      "epoch": 0.027717094483888587,
      "grad_norm": 0.06338495761156082,
      "kl": 0.03414264236926101,
      "learning_rate": 4.194736842105264e-06,
      "loss": -0.0002,
      "step": 203
    },
    {
      "clip_ratio": 0.0004032258002553135,
      "epoch": 0.02785363189513927,
      "grad_norm": 0.06049591302871704,
      "kl": 0.03434641310013831,
      "learning_rate": 4.189473684210527e-06,
      "loss": -0.0007,
      "step": 204
    },
    {
      "clip_ratio": 0.0007811779651092365,
      "completion_length": 117.34375,
      "epoch": 0.02799016930638995,
      "grad_norm": 0.09299098700284958,
      "kl": 0.03305593744153157,
      "learning_rate": 4.18421052631579e-06,
      "loss": 0.0004,
      "num_tokens": 575498.0,
      "reward": 0.4937500059604645,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.4937500059604645,
      "step": 205
    },
    {
      "clip_ratio": 0.0003324467979837209,
      "epoch": 0.028126706717640632,
      "grad_norm": 0.09308190643787384,
      "kl": 0.03269680918310769,
      "learning_rate": 4.178947368421053e-06,
      "loss": 0.0003,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.028263244128891318,
      "grad_norm": 0.08965054899454117,
      "kl": 0.03241226324462332,
      "learning_rate": 4.173684210526316e-06,
      "loss": -0.0006,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.028399781540142,
      "grad_norm": 0.07739660888910294,
      "kl": 0.03276632426423021,
      "learning_rate": 4.1684210526315794e-06,
      "loss": -0.0013,
      "step": 208
    },
    {
      "clip_ratio": 0.00019409938249737024,
      "completion_length": 123.375,
      "epoch": 0.02853631895139268,
      "grad_norm": 0.06903862208127975,
      "kl": 0.02725412551080808,
      "learning_rate": 4.163157894736843e-06,
      "loss": 0.0004,
      "num_tokens": 587282.0,
      "reward": 0.690625011920929,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.690625011920929,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.028672856362643363,
      "grad_norm": 0.06824494898319244,
      "kl": 0.027433462004410103,
      "learning_rate": 4.157894736842106e-06,
      "loss": 0.0002,
      "step": 210
    },
    {
      "clip_ratio": 0.0004374573181848973,
      "epoch": 0.02880939377389405,
      "grad_norm": 0.06868710368871689,
      "kl": 0.02734687612974085,
      "learning_rate": 4.152631578947369e-06,
      "loss": -0.0001,
      "step": 211
    },
    {
      "clip_ratio": 0.00023320894979406148,
      "epoch": 0.02894593118514473,
      "grad_norm": 0.0652066022157669,
      "kl": 0.027320355700794607,
      "learning_rate": 4.147368421052632e-06,
      "loss": -0.0008,
      "step": 212
    },
    {
      "clip_ratio": 0.00042826301068998873,
      "completion_length": 126.1875,
      "epoch": 0.029082468596395412,
      "grad_norm": 0.04128393158316612,
      "kl": 0.027315025363350287,
      "learning_rate": 4.142105263157895e-06,
      "loss": 0.0003,
      "num_tokens": 598840.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 213
    },
    {
      "clip_ratio": 0.0008565260213799775,
      "epoch": 0.029219006007646094,
      "grad_norm": 0.040705274790525436,
      "kl": 0.02696406643372029,
      "learning_rate": 4.136842105263158e-06,
      "loss": 0.0003,
      "step": 214
    },
    {
      "clip_ratio": 0.0004370629321783781,
      "epoch": 0.029355543418896776,
      "grad_norm": 0.0406440831720829,
      "kl": 0.02672984532546252,
      "learning_rate": 4.1315789473684216e-06,
      "loss": 0.0001,
      "step": 215
    },
    {
      "clip_ratio": 0.0006467944767791778,
      "epoch": 0.02949208083014746,
      "grad_norm": 0.039546553045511246,
      "kl": 0.026357230555731803,
      "learning_rate": 4.126315789473685e-06,
      "loss": -0.0001,
      "step": 216
    },
    {
      "clip_ratio": 0.00021551724057644606,
      "completion_length": 125.65625,
      "epoch": 0.029628618241398143,
      "grad_norm": 0.09606111794710159,
      "kl": 0.027327494870405644,
      "learning_rate": 4.121052631578948e-06,
      "loss": 0.0001,
      "num_tokens": 610221.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.029765155652648825,
      "grad_norm": 0.09475074708461761,
      "kl": 0.026967918034642935,
      "learning_rate": 4.115789473684211e-06,
      "loss": -0.0003,
      "step": 218
    },
    {
      "clip_ratio": 0.00021551724057644606,
      "epoch": 0.029901693063899507,
      "grad_norm": 0.09437484294176102,
      "kl": 0.02764826436759904,
      "learning_rate": 4.110526315789474e-06,
      "loss": -0.0009,
      "step": 219
    },
    {
      "clip_ratio": 0.00021551724057644606,
      "epoch": 0.030038230475150193,
      "grad_norm": 0.08844147622585297,
      "kl": 0.02761025112704374,
      "learning_rate": 4.105263157894737e-06,
      "loss": -0.0017,
      "step": 220
    },
    {
      "clip_ratio": 0.0004921259824186563,
      "completion_length": 117.5625,
      "epoch": 0.030174767886400874,
      "grad_norm": 0.08956988155841827,
      "kl": 0.027353109326213598,
      "learning_rate": 4.1e-06,
      "loss": 0.0005,
      "num_tokens": 621283.0,
      "reward": 0.746874988079071,
      "reward_std": 0.25110572576522827,
      "rewards/reward_fn": 0.746874988079071,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.030311305297651556,
      "grad_norm": 0.09050950407981873,
      "kl": 0.02785796401440166,
      "learning_rate": 4.094736842105264e-06,
      "loss": -0.0003,
      "step": 222
    },
    {
      "clip_ratio": 0.00039178448787424713,
      "epoch": 0.030447842708902238,
      "grad_norm": 0.08440433442592621,
      "kl": 0.0283550031890627,
      "learning_rate": 4.089473684210527e-06,
      "loss": -0.0006,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.030584380120152924,
      "grad_norm": 0.078850157558918,
      "kl": 0.029308805766049773,
      "learning_rate": 4.08421052631579e-06,
      "loss": -0.0015,
      "step": 224
    },
    {
      "clip_ratio": 0.000794308609329164,
      "completion_length": 112.1875,
      "epoch": 0.030720917531403606,
      "grad_norm": 0.09105496853590012,
      "kl": 0.02974900306435302,
      "learning_rate": 4.078947368421053e-06,
      "loss": 0.0002,
      "num_tokens": 631645.0,
      "reward": 0.746874988079071,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.746874988079071,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.030857454942654287,
      "grad_norm": 0.10003118962049484,
      "kl": 0.029050659737549722,
      "learning_rate": 4.073684210526316e-06,
      "loss": -0.0,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03099399235390497,
      "grad_norm": 0.08923932909965515,
      "kl": 0.02833891040063463,
      "learning_rate": 4.0684210526315795e-06,
      "loss": -0.0006,
      "step": 227
    },
    {
      "clip_ratio": 0.0015976974100340158,
      "epoch": 0.03113052976515565,
      "grad_norm": 0.07948505878448486,
      "kl": 0.02925992920063436,
      "learning_rate": 4.063157894736842e-06,
      "loss": -0.0015,
      "step": 228
    },
    {
      "clip_ratio": 0.0005824159306939691,
      "completion_length": 112.875,
      "epoch": 0.03126706717640634,
      "grad_norm": 0.08888618648052216,
      "kl": 0.026295807358110324,
      "learning_rate": 4.057894736842106e-06,
      "loss": 0.0004,
      "num_tokens": 642929.0,
      "reward": 0.578125,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.578125,
      "step": 229
    },
    {
      "clip_ratio": 0.001052564155543223,
      "epoch": 0.031403604587657015,
      "grad_norm": 0.08594765514135361,
      "kl": 0.026338903553551063,
      "learning_rate": 4.052631578947368e-06,
      "loss": -0.0002,
      "step": 230
    },
    {
      "clip_ratio": 0.0005595455877482891,
      "epoch": 0.0315401419989077,
      "grad_norm": 0.08271999657154083,
      "kl": 0.026568240544293076,
      "learning_rate": 4.047368421052632e-06,
      "loss": -0.0006,
      "step": 231
    },
    {
      "clip_ratio": 0.0005595455877482891,
      "epoch": 0.031676679410158386,
      "grad_norm": 0.07526501268148422,
      "kl": 0.026544391934294254,
      "learning_rate": 4.042105263157895e-06,
      "loss": -0.0014,
      "step": 232
    },
    {
      "clip_ratio": 0.0007640966796316206,
      "completion_length": 113.625,
      "epoch": 0.031813216821409064,
      "grad_norm": 0.0923089012503624,
      "kl": 0.024078507034573704,
      "learning_rate": 4.0368421052631585e-06,
      "loss": 0.0003,
      "num_tokens": 653789.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 233
    },
    {
      "clip_ratio": 0.0004877125611528754,
      "epoch": 0.03194975423265975,
      "grad_norm": 0.09331841766834259,
      "kl": 0.024048013438005,
      "learning_rate": 4.031578947368422e-06,
      "loss": -0.0001,
      "step": 234
    },
    {
      "clip_ratio": 0.00019290123600512743,
      "epoch": 0.032086291643910435,
      "grad_norm": 0.08464116603136063,
      "kl": 0.023768270009895787,
      "learning_rate": 4.026315789473684e-06,
      "loss": -0.0008,
      "step": 235
    },
    {
      "clip_ratio": 0.0004210034239804372,
      "epoch": 0.03222282905516111,
      "grad_norm": 0.07866553962230682,
      "kl": 0.023468335362849757,
      "learning_rate": 4.021052631578948e-06,
      "loss": -0.0016,
      "step": 236
    },
    {
      "clip_ratio": 0.00018274853937327862,
      "completion_length": 113.0,
      "epoch": 0.0323593664664118,
      "grad_norm": 0.06890566647052765,
      "kl": 0.02587902924278751,
      "learning_rate": 4.01578947368421e-06,
      "loss": 0.0004,
      "num_tokens": 664541.0,
      "reward": 0.4375,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.4375,
      "step": 237
    },
    {
      "clip_ratio": 0.0003633720916695893,
      "epoch": 0.03249590387766248,
      "grad_norm": 0.06925798207521439,
      "kl": 0.025461589102633297,
      "learning_rate": 4.010526315789474e-06,
      "loss": 0.0002,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03263244128891316,
      "grad_norm": 0.06528756767511368,
      "kl": 0.02574267768068239,
      "learning_rate": 4.005263157894737e-06,
      "loss": 0.0,
      "step": 239
    },
    {
      "clip_ratio": 0.00018274853937327862,
      "epoch": 0.03276897870016385,
      "grad_norm": 0.06375279277563095,
      "kl": 0.025849525845842436,
      "learning_rate": 4.000000000000001e-06,
      "loss": -0.0004,
      "step": 240
    },
    {
      "clip_ratio": 0.0008375860925298184,
      "completion_length": 106.09375,
      "epoch": 0.032905516111414526,
      "grad_norm": 0.0751417875289917,
      "kl": 0.027247032237937674,
      "learning_rate": 3.994736842105264e-06,
      "loss": 0.0004,
      "num_tokens": 674724.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03304205352266521,
      "grad_norm": 0.07292292267084122,
      "kl": 0.02736963256029412,
      "learning_rate": 3.989473684210526e-06,
      "loss": 0.0,
      "step": 242
    },
    {
      "clip_ratio": 0.0003033980610780418,
      "epoch": 0.03317859093391589,
      "grad_norm": 0.06825637817382812,
      "kl": 0.027891719597391784,
      "learning_rate": 3.98421052631579e-06,
      "loss": -0.0003,
      "step": 243
    },
    {
      "clip_ratio": 0.0003033980610780418,
      "epoch": 0.033315128345166575,
      "grad_norm": 0.06375395506620407,
      "kl": 0.0283044419484213,
      "learning_rate": 3.9789473684210525e-06,
      "loss": -0.0007,
      "step": 244
    },
    {
      "clip_ratio": 0.0005482456181198359,
      "completion_length": 124.5625,
      "epoch": 0.03345166575641726,
      "grad_norm": 0.09664054214954376,
      "kl": 0.028416587243555114,
      "learning_rate": 3.9736842105263165e-06,
      "loss": 0.0002,
      "num_tokens": 685466.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 245
    },
    {
      "clip_ratio": 0.0003923962212866172,
      "epoch": 0.03358820316766794,
      "grad_norm": 0.09948284178972244,
      "kl": 0.02821255396702327,
      "learning_rate": 3.968421052631579e-06,
      "loss": -0.0,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.033724740578918624,
      "grad_norm": 0.07397827506065369,
      "kl": 0.02847255891538225,
      "learning_rate": 3.963157894736843e-06,
      "loss": -0.0006,
      "step": 247
    },
    {
      "clip_ratio": 0.0004448878316907212,
      "epoch": 0.03386127799016931,
      "grad_norm": 0.07055655121803284,
      "kl": 0.028869623522041366,
      "learning_rate": 3.957894736842106e-06,
      "loss": -0.0011,
      "step": 248
    },
    {
      "clip_ratio": 0.0002765486715361476,
      "completion_length": 94.875,
      "epoch": 0.03399781540141999,
      "grad_norm": 0.052606552839279175,
      "kl": 0.020315775473136455,
      "learning_rate": 3.952631578947368e-06,
      "loss": 0.0004,
      "num_tokens": 696522.0,
      "reward": 0.831250011920929,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.831250011920929,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.034134352812670674,
      "grad_norm": 0.05249060317873955,
      "kl": 0.020589739870047197,
      "learning_rate": 3.947368421052632e-06,
      "loss": 0.0001,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03427089022392135,
      "grad_norm": 0.04982711747288704,
      "kl": 0.02040288085117936,
      "learning_rate": 3.942105263157895e-06,
      "loss": 0.0,
      "step": 251
    },
    {
      "clip_ratio": 0.0002765486715361476,
      "epoch": 0.03440742763517204,
      "grad_norm": 0.04824934899806976,
      "kl": 0.020667625038186088,
      "learning_rate": 3.936842105263159e-06,
      "loss": -0.0002,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 104.34375,
      "epoch": 0.03454396504642272,
      "grad_norm": 0.057229530066251755,
      "kl": 0.03170221848995425,
      "learning_rate": 3.931578947368421e-06,
      "loss": 0.0003,
      "num_tokens": 707233.0,
      "reward": 0.550000011920929,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.550000011920929,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0346805024576734,
      "grad_norm": 0.057189639657735825,
      "kl": 0.03171565095544793,
      "learning_rate": 3.926315789473685e-06,
      "loss": 0.0,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.034817039868924086,
      "grad_norm": 0.05530928447842598,
      "kl": 0.030918381758965552,
      "learning_rate": 3.921052631578947e-06,
      "loss": -0.0002,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.034953577280174765,
      "grad_norm": 0.052288055419921875,
      "kl": 0.03107694143545814,
      "learning_rate": 3.9157894736842104e-06,
      "loss": -0.0004,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.34375,
      "epoch": 0.03509011469142545,
      "grad_norm": 0.0891910195350647,
      "kl": 0.023110102163627744,
      "learning_rate": 3.9105263157894744e-06,
      "loss": 0.0001,
      "num_tokens": 718136.0,
      "reward": 0.606249988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.606249988079071,
      "step": 257
    },
    {
      "clip_ratio": 0.00022163119865581393,
      "epoch": 0.035226652102676136,
      "grad_norm": 0.07997400313615799,
      "kl": 0.02337674022419378,
      "learning_rate": 3.905263157894737e-06,
      "loss": -0.0002,
      "step": 258
    },
    {
      "clip_ratio": 0.00020695364219136536,
      "epoch": 0.035363189513926814,
      "grad_norm": 0.0842948779463768,
      "kl": 0.022346210171235725,
      "learning_rate": 3.900000000000001e-06,
      "loss": -0.0006,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0354997269251775,
      "grad_norm": 0.07676080614328384,
      "kl": 0.02255020069424063,
      "learning_rate": 3.894736842105263e-06,
      "loss": -0.0012,
      "step": 260
    },
    {
      "clip_ratio": 0.0008840422669891268,
      "completion_length": 102.75,
      "epoch": 0.035636264336428185,
      "grad_norm": 0.12544368207454681,
      "kl": 0.02338628371944651,
      "learning_rate": 3.889473684210527e-06,
      "loss": 0.0005,
      "num_tokens": 728780.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 261
    },
    {
      "clip_ratio": 0.000634926778730005,
      "epoch": 0.03577280174767886,
      "grad_norm": 0.11735007166862488,
      "kl": 0.023411447560647503,
      "learning_rate": 3.884210526315789e-06,
      "loss": -0.0002,
      "step": 262
    },
    {
      "clip_ratio": 0.00048496242379769683,
      "epoch": 0.03590933915892955,
      "grad_norm": 0.11159281432628632,
      "kl": 0.023148178006522357,
      "learning_rate": 3.878947368421053e-06,
      "loss": -0.001,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03604587657018023,
      "grad_norm": 0.09994325041770935,
      "kl": 0.023549747798824683,
      "learning_rate": 3.873684210526316e-06,
      "loss": -0.0021,
      "step": 264
    },
    {
      "clip_ratio": 0.0004883110523223877,
      "completion_length": 115.15625,
      "epoch": 0.03618241398143091,
      "grad_norm": 0.07745163142681122,
      "kl": 0.026939595787553117,
      "learning_rate": 3.868421052631579e-06,
      "loss": 0.0003,
      "num_tokens": 739625.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 265
    },
    {
      "clip_ratio": 0.00029761905898340046,
      "epoch": 0.0363189513926816,
      "grad_norm": 0.07724476605653763,
      "kl": 0.027008358330931515,
      "learning_rate": 3.863157894736843e-06,
      "loss": 0.0,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.036455488803932276,
      "grad_norm": 0.07575860619544983,
      "kl": 0.027253914857283235,
      "learning_rate": 3.857894736842105e-06,
      "loss": -0.0004,
      "step": 267
    },
    {
      "clip_ratio": 0.0005524969019461423,
      "epoch": 0.03659202621518296,
      "grad_norm": 0.07534480094909668,
      "kl": 0.02772324366378598,
      "learning_rate": 3.852631578947369e-06,
      "loss": -0.0008,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.5625,
      "epoch": 0.03672856362643364,
      "grad_norm": 0.03475145250558853,
      "kl": 0.021824754105182365,
      "learning_rate": 3.8473684210526316e-06,
      "loss": 0.0002,
      "num_tokens": 750175.0,
      "reward": 0.690625011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.690625011920929,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.036865101037684325,
      "grad_norm": 0.03451026603579521,
      "kl": 0.02219120250083506,
      "learning_rate": 3.842105263157895e-06,
      "loss": 0.0002,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03700163844893501,
      "grad_norm": 0.03484285995364189,
      "kl": 0.022240082558710128,
      "learning_rate": 3.836842105263158e-06,
      "loss": 0.0002,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03713817586018569,
      "grad_norm": 0.035136640071868896,
      "kl": 0.022295867995126173,
      "learning_rate": 3.831578947368421e-06,
      "loss": -0.0001,
      "step": 272
    },
    {
      "clip_ratio": 0.00013412017142400146,
      "completion_length": 121.78125,
      "epoch": 0.037274713271436374,
      "grad_norm": 0.09162586182355881,
      "kl": 0.02208938094554469,
      "learning_rate": 3.826315789473685e-06,
      "loss": 0.0,
      "num_tokens": 761652.0,
      "reward": 0.887499988079071,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.887499988079071,
      "step": 273
    },
    {
      "clip_ratio": 0.00025201612152159214,
      "epoch": 0.03741125068268706,
      "grad_norm": 0.08879063278436661,
      "kl": 0.02217887411825359,
      "learning_rate": 3.821052631578947e-06,
      "loss": -0.0002,
      "step": 274
    },
    {
      "clip_ratio": 0.0007802476466167718,
      "epoch": 0.03754778809393774,
      "grad_norm": 0.08000439405441284,
      "kl": 0.022196775797056034,
      "learning_rate": 3.815789473684211e-06,
      "loss": -0.0006,
      "step": 275
    },
    {
      "clip_ratio": 0.0007268308690981939,
      "epoch": 0.037684325505188423,
      "grad_norm": 0.06881428509950638,
      "kl": 0.02263434650376439,
      "learning_rate": 3.810526315789474e-06,
      "loss": -0.0013,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.28125,
      "epoch": 0.0378208629164391,
      "grad_norm": 0.05088047683238983,
      "kl": 0.02204109003650956,
      "learning_rate": 3.805263157894737e-06,
      "loss": 0.0003,
      "num_tokens": 773309.0,
      "reward": 0.859375,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.859375,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03795740032768979,
      "grad_norm": 0.049479130655527115,
      "kl": 0.022138332104077563,
      "learning_rate": 3.8000000000000005e-06,
      "loss": 0.0,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03809393773894047,
      "grad_norm": 0.0442269891500473,
      "kl": 0.022356969930115156,
      "learning_rate": 3.794736842105263e-06,
      "loss": -0.0001,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03823047515019115,
      "grad_norm": 0.04249510541558266,
      "kl": 0.022337044501909986,
      "learning_rate": 3.789473684210527e-06,
      "loss": -0.0003,
      "step": 280
    },
    {
      "clip_ratio": 0.0005657151050399989,
      "completion_length": 113.625,
      "epoch": 0.038367012561441836,
      "grad_norm": 0.07635533064603806,
      "kl": 0.023303329100599512,
      "learning_rate": 3.7842105263157895e-06,
      "loss": 0.0003,
      "num_tokens": 784297.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 281
    },
    {
      "clip_ratio": 0.00027901786961592734,
      "epoch": 0.038503549972692515,
      "grad_norm": 0.0776119977235794,
      "kl": 0.02313061192398891,
      "learning_rate": 3.778947368421053e-06,
      "loss": 0.0002,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0386400873839432,
      "grad_norm": 0.0766444057226181,
      "kl": 0.02328120105084963,
      "learning_rate": 3.773684210526316e-06,
      "loss": -0.0003,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.038776624795193886,
      "grad_norm": 0.0710226446390152,
      "kl": 0.023574423365062103,
      "learning_rate": 3.768421052631579e-06,
      "loss": -0.001,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.0625,
      "epoch": 0.038913162206444564,
      "grad_norm": 0.061644088476896286,
      "kl": 0.02391217538388446,
      "learning_rate": 3.7631578947368426e-06,
      "loss": 0.0003,
      "num_tokens": 795303.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03904969961769525,
      "grad_norm": 0.0605340413749218,
      "kl": 0.0234130252210889,
      "learning_rate": 3.7578947368421053e-06,
      "loss": 0.0001,
      "step": 286
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.03918623702894593,
      "grad_norm": 0.060991305857896805,
      "kl": 0.02373662660829723,
      "learning_rate": 3.752631578947369e-06,
      "loss": 0.0,
      "step": 287
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.03932277444019661,
      "grad_norm": 0.054490797221660614,
      "kl": 0.02412348723737523,
      "learning_rate": 3.7473684210526317e-06,
      "loss": -0.0006,
      "step": 288
    },
    {
      "clip_ratio": 0.00044014083687216043,
      "completion_length": 122.59375,
      "epoch": 0.0394593118514473,
      "grad_norm": 0.07991161197423935,
      "kl": 0.022707521973643452,
      "learning_rate": 3.7421052631578953e-06,
      "loss": 0.0004,
      "num_tokens": 806314.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 289
    },
    {
      "clip_ratio": 0.00022007041843608022,
      "epoch": 0.03959584926269798,
      "grad_norm": 0.07932053506374359,
      "kl": 0.022475473844679072,
      "learning_rate": 3.736842105263158e-06,
      "loss": -0.0001,
      "step": 290
    },
    {
      "clip_ratio": 0.00022007041843608022,
      "epoch": 0.03973238667394866,
      "grad_norm": 0.07653678953647614,
      "kl": 0.022855573624838144,
      "learning_rate": 3.7315789473684216e-06,
      "loss": -0.0003,
      "step": 291
    },
    {
      "clip_ratio": 0.0003703108086483553,
      "epoch": 0.03986892408519935,
      "grad_norm": 0.06487143039703369,
      "kl": 0.023218183137942106,
      "learning_rate": 3.7263157894736848e-06,
      "loss": -0.0008,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.3125,
      "epoch": 0.040005461496450026,
      "grad_norm": 0.05640777572989464,
      "kl": 0.021730259235482663,
      "learning_rate": 3.7210526315789475e-06,
      "loss": 0.0003,
      "num_tokens": 817436.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 293
    },
    {
      "clip_ratio": 0.0005408743163570762,
      "epoch": 0.04014199890770071,
      "grad_norm": 0.05590509995818138,
      "kl": 0.021133837261004373,
      "learning_rate": 3.715789473684211e-06,
      "loss": 0.0003,
      "step": 294
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.04027853631895139,
      "grad_norm": 0.05611417070031166,
      "kl": 0.021910520212259144,
      "learning_rate": 3.710526315789474e-06,
      "loss": 0.0002,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.040415073730202075,
      "grad_norm": 0.05293579027056694,
      "kl": 0.021879399777390063,
      "learning_rate": 3.7052631578947374e-06,
      "loss": -0.0002,
      "step": 296
    },
    {
      "clip_ratio": 0.00021853146608918905,
      "completion_length": 119.3125,
      "epoch": 0.04055161114145276,
      "grad_norm": 0.08654630929231644,
      "kl": 0.022178779676323757,
      "learning_rate": 3.7e-06,
      "loss": 0.0,
      "num_tokens": 828762.0,
      "reward": 0.550000011920929,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.550000011920929,
      "step": 297
    },
    {
      "clip_ratio": 0.0004370629321783781,
      "epoch": 0.04068814855270344,
      "grad_norm": 0.08861983567476273,
      "kl": 0.022127668256871402,
      "learning_rate": 3.6947368421052637e-06,
      "loss": -0.0001,
      "step": 298
    },
    {
      "clip_ratio": 0.0004370629321783781,
      "epoch": 0.040824685963954124,
      "grad_norm": 0.08493945002555847,
      "kl": 0.021997908304911107,
      "learning_rate": 3.6894736842105265e-06,
      "loss": -0.0006,
      "step": 299
    },
    {
      "clip_ratio": 0.00021853146608918905,
      "epoch": 0.0409612233752048,
      "grad_norm": 0.07281088829040527,
      "kl": 0.02197305968729779,
      "learning_rate": 3.6842105263157896e-06,
      "loss": -0.0013,
      "step": 300
    },
    {
      "clip_ratio": 0.0004409907996887341,
      "completion_length": 113.25,
      "epoch": 0.04109776078645549,
      "grad_norm": 0.07107827067375183,
      "kl": 0.022476364712929353,
      "learning_rate": 3.6789473684210532e-06,
      "loss": 0.0002,
      "num_tokens": 840026.0,
      "reward": 0.859375,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.859375,
      "step": 301
    },
    {
      "clip_ratio": 0.0004409907996887341,
      "epoch": 0.04123429819770617,
      "grad_norm": 0.06990018486976624,
      "kl": 0.02247147186426446,
      "learning_rate": 3.673684210526316e-06,
      "loss": 0.0001,
      "step": 302
    },
    {
      "clip_ratio": 0.0004409907996887341,
      "epoch": 0.04137083560895685,
      "grad_norm": 0.07051187008619308,
      "kl": 0.022447484341682866,
      "learning_rate": 3.6684210526315796e-06,
      "loss": -0.0002,
      "step": 303
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "epoch": 0.04150737302020754,
      "grad_norm": 0.06450870633125305,
      "kl": 0.022808179928688332,
      "learning_rate": 3.6631578947368423e-06,
      "loss": -0.0008,
      "step": 304
    },
    {
      "completion_length": 99.625,
      "epoch": 0.04164391043145822,
      "grad_norm": 0.0,
      "learning_rate": 3.657894736842106e-06,
      "loss": 0.0,
      "num_tokens": 849910.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/reward_fn": 1.0,
      "step": 305
    },
    {
      "epoch": 0.0417804478427089,
      "grad_norm": 0.0,
      "learning_rate": 3.6526315789473686e-06,
      "loss": 0.0,
      "step": 306
    },
    {
      "epoch": 0.041916985253959586,
      "grad_norm": 0.0,
      "learning_rate": 3.6473684210526318e-06,
      "loss": 0.0,
      "step": 307
    },
    {
      "epoch": 0.042053522665210265,
      "grad_norm": 0.0,
      "learning_rate": 3.642105263157895e-06,
      "loss": 0.0,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.5625,
      "epoch": 0.04219006007646095,
      "grad_norm": 0.04516865313053131,
      "kl": 0.023450861306628212,
      "learning_rate": 3.636842105263158e-06,
      "loss": 0.0002,
      "num_tokens": 861076.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.042326597487711635,
      "grad_norm": 0.044763803482055664,
      "kl": 0.023947950976435095,
      "learning_rate": 3.6315789473684217e-06,
      "loss": 0.0001,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.042463134898962314,
      "grad_norm": 0.045122891664505005,
      "kl": 0.02412795228883624,
      "learning_rate": 3.6263157894736844e-06,
      "loss": -0.0001,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.042599672310213,
      "grad_norm": 0.04445788636803627,
      "kl": 0.02396708173910156,
      "learning_rate": 3.621052631578948e-06,
      "loss": -0.0002,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 105.5625,
      "epoch": 0.04273620972146368,
      "grad_norm": 0.127179816365242,
      "kl": 0.028220025211339816,
      "learning_rate": 3.6157894736842108e-06,
      "loss": 0.0005,
      "num_tokens": 871366.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04287274713271436,
      "grad_norm": 0.08750166743993759,
      "kl": 0.027303016162477434,
      "learning_rate": 3.610526315789474e-06,
      "loss": 0.0,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04300928454396505,
      "grad_norm": 0.08059285581111908,
      "kl": 0.027014906285330653,
      "learning_rate": 3.605263157894737e-06,
      "loss": -0.0003,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04314582195521573,
      "grad_norm": 0.07490835338830948,
      "kl": 0.02650223023374565,
      "learning_rate": 3.6000000000000003e-06,
      "loss": -0.001,
      "step": 316
    },
    {
      "clip_ratio": 0.00024224806111305952,
      "completion_length": 124.59375,
      "epoch": 0.04328235936646641,
      "grad_norm": 0.08907640725374222,
      "kl": 0.022105894546257332,
      "learning_rate": 3.5947368421052634e-06,
      "loss": -0.0,
      "num_tokens": 883057.0,
      "reward": 0.859375,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.859375,
      "step": 317
    },
    {
      "clip_ratio": 0.000507078570080921,
      "epoch": 0.0434188967777171,
      "grad_norm": 0.08827097713947296,
      "kl": 0.021878067345824093,
      "learning_rate": 3.5894736842105266e-06,
      "loss": -0.0001,
      "step": 318
    },
    {
      "clip_ratio": 0.00024224806111305952,
      "epoch": 0.043555434188967776,
      "grad_norm": 0.08430935442447662,
      "kl": 0.02233501960290596,
      "learning_rate": 3.58421052631579e-06,
      "loss": -0.0008,
      "step": 319
    },
    {
      "clip_ratio": 0.0010449249821249396,
      "epoch": 0.04369197160021846,
      "grad_norm": 0.07642368972301483,
      "kl": 0.022479526698589325,
      "learning_rate": 3.578947368421053e-06,
      "loss": -0.0013,
      "step": 320
    },
    {
      "clip_ratio": 0.0004036639875266701,
      "completion_length": 128.375,
      "epoch": 0.04382850901146914,
      "grad_norm": 0.08837485313415527,
      "kl": 0.029423846543068066,
      "learning_rate": 3.5736842105263157e-06,
      "loss": 0.0001,
      "num_tokens": 894553.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 321
    },
    {
      "clip_ratio": 0.0004036639875266701,
      "epoch": 0.043965046422719825,
      "grad_norm": 0.08810171484947205,
      "kl": 0.028873553121229634,
      "learning_rate": 3.5684210526315792e-06,
      "loss": 0.0001,
      "step": 322
    },
    {
      "clip_ratio": 0.00042971508810296655,
      "epoch": 0.04410158383397051,
      "grad_norm": 0.0870831087231636,
      "kl": 0.0294377136597177,
      "learning_rate": 3.5631578947368424e-06,
      "loss": -0.0006,
      "step": 323
    },
    {
      "clip_ratio": 0.0004036639875266701,
      "epoch": 0.04423812124522119,
      "grad_norm": 0.08122850209474564,
      "kl": 0.029522280019591562,
      "learning_rate": 3.5578947368421056e-06,
      "loss": -0.0011,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 100.96875,
      "epoch": 0.044374658656471874,
      "grad_norm": 0.08125916123390198,
      "kl": 0.02477030295995064,
      "learning_rate": 3.5526315789473687e-06,
      "loss": 0.0002,
      "num_tokens": 904832.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 325
    },
    {
      "clip_ratio": 0.0003188775444868952,
      "epoch": 0.04451119606772255,
      "grad_norm": 0.08092335611581802,
      "kl": 0.025089218630455434,
      "learning_rate": 3.5473684210526323e-06,
      "loss": 0.0002,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04464773347897324,
      "grad_norm": 0.07883549481630325,
      "kl": 0.024627194390632212,
      "learning_rate": 3.542105263157895e-06,
      "loss": -0.0005,
      "step": 327
    },
    {
      "clip_ratio": 0.00020032051543239504,
      "epoch": 0.04478427089022392,
      "grad_norm": 0.07583944499492645,
      "kl": 0.024948682344984263,
      "learning_rate": 3.536842105263158e-06,
      "loss": -0.001,
      "step": 328
    },
    {
      "clip_ratio": 0.0008947963651735336,
      "completion_length": 113.46875,
      "epoch": 0.0449208083014746,
      "grad_norm": 0.09001241624355316,
      "kl": 0.023796884925104678,
      "learning_rate": 3.5315789473684214e-06,
      "loss": 0.0,
      "num_tokens": 915583.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 329
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "epoch": 0.04505734571272529,
      "grad_norm": 0.09002924710512161,
      "kl": 0.023522302479250357,
      "learning_rate": 3.5263157894736846e-06,
      "loss": -0.0003,
      "step": 330
    },
    {
      "clip_ratio": 0.0005853904294781387,
      "epoch": 0.04519388312397597,
      "grad_norm": 0.0916908010840416,
      "kl": 0.023461372969904914,
      "learning_rate": 3.5210526315789477e-06,
      "loss": -0.0006,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04533042053522665,
      "grad_norm": 0.07961533963680267,
      "kl": 0.023377142468234524,
      "learning_rate": 3.515789473684211e-06,
      "loss": -0.0014,
      "step": 332
    },
    {
      "clip_ratio": 0.00102028384571895,
      "completion_length": 118.96875,
      "epoch": 0.045466957946477336,
      "grad_norm": 0.06798560172319412,
      "kl": 0.023123418592149392,
      "learning_rate": 3.510526315789474e-06,
      "loss": 0.0004,
      "num_tokens": 926398.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 333
    },
    {
      "clip_ratio": 0.001159263076260686,
      "epoch": 0.045603495357728015,
      "grad_norm": 0.10703027248382568,
      "kl": 0.02294311163132079,
      "learning_rate": 3.505263157894737e-06,
      "loss": 0.0001,
      "step": 334
    },
    {
      "clip_ratio": 0.00023148147738538682,
      "epoch": 0.0457400327689787,
      "grad_norm": 0.07149892300367355,
      "kl": 0.024184314184822142,
      "learning_rate": 3.5e-06,
      "loss": 0.0001,
      "step": 335
    },
    {
      "clip_ratio": 0.0004500129434745759,
      "epoch": 0.045876570180229385,
      "grad_norm": 0.0723360925912857,
      "kl": 0.02370249154046178,
      "learning_rate": 3.4947368421052635e-06,
      "loss": -0.0003,
      "step": 336
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "completion_length": 137.6875,
      "epoch": 0.046013107591480064,
      "grad_norm": 0.08149459958076477,
      "kl": 0.020294574671424925,
      "learning_rate": 3.4894736842105263e-06,
      "loss": 0.0001,
      "num_tokens": 937840.0,
      "reward": 0.578125,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.578125,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04614964500273075,
      "grad_norm": 0.07450702041387558,
      "kl": 0.020218727935571223,
      "learning_rate": 3.48421052631579e-06,
      "loss": -0.0,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04628618241398143,
      "grad_norm": 0.0721743181347847,
      "kl": 0.02070053394709248,
      "learning_rate": 3.478947368421053e-06,
      "loss": -0.0002,
      "step": 339
    },
    {
      "clip_ratio": 0.0008059149840846658,
      "epoch": 0.04642271982523211,
      "grad_norm": 0.06328802555799484,
      "kl": 0.02160324275610037,
      "learning_rate": 3.473684210526316e-06,
      "loss": -0.0007,
      "step": 340
    },
    {
      "clip_ratio": 0.000163612567121163,
      "completion_length": 139.5,
      "epoch": 0.0465592572364828,
      "grad_norm": 0.07692926377058029,
      "kl": 0.022069479804486036,
      "learning_rate": 3.4684210526315794e-06,
      "loss": -0.0001,
      "num_tokens": 949784.0,
      "reward": 0.4937500059604645,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.4937500059604645,
      "step": 341
    },
    {
      "clip_ratio": 0.0006625788664678112,
      "epoch": 0.04669579464773348,
      "grad_norm": 0.07928881049156189,
      "kl": 0.021967595210298896,
      "learning_rate": 3.463157894736842e-06,
      "loss": -0.0002,
      "step": 342
    },
    {
      "clip_ratio": 0.00040216218621935695,
      "epoch": 0.04683233205898416,
      "grad_norm": 0.07507841289043427,
      "kl": 0.02228320448193699,
      "learning_rate": 3.4578947368421057e-06,
      "loss": -0.0007,
      "step": 343
    },
    {
      "clip_ratio": 0.000163612567121163,
      "epoch": 0.04696886947023485,
      "grad_norm": 0.06530670821666718,
      "kl": 0.023109311325242743,
      "learning_rate": 3.4526315789473684e-06,
      "loss": -0.001,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 115.21875,
      "epoch": 0.047105406881485526,
      "grad_norm": 0.058630239218473434,
      "kl": 0.027992002695100382,
      "learning_rate": 3.447368421052632e-06,
      "loss": 0.0004,
      "num_tokens": 960135.0,
      "reward": 0.690625011920929,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.690625011920929,
      "step": 345
    },
    {
      "clip_ratio": 0.0006200397037900984,
      "epoch": 0.04724194429273621,
      "grad_norm": 0.058566391468048096,
      "kl": 0.02831086376681924,
      "learning_rate": 3.4421052631578947e-06,
      "loss": 0.0003,
      "step": 346
    },
    {
      "clip_ratio": 0.000590555282542482,
      "epoch": 0.04737848170398689,
      "grad_norm": 0.05969183146953583,
      "kl": 0.028586817148607224,
      "learning_rate": 3.4368421052631583e-06,
      "loss": 0.0,
      "step": 347
    },
    {
      "clip_ratio": 0.00046267209108918905,
      "epoch": 0.047515019115237575,
      "grad_norm": 0.05286189541220665,
      "kl": 0.02830476386588998,
      "learning_rate": 3.4315789473684215e-06,
      "loss": -0.0003,
      "step": 348
    },
    {
      "clip_ratio": 0.00027901786961592734,
      "completion_length": 126.96875,
      "epoch": 0.04765155652648826,
      "grad_norm": 0.08239299803972244,
      "kl": 0.02313562305062078,
      "learning_rate": 3.4263157894736842e-06,
      "loss": 0.0003,
      "num_tokens": 971494.0,
      "reward": 0.35312503576278687,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.35312503576278687,
      "step": 349
    },
    {
      "clip_ratio": 0.0004470286221476272,
      "epoch": 0.04778809393773894,
      "grad_norm": 0.08283380419015884,
      "kl": 0.02339187340112403,
      "learning_rate": 3.421052631578948e-06,
      "loss": 0.0,
      "step": 350
    },
    {
      "clip_ratio": 0.000615039374679327,
      "epoch": 0.047924631348989624,
      "grad_norm": 0.07904310524463654,
      "kl": 0.02403911092551425,
      "learning_rate": 3.4157894736842106e-06,
      "loss": -0.0004,
      "step": 351
    },
    {
      "clip_ratio": 0.0004470286221476272,
      "epoch": 0.0480611687602403,
      "grad_norm": 0.0709666758775711,
      "kl": 0.024250175367342308,
      "learning_rate": 3.410526315789474e-06,
      "loss": -0.0009,
      "step": 352
    },
    {
      "clip_ratio": 0.00040849673678167164,
      "completion_length": 135.0625,
      "epoch": 0.04819770617149099,
      "grad_norm": 0.09687653183937073,
      "kl": 0.023374153446638957,
      "learning_rate": 3.405263157894737e-06,
      "loss": 0.0002,
      "num_tokens": 983216.0,
      "reward": 0.550000011920929,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.550000011920929,
      "step": 353
    },
    {
      "clip_ratio": 0.0009377389069413766,
      "epoch": 0.04833424358274167,
      "grad_norm": 0.09615480154752731,
      "kl": 0.02371872030198574,
      "learning_rate": 3.4000000000000005e-06,
      "loss": 0.0002,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04847078099399235,
      "grad_norm": 0.09314487129449844,
      "kl": 0.023429605411365628,
      "learning_rate": 3.3947368421052636e-06,
      "loss": -0.0006,
      "step": 355
    },
    {
      "clip_ratio": 0.000633919466054067,
      "epoch": 0.04860731840524304,
      "grad_norm": 0.08851853013038635,
      "kl": 0.023448748368537053,
      "learning_rate": 3.3894736842105264e-06,
      "loss": -0.0014,
      "step": 356
    },
    {
      "clip_ratio": 0.00020424836839083582,
      "completion_length": 133.375,
      "epoch": 0.04874385581649372,
      "grad_norm": 0.04887694865465164,
      "kl": 0.02090101721114479,
      "learning_rate": 3.38421052631579e-06,
      "loss": 0.0004,
      "num_tokens": 994928.0,
      "reward": 0.4937500059604645,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.4937500059604645,
      "step": 357
    },
    {
      "clip_ratio": 0.0004392107803141698,
      "epoch": 0.0488803932277444,
      "grad_norm": 0.047825887799263,
      "kl": 0.021263578528305516,
      "learning_rate": 3.3789473684210527e-06,
      "loss": 0.0001,
      "step": 358
    },
    {
      "clip_ratio": 0.00020424836839083582,
      "epoch": 0.049016930638995086,
      "grad_norm": 0.04860318452119827,
      "kl": 0.021143519785255194,
      "learning_rate": 3.3736842105263163e-06,
      "loss": 0.0001,
      "step": 359
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.049153468050245765,
      "grad_norm": 0.046374525874853134,
      "kl": 0.020965132425772026,
      "learning_rate": 3.368421052631579e-06,
      "loss": -0.0001,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.90625,
      "epoch": 0.04929000546149645,
      "grad_norm": 0.0487712062895298,
      "kl": 0.02792163222329691,
      "learning_rate": 3.3631578947368426e-06,
      "loss": 0.0002,
      "num_tokens": 1006129.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.049426542872747135,
      "grad_norm": 0.04795907065272331,
      "kl": 0.027323424263158813,
      "learning_rate": 3.3578947368421054e-06,
      "loss": 0.0003,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.049563080283997814,
      "grad_norm": 0.047760266810655594,
      "kl": 0.02758122145314701,
      "learning_rate": 3.3526315789473685e-06,
      "loss": 0.0001,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0496996176952485,
      "grad_norm": 0.04801912233233452,
      "kl": 0.0270579831849318,
      "learning_rate": 3.347368421052632e-06,
      "loss": -0.0001,
      "step": 364
    },
    {
      "clip_ratio": 0.0001849112450145185,
      "completion_length": 120.8125,
      "epoch": 0.04983615510649918,
      "grad_norm": 0.05475917458534241,
      "kl": 0.020006455102702603,
      "learning_rate": 3.342105263157895e-06,
      "loss": 0.0001,
      "num_tokens": 1016995.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 365
    },
    {
      "clip_ratio": 0.0001849112450145185,
      "epoch": 0.04997269251774986,
      "grad_norm": 0.05398503318428993,
      "kl": 0.01969250911497511,
      "learning_rate": 3.3368421052631584e-06,
      "loss": 0.0001,
      "step": 366
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.05010922992900055,
      "grad_norm": 0.053278569132089615,
      "kl": 0.01969363953685388,
      "learning_rate": 3.331578947368421e-06,
      "loss": -0.0002,
      "step": 367
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.05024576734025123,
      "grad_norm": 0.05154409632086754,
      "kl": 0.01971426818636246,
      "learning_rate": 3.3263157894736848e-06,
      "loss": -0.0004,
      "step": 368
    },
    {
      "clip_ratio": 0.00048449612222611904,
      "completion_length": 127.625,
      "epoch": 0.05038230475150191,
      "grad_norm": 0.08452387899160385,
      "kl": 0.020907113037537783,
      "learning_rate": 3.3210526315789475e-06,
      "loss": 0.0004,
      "num_tokens": 1028511.0,
      "reward": 0.49375003576278687,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.49375003576278687,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0505188421627526,
      "grad_norm": 0.08542630821466446,
      "kl": 0.020866161328740418,
      "learning_rate": 3.3157894736842107e-06,
      "loss": 0.0002,
      "step": 370
    },
    {
      "clip_ratio": 0.0003730945463757962,
      "epoch": 0.050655379574003276,
      "grad_norm": 0.08319123089313507,
      "kl": 0.021059497754322365,
      "learning_rate": 3.310526315789474e-06,
      "loss": -0.0004,
      "step": 371
    },
    {
      "clip_ratio": 0.0005486206209752709,
      "epoch": 0.05079191698525396,
      "grad_norm": 0.08353553712368011,
      "kl": 0.021253676881315187,
      "learning_rate": 3.305263157894737e-06,
      "loss": -0.001,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.03125,
      "epoch": 0.05092845439650464,
      "grad_norm": 0.08390096575021744,
      "kl": 0.022265749430516735,
      "learning_rate": 3.3000000000000006e-06,
      "loss": 0.0001,
      "num_tokens": 1039464.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.051064991807755325,
      "grad_norm": 0.08395753800868988,
      "kl": 0.022902959695784375,
      "learning_rate": 3.2947368421052633e-06,
      "loss": -0.0,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05120152921900601,
      "grad_norm": 0.07426461577415466,
      "kl": 0.02265952795278281,
      "learning_rate": 3.289473684210527e-06,
      "loss": -0.0004,
      "step": 375
    },
    {
      "clip_ratio": 0.00038479387876577675,
      "epoch": 0.05133806663025669,
      "grad_norm": 0.0708777979016304,
      "kl": 0.02249322272837162,
      "learning_rate": 3.2842105263157897e-06,
      "loss": -0.0009,
      "step": 376
    },
    {
      "clip_ratio": 9.61538462433964e-05,
      "completion_length": 153.9375,
      "epoch": 0.051474604041507374,
      "grad_norm": 0.08031871914863586,
      "kl": 0.02048320347967092,
      "learning_rate": 3.278947368421053e-06,
      "loss": 0.0,
      "num_tokens": 1052498.0,
      "reward": 0.49375003576278687,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.49375003576278687,
      "step": 377
    },
    {
      "clip_ratio": 0.00018221575010102242,
      "epoch": 0.05161114145275805,
      "grad_norm": 0.07619955390691757,
      "kl": 0.020257426964235492,
      "learning_rate": 3.273684210526316e-06,
      "loss": -0.0001,
      "step": 378
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "epoch": 0.05174767886400874,
      "grad_norm": 0.07639969140291214,
      "kl": 0.02037222741637379,
      "learning_rate": 3.268421052631579e-06,
      "loss": -0.0003,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05188421627525942,
      "grad_norm": 0.06297897547483444,
      "kl": 0.021020412095822394,
      "learning_rate": 3.2631578947368423e-06,
      "loss": -0.0007,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.09375,
      "epoch": 0.0520207536865101,
      "grad_norm": 0.041881829500198364,
      "kl": 0.022837618802441284,
      "learning_rate": 3.2578947368421055e-06,
      "loss": 0.0002,
      "num_tokens": 1063853.0,
      "reward": 0.71875,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.71875,
      "step": 381
    },
    {
      "clip_ratio": 0.00014810427092015743,
      "epoch": 0.05215729109776079,
      "grad_norm": 0.04161300137639046,
      "kl": 0.022631432017078623,
      "learning_rate": 3.252631578947369e-06,
      "loss": 0.0001,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05229382850901147,
      "grad_norm": 0.04173358157277107,
      "kl": 0.023325582704273984,
      "learning_rate": 3.247368421052632e-06,
      "loss": 0.0,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05243036592026215,
      "grad_norm": 0.03905336186289787,
      "kl": 0.022985702089499682,
      "learning_rate": 3.2421052631578945e-06,
      "loss": -0.0002,
      "step": 384
    },
    {
      "clip_ratio": 0.00015394088404718786,
      "completion_length": 110.125,
      "epoch": 0.052566903331512836,
      "grad_norm": 0.07995563000440598,
      "kl": 0.020230403955793008,
      "learning_rate": 3.236842105263158e-06,
      "loss": 0.0002,
      "num_tokens": 1074037.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 385
    },
    {
      "clip_ratio": 0.00015394088404718786,
      "epoch": 0.052703440742763515,
      "grad_norm": 0.08091157674789429,
      "kl": 0.02018303626391571,
      "learning_rate": 3.2315789473684213e-06,
      "loss": -0.0,
      "step": 386
    },
    {
      "clip_ratio": 0.00015394088404718786,
      "epoch": 0.0528399781540142,
      "grad_norm": 0.0798361599445343,
      "kl": 0.020285903723561205,
      "learning_rate": 3.2263157894736845e-06,
      "loss": -0.0005,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.052976515565264885,
      "grad_norm": 0.07278761267662048,
      "kl": 0.02051667576597538,
      "learning_rate": 3.2210526315789476e-06,
      "loss": -0.0011,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 107.21875,
      "epoch": 0.053113052976515564,
      "grad_norm": 0.07617516070604324,
      "kl": 0.028143407515017316,
      "learning_rate": 3.215789473684211e-06,
      "loss": 0.0003,
      "num_tokens": 1084844.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05324959038776625,
      "grad_norm": 0.07584980130195618,
      "kl": 0.02738626574864611,
      "learning_rate": 3.210526315789474e-06,
      "loss": 0.0003,
      "step": 390
    },
    {
      "clip_ratio": 0.0005663055635523051,
      "epoch": 0.05338612779901693,
      "grad_norm": 0.07561825215816498,
      "kl": 0.027368494658730924,
      "learning_rate": 3.2052631578947367e-06,
      "loss": -0.0001,
      "step": 391
    },
    {
      "clip_ratio": 0.0005663055635523051,
      "epoch": 0.05352266521026761,
      "grad_norm": 0.06851368397474289,
      "kl": 0.027929118426982313,
      "learning_rate": 3.2000000000000003e-06,
      "loss": -0.0006,
      "step": 392
    },
    {
      "clip_ratio": 0.00021853146608918905,
      "completion_length": 120.71875,
      "epoch": 0.0536592026215183,
      "grad_norm": 0.058026522397994995,
      "kl": 0.025286480551585555,
      "learning_rate": 3.1947368421052634e-06,
      "loss": 0.0001,
      "num_tokens": 1096015.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 393
    },
    {
      "clip_ratio": 0.00021853146608918905,
      "epoch": 0.05379574003276898,
      "grad_norm": 0.05753491073846817,
      "kl": 0.0260978079168126,
      "learning_rate": 3.1894736842105266e-06,
      "loss": 0.0,
      "step": 394
    },
    {
      "clip_ratio": 0.00021853146608918905,
      "epoch": 0.05393227744401966,
      "grad_norm": 0.05408268794417381,
      "kl": 0.026300053868908435,
      "learning_rate": 3.1842105263157898e-06,
      "loss": -0.0002,
      "step": 395
    },
    {
      "clip_ratio": 0.00020424836839083582,
      "epoch": 0.05406881485527035,
      "grad_norm": 0.04638107120990753,
      "kl": 0.02564926413469948,
      "learning_rate": 3.178947368421053e-06,
      "loss": -0.0005,
      "step": 396
    },
    {
      "clip_ratio": 0.00035511364694684744,
      "completion_length": 102.875,
      "epoch": 0.054205352266521026,
      "grad_norm": 0.08631628006696701,
      "kl": 0.02725237447884865,
      "learning_rate": 3.173684210526316e-06,
      "loss": 0.0004,
      "num_tokens": 1106459.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05434188967777171,
      "grad_norm": 0.0849158763885498,
      "kl": 0.02785361756104976,
      "learning_rate": 3.168421052631579e-06,
      "loss": 0.0001,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05447842708902239,
      "grad_norm": 0.08003275096416473,
      "kl": 0.027714760246453807,
      "learning_rate": 3.1631578947368424e-06,
      "loss": -0.0005,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.054614964500273075,
      "grad_norm": 0.06917204707860947,
      "kl": 0.027757962932810187,
      "learning_rate": 3.157894736842105e-06,
      "loss": -0.0008,
      "step": 400
    },
    {
      "clip_ratio": 0.00020833333837799728,
      "completion_length": 114.53125,
      "epoch": 0.05475150191152376,
      "grad_norm": 0.054499052464962006,
      "kl": 0.02256640234554652,
      "learning_rate": 3.1526315789473688e-06,
      "loss": 0.0001,
      "num_tokens": 1117220.0,
      "reward": 0.606249988079071,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.606249988079071,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05488803932277444,
      "grad_norm": 0.054717376828193665,
      "kl": 0.022691508871503174,
      "learning_rate": 3.147368421052632e-06,
      "loss": -0.0,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.055024576734025124,
      "grad_norm": 0.053422003984451294,
      "kl": 0.023062629101332277,
      "learning_rate": 3.142105263157895e-06,
      "loss": -0.0002,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0551611141452758,
      "grad_norm": 0.05216868966817856,
      "kl": 0.022557226198841818,
      "learning_rate": 3.1368421052631582e-06,
      "loss": -0.0006,
      "step": 404
    },
    {
      "completion_length": 115.21875,
      "epoch": 0.05529765155652649,
      "grad_norm": 0.0,
      "learning_rate": 3.131578947368421e-06,
      "loss": 0.0,
      "num_tokens": 1128051.0,
      "reward": 0.550000011920929,
      "reward_std": 0.0,
      "rewards/reward_fn": 0.550000011920929,
      "step": 405
    },
    {
      "epoch": 0.05543418896777717,
      "grad_norm": 0.0,
      "learning_rate": 3.1263157894736846e-06,
      "loss": 0.0,
      "step": 406
    },
    {
      "epoch": 0.05557072637902785,
      "grad_norm": 0.0,
      "learning_rate": 3.1210526315789473e-06,
      "loss": 0.0,
      "step": 407
    },
    {
      "epoch": 0.05570726379027854,
      "grad_norm": 0.0,
      "learning_rate": 3.115789473684211e-06,
      "loss": 0.0,
      "step": 408
    },
    {
      "clip_ratio": 0.0004936508776154369,
      "completion_length": 120.84375,
      "epoch": 0.05584380120152922,
      "grad_norm": 0.09440712630748749,
      "kl": 0.029624507413245738,
      "learning_rate": 3.1105263157894736e-06,
      "loss": 0.0005,
      "num_tokens": 1139234.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 409
    },
    {
      "clip_ratio": 0.0004337021673563868,
      "epoch": 0.0559803386127799,
      "grad_norm": 0.09525737166404724,
      "kl": 0.02997314796084538,
      "learning_rate": 3.1052631578947372e-06,
      "loss": 0.0003,
      "step": 410
    },
    {
      "clip_ratio": 0.000668147680698894,
      "epoch": 0.056116876024030586,
      "grad_norm": 0.09334629029035568,
      "kl": 0.029849036916857585,
      "learning_rate": 3.1000000000000004e-06,
      "loss": -0.0002,
      "step": 411
    },
    {
      "clip_ratio": 0.0006625162932323292,
      "epoch": 0.056253413435281265,
      "grad_norm": 0.09088991582393646,
      "kl": 0.029855199943995103,
      "learning_rate": 3.094736842105263e-06,
      "loss": -0.0009,
      "step": 412
    },
    {
      "clip_ratio": 0.00026709403027780354,
      "completion_length": 121.4375,
      "epoch": 0.05638995084653195,
      "grad_norm": 0.08999986946582794,
      "kl": 0.0211114841804374,
      "learning_rate": 3.0894736842105267e-06,
      "loss": 0.0001,
      "num_tokens": 1150204.0,
      "reward": 0.40937501192092896,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.40937501192092896,
      "step": 413
    },
    {
      "clip_ratio": 0.00044984256965108216,
      "epoch": 0.056526488257782635,
      "grad_norm": 0.0902203768491745,
      "kl": 0.021156518894713372,
      "learning_rate": 3.0842105263157895e-06,
      "loss": -0.0,
      "step": 414
    },
    {
      "clip_ratio": 0.000660560792312026,
      "epoch": 0.056663025669033314,
      "grad_norm": 0.08662670105695724,
      "kl": 0.021238182176603004,
      "learning_rate": 3.078947368421053e-06,
      "loss": -0.0004,
      "step": 415
    },
    {
      "clip_ratio": 0.0004912797885481268,
      "epoch": 0.056799563080284,
      "grad_norm": 0.08425270766019821,
      "kl": 0.021560683904681355,
      "learning_rate": 3.0736842105263158e-06,
      "loss": -0.0013,
      "step": 416
    },
    {
      "clip_ratio": 0.000488749414216727,
      "completion_length": 121.5,
      "epoch": 0.05693610049153468,
      "grad_norm": 0.07870546728372574,
      "kl": 0.02753204884356819,
      "learning_rate": 3.0684210526315794e-06,
      "loss": 0.0005,
      "num_tokens": 1161660.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 417
    },
    {
      "clip_ratio": 0.00045018277887720615,
      "epoch": 0.05707263790278536,
      "grad_norm": 0.0780278816819191,
      "kl": 0.02779330688645132,
      "learning_rate": 3.0631578947368425e-06,
      "loss": 0.0004,
      "step": 418
    },
    {
      "clip_ratio": 0.0009154355502687395,
      "epoch": 0.05720917531403605,
      "grad_norm": 0.07824473083019257,
      "kl": 0.027306273754220456,
      "learning_rate": 3.0578947368421053e-06,
      "loss": 0.0,
      "step": 419
    },
    {
      "clip_ratio": 0.0002736291062319651,
      "epoch": 0.05734571272528673,
      "grad_norm": 0.0745638981461525,
      "kl": 0.027793304558144882,
      "learning_rate": 3.052631578947369e-06,
      "loss": -0.0006,
      "step": 420
    },
    {
      "clip_ratio": 0.0003396739193703979,
      "completion_length": 103.65625,
      "epoch": 0.05748225013653741,
      "grad_norm": 0.042599353939294815,
      "kl": 0.028534257464343682,
      "learning_rate": 3.0473684210526316e-06,
      "loss": 0.0003,
      "num_tokens": 1171885.0,
      "reward": 0.578125,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.578125,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0576187875477881,
      "grad_norm": 0.042493246495723724,
      "kl": 0.028584119427250698,
      "learning_rate": 3.042105263157895e-06,
      "loss": 0.0002,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.057755324959038776,
      "grad_norm": 0.042571090161800385,
      "kl": 0.02841770788654685,
      "learning_rate": 3.036842105263158e-06,
      "loss": 0.0001,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05789186237028946,
      "grad_norm": 0.042970895767211914,
      "kl": 0.028437837143428624,
      "learning_rate": 3.0315789473684215e-06,
      "loss": -0.0001,
      "step": 424
    },
    {
      "clip_ratio": 0.0006481033633463085,
      "completion_length": 119.5,
      "epoch": 0.05802839978154014,
      "grad_norm": 0.08428654819726944,
      "kl": 0.03005171535187401,
      "learning_rate": 3.0263157894736843e-06,
      "loss": 0.0003,
      "num_tokens": 1182569.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 425
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.058164937192790825,
      "grad_norm": 0.08224932104349136,
      "kl": 0.029877902969019488,
      "learning_rate": 3.0210526315789474e-06,
      "loss": 0.0,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05830147460404151,
      "grad_norm": 0.0772838145494461,
      "kl": 0.02998991316417232,
      "learning_rate": 3.015789473684211e-06,
      "loss": -0.0003,
      "step": 427
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.05843801201529219,
      "grad_norm": 0.06520218402147293,
      "kl": 0.029636650171596557,
      "learning_rate": 3.0105263157894737e-06,
      "loss": -0.0008,
      "step": 428
    },
    {
      "clip_ratio": 0.00034340660204179585,
      "completion_length": 110.90625,
      "epoch": 0.058574549426542874,
      "grad_norm": 0.05254102498292923,
      "kl": 0.02795352225075476,
      "learning_rate": 3.0052631578947373e-06,
      "loss": 0.0004,
      "num_tokens": 1192970.0,
      "reward": 0.859375,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.859375,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05871108683779355,
      "grad_norm": 0.05203205719590187,
      "kl": 0.027702660532668233,
      "learning_rate": 3e-06,
      "loss": 0.0003,
      "step": 430
    },
    {
      "clip_ratio": 0.00034340660204179585,
      "epoch": 0.05884762424904424,
      "grad_norm": 0.050078123807907104,
      "kl": 0.027325164119247347,
      "learning_rate": 2.9947368421052637e-06,
      "loss": 0.0001,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05898416166029492,
      "grad_norm": 0.04930827394127846,
      "kl": 0.028029807755956426,
      "learning_rate": 2.9894736842105264e-06,
      "loss": -0.0001,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 111.625,
      "epoch": 0.0591206990715456,
      "grad_norm": 0.11645514518022537,
      "kl": 0.029349218850256875,
      "learning_rate": 2.9842105263157896e-06,
      "loss": 0.0004,
      "num_tokens": 1203310.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.606249988079071,
      "step": 433
    },
    {
      "clip_ratio": 0.0003522308688843623,
      "epoch": 0.05925723648279629,
      "grad_norm": 0.1155751571059227,
      "kl": 0.029482927988283336,
      "learning_rate": 2.9789473684210527e-06,
      "loss": -0.0003,
      "step": 434
    },
    {
      "clip_ratio": 0.0015326489083236083,
      "epoch": 0.05939377389404697,
      "grad_norm": 0.1054697334766388,
      "kl": 0.02898168022511527,
      "learning_rate": 2.973684210526316e-06,
      "loss": -0.0008,
      "step": 435
    },
    {
      "clip_ratio": 0.0028316913376329467,
      "epoch": 0.05953031130529765,
      "grad_norm": 0.08968142420053482,
      "kl": 0.02963988418923691,
      "learning_rate": 2.9684210526315795e-06,
      "loss": -0.0016,
      "step": 436
    },
    {
      "clip_ratio": 0.00034131205757148564,
      "completion_length": 124.125,
      "epoch": 0.059666848716548336,
      "grad_norm": 0.11992131173610687,
      "kl": 0.02362085011554882,
      "learning_rate": 2.9631578947368422e-06,
      "loss": 0.0004,
      "num_tokens": 1215018.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.35490378737449646,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.059803386127799014,
      "grad_norm": 0.10817261785268784,
      "kl": 0.023265416646609083,
      "learning_rate": 2.957894736842106e-06,
      "loss": -0.0002,
      "step": 438
    },
    {
      "clip_ratio": 0.00013297871919348836,
      "epoch": 0.0599399235390497,
      "grad_norm": 0.10226042568683624,
      "kl": 0.023465301826945506,
      "learning_rate": 2.9526315789473685e-06,
      "loss": -0.0006,
      "step": 439
    },
    {
      "clip_ratio": 0.0010071903670905158,
      "epoch": 0.060076460950300385,
      "grad_norm": 0.09243166446685791,
      "kl": 0.02359252143651247,
      "learning_rate": 2.9473684210526317e-06,
      "loss": -0.0013,
      "step": 440
    },
    {
      "clip_ratio": 0.00022007041843608022,
      "completion_length": 105.8125,
      "epoch": 0.060212998361551064,
      "grad_norm": 0.09433529525995255,
      "kl": 0.02786622042185627,
      "learning_rate": 2.942105263157895e-06,
      "loss": 0.0003,
      "num_tokens": 1225400.0,
      "reward": 0.71875,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.71875,
      "step": 441
    },
    {
      "clip_ratio": 0.0003289473825134337,
      "epoch": 0.06034953577280175,
      "grad_norm": 0.09359579533338547,
      "kl": 0.02748560524196364,
      "learning_rate": 2.936842105263158e-06,
      "loss": 0.0002,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06048607318405243,
      "grad_norm": 0.09082359075546265,
      "kl": 0.027210222935536876,
      "learning_rate": 2.931578947368421e-06,
      "loss": -0.0006,
      "step": 443
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.06062261059530311,
      "grad_norm": 0.08561991155147552,
      "kl": 0.02759760251501575,
      "learning_rate": 2.9263157894736844e-06,
      "loss": -0.0013,
      "step": 444
    },
    {
      "clip_ratio": 0.00017959770048037171,
      "completion_length": 121.34375,
      "epoch": 0.0607591480065538,
      "grad_norm": 0.08825256675481796,
      "kl": 0.023633092801901512,
      "learning_rate": 2.921052631578948e-06,
      "loss": 0.0003,
      "num_tokens": 1236679.0,
      "reward": 0.6624999642372131,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.6624999642372131,
      "step": 445
    },
    {
      "clip_ratio": 0.00039966811891645193,
      "epoch": 0.060895685417804477,
      "grad_norm": 0.08804938942193985,
      "kl": 0.02338150296418462,
      "learning_rate": 2.9157894736842107e-06,
      "loss": -0.0001,
      "step": 446
    },
    {
      "clip_ratio": 0.0005587452324107289,
      "epoch": 0.06103222282905516,
      "grad_norm": 0.08598316460847855,
      "kl": 0.023485250319936313,
      "learning_rate": 2.9105263157894743e-06,
      "loss": -0.0003,
      "step": 447
    },
    {
      "clip_ratio": 0.0005587452324107289,
      "epoch": 0.06116876024030585,
      "grad_norm": 0.08363504707813263,
      "kl": 0.023482873002649285,
      "learning_rate": 2.905263157894737e-06,
      "loss": -0.0009,
      "step": 448
    },
    {
      "clip_ratio": 0.000234962411923334,
      "completion_length": 128.09375,
      "epoch": 0.061305297651556526,
      "grad_norm": 0.09320642799139023,
      "kl": 0.025599762244382873,
      "learning_rate": 2.9e-06,
      "loss": 0.0002,
      "num_tokens": 1247894.0,
      "reward": 0.746874988079071,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.746874988079071,
      "step": 449
    },
    {
      "clip_ratio": 0.000234962411923334,
      "epoch": 0.06144183506280721,
      "grad_norm": 0.0868094265460968,
      "kl": 0.02552194581949152,
      "learning_rate": 2.8947368421052634e-06,
      "loss": -0.0001,
      "step": 450
    },
    {
      "clip_ratio": 0.000234962411923334,
      "epoch": 0.06157837247405789,
      "grad_norm": 0.07934384793043137,
      "kl": 0.024943586031440645,
      "learning_rate": 2.8894736842105265e-06,
      "loss": -0.0005,
      "step": 451
    },
    {
      "clip_ratio": 0.000234962411923334,
      "epoch": 0.061714909885308575,
      "grad_norm": 0.07852542400360107,
      "kl": 0.024750137235969305,
      "learning_rate": 2.88421052631579e-06,
      "loss": -0.0009,
      "step": 452
    },
    {
      "clip_ratio": 0.0010345236223656684,
      "completion_length": 118.21875,
      "epoch": 0.06185144729655926,
      "grad_norm": 0.12858963012695312,
      "kl": 0.02785053660045378,
      "learning_rate": 2.878947368421053e-06,
      "loss": 0.0006,
      "num_tokens": 1258989.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.40245187282562256,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 453
    },
    {
      "clip_ratio": 0.0012061969318892807,
      "epoch": 0.06198798470780994,
      "grad_norm": 0.12805882096290588,
      "kl": 0.02756309215328656,
      "learning_rate": 2.8736842105263164e-06,
      "loss": 0.0001,
      "step": 454
    },
    {
      "clip_ratio": 0.0009005931788124144,
      "epoch": 0.062124522119060624,
      "grad_norm": 0.12524963915348053,
      "kl": 0.027533066226169467,
      "learning_rate": 2.868421052631579e-06,
      "loss": -0.0012,
      "step": 455
    },
    {
      "clip_ratio": 0.0008774404122959822,
      "epoch": 0.0622610595303113,
      "grad_norm": 0.11410316824913025,
      "kl": 0.028351856861263514,
      "learning_rate": 2.8631578947368423e-06,
      "loss": -0.002,
      "step": 456
    },
    {
      "clip_ratio": 0.00044964029802940786,
      "completion_length": 119.75,
      "epoch": 0.06239759694156199,
      "grad_norm": 0.0884980857372284,
      "kl": 0.02622422287822701,
      "learning_rate": 2.8578947368421055e-06,
      "loss": 0.0003,
      "num_tokens": 1269833.0,
      "reward": 0.690625011920929,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.690625011920929,
      "step": 457
    },
    {
      "clip_ratio": 0.0004271467769285664,
      "epoch": 0.06253413435281267,
      "grad_norm": 0.08802725374698639,
      "kl": 0.0259568098699674,
      "learning_rate": 2.8526315789473687e-06,
      "loss": -0.0,
      "step": 458
    },
    {
      "clip_ratio": 0.0007370132370851934,
      "epoch": 0.06267067176406335,
      "grad_norm": 0.07317302376031876,
      "kl": 0.0255898576870095,
      "learning_rate": 2.847368421052632e-06,
      "loss": -0.0004,
      "step": 459
    },
    {
      "clip_ratio": 0.0002281021879753098,
      "epoch": 0.06280720917531403,
      "grad_norm": 0.07119499146938324,
      "kl": 0.025962556974263862,
      "learning_rate": 2.842105263157895e-06,
      "loss": -0.0009,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 104.9375,
      "epoch": 0.06294374658656472,
      "grad_norm": 0.07267936319112778,
      "kl": 0.030646027327748016,
      "learning_rate": 2.8368421052631586e-06,
      "loss": 0.0002,
      "num_tokens": 1280727.0,
      "reward": 0.7750000357627869,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7750000357627869,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0630802839978154,
      "grad_norm": 0.07068201154470444,
      "kl": 0.030932883906643838,
      "learning_rate": 2.8315789473684213e-06,
      "loss": 0.0001,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06321682140906608,
      "grad_norm": 0.06675128638744354,
      "kl": 0.03135033437865786,
      "learning_rate": 2.826315789473684e-06,
      "loss": -0.0003,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06335335882031677,
      "grad_norm": 0.060258980840444565,
      "kl": 0.031988496135454625,
      "learning_rate": 2.8210526315789476e-06,
      "loss": -0.0004,
      "step": 464
    },
    {
      "clip_ratio": 0.0005117313994560391,
      "completion_length": 122.84375,
      "epoch": 0.06348989623156745,
      "grad_norm": 0.07553315907716751,
      "kl": 0.03090066317236051,
      "learning_rate": 2.815789473684211e-06,
      "loss": 0.0003,
      "num_tokens": 1291590.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 465
    },
    {
      "clip_ratio": 0.0006195175519678742,
      "epoch": 0.06362643364281813,
      "grad_norm": 0.07488815486431122,
      "kl": 0.031175631389487535,
      "learning_rate": 2.810526315789474e-06,
      "loss": 0.0002,
      "step": 466
    },
    {
      "clip_ratio": 0.00039545909385196865,
      "epoch": 0.06376297105406882,
      "grad_norm": 0.07623933255672455,
      "kl": 0.030218343308661133,
      "learning_rate": 2.805263157894737e-06,
      "loss": -0.0003,
      "step": 467
    },
    {
      "clip_ratio": 0.00041963320109061897,
      "epoch": 0.0638995084653195,
      "grad_norm": 0.06566479057073593,
      "kl": 0.030176448373822495,
      "learning_rate": 2.8000000000000003e-06,
      "loss": -0.0006,
      "step": 468
    },
    {
      "clip_ratio": 0.000202922077733092,
      "completion_length": 143.6875,
      "epoch": 0.06403604587657018,
      "grad_norm": 0.07517211139202118,
      "kl": 0.026966167468344793,
      "learning_rate": 2.7947368421052635e-06,
      "loss": 0.0001,
      "num_tokens": 1303640.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 469
    },
    {
      "clip_ratio": 0.0011654012050712481,
      "epoch": 0.06417258328782087,
      "grad_norm": 0.07574381679296494,
      "kl": 0.026625617261743173,
      "learning_rate": 2.789473684210526e-06,
      "loss": -0.0001,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06430912069907155,
      "grad_norm": 0.06845681369304657,
      "kl": 0.027047655457863584,
      "learning_rate": 2.7842105263157898e-06,
      "loss": -0.0004,
      "step": 471
    },
    {
      "clip_ratio": 0.0004459502233657986,
      "epoch": 0.06444565811032223,
      "grad_norm": 0.06472209841012955,
      "kl": 0.02748362798593007,
      "learning_rate": 2.7789473684210525e-06,
      "loss": -0.0007,
      "step": 472
    },
    {
      "clip_ratio": 0.00017265194037463516,
      "completion_length": 100.0,
      "epoch": 0.0645821955215729,
      "grad_norm": 0.08828121423721313,
      "kl": 0.029740739322733134,
      "learning_rate": 2.773684210526316e-06,
      "loss": 0.0004,
      "num_tokens": 1314068.0,
      "reward": 0.859375,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.859375,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0647187329328236,
      "grad_norm": 0.08898042142391205,
      "kl": 0.029690030263736844,
      "learning_rate": 2.7684210526315793e-06,
      "loss": 0.0001,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06485527034407428,
      "grad_norm": 0.08371194452047348,
      "kl": 0.030416715890169144,
      "learning_rate": 2.7631578947368424e-06,
      "loss": -0.0003,
      "step": 475
    },
    {
      "clip_ratio": 0.0003927223588107154,
      "epoch": 0.06499180775532495,
      "grad_norm": 0.08132834732532501,
      "kl": 0.03134842653525993,
      "learning_rate": 2.7578947368421056e-06,
      "loss": -0.0009,
      "step": 476
    },
    {
      "completion_length": 111.34375,
      "epoch": 0.06512834516657565,
      "grad_norm": 0.0,
      "learning_rate": 2.7526315789473683e-06,
      "loss": 0.0,
      "num_tokens": 1325395.0,
      "reward": 0.887499988079071,
      "reward_std": 0.0,
      "rewards/reward_fn": 0.887499988079071,
      "step": 477
    },
    {
      "epoch": 0.06526488257782632,
      "grad_norm": 0.0,
      "learning_rate": 2.747368421052632e-06,
      "loss": 0.0,
      "step": 478
    },
    {
      "epoch": 0.065401419989077,
      "grad_norm": 0.0,
      "learning_rate": 2.7421052631578947e-06,
      "loss": 0.0,
      "step": 479
    },
    {
      "epoch": 0.0655379574003277,
      "grad_norm": 0.0,
      "learning_rate": 2.7368421052631583e-06,
      "loss": 0.0,
      "step": 480
    },
    {
      "clip_ratio": 0.0008182729070540518,
      "completion_length": 122.4375,
      "epoch": 0.06567449481157837,
      "grad_norm": 0.1075565367937088,
      "kl": 0.03270941000664607,
      "learning_rate": 2.7315789473684214e-06,
      "loss": 0.0004,
      "num_tokens": 1336833.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2511056959629059,
      "rewards/reward_fn": 0.690625011920929,
      "step": 481
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.06581103222282905,
      "grad_norm": 0.10678515583276749,
      "kl": 0.03262783802347258,
      "learning_rate": 2.7263157894736846e-06,
      "loss": 0.0003,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06594756963407974,
      "grad_norm": 0.10226035863161087,
      "kl": 0.03304731094976887,
      "learning_rate": 2.7210526315789478e-06,
      "loss": -0.0003,
      "step": 483
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "epoch": 0.06608410704533042,
      "grad_norm": 0.09197822958230972,
      "kl": 0.03369238827144727,
      "learning_rate": 2.7157894736842105e-06,
      "loss": -0.001,
      "step": 484
    },
    {
      "clip_ratio": 0.00024801588733680546,
      "completion_length": 105.90625,
      "epoch": 0.0662206444565811,
      "grad_norm": 0.10018602758646011,
      "kl": 0.03327567869564518,
      "learning_rate": 2.710526315789474e-06,
      "loss": 0.0004,
      "num_tokens": 1346950.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06635718186783178,
      "grad_norm": 0.09925264865159988,
      "kl": 0.032918925397098064,
      "learning_rate": 2.705263157894737e-06,
      "loss": 0.0003,
      "step": 486
    },
    {
      "clip_ratio": 0.0005063201388111338,
      "epoch": 0.06649371927908247,
      "grad_norm": 0.0975351333618164,
      "kl": 0.03307101898826659,
      "learning_rate": 2.7000000000000004e-06,
      "loss": -0.0003,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06663025669033315,
      "grad_norm": 0.0953407809138298,
      "kl": 0.03223849239293486,
      "learning_rate": 2.694736842105263e-06,
      "loss": -0.001,
      "step": 488
    },
    {
      "clip_ratio": 0.0003289473825134337,
      "completion_length": 120.3125,
      "epoch": 0.06676679410158383,
      "grad_norm": 0.09660880267620087,
      "kl": 0.03435020908364095,
      "learning_rate": 2.6894736842105267e-06,
      "loss": 0.0004,
      "num_tokens": 1357580.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06690333151283452,
      "grad_norm": 0.09672436863183975,
      "kl": 0.03443722132942639,
      "learning_rate": 2.68421052631579e-06,
      "loss": 0.0002,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0670398689240852,
      "grad_norm": 0.0930391252040863,
      "kl": 0.0339292844873853,
      "learning_rate": 2.6789473684210526e-06,
      "loss": -0.0002,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06717640633533588,
      "grad_norm": 0.08958112448453903,
      "kl": 0.03364733717171475,
      "learning_rate": 2.6736842105263162e-06,
      "loss": -0.0008,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 112.125,
      "epoch": 0.06731294374658657,
      "grad_norm": 0.05487014725804329,
      "kl": 0.0318443511496298,
      "learning_rate": 2.668421052631579e-06,
      "loss": 0.0003,
      "num_tokens": 1368584.0,
      "reward": 0.690625011920929,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.690625011920929,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06744948115783725,
      "grad_norm": 0.054498445242643356,
      "kl": 0.0327269172121305,
      "learning_rate": 2.6631578947368426e-06,
      "loss": 0.0001,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06758601856908793,
      "grad_norm": 0.053584616631269455,
      "kl": 0.03137458971468732,
      "learning_rate": 2.6578947368421053e-06,
      "loss": -0.0001,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06772255598033862,
      "grad_norm": 0.05226173624396324,
      "kl": 0.03136958571849391,
      "learning_rate": 2.652631578947369e-06,
      "loss": -0.0002,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 108.84375,
      "epoch": 0.0678590933915893,
      "grad_norm": 0.10085256397724152,
      "kl": 0.028638561692787334,
      "learning_rate": 2.6473684210526316e-06,
      "loss": 0.0002,
      "num_tokens": 1379079.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 497
    },
    {
      "clip_ratio": 0.000477544468594715,
      "epoch": 0.06799563080283998,
      "grad_norm": 0.09922110289335251,
      "kl": 0.028383127588313073,
      "learning_rate": 2.6421052631578948e-06,
      "loss": -0.0001,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06813216821409065,
      "grad_norm": 0.09748240560293198,
      "kl": 0.028070454631233588,
      "learning_rate": 2.6368421052631584e-06,
      "loss": -0.0007,
      "step": 499
    },
    {
      "clip_ratio": 0.00023148147738538682,
      "epoch": 0.06826870562534135,
      "grad_norm": 0.08522222936153412,
      "kl": 0.027039629960199818,
      "learning_rate": 2.631578947368421e-06,
      "loss": -0.0012,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.59375,
      "epoch": 0.06840524303659203,
      "grad_norm": 0.05568603426218033,
      "kl": 0.029449922149069607,
      "learning_rate": 2.6263157894736847e-06,
      "loss": 0.0003,
      "num_tokens": 1389594.0,
      "reward": 0.8875000476837158,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.8875000476837158,
      "step": 501
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0685417804478427,
      "grad_norm": 0.05571011081337929,
      "kl": 0.029838646500138566,
      "learning_rate": 2.6210526315789474e-06,
      "loss": 0.0002,
      "step": 502
    },
    {
      "clip_ratio": 0.00016622339899186045,
      "epoch": 0.0686783178590934,
      "grad_norm": 0.055335305631160736,
      "kl": 0.028605365252587944,
      "learning_rate": 2.615789473684211e-06,
      "loss": 0.0001,
      "step": 503
    },
    {
      "clip_ratio": 0.0001806358341127634,
      "epoch": 0.06881485527034407,
      "grad_norm": 0.053467657417058945,
      "kl": 0.02818925876636058,
      "learning_rate": 2.6105263157894738e-06,
      "loss": -0.0001,
      "step": 504
    },
    {
      "clip_ratio": 0.00041107917786575854,
      "completion_length": 112.375,
      "epoch": 0.06895139268159475,
      "grad_norm": 0.07417155802249908,
      "kl": 0.023815288412151858,
      "learning_rate": 2.605263157894737e-06,
      "loss": 0.0004,
      "num_tokens": 1399702.0,
      "reward": 0.859375,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.859375,
      "step": 505
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06908793009284545,
      "grad_norm": 0.07561016082763672,
      "kl": 0.02323581191012636,
      "learning_rate": 2.6e-06,
      "loss": 0.0002,
      "step": 506
    },
    {
      "clip_ratio": 0.0004826326767215505,
      "epoch": 0.06922446750409612,
      "grad_norm": 0.0658431127667427,
      "kl": 0.02391452860319987,
      "learning_rate": 2.5947368421052633e-06,
      "loss": -0.0002,
      "step": 507
    },
    {
      "clip_ratio": 0.00017959770048037171,
      "epoch": 0.0693610049153468,
      "grad_norm": 0.06122962757945061,
      "kl": 0.023273998143849894,
      "learning_rate": 2.589473684210527e-06,
      "loss": -0.0004,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 111.09375,
      "epoch": 0.0694975423265975,
      "grad_norm": 0.08124542981386185,
      "kl": 0.026170960336457938,
      "learning_rate": 2.5842105263157896e-06,
      "loss": 0.0002,
      "num_tokens": 1410113.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 509
    },
    {
      "clip_ratio": 0.0004949534049956128,
      "epoch": 0.06963407973784817,
      "grad_norm": 0.07992769777774811,
      "kl": 0.026557496574241668,
      "learning_rate": 2.578947368421053e-06,
      "loss": 0.0,
      "step": 510
    },
    {
      "clip_ratio": 0.00027173911803402007,
      "epoch": 0.06977061714909885,
      "grad_norm": 0.0765228420495987,
      "kl": 0.02642540223314427,
      "learning_rate": 2.573684210526316e-06,
      "loss": -0.0005,
      "step": 511
    },
    {
      "clip_ratio": 0.00027173911803402007,
      "epoch": 0.06990715456034953,
      "grad_norm": 0.07589668780565262,
      "kl": 0.026321951998397708,
      "learning_rate": 2.568421052631579e-06,
      "loss": -0.0008,
      "step": 512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.25,
      "epoch": 0.07004369197160022,
      "grad_norm": 0.06559047847986221,
      "kl": 0.026270317437592894,
      "learning_rate": 2.5631578947368422e-06,
      "loss": 0.0003,
      "num_tokens": 1421669.0,
      "reward": 0.746874988079071,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.746874988079071,
      "step": 513
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0701802293828509,
      "grad_norm": 0.06494513899087906,
      "kl": 0.026335061265854165,
      "learning_rate": 2.5578947368421054e-06,
      "loss": 0.0002,
      "step": 514
    },
    {
      "clip_ratio": 0.00020695364219136536,
      "epoch": 0.07031676679410158,
      "grad_norm": 0.06433810293674469,
      "kl": 0.027359777392121032,
      "learning_rate": 2.552631578947369e-06,
      "loss": -0.0001,
      "step": 515
    },
    {
      "clip_ratio": 0.00020695364219136536,
      "epoch": 0.07045330420535227,
      "grad_norm": 0.06409961730241776,
      "kl": 0.026299011806258932,
      "learning_rate": 2.5473684210526317e-06,
      "loss": -0.0002,
      "step": 516
    },
    {
      "clip_ratio": 0.0007239816040964797,
      "completion_length": 146.0625,
      "epoch": 0.07058984161660295,
      "grad_norm": 0.10985471308231354,
      "kl": 0.028434559819288552,
      "learning_rate": 2.5421052631578953e-06,
      "loss": 0.0,
      "num_tokens": 1433875.0,
      "reward": 0.578125,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.578125,
      "step": 517
    },
    {
      "clip_ratio": 0.0003907657810486853,
      "epoch": 0.07072637902785363,
      "grad_norm": 0.08845436573028564,
      "kl": 0.029101567299221642,
      "learning_rate": 2.536842105263158e-06,
      "loss": -0.0001,
      "step": 518
    },
    {
      "clip_ratio": 0.0006556590233230963,
      "epoch": 0.07086291643910432,
      "grad_norm": 0.0867236852645874,
      "kl": 0.028989310667384416,
      "learning_rate": 2.5315789473684212e-06,
      "loss": -0.0004,
      "step": 519
    },
    {
      "clip_ratio": 0.0002281021879753098,
      "epoch": 0.070999453850355,
      "grad_norm": 0.08522820472717285,
      "kl": 0.029066309274639934,
      "learning_rate": 2.5263157894736844e-06,
      "loss": -0.001,
      "step": 520
    },
    {
      "clip_ratio": 0.0005073317588539794,
      "completion_length": 137.53125,
      "epoch": 0.07113599126160568,
      "grad_norm": 0.08292447030544281,
      "kl": 0.02251457070815377,
      "learning_rate": 2.5210526315789475e-06,
      "loss": 0.0005,
      "num_tokens": 1444744.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.690625011920929,
      "step": 521
    },
    {
      "clip_ratio": 0.00021258502965793014,
      "epoch": 0.07127252867285637,
      "grad_norm": 0.0836159959435463,
      "kl": 0.022670210397336632,
      "learning_rate": 2.5157894736842107e-06,
      "loss": -0.0001,
      "step": 522
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07140906608410705,
      "grad_norm": 0.08502236008644104,
      "kl": 0.023073405434843153,
      "learning_rate": 2.510526315789474e-06,
      "loss": -0.0004,
      "step": 523
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07154560349535773,
      "grad_norm": 0.08420410007238388,
      "kl": 0.02333970883046277,
      "learning_rate": 2.5052631578947375e-06,
      "loss": -0.0008,
      "step": 524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.375,
      "epoch": 0.0716821409066084,
      "grad_norm": 0.06113371253013611,
      "kl": 0.028267830901313573,
      "learning_rate": 2.5e-06,
      "loss": 0.0002,
      "num_tokens": 1456204.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 525
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0718186783178591,
      "grad_norm": 0.058986373245716095,
      "kl": 0.029190031375037506,
      "learning_rate": 2.4947368421052634e-06,
      "loss": 0.0001,
      "step": 526
    },
    {
      "clip_ratio": 0.00019531250291038305,
      "epoch": 0.07195521572910978,
      "grad_norm": 0.056226328015327454,
      "kl": 0.029348920157644898,
      "learning_rate": 2.4894736842105265e-06,
      "loss": -0.0001,
      "step": 527
    },
    {
      "clip_ratio": 0.0003004807804245502,
      "epoch": 0.07209175314036045,
      "grad_norm": 0.06096241623163223,
      "kl": 0.029138524609152228,
      "learning_rate": 2.4842105263157897e-06,
      "loss": -0.0002,
      "step": 528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.0,
      "epoch": 0.07222829055161115,
      "grad_norm": 0.0643433928489685,
      "kl": 0.024482485474436544,
      "learning_rate": 2.478947368421053e-06,
      "loss": 0.0002,
      "num_tokens": 1468144.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 529
    },
    {
      "clip_ratio": 0.00013185653369873762,
      "epoch": 0.07236482796286182,
      "grad_norm": 0.06284752488136292,
      "kl": 0.024148478420102037,
      "learning_rate": 2.473684210526316e-06,
      "loss": 0.0001,
      "step": 530
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0725013653741125,
      "grad_norm": 0.055365268141031265,
      "kl": 0.02333252050448209,
      "learning_rate": 2.468421052631579e-06,
      "loss": 0.0,
      "step": 531
    },
    {
      "clip_ratio": 0.0004308253701310605,
      "epoch": 0.0726379027853632,
      "grad_norm": 0.046432189643383026,
      "kl": 0.023968403591425158,
      "learning_rate": 2.4631578947368424e-06,
      "loss": -0.0004,
      "step": 532
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "completion_length": 113.0,
      "epoch": 0.07277444019661387,
      "grad_norm": 0.10641506314277649,
      "kl": 0.028071990760508925,
      "learning_rate": 2.4578947368421055e-06,
      "loss": 0.0001,
      "num_tokens": 1479376.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 533
    },
    {
      "clip_ratio": 0.0004978458891855553,
      "epoch": 0.07291097760786455,
      "grad_norm": 0.1005944311618805,
      "kl": 0.026655245223082602,
      "learning_rate": 2.4526315789473687e-06,
      "loss": -0.0002,
      "step": 534
    },
    {
      "clip_ratio": 0.00044969827285967767,
      "epoch": 0.07304751501911524,
      "grad_norm": 0.10238411277532578,
      "kl": 0.02741476362280082,
      "learning_rate": 2.447368421052632e-06,
      "loss": -0.0003,
      "step": 535
    },
    {
      "clip_ratio": 0.0004633697681128979,
      "epoch": 0.07318405243036592,
      "grad_norm": 0.09010997414588928,
      "kl": 0.027337706153048202,
      "learning_rate": 2.442105263157895e-06,
      "loss": -0.001,
      "step": 536
    },
    {
      "clip_ratio": 0.0007482993096346036,
      "completion_length": 141.15625,
      "epoch": 0.0733205898416166,
      "grad_norm": 0.05969298258423805,
      "kl": 0.02480026413104497,
      "learning_rate": 2.436842105263158e-06,
      "loss": 0.0002,
      "num_tokens": 1491869.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 537
    },
    {
      "clip_ratio": 0.0005413736435002647,
      "epoch": 0.07345712725286728,
      "grad_norm": 0.06196073070168495,
      "kl": 0.024416016967734322,
      "learning_rate": 2.4315789473684213e-06,
      "loss": 0.0,
      "step": 538
    },
    {
      "clip_ratio": 0.0009388480830239132,
      "epoch": 0.07359366466411797,
      "grad_norm": 0.05955655127763748,
      "kl": 0.024354956054594368,
      "learning_rate": 2.4263157894736845e-06,
      "loss": -0.0001,
      "step": 539
    },
    {
      "clip_ratio": 0.00010557432688074186,
      "epoch": 0.07373020207536865,
      "grad_norm": 0.05420181155204773,
      "kl": 0.02480731243849732,
      "learning_rate": 2.4210526315789477e-06,
      "loss": -0.0004,
      "step": 540
    },
    {
      "clip_ratio": 0.0003166591050103307,
      "completion_length": 149.0,
      "epoch": 0.07386673948661933,
      "grad_norm": 0.0756562277674675,
      "kl": 0.024679992871824652,
      "learning_rate": 2.415789473684211e-06,
      "loss": 0.0,
      "num_tokens": 1504213.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.746874988079071,
      "step": 541
    },
    {
      "clip_ratio": 0.0001755617995513603,
      "epoch": 0.07400327689787002,
      "grad_norm": 0.07468439638614655,
      "kl": 0.024849631765391678,
      "learning_rate": 2.410526315789474e-06,
      "loss": -0.0001,
      "step": 542
    },
    {
      "clip_ratio": 0.0004983036633348092,
      "epoch": 0.0741398143091207,
      "grad_norm": 0.07378986477851868,
      "kl": 0.025112065923167393,
      "learning_rate": 2.4052631578947367e-06,
      "loss": -0.0003,
      "step": 543
    },
    {
      "clip_ratio": 0.0004942677624057978,
      "epoch": 0.07427635172037138,
      "grad_norm": 0.07501641660928726,
      "kl": 0.025077814352698624,
      "learning_rate": 2.4000000000000003e-06,
      "loss": -0.0006,
      "step": 544
    },
    {
      "clip_ratio": 0.0005071907507954165,
      "completion_length": 122.625,
      "epoch": 0.07441288913162207,
      "grad_norm": 0.07435806840658188,
      "kl": 0.02418363041942939,
      "learning_rate": 2.3947368421052635e-06,
      "loss": 0.0002,
      "num_tokens": 1515297.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 545
    },
    {
      "clip_ratio": 0.0008165966864908114,
      "epoch": 0.07454942654287275,
      "grad_norm": 0.07489731162786484,
      "kl": 0.025440962228458375,
      "learning_rate": 2.3894736842105266e-06,
      "loss": 0.0001,
      "step": 546
    },
    {
      "clip_ratio": 0.0005071907507954165,
      "epoch": 0.07468596395412343,
      "grad_norm": 0.06941622495651245,
      "kl": 0.025200976844644174,
      "learning_rate": 2.38421052631579e-06,
      "loss": -0.0002,
      "step": 547
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.07482250136537412,
      "grad_norm": 0.06859351694583893,
      "kl": 0.025588049815269187,
      "learning_rate": 2.378947368421053e-06,
      "loss": -0.0006,
      "step": 548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.15625,
      "epoch": 0.0749590387766248,
      "grad_norm": 0.06504392623901367,
      "kl": 0.025377440004376695,
      "learning_rate": 2.373684210526316e-06,
      "loss": 0.0002,
      "num_tokens": 1526610.0,
      "reward": 0.49375003576278687,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.49375003576278687,
      "step": 549
    },
    {
      "clip_ratio": 0.0004585849674185738,
      "epoch": 0.07509557618787548,
      "grad_norm": 0.06409519910812378,
      "kl": 0.025559124507708475,
      "learning_rate": 2.368421052631579e-06,
      "loss": 0.0002,
      "step": 550
    },
    {
      "clip_ratio": 0.00020833333837799728,
      "epoch": 0.07523211359912615,
      "grad_norm": 0.06408659368753433,
      "kl": 0.025702771148644388,
      "learning_rate": 2.363157894736842e-06,
      "loss": -0.0,
      "step": 551
    },
    {
      "clip_ratio": 0.0006567885575350374,
      "epoch": 0.07536865101037685,
      "grad_norm": 0.06450217217206955,
      "kl": 0.025717437063576654,
      "learning_rate": 2.357894736842105e-06,
      "loss": -0.0004,
      "step": 552
    },
    {
      "clip_ratio": 0.000879274244653061,
      "completion_length": 136.71875,
      "epoch": 0.07550518842162753,
      "grad_norm": 0.1316654086112976,
      "kl": 0.029629670636495575,
      "learning_rate": 2.352631578947369e-06,
      "loss": 0.0003,
      "num_tokens": 1538437.0,
      "reward": 0.625,
      "reward_std": 0.4086301922798157,
      "rewards/reward_fn": 0.625,
      "step": 553
    },
    {
      "clip_ratio": 0.000472686137072742,
      "epoch": 0.0756417258328782,
      "grad_norm": 0.1313183605670929,
      "kl": 0.029478842378011905,
      "learning_rate": 2.347368421052632e-06,
      "loss": -0.0003,
      "step": 554
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0757782632441289,
      "grad_norm": 0.12120455503463745,
      "kl": 0.029763573867967352,
      "learning_rate": 2.342105263157895e-06,
      "loss": -0.0009,
      "step": 555
    },
    {
      "clip_ratio": 0.00015703517419751734,
      "epoch": 0.07591480065537957,
      "grad_norm": 0.1141739934682846,
      "kl": 0.030053666734602302,
      "learning_rate": 2.3368421052631583e-06,
      "loss": -0.0016,
      "step": 556
    },
    {
      "clip_ratio": 0.0008116767785395496,
      "completion_length": 157.625,
      "epoch": 0.07605133806663025,
      "grad_norm": 0.08487264066934586,
      "kl": 0.023374435782898217,
      "learning_rate": 2.331578947368421e-06,
      "loss": 0.0003,
      "num_tokens": 1551185.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.746874988079071,
      "step": 557
    },
    {
      "clip_ratio": 0.0012086032365914434,
      "epoch": 0.07618787547788095,
      "grad_norm": 0.0859953910112381,
      "kl": 0.02305353432893753,
      "learning_rate": 2.326315789473684e-06,
      "loss": -0.0001,
      "step": 558
    },
    {
      "clip_ratio": 0.0003681144007714465,
      "epoch": 0.07632441288913162,
      "grad_norm": 0.08386844396591187,
      "kl": 0.023537778324680403,
      "learning_rate": 2.3210526315789473e-06,
      "loss": -0.0003,
      "step": 559
    },
    {
      "clip_ratio": 0.0010179669770877808,
      "epoch": 0.0764609503003823,
      "grad_norm": 0.0760241448879242,
      "kl": 0.023646087502129376,
      "learning_rate": 2.3157894736842105e-06,
      "loss": -0.0008,
      "step": 560
    },
    {
      "clip_ratio": 0.0006391926144715399,
      "completion_length": 139.71875,
      "epoch": 0.07659748771163298,
      "grad_norm": 0.04080972447991371,
      "kl": 0.022678025299683213,
      "learning_rate": 2.310526315789474e-06,
      "loss": 0.0003,
      "num_tokens": 1562616.0,
      "reward": 0.9125000238418579,
      "reward_std": 0.05863019451498985,
      "rewards/reward_fn": 0.9125000238418579,
      "step": 561
    },
    {
      "clip_ratio": 0.0006391926144715399,
      "epoch": 0.07673402512288367,
      "grad_norm": 0.039302002638578415,
      "kl": 0.022627048514550552,
      "learning_rate": 2.3052631578947373e-06,
      "loss": 0.0003,
      "step": 562
    },
    {
      "clip_ratio": 0.0005716463201679289,
      "epoch": 0.07687056253413435,
      "grad_norm": 0.03550795093178749,
      "kl": 0.02283687226008624,
      "learning_rate": 2.3000000000000004e-06,
      "loss": 0.0002,
      "step": 563
    },
    {
      "clip_ratio": 0.0003576610761228949,
      "epoch": 0.07700709994538503,
      "grad_norm": 0.035457585006952286,
      "kl": 0.02243449864909053,
      "learning_rate": 2.294736842105263e-06,
      "loss": 0.0002,
      "step": 564
    },
    {
      "clip_ratio": 0.00026483050896786153,
      "completion_length": 136.28125,
      "epoch": 0.07714363735663572,
      "grad_norm": 0.07386517524719238,
      "kl": 0.024913390225265175,
      "learning_rate": 2.2894736842105263e-06,
      "loss": 0.0003,
      "num_tokens": 1573761.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 565
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0772801747678864,
      "grad_norm": 0.07367141544818878,
      "kl": 0.024486958689521998,
      "learning_rate": 2.2842105263157895e-06,
      "loss": 0.0001,
      "step": 566
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07741671217913708,
      "grad_norm": 0.07088012993335724,
      "kl": 0.024814803357003257,
      "learning_rate": 2.2789473684210527e-06,
      "loss": -0.0002,
      "step": 567
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07755324959038777,
      "grad_norm": 0.07038354128599167,
      "kl": 0.02420603678910993,
      "learning_rate": 2.273684210526316e-06,
      "loss": -0.0004,
      "step": 568
    },
    {
      "clip_ratio": 0.0006139783363323659,
      "completion_length": 137.46875,
      "epoch": 0.07768978700163845,
      "grad_norm": 0.09878756105899811,
      "kl": 0.03216486514429562,
      "learning_rate": 2.268421052631579e-06,
      "loss": 0.0002,
      "num_tokens": 1585816.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.2511056959629059,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 569
    },
    {
      "clip_ratio": 0.0009651019354350865,
      "epoch": 0.07782632441288913,
      "grad_norm": 0.09815220534801483,
      "kl": 0.032273774879286066,
      "learning_rate": 2.2631578947368426e-06,
      "loss": -0.0,
      "step": 570
    },
    {
      "clip_ratio": 0.0011762505891965702,
      "epoch": 0.07796286182413982,
      "grad_norm": 0.09831620007753372,
      "kl": 0.03218546602874994,
      "learning_rate": 2.2578947368421057e-06,
      "loss": -0.0005,
      "step": 571
    },
    {
      "clip_ratio": 0.0012266866688150913,
      "epoch": 0.0780993992353905,
      "grad_norm": 0.0923735499382019,
      "kl": 0.032335673458874226,
      "learning_rate": 2.2526315789473685e-06,
      "loss": -0.0007,
      "step": 572
    },
    {
      "clip_ratio": 0.0005346590478438884,
      "completion_length": 128.40625,
      "epoch": 0.07823593664664118,
      "grad_norm": 0.07317294925451279,
      "kl": 0.031242430064594373,
      "learning_rate": 2.2473684210526316e-06,
      "loss": 0.0003,
      "num_tokens": 1597009.0,
      "reward": 0.9156249761581421,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.9156249761581421,
      "step": 573
    },
    {
      "clip_ratio": 0.0001516990305390209,
      "epoch": 0.07837247405789186,
      "grad_norm": 0.0728185847401619,
      "kl": 0.03100292361341417,
      "learning_rate": 2.242105263157895e-06,
      "loss": 0.0,
      "step": 574
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07850901146914255,
      "grad_norm": 0.07220152020454407,
      "kl": 0.03159632673487067,
      "learning_rate": 2.236842105263158e-06,
      "loss": -0.0002,
      "step": 575
    },
    {
      "clip_ratio": 0.0003676470660138875,
      "epoch": 0.07864554888039323,
      "grad_norm": 0.0730409100651741,
      "kl": 0.03174605511594564,
      "learning_rate": 2.231578947368421e-06,
      "loss": -0.0004,
      "step": 576
    },
    {
      "clip_ratio": 0.0003221649385523051,
      "completion_length": 134.53125,
      "epoch": 0.0787820862916439,
      "grad_norm": 0.08895674347877502,
      "kl": 0.027897415682673454,
      "learning_rate": 2.2263157894736843e-06,
      "loss": 0.0004,
      "num_tokens": 1608962.0,
      "reward": 0.578125,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.578125,
      "step": 577
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0789186237028946,
      "grad_norm": 0.0873054563999176,
      "kl": 0.027481836776132695,
      "learning_rate": 2.221052631578948e-06,
      "loss": 0.0001,
      "step": 578
    },
    {
      "clip_ratio": 0.0005704920913558453,
      "epoch": 0.07905516111414528,
      "grad_norm": 0.1051458939909935,
      "kl": 0.0269019435509108,
      "learning_rate": 2.2157894736842106e-06,
      "loss": -0.0002,
      "step": 579
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07919169852539595,
      "grad_norm": 0.08494027704000473,
      "kl": 0.026989290417986922,
      "learning_rate": 2.2105263157894738e-06,
      "loss": -0.0007,
      "step": 580
    },
    {
      "clip_ratio": 0.0010808736260514706,
      "completion_length": 147.8125,
      "epoch": 0.07932823593664665,
      "grad_norm": 0.08235741406679153,
      "kl": 0.025487731574685313,
      "learning_rate": 2.205263157894737e-06,
      "loss": 0.0005,
      "num_tokens": 1621644.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.2337018847465515,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 581
    },
    {
      "clip_ratio": 0.000671143236104399,
      "epoch": 0.07946477334789732,
      "grad_norm": 0.08296383172273636,
      "kl": 0.02604247519047931,
      "learning_rate": 2.2e-06,
      "loss": 0.0002,
      "step": 582
    },
    {
      "clip_ratio": 0.00048515741946175694,
      "epoch": 0.079601310759148,
      "grad_norm": 0.08231493830680847,
      "kl": 0.025760373420780525,
      "learning_rate": 2.1947368421052633e-06,
      "loss": -0.0001,
      "step": 583
    },
    {
      "clip_ratio": 0.00028742906579282135,
      "epoch": 0.0797378481703987,
      "grad_norm": 0.08307528495788574,
      "kl": 0.026076595226186328,
      "learning_rate": 2.1894736842105264e-06,
      "loss": -0.0005,
      "step": 584
    },
    {
      "clip_ratio": 0.00041828946268651634,
      "completion_length": 152.0625,
      "epoch": 0.07987438558164937,
      "grad_norm": 0.07609320431947708,
      "kl": 0.027080167375970632,
      "learning_rate": 2.1842105263157896e-06,
      "loss": 0.0002,
      "num_tokens": 1633538.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 585
    },
    {
      "clip_ratio": 0.00020424836839083582,
      "epoch": 0.08001092299290005,
      "grad_norm": 0.07597316056489944,
      "kl": 0.027335600112564862,
      "learning_rate": 2.1789473684210528e-06,
      "loss": 0.0001,
      "step": 586
    },
    {
      "clip_ratio": 0.00016711230273358524,
      "epoch": 0.08014746040415073,
      "grad_norm": 0.0741892158985138,
      "kl": 0.027281122165732086,
      "learning_rate": 2.173684210526316e-06,
      "loss": -0.0001,
      "step": 587
    },
    {
      "clip_ratio": 0.000540279594133608,
      "epoch": 0.08028399781540142,
      "grad_norm": 0.0735078901052475,
      "kl": 0.027697552199242637,
      "learning_rate": 2.168421052631579e-06,
      "loss": -0.0003,
      "step": 588
    },
    {
      "clip_ratio": 0.0004656559322029352,
      "completion_length": 130.5,
      "epoch": 0.0804205352266521,
      "grad_norm": 0.0796666294336319,
      "kl": 0.028533839125884697,
      "learning_rate": 2.1631578947368423e-06,
      "loss": 0.0003,
      "num_tokens": 1645442.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 589
    },
    {
      "clip_ratio": 0.00030637255986221135,
      "epoch": 0.08055707263790278,
      "grad_norm": 0.08065035194158554,
      "kl": 0.02866685314802453,
      "learning_rate": 2.1578947368421054e-06,
      "loss": 0.0001,
      "step": 590
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08069361004915347,
      "grad_norm": 0.08036067336797714,
      "kl": 0.029090011259540915,
      "learning_rate": 2.1526315789473686e-06,
      "loss": -0.0003,
      "step": 591
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08083014746040415,
      "grad_norm": 0.07513322681188583,
      "kl": 0.028922376601258293,
      "learning_rate": 2.1473684210526317e-06,
      "loss": -0.0006,
      "step": 592
    },
    {
      "clip_ratio": 0.0009937883296515793,
      "completion_length": 136.40625,
      "epoch": 0.08096668487165483,
      "grad_norm": 0.10887999832630157,
      "kl": 0.025917158054653555,
      "learning_rate": 2.142105263157895e-06,
      "loss": 0.0004,
      "num_tokens": 1657179.0,
      "reward": 0.71875,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.71875,
      "step": 593
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08110322228290552,
      "grad_norm": 0.10297057032585144,
      "kl": 0.025241338269552216,
      "learning_rate": 2.136842105263158e-06,
      "loss": 0.0001,
      "step": 594
    },
    {
      "clip_ratio": 0.0005030487664043903,
      "epoch": 0.0812397596941562,
      "grad_norm": 0.09531421959400177,
      "kl": 0.026220592786557972,
      "learning_rate": 2.1315789473684212e-06,
      "loss": -0.0004,
      "step": 595
    },
    {
      "clip_ratio": 0.00044461380457505584,
      "epoch": 0.08137629710540688,
      "grad_norm": 0.08854269981384277,
      "kl": 0.02591939302510582,
      "learning_rate": 2.1263157894736844e-06,
      "loss": -0.0008,
      "step": 596
    },
    {
      "clip_ratio": 0.00040286114381160587,
      "completion_length": 138.15625,
      "epoch": 0.08151283451665757,
      "grad_norm": 0.08875960856676102,
      "kl": 0.026444266317412257,
      "learning_rate": 2.1210526315789476e-06,
      "loss": -0.0002,
      "num_tokens": 1669548.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.606249988079071,
      "step": 597
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08164937192790825,
      "grad_norm": 0.08832712471485138,
      "kl": 0.02660306473262608,
      "learning_rate": 2.1157894736842107e-06,
      "loss": -0.0003,
      "step": 598
    },
    {
      "clip_ratio": 0.00022163119865581393,
      "epoch": 0.08178590933915893,
      "grad_norm": 0.08737774938344955,
      "kl": 0.02694275981048122,
      "learning_rate": 2.110526315789474e-06,
      "loss": -0.0007,
      "step": 599
    },
    {
      "clip_ratio": 0.00033734142198227346,
      "epoch": 0.0819224467504096,
      "grad_norm": 0.08558928221464157,
      "kl": 0.02657399495365098,
      "learning_rate": 2.105263157894737e-06,
      "loss": -0.001,
      "step": 600
    },
    {
      "clip_ratio": 0.0001188212918350473,
      "completion_length": 128.34375,
      "epoch": 0.0820589841616603,
      "grad_norm": 0.06710242480039597,
      "kl": 0.030188038130290806,
      "learning_rate": 2.1000000000000002e-06,
      "loss": 0.0003,
      "num_tokens": 1680927.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 601
    },
    {
      "clip_ratio": 0.00032577493402641267,
      "epoch": 0.08219552157291098,
      "grad_norm": 0.06653324514627457,
      "kl": 0.030682684533530846,
      "learning_rate": 2.0947368421052634e-06,
      "loss": 0.0003,
      "step": 602
    },
    {
      "clip_ratio": 0.00046111433766782284,
      "epoch": 0.08233205898416165,
      "grad_norm": 0.06612059473991394,
      "kl": 0.030843880609609187,
      "learning_rate": 2.0894736842105266e-06,
      "loss": 0.0002,
      "step": 603
    },
    {
      "clip_ratio": 0.00020695364219136536,
      "epoch": 0.08246859639541235,
      "grad_norm": 0.06393194198608398,
      "kl": 0.031093561818124726,
      "learning_rate": 2.0842105263157897e-06,
      "loss": -0.0001,
      "step": 604
    },
    {
      "clip_ratio": 0.000244140625,
      "completion_length": 128.9375,
      "epoch": 0.08260513380666303,
      "grad_norm": 0.06816224753856659,
      "kl": 0.026418528461363167,
      "learning_rate": 2.078947368421053e-06,
      "loss": 0.0003,
      "num_tokens": 1691557.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 605
    },
    {
      "clip_ratio": 0.0008448271255474538,
      "epoch": 0.0827416712179137,
      "grad_norm": 0.06632911413908005,
      "kl": 0.0269781686947681,
      "learning_rate": 2.073684210526316e-06,
      "loss": 0.0003,
      "step": 606
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.0828782086291644,
      "grad_norm": 0.06394495069980621,
      "kl": 0.026145196170546114,
      "learning_rate": 2.068421052631579e-06,
      "loss": -0.0001,
      "step": 607
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.08301474604041507,
      "grad_norm": 0.06764407455921173,
      "kl": 0.026354748668381944,
      "learning_rate": 2.0631578947368424e-06,
      "loss": -0.0002,
      "step": 608
    },
    {
      "clip_ratio": 0.0009424716117791831,
      "completion_length": 151.625,
      "epoch": 0.08315128345166575,
      "grad_norm": 0.08172643929719925,
      "kl": 0.026212202908936888,
      "learning_rate": 2.0578947368421055e-06,
      "loss": 0.0005,
      "num_tokens": 1703877.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 609
    },
    {
      "clip_ratio": 0.0007314500107895583,
      "epoch": 0.08328782086291645,
      "grad_norm": 0.08089722692966461,
      "kl": 0.0268472250609193,
      "learning_rate": 2.0526315789473687e-06,
      "loss": 0.0004,
      "step": 610
    },
    {
      "clip_ratio": 0.0004393939452711493,
      "epoch": 0.08342435827416712,
      "grad_norm": 0.08089084923267365,
      "kl": 0.02699510075035505,
      "learning_rate": 2.047368421052632e-06,
      "loss": 0.0,
      "step": 611
    },
    {
      "clip_ratio": 0.0006021543667884544,
      "epoch": 0.0835608956854178,
      "grad_norm": 0.08019546419382095,
      "kl": 0.026255997800035402,
      "learning_rate": 2.042105263157895e-06,
      "loss": -0.0005,
      "step": 612
    },
    {
      "clip_ratio": 0.000810613390058279,
      "completion_length": 124.96875,
      "epoch": 0.08369743309666848,
      "grad_norm": 0.09329909086227417,
      "kl": 0.026048675848869607,
      "learning_rate": 2.036842105263158e-06,
      "loss": 0.0001,
      "num_tokens": 1714476.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.25980761647224426,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 613
    },
    {
      "clip_ratio": 0.0012707302666967735,
      "epoch": 0.08383397050791917,
      "grad_norm": 0.09495878219604492,
      "kl": 0.025636921141995117,
      "learning_rate": 2.031578947368421e-06,
      "loss": 0.0002,
      "step": 614
    },
    {
      "clip_ratio": 0.0001707650226308033,
      "epoch": 0.08397050791916985,
      "grad_norm": 0.09412883967161179,
      "kl": 0.025533305306453258,
      "learning_rate": 2.026315789473684e-06,
      "loss": -0.0005,
      "step": 615
    },
    {
      "clip_ratio": 0.00100363623641897,
      "epoch": 0.08410704533042053,
      "grad_norm": 0.08954595029354095,
      "kl": 0.02580202795797959,
      "learning_rate": 2.0210526315789477e-06,
      "loss": -0.0008,
      "step": 616
    },
    {
      "clip_ratio": 0.0007753242534818128,
      "completion_length": 168.53125,
      "epoch": 0.08424358274167122,
      "grad_norm": 0.07307417690753937,
      "kl": 0.026822772080777213,
      "learning_rate": 2.015789473684211e-06,
      "loss": 0.0002,
      "num_tokens": 1727825.0,
      "reward": 0.578125,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.578125,
      "step": 617
    },
    {
      "clip_ratio": 0.0007655591471120715,
      "epoch": 0.0843801201529219,
      "grad_norm": 0.0729583203792572,
      "kl": 0.02734056010376662,
      "learning_rate": 2.010526315789474e-06,
      "loss": 0.0002,
      "step": 618
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08451665756417258,
      "grad_norm": 0.07249397039413452,
      "kl": 0.027082344196969643,
      "learning_rate": 2.005263157894737e-06,
      "loss": -0.0001,
      "step": 619
    },
    {
      "clip_ratio": 0.0004055017634527758,
      "epoch": 0.08465319497542327,
      "grad_norm": 0.06983788311481476,
      "kl": 0.027317771135130897,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0003,
      "step": 620
    },
    {
      "clip_ratio": 0.00041753742698347196,
      "completion_length": 153.78125,
      "epoch": 0.08478973238667395,
      "grad_norm": 0.0808953270316124,
      "kl": 0.028358329698676243,
      "learning_rate": 1.994736842105263e-06,
      "loss": 0.0004,
      "num_tokens": 1740298.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 621
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.08492626979792463,
      "grad_norm": 0.08027170598506927,
      "kl": 0.02871102059725672,
      "learning_rate": 1.9894736842105262e-06,
      "loss": 0.0,
      "step": 622
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.08506280720917532,
      "grad_norm": 0.07950595021247864,
      "kl": 0.02856921256170608,
      "learning_rate": 1.9842105263157894e-06,
      "loss": -0.0001,
      "step": 623
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.085199344620426,
      "grad_norm": 0.07559674233198166,
      "kl": 0.02868326409952715,
      "learning_rate": 1.978947368421053e-06,
      "loss": -0.0006,
      "step": 624
    },
    {
      "clip_ratio": 0.00043604651000350714,
      "completion_length": 137.78125,
      "epoch": 0.08533588203167668,
      "grad_norm": 0.03387494385242462,
      "kl": 0.025424274819670245,
      "learning_rate": 1.973684210526316e-06,
      "loss": 0.0002,
      "num_tokens": 1752347.0,
      "reward": 0.690625011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.690625011920929,
      "step": 625
    },
    {
      "clip_ratio": 0.0005163294408703223,
      "epoch": 0.08547241944292736,
      "grad_norm": 0.03339214622974396,
      "kl": 0.025453351787291467,
      "learning_rate": 1.9684210526315793e-06,
      "loss": 0.0002,
      "step": 626
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08560895685417805,
      "grad_norm": 0.033701345324516296,
      "kl": 0.025396857847226784,
      "learning_rate": 1.9631578947368425e-06,
      "loss": 0.0002,
      "step": 627
    },
    {
      "clip_ratio": 0.0001453488366678357,
      "epoch": 0.08574549426542873,
      "grad_norm": 0.03361152485013008,
      "kl": 0.02518440838321112,
      "learning_rate": 1.9578947368421052e-06,
      "loss": 0.0,
      "step": 628
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "completion_length": 124.375,
      "epoch": 0.0858820316766794,
      "grad_norm": 0.09590297192335129,
      "kl": 0.03190047410316765,
      "learning_rate": 1.9526315789473684e-06,
      "loss": -0.0001,
      "num_tokens": 1763875.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 629
    },
    {
      "clip_ratio": 0.0007283711893251166,
      "epoch": 0.0860185690879301,
      "grad_norm": 0.09830604493618011,
      "kl": 0.03203440588549711,
      "learning_rate": 1.9473684210526315e-06,
      "loss": -0.0,
      "step": 630
    },
    {
      "clip_ratio": 0.0004833552229683846,
      "epoch": 0.08615510649918078,
      "grad_norm": 0.09596423804759979,
      "kl": 0.03181770283845253,
      "learning_rate": 1.9421052631578947e-06,
      "loss": -0.0004,
      "step": 631
    },
    {
      "clip_ratio": 0.0006497451540781185,
      "epoch": 0.08629164391043145,
      "grad_norm": 0.08988839387893677,
      "kl": 0.031553027947666124,
      "learning_rate": 1.936842105263158e-06,
      "loss": -0.0008,
      "step": 632
    },
    {
      "clip_ratio": 0.0003396739193703979,
      "completion_length": 139.6875,
      "epoch": 0.08642818132168215,
      "grad_norm": 0.11786044389009476,
      "kl": 0.029205120386905037,
      "learning_rate": 1.9315789473684215e-06,
      "loss": 0.0001,
      "num_tokens": 1775645.0,
      "reward": 0.690625011920929,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.690625011920929,
      "step": 633
    },
    {
      "clip_ratio": 0.0007215468504000455,
      "epoch": 0.08656471873293282,
      "grad_norm": 0.117555171251297,
      "kl": 0.029039909320999868,
      "learning_rate": 1.9263157894736846e-06,
      "loss": 0.0,
      "step": 634
    },
    {
      "clip_ratio": 0.00045773174497298896,
      "epoch": 0.0867012561441835,
      "grad_norm": 0.11143205314874649,
      "kl": 0.029505190832423978,
      "learning_rate": 1.9210526315789474e-06,
      "loss": -0.0005,
      "step": 635
    },
    {
      "clip_ratio": 0.0006555165600730106,
      "epoch": 0.0868377935554342,
      "grad_norm": 0.10721877217292786,
      "kl": 0.02906324692594353,
      "learning_rate": 1.9157894736842105e-06,
      "loss": -0.0012,
      "step": 636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.125,
      "epoch": 0.08697433096668487,
      "grad_norm": 0.0816088318824768,
      "kl": 0.03337540605571121,
      "learning_rate": 1.9105263157894737e-06,
      "loss": 0.0002,
      "num_tokens": 1787193.0,
      "reward": 0.578125,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.578125,
      "step": 637
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08711086837793555,
      "grad_norm": 0.08079145103693008,
      "kl": 0.03298256819834933,
      "learning_rate": 1.905263157894737e-06,
      "loss": 0.0002,
      "step": 638
    },
    {
      "clip_ratio": 0.00028935185400769114,
      "epoch": 0.08724740578918623,
      "grad_norm": 0.07375474274158478,
      "kl": 0.03400870703626424,
      "learning_rate": 1.9000000000000002e-06,
      "loss": -0.0002,
      "step": 639
    },
    {
      "clip_ratio": 0.00014952152559999377,
      "epoch": 0.08738394320043692,
      "grad_norm": 0.07169979810714722,
      "kl": 0.0339630561938975,
      "learning_rate": 1.8947368421052634e-06,
      "loss": -0.0003,
      "step": 640
    },
    {
      "clip_ratio": 0.00046546234807465225,
      "completion_length": 129.09375,
      "epoch": 0.0875204806116876,
      "grad_norm": 0.0885995551943779,
      "kl": 0.03377768350765109,
      "learning_rate": 1.8894736842105266e-06,
      "loss": 0.0002,
      "num_tokens": 1798452.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 641
    },
    {
      "clip_ratio": 0.0004482519725570455,
      "epoch": 0.08765701802293828,
      "grad_norm": 0.0879860669374466,
      "kl": 0.0345135809329804,
      "learning_rate": 1.8842105263157895e-06,
      "loss": 0.0,
      "step": 642
    },
    {
      "clip_ratio": 0.00014810427092015743,
      "epoch": 0.08779355543418897,
      "grad_norm": 0.08827782422304153,
      "kl": 0.03430288049275987,
      "learning_rate": 1.8789473684210527e-06,
      "loss": -0.0001,
      "step": 643
    },
    {
      "clip_ratio": 0.0009655518224462867,
      "epoch": 0.08793009284543965,
      "grad_norm": 0.0869118869304657,
      "kl": 0.03385839582188055,
      "learning_rate": 1.8736842105263158e-06,
      "loss": -0.0003,
      "step": 644
    },
    {
      "clip_ratio": 0.001409449425409548,
      "completion_length": 128.125,
      "epoch": 0.08806663025669033,
      "grad_norm": 0.0921865850687027,
      "kl": 0.033534372836584225,
      "learning_rate": 1.868421052631579e-06,
      "loss": 0.0005,
      "num_tokens": 1809316.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 645
    },
    {
      "clip_ratio": 0.0006227420526556671,
      "epoch": 0.08820316766794102,
      "grad_norm": 0.09347839653491974,
      "kl": 0.03369360527722165,
      "learning_rate": 1.8631578947368424e-06,
      "loss": 0.0,
      "step": 646
    },
    {
      "clip_ratio": 0.0008524778095306829,
      "epoch": 0.0883397050791917,
      "grad_norm": 0.08786574006080627,
      "kl": 0.03364496599533595,
      "learning_rate": 1.8578947368421055e-06,
      "loss": -0.0002,
      "step": 647
    },
    {
      "clip_ratio": 0.0004562043759506196,
      "epoch": 0.08847624249044238,
      "grad_norm": 0.08518879115581512,
      "kl": 0.03309225430712104,
      "learning_rate": 1.8526315789473687e-06,
      "loss": -0.0008,
      "step": 648
    },
    {
      "clip_ratio": 0.0007344547484535724,
      "completion_length": 130.65625,
      "epoch": 0.08861277990169307,
      "grad_norm": 0.093144990503788,
      "kl": 0.037557946430752054,
      "learning_rate": 1.8473684210526319e-06,
      "loss": 0.0008,
      "num_tokens": 1820961.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 649
    },
    {
      "clip_ratio": 0.0004762095195474103,
      "epoch": 0.08874931731294375,
      "grad_norm": 0.09195293486118317,
      "kl": 0.03590751811861992,
      "learning_rate": 1.8421052631578948e-06,
      "loss": 0.0003,
      "step": 650
    },
    {
      "clip_ratio": 0.000642394064925611,
      "epoch": 0.08888585472419443,
      "grad_norm": 0.09232187271118164,
      "kl": 0.03571216404088773,
      "learning_rate": 1.836842105263158e-06,
      "loss": 0.0003,
      "step": 651
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0890223921354451,
      "grad_norm": 0.08797423541545868,
      "kl": 0.03472030599368736,
      "learning_rate": 1.8315789473684211e-06,
      "loss": -0.0004,
      "step": 652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.21875,
      "epoch": 0.0891589295466958,
      "grad_norm": 0.04103375971317291,
      "kl": 0.03244091695523821,
      "learning_rate": 1.8263157894736843e-06,
      "loss": 0.0003,
      "num_tokens": 1833080.0,
      "reward": 0.71875,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.71875,
      "step": 653
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08929546695794648,
      "grad_norm": 0.04076002910733223,
      "kl": 0.03265755213215016,
      "learning_rate": 1.8210526315789475e-06,
      "loss": 0.0003,
      "step": 654
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08943200436919715,
      "grad_norm": 0.04100827872753143,
      "kl": 0.03254827967612073,
      "learning_rate": 1.8157894736842109e-06,
      "loss": 0.0002,
      "step": 655
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08956854178044785,
      "grad_norm": 0.04075401648879051,
      "kl": 0.03307641306309961,
      "learning_rate": 1.810526315789474e-06,
      "loss": 0.0,
      "step": 656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.78125,
      "epoch": 0.08970507919169853,
      "grad_norm": 0.03296128287911415,
      "kl": 0.027622340683592483,
      "learning_rate": 1.805263157894737e-06,
      "loss": 0.0001,
      "num_tokens": 1845289.0,
      "reward": 0.859375,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.859375,
      "step": 657
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0898416166029492,
      "grad_norm": 0.03176380693912506,
      "kl": 0.027481299359351397,
      "learning_rate": 1.8000000000000001e-06,
      "loss": 0.0001,
      "step": 658
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0899781540141999,
      "grad_norm": 0.031271521002054214,
      "kl": 0.02771525498246774,
      "learning_rate": 1.7947368421052633e-06,
      "loss": 0.0001,
      "step": 659
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09011469142545057,
      "grad_norm": 0.03006802499294281,
      "kl": 0.027794133289717138,
      "learning_rate": 1.7894736842105265e-06,
      "loss": 0.0,
      "step": 660
    },
    {
      "clip_ratio": 0.00037045805947855115,
      "completion_length": 144.34375,
      "epoch": 0.09025122883670125,
      "grad_norm": 0.07770948857069016,
      "kl": 0.032641842582961544,
      "learning_rate": 1.7842105263157896e-06,
      "loss": 0.0002,
      "num_tokens": 1857092.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 661
    },
    {
      "clip_ratio": 0.00035511364694684744,
      "epoch": 0.09038776624795195,
      "grad_norm": 0.08898326009511948,
      "kl": 0.03239144055987708,
      "learning_rate": 1.7789473684210528e-06,
      "loss": 0.0003,
      "step": 662
    },
    {
      "clip_ratio": 0.0004056590114487335,
      "epoch": 0.09052430365920262,
      "grad_norm": 0.07720089703798294,
      "kl": 0.03239328300696798,
      "learning_rate": 1.7736842105263162e-06,
      "loss": -0.0001,
      "step": 663
    },
    {
      "clip_ratio": 0.0003613803564803675,
      "epoch": 0.0906608410704533,
      "grad_norm": 0.0780443400144577,
      "kl": 0.03282105067046359,
      "learning_rate": 1.768421052631579e-06,
      "loss": -0.0002,
      "step": 664
    },
    {
      "clip_ratio": 0.00037165620597079396,
      "completion_length": 142.3125,
      "epoch": 0.09079737848170398,
      "grad_norm": 0.0875195860862732,
      "kl": 0.032956337759969756,
      "learning_rate": 1.7631578947368423e-06,
      "loss": 0.0003,
      "num_tokens": 1868814.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.746874988079071,
      "step": 665
    },
    {
      "clip_ratio": 0.0006258496141526848,
      "epoch": 0.09093391589295467,
      "grad_norm": 0.08770208060741425,
      "kl": 0.032618753350107,
      "learning_rate": 1.7578947368421054e-06,
      "loss": -0.0001,
      "step": 666
    },
    {
      "clip_ratio": 0.00048392938333563507,
      "epoch": 0.09107045330420535,
      "grad_norm": 0.086226686835289,
      "kl": 0.03322019218467176,
      "learning_rate": 1.7526315789473686e-06,
      "loss": -0.0,
      "step": 667
    },
    {
      "clip_ratio": 0.0005311927088769153,
      "epoch": 0.09120699071545603,
      "grad_norm": 0.08620792627334595,
      "kl": 0.032796544779557735,
      "learning_rate": 1.7473684210526318e-06,
      "loss": -0.0005,
      "step": 668
    },
    {
      "clip_ratio": 0.00016891892300918698,
      "completion_length": 152.03125,
      "epoch": 0.09134352812670672,
      "grad_norm": 0.06065315380692482,
      "kl": 0.030359022639459,
      "learning_rate": 1.742105263157895e-06,
      "loss": 0.0004,
      "num_tokens": 1881131.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 669
    },
    {
      "clip_ratio": 0.0001806358341127634,
      "epoch": 0.0914800655379574,
      "grad_norm": 0.060918476432561874,
      "kl": 0.029836400091880932,
      "learning_rate": 1.736842105263158e-06,
      "loss": 0.0003,
      "step": 670
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09161660294920808,
      "grad_norm": 0.06018917262554169,
      "kl": 0.029947946954052895,
      "learning_rate": 1.731578947368421e-06,
      "loss": 0.0,
      "step": 671
    },
    {
      "clip_ratio": 0.0001806358341127634,
      "epoch": 0.09175314036045877,
      "grad_norm": 0.06113429367542267,
      "kl": 0.02997833924018778,
      "learning_rate": 1.7263157894736842e-06,
      "loss": -0.0001,
      "step": 672
    },
    {
      "clip_ratio": 0.0010184195562032983,
      "completion_length": 149.71875,
      "epoch": 0.09188967777170945,
      "grad_norm": 0.11531484127044678,
      "kl": 0.029697615158511326,
      "learning_rate": 1.7210526315789474e-06,
      "loss": 0.0002,
      "num_tokens": 1893158.0,
      "reward": 0.6031249761581421,
      "reward_std": 0.42610567808151245,
      "rewards/reward_fn": 0.6031249761581421,
      "step": 673
    },
    {
      "clip_ratio": 0.0008683160558575764,
      "epoch": 0.09202621518296013,
      "grad_norm": 0.11043091863393784,
      "kl": 0.031451030779862776,
      "learning_rate": 1.7157894736842107e-06,
      "loss": 0.0,
      "step": 674
    },
    {
      "clip_ratio": 0.0016366316704079509,
      "epoch": 0.09216275259421082,
      "grad_norm": 0.11153166741132736,
      "kl": 0.030584218620788306,
      "learning_rate": 1.710526315789474e-06,
      "loss": -0.0003,
      "step": 675
    },
    {
      "clip_ratio": 0.0007215101504698396,
      "epoch": 0.0922992900054615,
      "grad_norm": 0.10511306673288345,
      "kl": 0.030299391684820876,
      "learning_rate": 1.705263157894737e-06,
      "loss": -0.0009,
      "step": 676
    },
    {
      "clip_ratio": 0.00036520417779684067,
      "completion_length": 150.875,
      "epoch": 0.09243582741671218,
      "grad_norm": 0.10060536116361618,
      "kl": 0.03141664047143422,
      "learning_rate": 1.7000000000000002e-06,
      "loss": 0.0006,
      "num_tokens": 1905082.0,
      "reward": 0.71875,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.71875,
      "step": 677
    },
    {
      "clip_ratio": 0.0008521379495505244,
      "epoch": 0.09257236482796286,
      "grad_norm": 0.0980759933590889,
      "kl": 0.03153267546440475,
      "learning_rate": 1.6947368421052632e-06,
      "loss": 0.0001,
      "step": 678
    },
    {
      "clip_ratio": 0.0003382211725693196,
      "epoch": 0.09270890223921355,
      "grad_norm": 0.09468948096036911,
      "kl": 0.03160182334249839,
      "learning_rate": 1.6894736842105264e-06,
      "loss": -0.0004,
      "step": 679
    },
    {
      "clip_ratio": 0.0003924800839740783,
      "epoch": 0.09284543965046423,
      "grad_norm": 0.09634365886449814,
      "kl": 0.03162823195452802,
      "learning_rate": 1.6842105263157895e-06,
      "loss": -0.0007,
      "step": 680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.28125,
      "epoch": 0.0929819770617149,
      "grad_norm": 0.08576124906539917,
      "kl": 0.029789241671096534,
      "learning_rate": 1.6789473684210527e-06,
      "loss": 0.0004,
      "num_tokens": 1915887.0,
      "reward": 0.9156249761581421,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.9156249761581421,
      "step": 681
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0931185144729656,
      "grad_norm": 0.08568418025970459,
      "kl": 0.02953265013638884,
      "learning_rate": 1.673684210526316e-06,
      "loss": 0.0003,
      "step": 682
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09325505188421628,
      "grad_norm": 0.08653020858764648,
      "kl": 0.030135785636957735,
      "learning_rate": 1.6684210526315792e-06,
      "loss": 0.0001,
      "step": 683
    },
    {
      "clip_ratio": 0.0004459502233657986,
      "epoch": 0.09339158929546695,
      "grad_norm": 0.07059013098478317,
      "kl": 0.03075120344874449,
      "learning_rate": 1.6631578947368424e-06,
      "loss": -0.0001,
      "step": 684
    },
    {
      "clip_ratio": 0.0005163708701729774,
      "completion_length": 137.875,
      "epoch": 0.09352812670671765,
      "grad_norm": 0.0986505001783371,
      "kl": 0.03283812435984146,
      "learning_rate": 1.6578947368421053e-06,
      "loss": 0.0005,
      "num_tokens": 1926891.0,
      "reward": 0.578125,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.578125,
      "step": 685
    },
    {
      "clip_ratio": 0.0004849253600696102,
      "epoch": 0.09366466411796832,
      "grad_norm": 0.09848025441169739,
      "kl": 0.032939996555796824,
      "learning_rate": 1.6526315789473685e-06,
      "loss": 0.0002,
      "step": 686
    },
    {
      "clip_ratio": 0.0006144791841506958,
      "epoch": 0.093801201529219,
      "grad_norm": 0.09903288632631302,
      "kl": 0.033225755047169514,
      "learning_rate": 1.6473684210526317e-06,
      "loss": 0.0,
      "step": 687
    },
    {
      "clip_ratio": 0.0003739090752787888,
      "epoch": 0.0939377389404697,
      "grad_norm": 0.09462776780128479,
      "kl": 0.0339893525961088,
      "learning_rate": 1.6421052631578948e-06,
      "loss": -0.0003,
      "step": 688
    },
    {
      "clip_ratio": 0.0004387063963804394,
      "completion_length": 140.9375,
      "epoch": 0.09407427635172037,
      "grad_norm": 0.07126783579587936,
      "kl": 0.032499870838364586,
      "learning_rate": 1.636842105263158e-06,
      "loss": 0.0004,
      "num_tokens": 1938977.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 689
    },
    {
      "clip_ratio": 0.00033783784601837397,
      "epoch": 0.09421081376297105,
      "grad_norm": 0.0714046061038971,
      "kl": 0.03266788163455203,
      "learning_rate": 1.6315789473684212e-06,
      "loss": 0.0001,
      "step": 690
    },
    {
      "clip_ratio": 0.0003198851045453921,
      "epoch": 0.09434735117422173,
      "grad_norm": 0.07156651467084885,
      "kl": 0.03251979897322599,
      "learning_rate": 1.6263157894736845e-06,
      "loss": 0.0002,
      "step": 691
    },
    {
      "clip_ratio": 0.0002877402148442343,
      "epoch": 0.09448388858547242,
      "grad_norm": 0.07169118523597717,
      "kl": 0.03235316323116422,
      "learning_rate": 1.6210526315789473e-06,
      "loss": -0.0001,
      "step": 692
    },
    {
      "clip_ratio": 0.00025826445198617876,
      "completion_length": 133.65625,
      "epoch": 0.0946204259967231,
      "grad_norm": 0.0648774579167366,
      "kl": 0.03120983185363002,
      "learning_rate": 1.6157894736842106e-06,
      "loss": 0.0002,
      "num_tokens": 1950866.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 693
    },
    {
      "clip_ratio": 0.00025826445198617876,
      "epoch": 0.09475696340797378,
      "grad_norm": 0.06372489035129547,
      "kl": 0.0311219054274261,
      "learning_rate": 1.6105263157894738e-06,
      "loss": 0.0001,
      "step": 694
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09489350081922447,
      "grad_norm": 0.06404761970043182,
      "kl": 0.030508951924275607,
      "learning_rate": 1.605263157894737e-06,
      "loss": -0.0,
      "step": 695
    },
    {
      "clip_ratio": 0.0002281021879753098,
      "epoch": 0.09503003823047515,
      "grad_norm": 0.0647362545132637,
      "kl": 0.031124803936108947,
      "learning_rate": 1.6000000000000001e-06,
      "loss": -0.0001,
      "step": 696
    },
    {
      "clip_ratio": 0.0002855776983778924,
      "completion_length": 135.40625,
      "epoch": 0.09516657564172583,
      "grad_norm": 0.069391630589962,
      "kl": 0.03132415152504109,
      "learning_rate": 1.5947368421052633e-06,
      "loss": 0.0004,
      "num_tokens": 1962359.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 697
    },
    {
      "clip_ratio": 0.0005032864864915609,
      "epoch": 0.09530311305297652,
      "grad_norm": 0.07143551856279373,
      "kl": 0.03164268011460081,
      "learning_rate": 1.5894736842105265e-06,
      "loss": 0.0005,
      "step": 698
    },
    {
      "clip_ratio": 0.000163612567121163,
      "epoch": 0.0954396504642272,
      "grad_norm": 0.06680339574813843,
      "kl": 0.03151996951783076,
      "learning_rate": 1.5842105263157894e-06,
      "loss": 0.0002,
      "step": 699
    },
    {
      "clip_ratio": 0.000163612567121163,
      "epoch": 0.09557618787547788,
      "grad_norm": 0.0664854645729065,
      "kl": 0.03106894381926395,
      "learning_rate": 1.5789473684210526e-06,
      "loss": -0.0,
      "step": 700
    },
    {
      "clip_ratio": 0.0014787253021495417,
      "completion_length": 166.3125,
      "epoch": 0.09571272528672857,
      "grad_norm": 0.11290665715932846,
      "kl": 0.031161494320258498,
      "learning_rate": 1.573684210526316e-06,
      "loss": 0.0007,
      "num_tokens": 1975669.0,
      "reward": 0.578125,
      "reward_std": 0.36360570788383484,
      "rewards/reward_fn": 0.578125,
      "step": 701
    },
    {
      "clip_ratio": 0.000746544887078926,
      "epoch": 0.09584926269797925,
      "grad_norm": 0.11650830507278442,
      "kl": 0.03087172904633917,
      "learning_rate": 1.5684210526315791e-06,
      "loss": 0.0003,
      "step": 702
    },
    {
      "clip_ratio": 0.0007508412381866947,
      "epoch": 0.09598580010922993,
      "grad_norm": 0.11172571778297424,
      "kl": 0.03082931600511074,
      "learning_rate": 1.5631578947368423e-06,
      "loss": -0.0001,
      "step": 703
    },
    {
      "clip_ratio": 0.0008415376505581662,
      "epoch": 0.0961223375204806,
      "grad_norm": 0.11011453717947006,
      "kl": 0.030562146741431206,
      "learning_rate": 1.5578947368421054e-06,
      "loss": -0.0005,
      "step": 704
    },
    {
      "clip_ratio": 0.000691575522068888,
      "completion_length": 119.21875,
      "epoch": 0.0962588749317313,
      "grad_norm": 0.09140753000974655,
      "kl": 0.03985722205834463,
      "learning_rate": 1.5526315789473686e-06,
      "loss": 0.0005,
      "num_tokens": 1986412.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 705
    },
    {
      "clip_ratio": 0.00048280542250722647,
      "epoch": 0.09639541234298198,
      "grad_norm": 0.09256947040557861,
      "kl": 0.03949978741002269,
      "learning_rate": 1.5473684210526316e-06,
      "loss": 0.0003,
      "step": 706
    },
    {
      "clip_ratio": 0.0006655539618805051,
      "epoch": 0.09653194975423265,
      "grad_norm": 0.0919048860669136,
      "kl": 0.04003440952510573,
      "learning_rate": 1.5421052631578947e-06,
      "loss": 0.0001,
      "step": 707
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "epoch": 0.09666848716548335,
      "grad_norm": 0.08757032454013824,
      "kl": 0.0402066423848737,
      "learning_rate": 1.5368421052631579e-06,
      "loss": -0.0003,
      "step": 708
    },
    {
      "clip_ratio": 0.00018601190822664648,
      "completion_length": 131.5625,
      "epoch": 0.09680502457673403,
      "grad_norm": 0.04012110456824303,
      "kl": 0.026871845373534597,
      "learning_rate": 1.5315789473684213e-06,
      "loss": 0.0003,
      "num_tokens": 1997494.0,
      "reward": 0.690625011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.690625011920929,
      "step": 709
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0969415619879847,
      "grad_norm": 0.04035944491624832,
      "kl": 0.026953702850732952,
      "learning_rate": 1.5263157894736844e-06,
      "loss": 0.0002,
      "step": 710
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0970780993992354,
      "grad_norm": 0.040319595485925674,
      "kl": 0.026927403203444555,
      "learning_rate": 1.5210526315789476e-06,
      "loss": 0.0003,
      "step": 711
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09721463681048607,
      "grad_norm": 0.037202075123786926,
      "kl": 0.027284836760372855,
      "learning_rate": 1.5157894736842108e-06,
      "loss": 0.0002,
      "step": 712
    },
    {
      "clip_ratio": 0.0005803374951938167,
      "completion_length": 134.25,
      "epoch": 0.09735117422173675,
      "grad_norm": 0.08356423676013947,
      "kl": 0.028570065303938463,
      "learning_rate": 1.5105263157894737e-06,
      "loss": 0.0004,
      "num_tokens": 2008914.0,
      "reward": 0.887499988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.887499988079071,
      "step": 713
    },
    {
      "clip_ratio": 0.00037202381645329297,
      "epoch": 0.09748771163298744,
      "grad_norm": 0.0837133526802063,
      "kl": 0.029038649721769616,
      "learning_rate": 1.5052631578947369e-06,
      "loss": 0.0002,
      "step": 714
    },
    {
      "clip_ratio": 0.0002281021879753098,
      "epoch": 0.09762424904423812,
      "grad_norm": 0.0792241021990776,
      "kl": 0.029093143355567008,
      "learning_rate": 1.5e-06,
      "loss": 0.0003,
      "step": 715
    },
    {
      "clip_ratio": 0.00018601190822664648,
      "epoch": 0.0977607864554888,
      "grad_norm": 0.07729717344045639,
      "kl": 0.02941943213227205,
      "learning_rate": 1.4947368421052632e-06,
      "loss": -0.0002,
      "step": 716
    },
    {
      "clip_ratio": 0.0007804456472513266,
      "completion_length": 154.4375,
      "epoch": 0.09789732386673948,
      "grad_norm": 0.08151241391897202,
      "kl": 0.03156525699887425,
      "learning_rate": 1.4894736842105264e-06,
      "loss": 0.0002,
      "num_tokens": 2021000.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 717
    },
    {
      "clip_ratio": 0.00025540192291373387,
      "epoch": 0.09803386127799017,
      "grad_norm": 0.08159659057855606,
      "kl": 0.0318286495457869,
      "learning_rate": 1.4842105263157897e-06,
      "loss": 0.0,
      "step": 718
    },
    {
      "clip_ratio": 0.0005171415978111327,
      "epoch": 0.09817039868924085,
      "grad_norm": 0.08123067766427994,
      "kl": 0.031137815676629543,
      "learning_rate": 1.478947368421053e-06,
      "loss": -0.0001,
      "step": 719
    },
    {
      "clip_ratio": 0.0006710824818583205,
      "epoch": 0.09830693610049153,
      "grad_norm": 0.07915233820676804,
      "kl": 0.031273662752937526,
      "learning_rate": 1.4736842105263159e-06,
      "loss": -0.0004,
      "step": 720
    },
    {
      "clip_ratio": 0.00026709403027780354,
      "completion_length": 128.96875,
      "epoch": 0.09844347351174222,
      "grad_norm": 0.11582774668931961,
      "kl": 0.03666126274038106,
      "learning_rate": 1.468421052631579e-06,
      "loss": 0.0003,
      "num_tokens": 2032063.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 721
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "epoch": 0.0985800109229929,
      "grad_norm": 0.11533697694540024,
      "kl": 0.036149427643977106,
      "learning_rate": 1.4631578947368422e-06,
      "loss": 0.0,
      "step": 722
    },
    {
      "clip_ratio": 0.0005681818001903594,
      "epoch": 0.09871654833424358,
      "grad_norm": 0.11449582874774933,
      "kl": 0.035863583092577755,
      "learning_rate": 1.4578947368421053e-06,
      "loss": -0.0001,
      "step": 723
    },
    {
      "clip_ratio": 0.0015406785241793841,
      "epoch": 0.09885308574549427,
      "grad_norm": 0.1062321737408638,
      "kl": 0.03639277716865763,
      "learning_rate": 1.4526315789473685e-06,
      "loss": -0.0008,
      "step": 724
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "completion_length": 127.3125,
      "epoch": 0.09898962315674495,
      "grad_norm": 0.08646994829177856,
      "kl": 0.031235893722623587,
      "learning_rate": 1.4473684210526317e-06,
      "loss": 0.0006,
      "num_tokens": 2043593.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 725
    },
    {
      "clip_ratio": 0.00031565656536258757,
      "epoch": 0.09912616056799563,
      "grad_norm": 0.08576333522796631,
      "kl": 0.03090131061617285,
      "learning_rate": 1.442105263157895e-06,
      "loss": 0.0004,
      "step": 726
    },
    {
      "clip_ratio": 0.00015318627993110567,
      "epoch": 0.09926269797924632,
      "grad_norm": 0.08371236175298691,
      "kl": 0.03052604233380407,
      "learning_rate": 1.4368421052631582e-06,
      "loss": 0.0002,
      "step": 727
    },
    {
      "clip_ratio": 0.0003327839804114774,
      "epoch": 0.099399235390497,
      "grad_norm": 0.08282700181007385,
      "kl": 0.0304128993593622,
      "learning_rate": 1.4315789473684212e-06,
      "loss": -0.0003,
      "step": 728
    },
    {
      "clip_ratio": 0.0013521671062335372,
      "completion_length": 135.6875,
      "epoch": 0.09953577280174768,
      "grad_norm": 0.120718814432621,
      "kl": 0.030184058006852865,
      "learning_rate": 1.4263157894736843e-06,
      "loss": 0.0004,
      "num_tokens": 2055143.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.3374999761581421,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 729
    },
    {
      "clip_ratio": 0.00020559210679493845,
      "epoch": 0.09967231021299836,
      "grad_norm": 0.1224217414855957,
      "kl": 0.030380245007108897,
      "learning_rate": 1.4210526315789475e-06,
      "loss": -0.0002,
      "step": 730
    },
    {
      "clip_ratio": 0.0008518250251654536,
      "epoch": 0.09980884762424905,
      "grad_norm": 0.12106145918369293,
      "kl": 0.03084506734739989,
      "learning_rate": 1.4157894736842107e-06,
      "loss": -0.0005,
      "step": 731
    },
    {
      "clip_ratio": 0.0003453038807492703,
      "epoch": 0.09994538503549973,
      "grad_norm": 0.11929699033498764,
      "kl": 0.03096500746323727,
      "learning_rate": 1.4105263157894738e-06,
      "loss": -0.001,
      "step": 732
    },
    {
      "clip_ratio": 0.0001220703125,
      "completion_length": 123.625,
      "epoch": 0.1000819224467504,
      "grad_norm": 0.06932857632637024,
      "kl": 0.03866506487247534,
      "learning_rate": 1.405263157894737e-06,
      "loss": 0.0003,
      "num_tokens": 2066631.0,
      "reward": 0.9437500238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.9437500238418579,
      "step": 733
    },
    {
      "clip_ratio": 0.0004617442318703979,
      "epoch": 0.1002184598580011,
      "grad_norm": 0.06737175583839417,
      "kl": 0.0382203079352621,
      "learning_rate": 1.4000000000000001e-06,
      "loss": 0.0002,
      "step": 734
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.10035499726925177,
      "grad_norm": 0.06321597844362259,
      "kl": 0.0391845365811605,
      "learning_rate": 1.394736842105263e-06,
      "loss": 0.0,
      "step": 735
    },
    {
      "clip_ratio": 0.0001220703125,
      "epoch": 0.10049153468050245,
      "grad_norm": 0.060581691563129425,
      "kl": 0.039463951223297045,
      "learning_rate": 1.3894736842105263e-06,
      "loss": -0.0001,
      "step": 736
    },
    {
      "clip_ratio": 0.0004249223129590973,
      "completion_length": 114.3125,
      "epoch": 0.10062807209175315,
      "grad_norm": 0.06796769797801971,
      "kl": 0.03601354014244862,
      "learning_rate": 1.3842105263157896e-06,
      "loss": 0.0003,
      "num_tokens": 2078197.0,
      "reward": 0.859375,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.859375,
      "step": 737
    },
    {
      "clip_ratio": 0.00013185653369873762,
      "epoch": 0.10076460950300382,
      "grad_norm": 0.06704866886138916,
      "kl": 0.036752572166733444,
      "learning_rate": 1.3789473684210528e-06,
      "loss": 0.0004,
      "step": 738
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1009011469142545,
      "grad_norm": 0.06749145686626434,
      "kl": 0.036876781057799235,
      "learning_rate": 1.373684210526316e-06,
      "loss": 0.0003,
      "step": 739
    },
    {
      "clip_ratio": 0.00013185653369873762,
      "epoch": 0.1010376843255052,
      "grad_norm": 0.061885811388492584,
      "kl": 0.036570252443198115,
      "learning_rate": 1.3684210526315791e-06,
      "loss": 0.0001,
      "step": 740
    },
    {
      "clip_ratio": 0.0006059740553610027,
      "completion_length": 146.1875,
      "epoch": 0.10117422173675587,
      "grad_norm": 0.09527821838855743,
      "kl": 0.032047255721408874,
      "learning_rate": 1.3631578947368423e-06,
      "loss": 0.0003,
      "num_tokens": 2089871.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 741
    },
    {
      "clip_ratio": 0.0011411394516471773,
      "epoch": 0.10131075914800655,
      "grad_norm": 0.09487205743789673,
      "kl": 0.03190063979127444,
      "learning_rate": 1.3578947368421052e-06,
      "loss": 0.0002,
      "step": 742
    },
    {
      "clip_ratio": 0.0006642076768912375,
      "epoch": 0.10144729655925723,
      "grad_norm": 0.09328478574752808,
      "kl": 0.03218909970019013,
      "learning_rate": 1.3526315789473684e-06,
      "loss": 0.0,
      "step": 743
    },
    {
      "clip_ratio": 0.0008540581329725683,
      "epoch": 0.10158383397050792,
      "grad_norm": 0.09467902779579163,
      "kl": 0.03178860497428104,
      "learning_rate": 1.3473684210526316e-06,
      "loss": -0.0006,
      "step": 744
    },
    {
      "clip_ratio": 0.0014868229773128405,
      "completion_length": 132.3125,
      "epoch": 0.1017203713817586,
      "grad_norm": 0.12321406602859497,
      "kl": 0.03981124117854051,
      "learning_rate": 1.342105263157895e-06,
      "loss": 0.0006,
      "num_tokens": 2101149.0,
      "reward": 0.578125,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.578125,
      "step": 745
    },
    {
      "clip_ratio": 0.0005994522216496989,
      "epoch": 0.10185690879300928,
      "grad_norm": 0.11873092502355576,
      "kl": 0.039661494083702564,
      "learning_rate": 1.3368421052631581e-06,
      "loss": 0.0002,
      "step": 746
    },
    {
      "clip_ratio": 0.0002943657018477097,
      "epoch": 0.10199344620425997,
      "grad_norm": 0.12101374566555023,
      "kl": 0.039191018004203215,
      "learning_rate": 1.3315789473684213e-06,
      "loss": -0.0002,
      "step": 747
    },
    {
      "clip_ratio": 0.00041562868864275515,
      "epoch": 0.10212998361551065,
      "grad_norm": 0.11430719494819641,
      "kl": 0.03897344155120663,
      "learning_rate": 1.3263157894736844e-06,
      "loss": -0.0006,
      "step": 748
    },
    {
      "clip_ratio": 0.0008153396483976394,
      "completion_length": 129.5,
      "epoch": 0.10226652102676133,
      "grad_norm": 0.06621770560741425,
      "kl": 0.030582898791180924,
      "learning_rate": 1.3210526315789474e-06,
      "loss": 0.0004,
      "num_tokens": 2112669.0,
      "reward": 0.831250011920929,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.831250011920929,
      "step": 749
    },
    {
      "clip_ratio": 0.0005412168393377215,
      "epoch": 0.10240305843801202,
      "grad_norm": 0.06624240428209305,
      "kl": 0.03081196144921705,
      "learning_rate": 1.3157894736842106e-06,
      "loss": 0.0001,
      "step": 750
    },
    {
      "clip_ratio": 0.00026709403027780354,
      "epoch": 0.1025395958492627,
      "grad_norm": 0.0665903389453888,
      "kl": 0.030628347245510668,
      "learning_rate": 1.3105263157894737e-06,
      "loss": 0.0,
      "step": 751
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.10267613326051338,
      "grad_norm": 0.0640382319688797,
      "kl": 0.03100741034722887,
      "learning_rate": 1.3052631578947369e-06,
      "loss": -0.0002,
      "step": 752
    },
    {
      "clip_ratio": 0.0003129117249045521,
      "completion_length": 142.8125,
      "epoch": 0.10281267067176407,
      "grad_norm": 0.10943469405174255,
      "kl": 0.030879329569870606,
      "learning_rate": 1.3e-06,
      "loss": 0.0001,
      "num_tokens": 2124911.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 753
    },
    {
      "clip_ratio": 0.0004050493298564106,
      "epoch": 0.10294920808301475,
      "grad_norm": 0.11133232712745667,
      "kl": 0.03102087226579897,
      "learning_rate": 1.2947368421052634e-06,
      "loss": -0.0002,
      "step": 754
    },
    {
      "clip_ratio": 0.0006491899548564106,
      "epoch": 0.10308574549426543,
      "grad_norm": 0.10690458118915558,
      "kl": 0.030950611282605678,
      "learning_rate": 1.2894736842105266e-06,
      "loss": -0.0006,
      "step": 755
    },
    {
      "clip_ratio": 0.0012277475470909849,
      "epoch": 0.1032222829055161,
      "grad_norm": 0.10306268185377121,
      "kl": 0.030929109954740852,
      "learning_rate": 1.2842105263157895e-06,
      "loss": -0.0008,
      "step": 756
    },
    {
      "clip_ratio": 0.0001849112450145185,
      "completion_length": 136.25,
      "epoch": 0.1033588203167668,
      "grad_norm": 0.04018304869532585,
      "kl": 0.03246450412552804,
      "learning_rate": 1.2789473684210527e-06,
      "loss": 0.0003,
      "num_tokens": 2136827.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 757
    },
    {
      "clip_ratio": 0.00040654244367033243,
      "epoch": 0.10349535772801748,
      "grad_norm": 0.03952132910490036,
      "kl": 0.033268949860939756,
      "learning_rate": 1.2736842105263159e-06,
      "loss": 0.0003,
      "step": 758
    },
    {
      "clip_ratio": 0.0001849112450145185,
      "epoch": 0.10363189513926815,
      "grad_norm": 0.039509255439043045,
      "kl": 0.03303608257556334,
      "learning_rate": 1.268421052631579e-06,
      "loss": 0.0003,
      "step": 759
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10376843255051885,
      "grad_norm": 0.03916515037417412,
      "kl": 0.032561466010520235,
      "learning_rate": 1.2631578947368422e-06,
      "loss": 0.0003,
      "step": 760
    },
    {
      "clip_ratio": 0.0006341315165627748,
      "completion_length": 131.75,
      "epoch": 0.10390496996176952,
      "grad_norm": 0.08830566704273224,
      "kl": 0.03105318936286494,
      "learning_rate": 1.2578947368421054e-06,
      "loss": 0.0002,
      "num_tokens": 2148251.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.606249988079071,
      "step": 761
    },
    {
      "clip_ratio": 0.0009561895130900666,
      "epoch": 0.1040415073730202,
      "grad_norm": 0.08864033222198486,
      "kl": 0.03143153933342546,
      "learning_rate": 1.2526315789473687e-06,
      "loss": 0.0002,
      "step": 762
    },
    {
      "clip_ratio": 0.0012018284760415554,
      "epoch": 0.1041780447842709,
      "grad_norm": 0.08631853014230728,
      "kl": 0.03134011913789436,
      "learning_rate": 1.2473684210526317e-06,
      "loss": 0.0,
      "step": 763
    },
    {
      "clip_ratio": 0.0006564522045664489,
      "epoch": 0.10431458219552157,
      "grad_norm": 0.08750317245721817,
      "kl": 0.03160371968988329,
      "learning_rate": 1.2421052631578948e-06,
      "loss": -0.0002,
      "step": 764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.21875,
      "epoch": 0.10445111960677225,
      "grad_norm": 0.04507174715399742,
      "kl": 0.03025782248005271,
      "learning_rate": 1.236842105263158e-06,
      "loss": 0.0003,
      "num_tokens": 2159018.0,
      "reward": 0.971875011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.971875011920929,
      "step": 765
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10458765701802294,
      "grad_norm": 0.04543492570519447,
      "kl": 0.03106310815201141,
      "learning_rate": 1.2315789473684212e-06,
      "loss": 0.0004,
      "step": 766
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10472419442927362,
      "grad_norm": 0.04544907435774803,
      "kl": 0.030511865101289004,
      "learning_rate": 1.2263157894736843e-06,
      "loss": 0.0002,
      "step": 767
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1048607318405243,
      "grad_norm": 0.0443943589925766,
      "kl": 0.02988862877828069,
      "learning_rate": 1.2210526315789475e-06,
      "loss": 0.0002,
      "step": 768
    },
    {
      "clip_ratio": 0.0007646681769983843,
      "completion_length": 132.4375,
      "epoch": 0.10499726925177498,
      "grad_norm": 0.05918116122484207,
      "kl": 0.031367348274216056,
      "learning_rate": 1.2157894736842107e-06,
      "loss": 0.0004,
      "num_tokens": 2170364.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 769
    },
    {
      "clip_ratio": 0.000615662953350693,
      "epoch": 0.10513380666302567,
      "grad_norm": 0.058712657541036606,
      "kl": 0.031523832760285586,
      "learning_rate": 1.2105263157894738e-06,
      "loss": 0.0003,
      "step": 770
    },
    {
      "clip_ratio": 0.00041693481034599245,
      "epoch": 0.10527034407427635,
      "grad_norm": 0.05310717225074768,
      "kl": 0.03125297068618238,
      "learning_rate": 1.205263157894737e-06,
      "loss": 0.0002,
      "step": 771
    },
    {
      "clip_ratio": 0.000615662953350693,
      "epoch": 0.10540688148552703,
      "grad_norm": 0.05317603796720505,
      "kl": 0.031730295915622264,
      "learning_rate": 1.2000000000000002e-06,
      "loss": 0.0002,
      "step": 772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 115.84375,
      "epoch": 0.10554341889677772,
      "grad_norm": 0.08601640909910202,
      "kl": 0.036299167724791914,
      "learning_rate": 1.1947368421052633e-06,
      "loss": 0.0004,
      "num_tokens": 2181423.0,
      "reward": 0.550000011920929,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.550000011920929,
      "step": 773
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1056799563080284,
      "grad_norm": 0.08543814718723297,
      "kl": 0.03631689946632832,
      "learning_rate": 1.1894736842105265e-06,
      "loss": 0.0001,
      "step": 774
    },
    {
      "clip_ratio": 0.0004579307569656521,
      "epoch": 0.10581649371927908,
      "grad_norm": 0.0862925574183464,
      "kl": 0.03603148739784956,
      "learning_rate": 1.1842105263157894e-06,
      "loss": 0.0001,
      "step": 775
    },
    {
      "clip_ratio": 0.00023148147738538682,
      "epoch": 0.10595303113052977,
      "grad_norm": 0.08512796461582184,
      "kl": 0.03633074296521954,
      "learning_rate": 1.1789473684210526e-06,
      "loss": -0.0002,
      "step": 776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.15625,
      "epoch": 0.10608956854178045,
      "grad_norm": 0.04055812582373619,
      "kl": 0.03543747233925387,
      "learning_rate": 1.173684210526316e-06,
      "loss": 0.0003,
      "num_tokens": 2192324.0,
      "reward": 0.578125,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.578125,
      "step": 777
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10622610595303113,
      "grad_norm": 0.040661316365003586,
      "kl": 0.035565093508921564,
      "learning_rate": 1.1684210526315791e-06,
      "loss": 0.0002,
      "step": 778
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10636264336428182,
      "grad_norm": 0.04156764596700668,
      "kl": 0.0351781653589569,
      "learning_rate": 1.163157894736842e-06,
      "loss": 0.0002,
      "step": 779
    },
    {
      "clip_ratio": 0.00028669723542407155,
      "epoch": 0.1064991807755325,
      "grad_norm": 0.04101695492863655,
      "kl": 0.03485118242679164,
      "learning_rate": 1.1578947368421053e-06,
      "loss": 0.0002,
      "step": 780
    },
    {
      "clip_ratio": 0.0012449884961824864,
      "completion_length": 144.90625,
      "epoch": 0.10663571818678318,
      "grad_norm": 0.09758393466472626,
      "kl": 0.037451187847182155,
      "learning_rate": 1.1526315789473686e-06,
      "loss": 0.0005,
      "num_tokens": 2204517.0,
      "reward": 0.3812500238418579,
      "reward_std": 0.1948557198047638,
      "rewards/reward_fn": 0.3812500238418579,
      "step": 781
    },
    {
      "clip_ratio": 0.0007635208748979494,
      "epoch": 0.10677225559803386,
      "grad_norm": 0.09678144752979279,
      "kl": 0.03737691760761663,
      "learning_rate": 1.1473684210526316e-06,
      "loss": 0.0005,
      "step": 782
    },
    {
      "clip_ratio": 0.00034340660204179585,
      "epoch": 0.10690879300928455,
      "grad_norm": 0.0968538299202919,
      "kl": 0.03713806488667615,
      "learning_rate": 1.1421052631578947e-06,
      "loss": 0.0003,
      "step": 783
    },
    {
      "clip_ratio": 0.000750611608964391,
      "epoch": 0.10704533042053523,
      "grad_norm": 0.09521595388650894,
      "kl": 0.037653202831279486,
      "learning_rate": 1.136842105263158e-06,
      "loss": 0.0,
      "step": 784
    },
    {
      "clip_ratio": 0.0013667781895492226,
      "completion_length": 131.09375,
      "epoch": 0.1071818678317859,
      "grad_norm": 0.145091250538826,
      "kl": 0.03401468286756426,
      "learning_rate": 1.1315789473684213e-06,
      "loss": 0.0001,
      "num_tokens": 2216008.0,
      "reward": 0.746874988079071,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.746874988079071,
      "step": 785
    },
    {
      "clip_ratio": 0.0008191931410692632,
      "epoch": 0.1073184052430366,
      "grad_norm": 0.14306364953517914,
      "kl": 0.03373881638981402,
      "learning_rate": 1.1263157894736842e-06,
      "loss": -0.0003,
      "step": 786
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10745494265428727,
      "grad_norm": 0.1369042545557022,
      "kl": 0.03430832191952504,
      "learning_rate": 1.1210526315789474e-06,
      "loss": -0.0004,
      "step": 787
    },
    {
      "clip_ratio": 0.0005872874171473086,
      "epoch": 0.10759148006553795,
      "grad_norm": 0.1372627466917038,
      "kl": 0.03363573501701467,
      "learning_rate": 1.1157894736842106e-06,
      "loss": -0.0013,
      "step": 788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.40625,
      "epoch": 0.10772801747678865,
      "grad_norm": 0.05001708120107651,
      "kl": 0.03717817255528644,
      "learning_rate": 1.110526315789474e-06,
      "loss": 0.0003,
      "num_tokens": 2226477.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 789
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10786455488803932,
      "grad_norm": 0.047382958233356476,
      "kl": 0.0376498686382547,
      "learning_rate": 1.1052631578947369e-06,
      "loss": 0.0003,
      "step": 790
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10800109229929,
      "grad_norm": 0.048519086092710495,
      "kl": 0.03672667942009866,
      "learning_rate": 1.1e-06,
      "loss": 0.0003,
      "step": 791
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1081376297105407,
      "grad_norm": 0.04772576689720154,
      "kl": 0.03647302172612399,
      "learning_rate": 1.0947368421052632e-06,
      "loss": 0.0002,
      "step": 792
    },
    {
      "clip_ratio": 0.0006341192492982373,
      "completion_length": 131.53125,
      "epoch": 0.10827416712179137,
      "grad_norm": 0.08512444794178009,
      "kl": 0.028109061560826376,
      "learning_rate": 1.0894736842105264e-06,
      "loss": 0.0001,
      "num_tokens": 2238058.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 793
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10841070453304205,
      "grad_norm": 0.08351006358861923,
      "kl": 0.028101337986299768,
      "learning_rate": 1.0842105263157895e-06,
      "loss": 0.0001,
      "step": 794
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10854724194429273,
      "grad_norm": 0.08248769491910934,
      "kl": 0.028218604769790545,
      "learning_rate": 1.0789473684210527e-06,
      "loss": -0.0001,
      "step": 795
    },
    {
      "clip_ratio": 0.0003955696302000433,
      "epoch": 0.10868377935554342,
      "grad_norm": 0.08255155384540558,
      "kl": 0.028063812758773565,
      "learning_rate": 1.0736842105263159e-06,
      "loss": -0.0003,
      "step": 796
    },
    {
      "clip_ratio": 0.00044361942855175585,
      "completion_length": 125.34375,
      "epoch": 0.1088203167667941,
      "grad_norm": 0.09224527329206467,
      "kl": 0.033332623599562794,
      "learning_rate": 1.068421052631579e-06,
      "loss": 0.0005,
      "num_tokens": 2250205.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 797
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10895685417804478,
      "grad_norm": 0.09290765970945358,
      "kl": 0.033241317549254745,
      "learning_rate": 1.0631578947368422e-06,
      "loss": 0.0003,
      "step": 798
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10909339158929547,
      "grad_norm": 0.09002580493688583,
      "kl": 0.03237891127355397,
      "learning_rate": 1.0578947368421054e-06,
      "loss": 0.0,
      "step": 799
    },
    {
      "clip_ratio": 0.000240384615608491,
      "epoch": 0.10922992900054615,
      "grad_norm": 0.09006845206022263,
      "kl": 0.0326729342341423,
      "learning_rate": 1.0526315789473685e-06,
      "loss": -0.0002,
      "step": 800
    },
    {
      "clip_ratio": 0.0003324467979837209,
      "completion_length": 115.53125,
      "epoch": 0.10936646641179683,
      "grad_norm": 0.05152610316872597,
      "kl": 0.031394801451824605,
      "learning_rate": 1.0473684210526317e-06,
      "loss": 0.0005,
      "num_tokens": 2260810.0,
      "reward": 0.71875,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.71875,
      "step": 801
    },
    {
      "clip_ratio": 0.0003324467979837209,
      "epoch": 0.10950300382304752,
      "grad_norm": 0.051167577505111694,
      "kl": 0.03137708600843325,
      "learning_rate": 1.0421052631578949e-06,
      "loss": 0.0004,
      "step": 802
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1096395412342982,
      "grad_norm": 0.05100513994693756,
      "kl": 0.03141244366997853,
      "learning_rate": 1.036842105263158e-06,
      "loss": 0.0003,
      "step": 803
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10977607864554888,
      "grad_norm": 0.0502689927816391,
      "kl": 0.03105094050988555,
      "learning_rate": 1.0315789473684212e-06,
      "loss": 0.0002,
      "step": 804
    },
    {
      "clip_ratio": 0.0001990445889532566,
      "completion_length": 132.75,
      "epoch": 0.10991261605679957,
      "grad_norm": 0.06489317119121552,
      "kl": 0.034109927713871,
      "learning_rate": 1.0263157894736843e-06,
      "loss": 0.0004,
      "num_tokens": 2272318.0,
      "reward": 0.831250011920929,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.831250011920929,
      "step": 805
    },
    {
      "clip_ratio": 0.0004087761335540563,
      "epoch": 0.11004915346805025,
      "grad_norm": 0.058789148926734924,
      "kl": 0.03437470211065374,
      "learning_rate": 1.0210526315789475e-06,
      "loss": 0.0005,
      "step": 806
    },
    {
      "clip_ratio": 0.00020973154460079968,
      "epoch": 0.11018569087930093,
      "grad_norm": 0.06509160250425339,
      "kl": 0.034245212213136256,
      "learning_rate": 1.0157894736842105e-06,
      "loss": 0.0004,
      "step": 807
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1103222282905516,
      "grad_norm": 0.0659160166978836,
      "kl": 0.03385831121704541,
      "learning_rate": 1.0105263157894738e-06,
      "loss": 0.0002,
      "step": 808
    },
    {
      "clip_ratio": 0.0010227008606307209,
      "completion_length": 126.8125,
      "epoch": 0.1104587657018023,
      "grad_norm": 0.10646427422761917,
      "kl": 0.03514612466096878,
      "learning_rate": 1.005263157894737e-06,
      "loss": 0.0004,
      "num_tokens": 2283260.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 809
    },
    {
      "clip_ratio": 0.0009959485905710608,
      "epoch": 0.11059530311305298,
      "grad_norm": 0.10746331512928009,
      "kl": 0.03498730491264723,
      "learning_rate": 1.0000000000000002e-06,
      "loss": 0.0004,
      "step": 810
    },
    {
      "clip_ratio": 0.001203794716275297,
      "epoch": 0.11073184052430365,
      "grad_norm": 0.1056860014796257,
      "kl": 0.03553603668115102,
      "learning_rate": 9.947368421052631e-07,
      "loss": 0.0003,
      "step": 811
    },
    {
      "clip_ratio": 0.0003511235991027206,
      "epoch": 0.11086837793555435,
      "grad_norm": 0.1063576340675354,
      "kl": 0.034650685905944556,
      "learning_rate": 9.894736842105265e-07,
      "loss": -0.0003,
      "step": 812
    },
    {
      "clip_ratio": 0.0005857369105797261,
      "completion_length": 108.96875,
      "epoch": 0.11100491534680502,
      "grad_norm": 0.13227391242980957,
      "kl": 0.03337117409682833,
      "learning_rate": 9.842105263157897e-07,
      "loss": 0.0005,
      "num_tokens": 2294083.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.746874988079071,
      "step": 813
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.1111414527580557,
      "grad_norm": 0.12803682684898376,
      "kl": 0.032956252398435026,
      "learning_rate": 9.789473684210526e-07,
      "loss": -0.0,
      "step": 814
    },
    {
      "clip_ratio": 0.0008287565433420241,
      "epoch": 0.1112779901693064,
      "grad_norm": 0.12848548591136932,
      "kl": 0.032824013265781105,
      "learning_rate": 9.736842105263158e-07,
      "loss": -0.0,
      "step": 815
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.11141452758055707,
      "grad_norm": 0.11636755615472794,
      "kl": 0.032896427903324366,
      "learning_rate": 9.68421052631579e-07,
      "loss": -0.0004,
      "step": 816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.65625,
      "epoch": 0.11155106499180775,
      "grad_norm": 0.0969424620270729,
      "kl": 0.03230866280500777,
      "learning_rate": 9.631578947368423e-07,
      "loss": 0.0001,
      "num_tokens": 2304824.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 817
    },
    {
      "clip_ratio": 0.00019778481510002166,
      "epoch": 0.11168760240305844,
      "grad_norm": 0.09678807854652405,
      "kl": 0.032764478790340945,
      "learning_rate": 9.578947368421053e-07,
      "loss": 0.0002,
      "step": 818
    },
    {
      "clip_ratio": 0.00047374211135320365,
      "epoch": 0.11182413981430912,
      "grad_norm": 0.09999542683362961,
      "kl": 0.03256647096714005,
      "learning_rate": 9.526315789473685e-07,
      "loss": 0.0002,
      "step": 819
    },
    {
      "clip_ratio": 0.00047374211135320365,
      "epoch": 0.1119606772255598,
      "grad_norm": 0.0940428152680397,
      "kl": 0.03249253722606227,
      "learning_rate": 9.473684210526317e-07,
      "loss": -0.0002,
      "step": 820
    },
    {
      "clip_ratio": 0.0004139072843827307,
      "completion_length": 129.6875,
      "epoch": 0.11209721463681048,
      "grad_norm": 0.08134789764881134,
      "kl": 0.04074790241429582,
      "learning_rate": 9.421052631578948e-07,
      "loss": 0.0004,
      "num_tokens": 2315998.0,
      "reward": 0.746874988079071,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.746874988079071,
      "step": 821
    },
    {
      "clip_ratio": 0.0006293771584751084,
      "epoch": 0.11223375204806117,
      "grad_norm": 0.07964572310447693,
      "kl": 0.040875735925510526,
      "learning_rate": 9.368421052631579e-07,
      "loss": 0.0005,
      "step": 822
    },
    {
      "clip_ratio": 0.00020695364219136536,
      "epoch": 0.11237028945931185,
      "grad_norm": 0.07946870476007462,
      "kl": 0.0408237244701013,
      "learning_rate": 9.315789473684212e-07,
      "loss": 0.0001,
      "step": 823
    },
    {
      "clip_ratio": 0.00020695364219136536,
      "epoch": 0.11250682687056253,
      "grad_norm": 0.07891543954610825,
      "kl": 0.041721748071722686,
      "learning_rate": 9.263157894736844e-07,
      "loss": -0.0,
      "step": 824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.1875,
      "epoch": 0.11264336428181322,
      "grad_norm": 0.04926425591111183,
      "kl": 0.03286067905719392,
      "learning_rate": 9.210526315789474e-07,
      "loss": 0.0004,
      "num_tokens": 2327484.0,
      "reward": 0.9437500238418579,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.9437500238418579,
      "step": 825
    },
    {
      "clip_ratio": 0.00040524803625885397,
      "epoch": 0.1127799016930639,
      "grad_norm": 0.04810812696814537,
      "kl": 0.03193527160328813,
      "learning_rate": 9.157894736842106e-07,
      "loss": 0.0003,
      "step": 826
    },
    {
      "clip_ratio": 0.00042229730752296746,
      "epoch": 0.11291643910431458,
      "grad_norm": 0.04899256303906441,
      "kl": 0.03242765832692385,
      "learning_rate": 9.105263157894737e-07,
      "loss": 0.0003,
      "step": 827
    },
    {
      "clip_ratio": 0.0003544972714735195,
      "epoch": 0.11305297651556527,
      "grad_norm": 0.049238696694374084,
      "kl": 0.03237058556987904,
      "learning_rate": 9.05263157894737e-07,
      "loss": 0.0003,
      "step": 828
    },
    {
      "clip_ratio": 0.0005081300623714924,
      "completion_length": 125.4375,
      "epoch": 0.11318951392681595,
      "grad_norm": 0.13235945999622345,
      "kl": 0.03304180080885999,
      "learning_rate": 9.000000000000001e-07,
      "loss": 0.0007,
      "num_tokens": 2338158.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 829
    },
    {
      "clip_ratio": 0.0011491246259538457,
      "epoch": 0.11332605133806663,
      "grad_norm": 0.13015788793563843,
      "kl": 0.033487798733403906,
      "learning_rate": 8.947368421052632e-07,
      "loss": 0.0004,
      "step": 830
    },
    {
      "clip_ratio": 0.00040650405571796,
      "epoch": 0.11346258874931732,
      "grad_norm": 0.12753689289093018,
      "kl": 0.033373525278875604,
      "learning_rate": 8.894736842105264e-07,
      "loss": 0.0001,
      "step": 831
    },
    {
      "clip_ratio": 0.00022163119865581393,
      "epoch": 0.113599126160568,
      "grad_norm": 0.11997953057289124,
      "kl": 0.03349483574857004,
      "learning_rate": 8.842105263157895e-07,
      "loss": -0.0002,
      "step": 832
    },
    {
      "clip_ratio": 0.0005183477042010054,
      "completion_length": 131.71875,
      "epoch": 0.11373566357181868,
      "grad_norm": 0.1369657963514328,
      "kl": 0.03463814881979488,
      "learning_rate": 8.789473684210527e-07,
      "loss": 0.0006,
      "num_tokens": 2349773.0,
      "reward": 0.6343750357627869,
      "reward_std": 0.34620189666748047,
      "rewards/reward_fn": 0.6343750357627869,
      "step": 833
    },
    {
      "clip_ratio": 0.001799030287656933,
      "epoch": 0.11387220098306935,
      "grad_norm": 0.13694268465042114,
      "kl": 0.035131270095007494,
      "learning_rate": 8.736842105263159e-07,
      "loss": 0.0007,
      "step": 834
    },
    {
      "clip_ratio": 0.00014400921645574272,
      "epoch": 0.11400873839432005,
      "grad_norm": 0.12898842990398407,
      "kl": 0.034667212778003886,
      "learning_rate": 8.68421052631579e-07,
      "loss": 0.0001,
      "step": 835
    },
    {
      "clip_ratio": 0.0007648299215361476,
      "epoch": 0.11414527580557073,
      "grad_norm": 0.1323985755443573,
      "kl": 0.03442463994724676,
      "learning_rate": 8.631578947368421e-07,
      "loss": -0.0004,
      "step": 836
    },
    {
      "clip_ratio": 0.00035511364694684744,
      "completion_length": 134.71875,
      "epoch": 0.1142818132168214,
      "grad_norm": 0.07983018457889557,
      "kl": 0.03501651559781749,
      "learning_rate": 8.578947368421054e-07,
      "loss": 0.0003,
      "num_tokens": 2361200.0,
      "reward": 0.690625011920929,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.690625011920929,
      "step": 837
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1144183506280721,
      "grad_norm": 0.07975803315639496,
      "kl": 0.034936482581542805,
      "learning_rate": 8.526315789473685e-07,
      "loss": 0.0003,
      "step": 838
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11455488803932277,
      "grad_norm": 0.07610788196325302,
      "kl": 0.03467000252567232,
      "learning_rate": 8.473684210526316e-07,
      "loss": 0.0001,
      "step": 839
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11469142545057345,
      "grad_norm": 0.07802704721689224,
      "kl": 0.03485574647493195,
      "learning_rate": 8.421052631578948e-07,
      "loss": -0.0,
      "step": 840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.0625,
      "epoch": 0.11482796286182415,
      "grad_norm": 0.04737474024295807,
      "kl": 0.0320931714377366,
      "learning_rate": 8.36842105263158e-07,
      "loss": 0.0003,
      "num_tokens": 2372558.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 841
    },
    {
      "clip_ratio": 0.0001220703125,
      "epoch": 0.11496450027307482,
      "grad_norm": 0.0413704551756382,
      "kl": 0.03146939189173281,
      "learning_rate": 8.315789473684212e-07,
      "loss": 0.0002,
      "step": 842
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1151010376843255,
      "grad_norm": 0.041506968438625336,
      "kl": 0.03132388353697024,
      "learning_rate": 8.263157894736843e-07,
      "loss": 0.0002,
      "step": 843
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1152375750955762,
      "grad_norm": 0.038393907248973846,
      "kl": 0.031428944814251736,
      "learning_rate": 8.210526315789474e-07,
      "loss": 0.0001,
      "step": 844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.46875,
      "epoch": 0.11537411250682687,
      "grad_norm": 0.062188491225242615,
      "kl": 0.036099575518164784,
      "learning_rate": 8.157894736842106e-07,
      "loss": 0.0002,
      "num_tokens": 2384597.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 845
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "epoch": 0.11551064991807755,
      "grad_norm": 0.059262264519929886,
      "kl": 0.035657460655784234,
      "learning_rate": 8.105263157894736e-07,
      "loss": 0.0001,
      "step": 846
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "epoch": 0.11564718732932823,
      "grad_norm": 0.05964317545294762,
      "kl": 0.03621940390439704,
      "learning_rate": 8.052631578947369e-07,
      "loss": 0.0002,
      "step": 847
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11578372474057892,
      "grad_norm": 0.0571606270968914,
      "kl": 0.03622490944690071,
      "learning_rate": 8.000000000000001e-07,
      "loss": 0.0001,
      "step": 848
    },
    {
      "clip_ratio": 0.0009445661416975781,
      "completion_length": 135.84375,
      "epoch": 0.1159202621518296,
      "grad_norm": 0.07988615334033966,
      "kl": 0.03682940514408983,
      "learning_rate": 7.947368421052632e-07,
      "loss": 0.0005,
      "num_tokens": 2395576.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 849
    },
    {
      "clip_ratio": 0.000547244620975107,
      "epoch": 0.11605679956308028,
      "grad_norm": 0.08060329407453537,
      "kl": 0.03640499952598475,
      "learning_rate": 7.894736842105263e-07,
      "loss": 0.0004,
      "step": 850
    },
    {
      "clip_ratio": 0.0003555268340278417,
      "epoch": 0.11619333697433097,
      "grad_norm": 0.07949917018413544,
      "kl": 0.03623723401688039,
      "learning_rate": 7.842105263157896e-07,
      "loss": 0.0002,
      "step": 851
    },
    {
      "clip_ratio": 0.0009964463242795318,
      "epoch": 0.11632987438558165,
      "grad_norm": 0.07702900469303131,
      "kl": 0.036871126416372135,
      "learning_rate": 7.789473684210527e-07,
      "loss": 0.0003,
      "step": 852
    },
    {
      "clip_ratio": 0.0005859898083144799,
      "completion_length": 120.15625,
      "epoch": 0.11646641179683233,
      "grad_norm": 0.0983206108212471,
      "kl": 0.03288867976516485,
      "learning_rate": 7.736842105263158e-07,
      "loss": 0.0004,
      "num_tokens": 2406365.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.746874988079071,
      "step": 853
    },
    {
      "clip_ratio": 0.0006730413442710415,
      "epoch": 0.11660294920808302,
      "grad_norm": 0.09928011149168015,
      "kl": 0.032499166671186686,
      "learning_rate": 7.684210526315789e-07,
      "loss": 0.0002,
      "step": 854
    },
    {
      "clip_ratio": 0.0004937214544042945,
      "epoch": 0.1167394866193337,
      "grad_norm": 0.0939321517944336,
      "kl": 0.03273730678483844,
      "learning_rate": 7.631578947368422e-07,
      "loss": -0.0,
      "step": 855
    },
    {
      "clip_ratio": 0.00020833333837799728,
      "epoch": 0.11687602403058438,
      "grad_norm": 0.09672950953245163,
      "kl": 0.03311119574937038,
      "learning_rate": 7.578947368421054e-07,
      "loss": -0.0002,
      "step": 856
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "completion_length": 133.40625,
      "epoch": 0.11701256144183507,
      "grad_norm": 0.056489668786525726,
      "kl": 0.030313314258819446,
      "learning_rate": 7.526315789473684e-07,
      "loss": 0.0004,
      "num_tokens": 2417958.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 857
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11714909885308575,
      "grad_norm": 0.0562896803021431,
      "kl": 0.03055126141407527,
      "learning_rate": 7.473684210526316e-07,
      "loss": 0.0004,
      "step": 858
    },
    {
      "clip_ratio": 0.0003907771751983091,
      "epoch": 0.11728563626433643,
      "grad_norm": 0.05616592615842819,
      "kl": 0.030165656498866156,
      "learning_rate": 7.421052631578949e-07,
      "loss": 0.0004,
      "step": 859
    },
    {
      "clip_ratio": 0.0004477034672163427,
      "epoch": 0.1174221736755871,
      "grad_norm": 0.05474083498120308,
      "kl": 0.030640476441476494,
      "learning_rate": 7.368421052631579e-07,
      "loss": 0.0002,
      "step": 860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.6875,
      "epoch": 0.1175587110868378,
      "grad_norm": 0.10344516485929489,
      "kl": 0.037577818031422794,
      "learning_rate": 7.315789473684211e-07,
      "loss": 0.0003,
      "num_tokens": 2429480.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 861
    },
    {
      "clip_ratio": 0.00020032051543239504,
      "epoch": 0.11769524849808848,
      "grad_norm": 0.10587334632873535,
      "kl": 0.03769170871237293,
      "learning_rate": 7.263157894736843e-07,
      "loss": 0.0002,
      "step": 862
    },
    {
      "clip_ratio": 0.0004006410308647901,
      "epoch": 0.11783178590933915,
      "grad_norm": 0.10316218435764313,
      "kl": 0.037826200015842915,
      "learning_rate": 7.210526315789475e-07,
      "loss": 0.0001,
      "step": 863
    },
    {
      "clip_ratio": 0.0004485645913518965,
      "epoch": 0.11796832332058985,
      "grad_norm": 0.1059485375881195,
      "kl": 0.037548283056821674,
      "learning_rate": 7.157894736842106e-07,
      "loss": -0.0002,
      "step": 864
    },
    {
      "clip_ratio": 0.00033602150506339967,
      "completion_length": 132.25,
      "epoch": 0.11810486073184052,
      "grad_norm": 0.09281036257743835,
      "kl": 0.03125832715886645,
      "learning_rate": 7.105263157894737e-07,
      "loss": 0.0002,
      "num_tokens": 2441072.0,
      "reward": 0.46562501788139343,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.46562501788139343,
      "step": 865
    },
    {
      "clip_ratio": 0.0005000000237487257,
      "epoch": 0.1182413981430912,
      "grad_norm": 0.09430094808340073,
      "kl": 0.03168348016333766,
      "learning_rate": 7.052631578947369e-07,
      "loss": 0.0003,
      "step": 866
    },
    {
      "clip_ratio": 0.0002500000118743628,
      "epoch": 0.1183779355543419,
      "grad_norm": 0.0902150422334671,
      "kl": 0.03158555779373273,
      "learning_rate": 7.000000000000001e-07,
      "loss": 0.0002,
      "step": 867
    },
    {
      "clip_ratio": 0.00019654087373055518,
      "epoch": 0.11851447296559257,
      "grad_norm": 0.09070498496294022,
      "kl": 0.031180568737909198,
      "learning_rate": 6.947368421052631e-07,
      "loss": -0.0002,
      "step": 868
    },
    {
      "clip_ratio": 0.00033602150506339967,
      "completion_length": 110.90625,
      "epoch": 0.11865101037684325,
      "grad_norm": 0.06536060571670532,
      "kl": 0.031654038379201666,
      "learning_rate": 6.894736842105264e-07,
      "loss": 0.0004,
      "num_tokens": 2451289.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 869
    },
    {
      "clip_ratio": 0.0010303832968929783,
      "epoch": 0.11878754778809394,
      "grad_norm": 0.06535768508911133,
      "kl": 0.03135678684338927,
      "learning_rate": 6.842105263157896e-07,
      "loss": 0.0005,
      "step": 870
    },
    {
      "clip_ratio": 0.0011440572561696172,
      "epoch": 0.11892408519934462,
      "grad_norm": 0.06512617319822311,
      "kl": 0.031891329621430486,
      "learning_rate": 6.789473684210526e-07,
      "loss": 0.0004,
      "step": 871
    },
    {
      "clip_ratio": 0.001402321708155796,
      "epoch": 0.1190606226105953,
      "grad_norm": 0.06582646071910858,
      "kl": 0.03158884108415805,
      "learning_rate": 6.736842105263158e-07,
      "loss": 0.0004,
      "step": 872
    },
    {
      "clip_ratio": 0.00038580247201025486,
      "completion_length": 111.09375,
      "epoch": 0.11919716002184598,
      "grad_norm": 0.0955490916967392,
      "kl": 0.038142890349263325,
      "learning_rate": 6.684210526315791e-07,
      "loss": 0.0004,
      "num_tokens": 2462664.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 873
    },
    {
      "clip_ratio": 0.000405844155466184,
      "epoch": 0.11933369743309667,
      "grad_norm": 0.09661541134119034,
      "kl": 0.037784790445584804,
      "learning_rate": 6.631578947368422e-07,
      "loss": 0.0003,
      "step": 874
    },
    {
      "clip_ratio": 0.0006363249995047227,
      "epoch": 0.11947023484434735,
      "grad_norm": 0.0970974862575531,
      "kl": 0.03744145468226634,
      "learning_rate": 6.578947368421053e-07,
      "loss": 0.0003,
      "step": 875
    },
    {
      "clip_ratio": 0.0003878333227476105,
      "epoch": 0.11960677225559803,
      "grad_norm": 0.09769237041473389,
      "kl": 0.03685220205807127,
      "learning_rate": 6.526315789473684e-07,
      "loss": 0.0001,
      "step": 876
    },
    {
      "clip_ratio": 0.0003955696302000433,
      "completion_length": 116.625,
      "epoch": 0.11974330966684872,
      "grad_norm": 0.08843936026096344,
      "kl": 0.03245319478446618,
      "learning_rate": 6.473684210526317e-07,
      "loss": 0.0003,
      "num_tokens": 2473228.0,
      "reward": 0.859375,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.859375,
      "step": 877
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1198798470780994,
      "grad_norm": 0.08856622129678726,
      "kl": 0.032318223151378334,
      "learning_rate": 6.421052631578948e-07,
      "loss": 0.0002,
      "step": 878
    },
    {
      "clip_ratio": 0.00016191709437407553,
      "epoch": 0.12001638448935008,
      "grad_norm": 0.08873509615659714,
      "kl": 0.03196106932591647,
      "learning_rate": 6.368421052631579e-07,
      "loss": 0.0,
      "step": 879
    },
    {
      "clip_ratio": 0.0003955696302000433,
      "epoch": 0.12015292190060077,
      "grad_norm": 0.08736385405063629,
      "kl": 0.03180155347217806,
      "learning_rate": 6.315789473684211e-07,
      "loss": 0.0002,
      "step": 880
    },
    {
      "clip_ratio": 0.00028669723542407155,
      "completion_length": 144.90625,
      "epoch": 0.12028945931185145,
      "grad_norm": 0.0820450633764267,
      "kl": 0.037947153381537646,
      "learning_rate": 6.263157894736844e-07,
      "loss": 0.0004,
      "num_tokens": 2485273.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 881
    },
    {
      "clip_ratio": 0.0006903430767124519,
      "epoch": 0.12042599672310213,
      "grad_norm": 0.08431774377822876,
      "kl": 0.03700205200584605,
      "learning_rate": 6.210526315789474e-07,
      "loss": 0.0005,
      "step": 882
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12056253413435282,
      "grad_norm": 0.08097940683364868,
      "kl": 0.03734048001933843,
      "learning_rate": 6.157894736842106e-07,
      "loss": 0.0002,
      "step": 883
    },
    {
      "clip_ratio": 0.00020833333837799728,
      "epoch": 0.1206990715456035,
      "grad_norm": 0.0801195502281189,
      "kl": 0.037502437888178974,
      "learning_rate": 6.105263157894738e-07,
      "loss": 0.0002,
      "step": 884
    },
    {
      "clip_ratio": 0.0008994654635898769,
      "completion_length": 114.625,
      "epoch": 0.12083560895685418,
      "grad_norm": 0.09578239172697067,
      "kl": 0.03379804195719771,
      "learning_rate": 6.052631578947369e-07,
      "loss": 0.0004,
      "num_tokens": 2496109.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 885
    },
    {
      "clip_ratio": 0.0006313131307251751,
      "epoch": 0.12097214636810485,
      "grad_norm": 0.09582193195819855,
      "kl": 0.033672553807264194,
      "learning_rate": 6.000000000000001e-07,
      "loss": 0.0001,
      "step": 886
    },
    {
      "clip_ratio": 0.0006313131307251751,
      "epoch": 0.12110868377935555,
      "grad_norm": 0.09397005289793015,
      "kl": 0.03385620258632116,
      "learning_rate": 5.947368421052632e-07,
      "loss": 0.0002,
      "step": 887
    },
    {
      "clip_ratio": 0.00018825300503522158,
      "epoch": 0.12124522119060623,
      "grad_norm": 0.09514564275741577,
      "kl": 0.03451714990660548,
      "learning_rate": 5.894736842105263e-07,
      "loss": 0.0001,
      "step": 888
    },
    {
      "clip_ratio": 0.00015862943837419152,
      "completion_length": 124.9375,
      "epoch": 0.1213817586018569,
      "grad_norm": 0.08339914679527283,
      "kl": 0.037555598362814635,
      "learning_rate": 5.842105263157896e-07,
      "loss": 0.0004,
      "num_tokens": 2507443.0,
      "reward": 0.690625011920929,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.690625011920929,
      "step": 889
    },
    {
      "clip_ratio": 0.0005922052368987352,
      "epoch": 0.1215182960131076,
      "grad_norm": 0.08334753662347794,
      "kl": 0.03709682886255905,
      "learning_rate": 5.789473684210526e-07,
      "loss": 0.0002,
      "step": 890
    },
    {
      "clip_ratio": 0.0005697215965483338,
      "epoch": 0.12165483342435827,
      "grad_norm": 0.08741384744644165,
      "kl": 0.03697336750337854,
      "learning_rate": 5.736842105263158e-07,
      "loss": 0.0005,
      "step": 891
    },
    {
      "clip_ratio": 0.00018825300503522158,
      "epoch": 0.12179137083560895,
      "grad_norm": 0.08283984661102295,
      "kl": 0.03640045766951516,
      "learning_rate": 5.68421052631579e-07,
      "loss": 0.0,
      "step": 892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.8125,
      "epoch": 0.12192790824685965,
      "grad_norm": 0.10741044580936432,
      "kl": 0.03149555827258155,
      "learning_rate": 5.631578947368421e-07,
      "loss": 0.0003,
      "num_tokens": 2518249.0,
      "reward": 0.46562501788139343,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.46562501788139343,
      "step": 893
    },
    {
      "clip_ratio": 0.0006387395842466503,
      "epoch": 0.12206444565811032,
      "grad_norm": 0.10868413001298904,
      "kl": 0.03139789224951528,
      "learning_rate": 5.578947368421053e-07,
      "loss": 0.0002,
      "step": 894
    },
    {
      "clip_ratio": 0.00042229730752296746,
      "epoch": 0.122200983069361,
      "grad_norm": 0.1074204370379448,
      "kl": 0.03123481632792391,
      "learning_rate": 5.526315789473684e-07,
      "loss": 0.0001,
      "step": 895
    },
    {
      "clip_ratio": 0.00021114865376148373,
      "epoch": 0.1223375204806117,
      "grad_norm": 0.10637306421995163,
      "kl": 0.03106152272084728,
      "learning_rate": 5.473684210526316e-07,
      "loss": 0.0,
      "step": 896
    },
    {
      "clip_ratio": 0.00020032051543239504,
      "completion_length": 132.875,
      "epoch": 0.12247405789186237,
      "grad_norm": 0.06850402057170868,
      "kl": 0.03606277168728411,
      "learning_rate": 5.421052631578948e-07,
      "loss": 0.0003,
      "num_tokens": 2529705.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 897
    },
    {
      "clip_ratio": 0.00022321428696159273,
      "epoch": 0.12261059530311305,
      "grad_norm": 0.06820071488618851,
      "kl": 0.03548625635448843,
      "learning_rate": 5.368421052631579e-07,
      "loss": 0.0002,
      "step": 898
    },
    {
      "clip_ratio": 0.00020032051543239504,
      "epoch": 0.12274713271436373,
      "grad_norm": 0.0653720423579216,
      "kl": 0.03564649049076252,
      "learning_rate": 5.315789473684211e-07,
      "loss": 0.0002,
      "step": 899
    },
    {
      "clip_ratio": 0.00020032051543239504,
      "epoch": 0.12288367012561442,
      "grad_norm": 0.0647495910525322,
      "kl": 0.03579084068769589,
      "learning_rate": 5.263157894736843e-07,
      "loss": 0.0001,
      "step": 900
    },
    {
      "clip_ratio": 0.000712268622010015,
      "completion_length": 141.59375,
      "epoch": 0.1230202075368651,
      "grad_norm": 0.13548840582370758,
      "kl": 0.037526937725488096,
      "learning_rate": 5.210526315789474e-07,
      "loss": 0.0007,
      "num_tokens": 2541880.0,
      "reward": 0.49375003576278687,
      "reward_std": 0.30735570192337036,
      "rewards/reward_fn": 0.49375003576278687,
      "step": 901
    },
    {
      "clip_ratio": 0.001696178747806698,
      "epoch": 0.12315674494811578,
      "grad_norm": 0.13360342383384705,
      "kl": 0.03651306492974982,
      "learning_rate": 5.157894736842106e-07,
      "loss": 0.0005,
      "step": 902
    },
    {
      "clip_ratio": 0.0004890543350484222,
      "epoch": 0.12329328235936647,
      "grad_norm": 0.13228605687618256,
      "kl": 0.03699741544551216,
      "learning_rate": 5.105263157894738e-07,
      "loss": 0.0004,
      "step": 903
    },
    {
      "clip_ratio": 0.000613276133663021,
      "epoch": 0.12342981977061715,
      "grad_norm": 0.13343238830566406,
      "kl": 0.03710320312529802,
      "learning_rate": 5.052631578947369e-07,
      "loss": 0.0004,
      "step": 904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.75,
      "epoch": 0.12356635718186783,
      "grad_norm": 0.10324323177337646,
      "kl": 0.03289288666564971,
      "learning_rate": 5.000000000000001e-07,
      "loss": 0.0004,
      "num_tokens": 2552476.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 905
    },
    {
      "clip_ratio": 0.000510540179675445,
      "epoch": 0.12370289459311852,
      "grad_norm": 0.10418035089969635,
      "kl": 0.03300356210093014,
      "learning_rate": 4.947368421052632e-07,
      "loss": 0.0003,
      "step": 906
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "epoch": 0.1238394320043692,
      "grad_norm": 0.10630179196596146,
      "kl": 0.033201081794686615,
      "learning_rate": 4.894736842105263e-07,
      "loss": 0.0002,
      "step": 907
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12397596941561988,
      "grad_norm": 0.10277502238750458,
      "kl": 0.032689066021703184,
      "learning_rate": 4.842105263157895e-07,
      "loss": 0.0,
      "step": 908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.21875,
      "epoch": 0.12411250682687057,
      "grad_norm": 0.06736943870782852,
      "kl": 0.03596130615915172,
      "learning_rate": 4.789473684210526e-07,
      "loss": 0.0003,
      "num_tokens": 2563159.0,
      "reward": 0.9156249761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.9156249761581421,
      "step": 909
    },
    {
      "clip_ratio": 0.00018825300503522158,
      "epoch": 0.12424904423812125,
      "grad_norm": 0.06819276511669159,
      "kl": 0.03568592216470279,
      "learning_rate": 4.7368421052631585e-07,
      "loss": 0.0003,
      "step": 910
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12438558164937193,
      "grad_norm": 0.06739851832389832,
      "kl": 0.035914964973926544,
      "learning_rate": 4.6842105263157896e-07,
      "loss": 0.0002,
      "step": 911
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1245221190606226,
      "grad_norm": 0.0662432387471199,
      "kl": 0.03619830956449732,
      "learning_rate": 4.631578947368422e-07,
      "loss": 0.0002,
      "step": 912
    },
    {
      "clip_ratio": 0.0013353245303733274,
      "completion_length": 120.15625,
      "epoch": 0.1246586564718733,
      "grad_norm": 0.12909458577632904,
      "kl": 0.03360832063481212,
      "learning_rate": 4.578947368421053e-07,
      "loss": 0.0003,
      "num_tokens": 2574240.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.3374999761581421,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 913
    },
    {
      "clip_ratio": 0.0015245912363752723,
      "epoch": 0.12479519388312398,
      "grad_norm": 0.12952177226543427,
      "kl": 0.03388803228153847,
      "learning_rate": 4.526315789473685e-07,
      "loss": 0.0002,
      "step": 914
    },
    {
      "clip_ratio": 0.0013912470312789083,
      "epoch": 0.12493173129437465,
      "grad_norm": 0.13111548125743866,
      "kl": 0.03383615874918178,
      "learning_rate": 4.473684210526316e-07,
      "loss": 0.0,
      "step": 915
    },
    {
      "clip_ratio": 0.0012334425409790128,
      "epoch": 0.12506826870562535,
      "grad_norm": 0.1287565976381302,
      "kl": 0.03395371185615659,
      "learning_rate": 4.421052631578947e-07,
      "loss": 0.0,
      "step": 916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.53125,
      "epoch": 0.125204806116876,
      "grad_norm": 0.10184290260076523,
      "kl": 0.03474083298351616,
      "learning_rate": 4.3684210526315794e-07,
      "loss": 0.0001,
      "num_tokens": 2585617.0,
      "reward": 0.71875,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.71875,
      "step": 917
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "epoch": 0.1253413435281267,
      "grad_norm": 0.10121199488639832,
      "kl": 0.03448459054925479,
      "learning_rate": 4.3157894736842105e-07,
      "loss": 0.0002,
      "step": 918
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1254778809393774,
      "grad_norm": 0.09928610175848007,
      "kl": 0.034620841237483546,
      "learning_rate": 4.2631578947368427e-07,
      "loss": 0.0,
      "step": 919
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12561441835062806,
      "grad_norm": 0.10057465732097626,
      "kl": 0.03436506350408308,
      "learning_rate": 4.210526315789474e-07,
      "loss": 0.0,
      "step": 920
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "completion_length": 121.5,
      "epoch": 0.12575095576187875,
      "grad_norm": 0.03331659734249115,
      "kl": 0.033621530630625784,
      "learning_rate": 4.157894736842106e-07,
      "loss": 0.0004,
      "num_tokens": 2596589.0,
      "reward": 0.746874988079071,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.746874988079071,
      "step": 921
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "epoch": 0.12588749317312944,
      "grad_norm": 0.03316027671098709,
      "kl": 0.0338965390692465,
      "learning_rate": 4.105263157894737e-07,
      "loss": 0.0003,
      "step": 922
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "epoch": 0.1260240305843801,
      "grad_norm": 0.033771876245737076,
      "kl": 0.03396968066226691,
      "learning_rate": 4.052631578947368e-07,
      "loss": 0.0003,
      "step": 923
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1261605679956308,
      "grad_norm": 0.03320735692977905,
      "kl": 0.03369491477496922,
      "learning_rate": 4.0000000000000003e-07,
      "loss": 0.0002,
      "step": 924
    },
    {
      "clip_ratio": 0.0008623184985481203,
      "completion_length": 139.21875,
      "epoch": 0.1262971054068815,
      "grad_norm": 0.11683438718318939,
      "kl": 0.04034199519082904,
      "learning_rate": 3.9473684210526315e-07,
      "loss": 0.0005,
      "num_tokens": 2608672.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.289951890707016,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 925
    },
    {
      "clip_ratio": 0.0004209743201499805,
      "epoch": 0.12643364281813216,
      "grad_norm": 0.13628651201725006,
      "kl": 0.03980723247514106,
      "learning_rate": 3.8947368421052636e-07,
      "loss": 0.0005,
      "step": 926
    },
    {
      "clip_ratio": 0.0008091448398772627,
      "epoch": 0.12657018022938285,
      "grad_norm": 0.12596413493156433,
      "kl": 0.03970417007803917,
      "learning_rate": 3.8421052631578947e-07,
      "loss": 0.0005,
      "step": 927
    },
    {
      "clip_ratio": 0.001051957588060759,
      "epoch": 0.12670671764063354,
      "grad_norm": 0.11056431382894516,
      "kl": 0.039803695428417996,
      "learning_rate": 3.789473684210527e-07,
      "loss": 0.0005,
      "step": 928
    },
    {
      "clip_ratio": 0.0010233541834168136,
      "completion_length": 140.59375,
      "epoch": 0.1268432550518842,
      "grad_norm": 0.08445189893245697,
      "kl": 0.02938798555987887,
      "learning_rate": 3.736842105263158e-07,
      "loss": 0.0006,
      "num_tokens": 2620411.0,
      "reward": 0.578125,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.578125,
      "step": 929
    },
    {
      "clip_ratio": 0.00040687595901545137,
      "epoch": 0.1269797924631349,
      "grad_norm": 0.08192367851734161,
      "kl": 0.029562115902081132,
      "learning_rate": 3.6842105263157896e-07,
      "loss": 0.0003,
      "step": 930
    },
    {
      "clip_ratio": 0.00026483050896786153,
      "epoch": 0.1271163298743856,
      "grad_norm": 0.08354966342449188,
      "kl": 0.029616149666253477,
      "learning_rate": 3.6315789473684213e-07,
      "loss": 0.0004,
      "step": 931
    },
    {
      "clip_ratio": 0.0008278935856651515,
      "epoch": 0.12725286728563626,
      "grad_norm": 0.08334451168775558,
      "kl": 0.029312406637473032,
      "learning_rate": 3.578947368421053e-07,
      "loss": 0.0004,
      "step": 932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.34375,
      "epoch": 0.12738940469688695,
      "grad_norm": 0.10578613728284836,
      "kl": 0.028767388517735526,
      "learning_rate": 3.5263157894736846e-07,
      "loss": -0.0001,
      "num_tokens": 2631774.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 933
    },
    {
      "clip_ratio": 0.0002500000118743628,
      "epoch": 0.12752594210813764,
      "grad_norm": 0.10861129313707352,
      "kl": 0.02848425370757468,
      "learning_rate": 3.4736842105263157e-07,
      "loss": 0.0001,
      "step": 934
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1276624795193883,
      "grad_norm": 0.09988290071487427,
      "kl": 0.028927168430527672,
      "learning_rate": 3.421052631578948e-07,
      "loss": -0.0001,
      "step": 935
    },
    {
      "clip_ratio": 0.0002500000118743628,
      "epoch": 0.127799016930639,
      "grad_norm": 0.10976044088602066,
      "kl": 0.029052481811959296,
      "learning_rate": 3.368421052631579e-07,
      "loss": -0.0003,
      "step": 936
    },
    {
      "clip_ratio": 0.00046764573198743165,
      "completion_length": 149.4375,
      "epoch": 0.1279355543418897,
      "grad_norm": 0.1032148227095604,
      "kl": 0.030085035221418366,
      "learning_rate": 3.315789473684211e-07,
      "loss": 0.0006,
      "num_tokens": 2643816.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.35490381717681885,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 937
    },
    {
      "clip_ratio": 0.0008250703540397808,
      "epoch": 0.12807209175314035,
      "grad_norm": 0.10333816707134247,
      "kl": 0.029980021994560957,
      "learning_rate": 3.263157894736842e-07,
      "loss": 0.0004,
      "step": 938
    },
    {
      "clip_ratio": 0.0009608945401851088,
      "epoch": 0.12820862916439105,
      "grad_norm": 0.10449221730232239,
      "kl": 0.030026076681679115,
      "learning_rate": 3.210526315789474e-07,
      "loss": 0.0004,
      "step": 939
    },
    {
      "clip_ratio": 0.0008269436075352132,
      "epoch": 0.12834516657564174,
      "grad_norm": 0.10373935848474503,
      "kl": 0.02982323686592281,
      "learning_rate": 3.1578947368421055e-07,
      "loss": 0.0004,
      "step": 940
    },
    {
      "clip_ratio": 0.000616618781350553,
      "completion_length": 114.5,
      "epoch": 0.1284817039868924,
      "grad_norm": 0.11087680608034134,
      "kl": 0.0374185201653745,
      "learning_rate": 3.105263157894737e-07,
      "loss": 0.0002,
      "num_tokens": 2654592.0,
      "reward": 0.887499988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.887499988079071,
      "step": 941
    },
    {
      "clip_ratio": 0.00026939655072055757,
      "epoch": 0.1286182413981431,
      "grad_norm": 0.12226645648479462,
      "kl": 0.036657080316217616,
      "learning_rate": 3.052631578947369e-07,
      "loss": 0.0001,
      "step": 942
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12875477880939376,
      "grad_norm": 0.0905148908495903,
      "kl": 0.03605576854897663,
      "learning_rate": 3.0000000000000004e-07,
      "loss": 0.0003,
      "step": 943
    },
    {
      "clip_ratio": 0.00026939655072055757,
      "epoch": 0.12889131622064445,
      "grad_norm": 0.09072388708591461,
      "kl": 0.0357032545725815,
      "learning_rate": 2.9473684210526315e-07,
      "loss": 0.0001,
      "step": 944
    },
    {
      "clip_ratio": 0.0003004807804245502,
      "completion_length": 125.3125,
      "epoch": 0.12902785363189515,
      "grad_norm": 0.1038588210940361,
      "kl": 0.03439449705183506,
      "learning_rate": 2.894736842105263e-07,
      "loss": 0.0003,
      "num_tokens": 2665466.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 945
    },
    {
      "clip_ratio": 0.0007869734690757468,
      "epoch": 0.1291643910431458,
      "grad_norm": 0.10634484887123108,
      "kl": 0.035002676682779565,
      "learning_rate": 2.842105263157895e-07,
      "loss": 0.0002,
      "step": 946
    },
    {
      "clip_ratio": 0.000922937979339622,
      "epoch": 0.1293009284543965,
      "grad_norm": 0.10326269268989563,
      "kl": 0.03447505127405748,
      "learning_rate": 2.7894736842105264e-07,
      "loss": 0.0001,
      "step": 947
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1294374658656472,
      "grad_norm": 0.1021365150809288,
      "kl": 0.03501308287377469,
      "learning_rate": 2.736842105263158e-07,
      "loss": -0.0001,
      "step": 948
    },
    {
      "clip_ratio": 0.00013469827536027879,
      "completion_length": 138.09375,
      "epoch": 0.12957400327689786,
      "grad_norm": 0.07968829572200775,
      "kl": 0.030057171738008037,
      "learning_rate": 2.6842105263157897e-07,
      "loss": 0.0003,
      "num_tokens": 2677229.0,
      "reward": 0.8875000476837158,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.8875000476837158,
      "step": 949
    },
    {
      "clip_ratio": 0.0006722042598994449,
      "epoch": 0.12971054068814855,
      "grad_norm": 0.0800618827342987,
      "kl": 0.029572369530797005,
      "learning_rate": 2.6315789473684213e-07,
      "loss": 0.0004,
      "step": 950
    },
    {
      "clip_ratio": 0.0005265824729576707,
      "epoch": 0.12984707809939924,
      "grad_norm": 0.08067598938941956,
      "kl": 0.029750974994385615,
      "learning_rate": 2.578947368421053e-07,
      "loss": 0.0002,
      "step": 951
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1299836155106499,
      "grad_norm": 0.07985977828502655,
      "kl": 0.02976464355015196,
      "learning_rate": 2.5263157894736846e-07,
      "loss": 0.0002,
      "step": 952
    },
    {
      "clip_ratio": 0.0028174469189252704,
      "completion_length": 130.96875,
      "epoch": 0.1301201529219006,
      "grad_norm": 2.5399577617645264,
      "kl": 0.13722280028741807,
      "learning_rate": 2.473684210526316e-07,
      "loss": 0.0026,
      "num_tokens": 2688792.0,
      "reward": 0.546875,
      "reward_std": 0.3111537992954254,
      "rewards/reward_fn": 0.546875,
      "step": 953
    },
    {
      "clip_ratio": 0.0007530120201408863,
      "epoch": 0.1302566903331513,
      "grad_norm": 0.9123467206954956,
      "kl": 0.0975925590028055,
      "learning_rate": 2.4210526315789473e-07,
      "loss": 0.0007,
      "step": 954
    },
    {
      "clip_ratio": 0.0003077960864175111,
      "epoch": 0.13039322774440196,
      "grad_norm": 0.7623748779296875,
      "kl": 0.11674012421281077,
      "learning_rate": 2.3684210526315792e-07,
      "loss": 0.0004,
      "step": 955
    },
    {
      "clip_ratio": 0.0006403644947567955,
      "epoch": 0.13052976515565265,
      "grad_norm": 0.6193297505378723,
      "kl": 0.09545972407795489,
      "learning_rate": 2.315789473684211e-07,
      "loss": 0.0002,
      "step": 956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.96875,
      "epoch": 0.13066630256690334,
      "grad_norm": 0.07178813219070435,
      "kl": 0.03629979118704796,
      "learning_rate": 2.2631578947368425e-07,
      "loss": 0.0002,
      "num_tokens": 2699387.0,
      "reward": 0.9437500238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.9437500238418579,
      "step": 957
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.130802839978154,
      "grad_norm": 0.07103423029184341,
      "kl": 0.03656398045131937,
      "learning_rate": 2.2105263157894736e-07,
      "loss": 0.0004,
      "step": 958
    },
    {
      "clip_ratio": 0.00025826445198617876,
      "epoch": 0.1309393773894047,
      "grad_norm": 0.0723857656121254,
      "kl": 0.03695206393604167,
      "learning_rate": 2.1578947368421053e-07,
      "loss": 0.0002,
      "step": 959
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1310759148006554,
      "grad_norm": 0.07231997698545456,
      "kl": 0.0363530462491326,
      "learning_rate": 2.105263157894737e-07,
      "loss": 0.0002,
      "step": 960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.84375,
      "epoch": 0.13121245221190606,
      "grad_norm": 0.09480443596839905,
      "kl": 0.027393472104449756,
      "learning_rate": 2.0526315789473685e-07,
      "loss": 0.0002,
      "num_tokens": 2710114.0,
      "reward": 0.887499988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.887499988079071,
      "step": 961
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.13134898962315675,
      "grad_norm": 0.09404343366622925,
      "kl": 0.027128341323987115,
      "learning_rate": 2.0000000000000002e-07,
      "loss": 0.0001,
      "step": 962
    },
    {
      "clip_ratio": 0.0008934933575801551,
      "epoch": 0.13148552703440744,
      "grad_norm": 0.09405073523521423,
      "kl": 0.027535340806934983,
      "learning_rate": 1.9473684210526318e-07,
      "loss": 0.0001,
      "step": 963
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1316220644456581,
      "grad_norm": 0.08958923071622849,
      "kl": 0.027381854291888885,
      "learning_rate": 1.8947368421052634e-07,
      "loss": -0.0001,
      "step": 964
    },
    {
      "clip_ratio": 0.00042271205165889114,
      "completion_length": 123.78125,
      "epoch": 0.1317586018569088,
      "grad_norm": 0.09731235355138779,
      "kl": 0.02951424481580034,
      "learning_rate": 1.8421052631578948e-07,
      "loss": 0.0003,
      "num_tokens": 2721579.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 965
    },
    {
      "clip_ratio": 0.0006630534771829844,
      "epoch": 0.1318951392681595,
      "grad_norm": 0.0930672436952591,
      "kl": 0.029122414154699072,
      "learning_rate": 1.7894736842105265e-07,
      "loss": 0.0003,
      "step": 966
    },
    {
      "clip_ratio": 0.0005357142654247582,
      "epoch": 0.13203167667941015,
      "grad_norm": 0.09273795038461685,
      "kl": 0.02928502397844568,
      "learning_rate": 1.7368421052631578e-07,
      "loss": 0.0004,
      "step": 967
    },
    {
      "clip_ratio": 0.0007173645281000063,
      "epoch": 0.13216821409066085,
      "grad_norm": 0.09362951666116714,
      "kl": 0.029012724291533232,
      "learning_rate": 1.6842105263157895e-07,
      "loss": 0.0003,
      "step": 968
    },
    {
      "clip_ratio": 0.0003330294566694647,
      "completion_length": 126.40625,
      "epoch": 0.1323047515019115,
      "grad_norm": 0.08251232653856277,
      "kl": 0.03195799436070956,
      "learning_rate": 1.631578947368421e-07,
      "loss": 0.0005,
      "num_tokens": 2732936.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 969
    },
    {
      "clip_ratio": 0.00035325744829606265,
      "epoch": 0.1324412889131622,
      "grad_norm": 0.08156436681747437,
      "kl": 0.031294897315092385,
      "learning_rate": 1.5789473684210527e-07,
      "loss": 0.0002,
      "step": 970
    },
    {
      "clip_ratio": 0.0004885020898655057,
      "epoch": 0.1325778263244129,
      "grad_norm": 0.08326704800128937,
      "kl": 0.03114733239635825,
      "learning_rate": 1.5263157894736844e-07,
      "loss": 0.0002,
      "step": 971
    },
    {
      "clip_ratio": 0.0011412299354560673,
      "epoch": 0.13271436373566356,
      "grad_norm": 0.08228829503059387,
      "kl": 0.03161389287561178,
      "learning_rate": 1.4736842105263158e-07,
      "loss": 0.0003,
      "step": 972
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "completion_length": 118.3125,
      "epoch": 0.13285090114691425,
      "grad_norm": 0.11592817306518555,
      "kl": 0.026773742283694446,
      "learning_rate": 1.4210526315789474e-07,
      "loss": 0.0001,
      "num_tokens": 2743618.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 973
    },
    {
      "clip_ratio": 0.0005201858002692461,
      "epoch": 0.13298743855816494,
      "grad_norm": 0.11682640016078949,
      "kl": 0.027475335708004422,
      "learning_rate": 1.368421052631579e-07,
      "loss": 0.0002,
      "step": 974
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "epoch": 0.1331239759694156,
      "grad_norm": 0.11784546077251434,
      "kl": 0.027234470340772532,
      "learning_rate": 1.3157894736842107e-07,
      "loss": 0.0001,
      "step": 975
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "epoch": 0.1332605133806663,
      "grad_norm": 0.10642171651124954,
      "kl": 0.027347801194991916,
      "learning_rate": 1.2631578947368423e-07,
      "loss": -0.0001,
      "step": 976
    },
    {
      "clip_ratio": 0.0009640677599236369,
      "completion_length": 122.6875,
      "epoch": 0.133397050791917,
      "grad_norm": 0.13537083566188812,
      "kl": 0.02874184341635555,
      "learning_rate": 1.2105263157894737e-07,
      "loss": 0.0002,
      "num_tokens": 2754564.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.4587019085884094,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 977
    },
    {
      "clip_ratio": 0.0013094727764837444,
      "epoch": 0.13353358820316766,
      "grad_norm": 0.13505354523658752,
      "kl": 0.028210823540575802,
      "learning_rate": 1.1578947368421054e-07,
      "loss": 0.0003,
      "step": 978
    },
    {
      "clip_ratio": 0.0004968734428985044,
      "epoch": 0.13367012561441835,
      "grad_norm": 0.13402731716632843,
      "kl": 0.02882817276986316,
      "learning_rate": 1.1052631578947368e-07,
      "loss": 0.0001,
      "step": 979
    },
    {
      "clip_ratio": 0.0010141732927877456,
      "epoch": 0.13380666302566904,
      "grad_norm": 0.13402649760246277,
      "kl": 0.027990734379272908,
      "learning_rate": 1.0526315789473685e-07,
      "loss": 0.0001,
      "step": 980
    },
    {
      "clip_ratio": 0.0004788851802004501,
      "completion_length": 127.625,
      "epoch": 0.1339432004369197,
      "grad_norm": 0.08159787952899933,
      "kl": 0.028808709408622235,
      "learning_rate": 1.0000000000000001e-07,
      "loss": -0.0,
      "num_tokens": 2765624.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 981
    },
    {
      "clip_ratio": 0.0002540650311857462,
      "epoch": 0.1340797378481704,
      "grad_norm": 0.08206771314144135,
      "kl": 0.029088294133543968,
      "learning_rate": 9.473684210526317e-08,
      "loss": 0.0002,
      "step": 982
    },
    {
      "clip_ratio": 0.0002540650311857462,
      "epoch": 0.1342162752594211,
      "grad_norm": 0.08167807012796402,
      "kl": 0.028653373738052323,
      "learning_rate": 8.947368421052632e-08,
      "loss": 0.0003,
      "step": 983
    },
    {
      "clip_ratio": 0.0004788851802004501,
      "epoch": 0.13435281267067176,
      "grad_norm": 0.08144602179527283,
      "kl": 0.028931573644513264,
      "learning_rate": 8.421052631578947e-08,
      "loss": 0.0001,
      "step": 984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.25,
      "epoch": 0.13448935008192245,
      "grad_norm": 0.06735821813344955,
      "kl": 0.029827186255715787,
      "learning_rate": 7.894736842105264e-08,
      "loss": 0.0004,
      "num_tokens": 2775960.0,
      "reward": 0.859375,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.859375,
      "step": 985
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.13462588749317314,
      "grad_norm": 0.06738390773534775,
      "kl": 0.02973687121993862,
      "learning_rate": 7.368421052631579e-08,
      "loss": 0.0004,
      "step": 986
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1347624249044238,
      "grad_norm": 0.06684897094964981,
      "kl": 0.029811298853019252,
      "learning_rate": 6.842105263157895e-08,
      "loss": 0.0003,
      "step": 987
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1348989623156745,
      "grad_norm": 0.06784624606370926,
      "kl": 0.029368669900577515,
      "learning_rate": 6.315789473684211e-08,
      "loss": 0.0004,
      "step": 988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.90625,
      "epoch": 0.1350354997269252,
      "grad_norm": 0.08901333808898926,
      "kl": 0.032253742014290765,
      "learning_rate": 5.789473684210527e-08,
      "loss": 0.0002,
      "num_tokens": 2787289.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 989
    },
    {
      "clip_ratio": 0.0004521122755249962,
      "epoch": 0.13517203713817585,
      "grad_norm": 0.08875369280576706,
      "kl": 0.03244263178203255,
      "learning_rate": 5.263157894736842e-08,
      "loss": 0.0004,
      "step": 990
    },
    {
      "clip_ratio": 0.000534784237970598,
      "epoch": 0.13530857454942655,
      "grad_norm": 0.08818361163139343,
      "kl": 0.032299379439791664,
      "learning_rate": 4.7368421052631586e-08,
      "loss": 0.0004,
      "step": 991
    },
    {
      "clip_ratio": 0.0009257595957024023,
      "epoch": 0.13544511196067724,
      "grad_norm": 0.0888131707906723,
      "kl": 0.03237587565672584,
      "learning_rate": 4.2105263157894737e-08,
      "loss": 0.0002,
      "step": 992
    },
    {
      "clip_ratio": 0.000294811325147748,
      "completion_length": 124.8125,
      "epoch": 0.1355816493719279,
      "grad_norm": 0.10416112095117569,
      "kl": 0.029586980352178216,
      "learning_rate": 3.6842105263157894e-08,
      "loss": 0.0004,
      "num_tokens": 2798919.0,
      "reward": 0.71875,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.71875,
      "step": 993
    },
    {
      "clip_ratio": 0.0009029616194311529,
      "epoch": 0.1357181867831786,
      "grad_norm": 0.10409634560346603,
      "kl": 0.029650241776835173,
      "learning_rate": 3.157894736842106e-08,
      "loss": 0.0004,
      "step": 994
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.13585472419442926,
      "grad_norm": 0.10347838699817657,
      "kl": 0.02986885284190066,
      "learning_rate": 2.631578947368421e-08,
      "loss": 0.0003,
      "step": 995
    },
    {
      "clip_ratio": 0.00047837117745075375,
      "epoch": 0.13599126160567995,
      "grad_norm": 0.10247468948364258,
      "kl": 0.029298015346284956,
      "learning_rate": 2.1052631578947368e-08,
      "loss": 0.0006,
      "step": 996
    },
    {
      "completion_length": 119.46875,
      "epoch": 0.13612779901693065,
      "grad_norm": 0.0,
      "learning_rate": 1.578947368421053e-08,
      "loss": 0.0,
      "num_tokens": 2810074.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.0,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 997
    },
    {
      "epoch": 0.1362643364281813,
      "grad_norm": 0.0,
      "learning_rate": 1.0526315789473684e-08,
      "loss": 0.0,
      "step": 998
    },
    {
      "epoch": 0.136400873839432,
      "grad_norm": 0.0,
      "learning_rate": 5.263157894736842e-09,
      "loss": 0.0,
      "step": 999
    },
    {
      "epoch": 0.1365374112506827,
      "grad_norm": 0.0,
      "learning_rate": 0.0,
      "loss": 0.0,
      "step": 1000
    }
  ],
  "logging_steps": 1,
  "max_steps": 1000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 250,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 32,
  "trial_name": null,
  "trial_params": null
}