{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.1365374112506827,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0625,
      "epoch": 0.00013653741125068268,
      "grad_norm": 0.02207290008664131,
      "kl": 0.0006317789470813295,
      "learning_rate": 1.0000000000000001e-07,
      "loss": -0.0002,
      "num_tokens": 11046.0,
      "reward": 0.7437499761581421,
      "reward_std": 0.19240379333496094,
      "rewards/reward_fn": 0.7437499761581421,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00027307482250136535,
      "grad_norm": 0.02206706441938877,
      "kl": 0.0006317789470813295,
      "learning_rate": 2.0000000000000002e-07,
      "loss": -0.0002,
      "step": 2
    },
    {
      "clip_ratio": 0.0005360348441172391,
      "epoch": 0.00040961223375204805,
      "grad_norm": 0.022166570648550987,
      "kl": 0.0006522277640215179,
      "learning_rate": 3.0000000000000004e-07,
      "loss": 0.0001,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005461496450027307,
      "grad_norm": 0.021423812955617905,
      "kl": 0.0006513863058899005,
      "learning_rate": 4.0000000000000003e-07,
      "loss": -0.0002,
      "step": 4
    },
    {
      "clip_ratio": 0.0005387133569456637,
      "completion_length": 122.125,
      "epoch": 0.0006826870562534135,
      "grad_norm": 0.02105766348540783,
      "kl": 0.0005707805985366576,
      "learning_rate": 5.000000000000001e-07,
      "loss": -0.0,
      "num_tokens": 21978.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 5
    },
    {
      "clip_ratio": 0.0004984622646588832,
      "epoch": 0.0008192244675040961,
      "grad_norm": 0.021098464727401733,
      "kl": 0.0005492573363881093,
      "learning_rate": 6.000000000000001e-07,
      "loss": -0.0001,
      "step": 6
    },
    {
      "clip_ratio": 0.0007210461335489526,
      "epoch": 0.0009557618787547789,
      "grad_norm": 0.020958311855793,
      "kl": 0.0004925800985802198,
      "learning_rate": 7.000000000000001e-07,
      "loss": -0.0,
      "step": 7
    },
    {
      "clip_ratio": 0.00021701389050576836,
      "epoch": 0.0010922992900054614,
      "grad_norm": 0.021792054176330566,
      "kl": 0.0005863340506948589,
      "learning_rate": 8.000000000000001e-07,
      "loss": -0.0,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.21875,
      "epoch": 0.0012288367012561442,
      "grad_norm": 0.025502244010567665,
      "kl": 0.0005780151559520164,
      "learning_rate": 9.000000000000001e-07,
      "loss": -0.0001,
      "num_tokens": 34097.0,
      "reward": 0.746874988079071,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.746874988079071,
      "step": 9
    },
    {
      "clip_ratio": 0.00046690867748111486,
      "epoch": 0.001365374112506827,
      "grad_norm": 0.025376036763191223,
      "kl": 0.0005691521364497021,
      "learning_rate": 1.0000000000000002e-06,
      "loss": -0.0001,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015019115237575095,
      "grad_norm": 0.025907186791300774,
      "kl": 0.000522814968462626,
      "learning_rate": 1.1e-06,
      "loss": 0.0004,
      "step": 11
    },
    {
      "clip_ratio": 0.00013646288425661623,
      "epoch": 0.0016384489350081922,
      "grad_norm": 0.025343747809529305,
      "kl": 0.000557012675471924,
      "learning_rate": 1.2000000000000002e-06,
      "loss": 0.0002,
      "step": 12
    },
    {
      "clip_ratio": 0.0009560473263263702,
      "completion_length": 146.15625,
      "epoch": 0.001774986346258875,
      "grad_norm": 0.023333657532930374,
      "kl": 0.0005626363158626191,
      "learning_rate": 1.3e-06,
      "loss": 0.0,
      "num_tokens": 45898.0,
      "reward": 0.6312500238418579,
      "reward_std": 0.2836301922798157,
      "rewards/reward_fn": 0.6312500238418579,
      "step": 13
    },
    {
      "clip_ratio": 0.0008510874467901886,
      "epoch": 0.0019115237575095577,
      "grad_norm": 0.022622594609856606,
      "kl": 0.0005286783398332773,
      "learning_rate": 1.4000000000000001e-06,
      "loss": -0.0,
      "step": 14
    },
    {
      "clip_ratio": 0.0003033980610780418,
      "epoch": 0.0020480611687602405,
      "grad_norm": 0.022668462246656418,
      "kl": 0.0005744425789089291,
      "learning_rate": 1.5e-06,
      "loss": -0.0002,
      "step": 15
    },
    {
      "clip_ratio": 0.0005991465295664966,
      "epoch": 0.002184598580010923,
      "grad_norm": 0.02338634803891182,
      "kl": 0.0005220320081207319,
      "learning_rate": 1.6000000000000001e-06,
      "loss": -0.0001,
      "step": 16
    },
    {
      "clip_ratio": 0.0001270325155928731,
      "completion_length": 127.4375,
      "epoch": 0.0023211359912616056,
      "grad_norm": 0.023932231590151787,
      "kl": 0.0005799778509754105,
      "learning_rate": 1.7000000000000002e-06,
      "loss": 0.0001,
      "num_tokens": 57488.0,
      "reward": 0.606249988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.606249988079071,
      "step": 17
    },
    {
      "clip_ratio": 0.0007593779591843486,
      "epoch": 0.0024576734025122883,
      "grad_norm": 0.023600488901138306,
      "kl": 0.0005501173236552859,
      "learning_rate": 1.8000000000000001e-06,
      "loss": -0.0001,
      "step": 18
    },
    {
      "clip_ratio": 0.0002500000118743628,
      "epoch": 0.002594210813762971,
      "grad_norm": 0.024099135771393776,
      "kl": 0.0005574463702942012,
      "learning_rate": 1.9000000000000002e-06,
      "loss": -0.0,
      "step": 19
    },
    {
      "clip_ratio": 0.0008590422658016905,
      "epoch": 0.002730748225013654,
      "grad_norm": 0.024080658331513405,
      "kl": 0.0005742711964558112,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0001,
      "step": 20
    },
    {
      "clip_ratio": 0.0015966849750839174,
      "completion_length": 176.875,
      "epoch": 0.0028672856362643366,
      "grad_norm": 0.01954622007906437,
      "kl": 0.0005687026732630329,
      "learning_rate": 2.1000000000000002e-06,
      "loss": 0.0001,
      "num_tokens": 70192.0,
      "reward": 0.4906250238418579,
      "reward_std": 0.23124998807907104,
      "rewards/reward_fn": 0.4906250238418579,
      "step": 21
    },
    {
      "clip_ratio": 0.0005851916066603735,
      "epoch": 0.003003823047515019,
      "grad_norm": 0.01967364363372326,
      "kl": 0.0005616483749690815,
      "learning_rate": 2.2e-06,
      "loss": 0.0,
      "step": 22
    },
    {
      "clip_ratio": 0.0010330898803658783,
      "epoch": 0.0031403604587657017,
      "grad_norm": 0.019373122602701187,
      "kl": 0.0005907249005758786,
      "learning_rate": 2.3000000000000004e-06,
      "loss": 0.0002,
      "step": 23
    },
    {
      "clip_ratio": 0.0009514934354228899,
      "epoch": 0.0032768978700163844,
      "grad_norm": 0.01972307451069355,
      "kl": 0.0005845337018399732,
      "learning_rate": 2.4000000000000003e-06,
      "loss": -0.0,
      "step": 24
    },
    {
      "clip_ratio": 0.0009890664659906179,
      "completion_length": 130.21875,
      "epoch": 0.003413435281267067,
      "grad_norm": 0.02998182363808155,
      "kl": 0.0005863008900632849,
      "learning_rate": 2.5e-06,
      "loss": 0.0002,
      "num_tokens": 81299.0,
      "reward": 0.578125,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.578125,
      "step": 25
    },
    {
      "clip_ratio": 0.000487977362354286,
      "epoch": 0.00354997269251775,
      "grad_norm": 0.030615828931331635,
      "kl": 0.0005413657777353365,
      "learning_rate": 2.6e-06,
      "loss": 0.0001,
      "step": 26
    },
    {
      "clip_ratio": 0.000487977362354286,
      "epoch": 0.0036865101037684327,
      "grad_norm": 0.030381930992007256,
      "kl": 0.00050132268370362,
      "learning_rate": 2.7000000000000004e-06,
      "loss": 0.0001,
      "step": 27
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.0038230475150191155,
      "grad_norm": 0.03030545637011528,
      "kl": 0.0005266097541607451,
      "learning_rate": 2.8000000000000003e-06,
      "loss": -0.0001,
      "step": 28
    },
    {
      "clip_ratio": 0.00014671361714135855,
      "completion_length": 128.8125,
      "epoch": 0.003959584926269798,
      "grad_norm": 0.017806528136134148,
      "kl": 0.0006746522994944826,
      "learning_rate": 2.9e-06,
      "loss": -0.0001,
      "num_tokens": 92649.0,
      "reward": 0.5437500476837158,
      "reward_std": 0.13712069392204285,
      "rewards/reward_fn": 0.5437500476837158,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004096122337520481,
      "grad_norm": 0.017719408497214317,
      "kl": 0.0006834448263361992,
      "learning_rate": 3e-06,
      "loss": -0.0001,
      "step": 30
    },
    {
      "clip_ratio": 0.00031565656536258757,
      "epoch": 0.004232659748771164,
      "grad_norm": 0.0178105216473341,
      "kl": 0.000749060689486214,
      "learning_rate": 3.1000000000000004e-06,
      "loss": -0.0001,
      "step": 31
    },
    {
      "clip_ratio": 0.00027901786961592734,
      "epoch": 0.004369197160021846,
      "grad_norm": 0.018075253814458847,
      "kl": 0.0006366256338878884,
      "learning_rate": 3.2000000000000003e-06,
      "loss": 0.0,
      "step": 32
    },
    {
      "clip_ratio": 0.00016108246927615255,
      "completion_length": 137.84375,
      "epoch": 0.004505734571272528,
      "grad_norm": 0.01944643445312977,
      "kl": 0.0006676609509668197,
      "learning_rate": 3.3000000000000006e-06,
      "loss": 0.0,
      "num_tokens": 104856.0,
      "reward": 0.659375011920929,
      "reward_std": 0.11874999850988388,
      "rewards/reward_fn": 0.659375011920929,
      "step": 33
    },
    {
      "clip_ratio": 0.0006088621739763767,
      "epoch": 0.004642271982523211,
      "grad_norm": 0.01900452747941017,
      "kl": 0.0006304297803580994,
      "learning_rate": 3.4000000000000005e-06,
      "loss": 0.0002,
      "step": 34
    },
    {
      "clip_ratio": 0.0007152466860134155,
      "epoch": 0.004778809393773894,
      "grad_norm": 0.019996952265501022,
      "kl": 0.0006105903066782048,
      "learning_rate": 3.5e-06,
      "loss": 0.0002,
      "step": 35
    },
    {
      "clip_ratio": 0.0005187058122828603,
      "epoch": 0.004915346805024577,
      "grad_norm": 0.0178091861307621,
      "kl": 0.0006019014399498701,
      "learning_rate": 3.6000000000000003e-06,
      "loss": -0.0001,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.96875,
      "epoch": 0.005051884216275259,
      "grad_norm": 0.00929734855890274,
      "kl": 0.0005888168434466934,
      "learning_rate": 3.7e-06,
      "loss": -0.0001,
      "num_tokens": 116419.0,
      "reward": 0.746874988079071,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.746874988079071,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005188421627525942,
      "grad_norm": 0.009237742982804775,
      "kl": 0.0005845859432156431,
      "learning_rate": 3.8000000000000005e-06,
      "loss": -0.0,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005324959038776625,
      "grad_norm": 0.009027848951518536,
      "kl": 0.000577758685722074,
      "learning_rate": 3.900000000000001e-06,
      "loss": -0.0001,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005461496450027308,
      "grad_norm": 0.00940747931599617,
      "kl": 0.0005938013127888553,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0,
      "step": 40
    },
    {
      "clip_ratio": 0.0003538882592692971,
      "completion_length": 148.375,
      "epoch": 0.00559803386127799,
      "grad_norm": 0.0165647454559803,
      "kl": 0.0005366879777284339,
      "learning_rate": 4.1e-06,
      "loss": 0.0001,
      "num_tokens": 128235.0,
      "reward": 0.8562500476837158,
      "reward_std": 0.1711301952600479,
      "rewards/reward_fn": 0.8562500476837158,
      "step": 41
    },
    {
      "clip_ratio": 0.0003231143855373375,
      "epoch": 0.005734571272528673,
      "grad_norm": 0.016383424401283264,
      "kl": 0.0005500864490386448,
      "learning_rate": 4.2000000000000004e-06,
      "loss": -0.0001,
      "step": 42
    },
    {
      "clip_ratio": 0.00040967948007164523,
      "epoch": 0.005871108683779356,
      "grad_norm": 0.016405031085014343,
      "kl": 0.0005530345652005053,
      "learning_rate": 4.3e-06,
      "loss": 0.0,
      "step": 43
    },
    {
      "clip_ratio": 0.00032404610828962177,
      "epoch": 0.006007646095030038,
      "grad_norm": 0.016346527263522148,
      "kl": 0.0005656518319483439,
      "learning_rate": 4.4e-06,
      "loss": 0.0,
      "step": 44
    },
    {
      "clip_ratio": 0.0004112554161110893,
      "completion_length": 137.0,
      "epoch": 0.006144183506280721,
      "grad_norm": 0.01740082912147045,
      "kl": 0.000573518580495147,
      "learning_rate": 4.5e-06,
      "loss": -0.0,
      "num_tokens": 140159.0,
      "reward": 0.518750011920929,
      "reward_std": 0.12358209490776062,
      "rewards/reward_fn": 0.518750011920929,
      "step": 45
    },
    {
      "clip_ratio": 0.000101461038866546,
      "epoch": 0.006280720917531403,
      "grad_norm": 0.017511451616883278,
      "kl": 0.0004865345692905976,
      "learning_rate": 4.600000000000001e-06,
      "loss": 0.0001,
      "step": 46
    },
    {
      "clip_ratio": 0.0003097943772445433,
      "epoch": 0.006417258328782086,
      "grad_norm": 0.017639311030507088,
      "kl": 0.0005538275008802884,
      "learning_rate": 4.7e-06,
      "loss": 0.0,
      "step": 47
    },
    {
      "clip_ratio": 0.00045951134961796924,
      "epoch": 0.006553795740032769,
      "grad_norm": 0.017566056922078133,
      "kl": 0.0005790123971110006,
      "learning_rate": 4.800000000000001e-06,
      "loss": 0.0,
      "step": 48
    },
    {
      "clip_ratio": 0.0002090300986310467,
      "completion_length": 159.25,
      "epoch": 0.006690333151283452,
      "grad_norm": 0.01749393157660961,
      "kl": 0.0005347986639208102,
      "learning_rate": 4.9000000000000005e-06,
      "loss": -0.0,
      "num_tokens": 152155.0,
      "reward": 0.4625000059604645,
      "reward_std": 0.17499999701976776,
      "rewards/reward_fn": 0.4625000059604645,
      "step": 49
    },
    {
      "clip_ratio": 0.00040434260154142976,
      "epoch": 0.006826870562534134,
      "grad_norm": 0.017458247020840645,
      "kl": 0.0005386869147514517,
      "learning_rate": 5e-06,
      "loss": 0.0001,
      "step": 50
    },
    {
      "clip_ratio": 0.0007310814908123575,
      "epoch": 0.006963407973784817,
      "grad_norm": 0.017301850020885468,
      "kl": 0.0006885109469294548,
      "learning_rate": 4.9947368421052636e-06,
      "loss": 0.0,
      "step": 51
    },
    {
      "clip_ratio": 0.00043346243182895705,
      "epoch": 0.0070999453850355,
      "grad_norm": 0.01740550808608532,
      "kl": 0.0005350831738724082,
      "learning_rate": 4.989473684210527e-06,
      "loss": 0.0001,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 154.03125,
      "epoch": 0.007236482796286183,
      "grad_norm": 0.028556713834404945,
      "kl": 0.0005652442723658169,
      "learning_rate": 4.98421052631579e-06,
      "loss": 0.0002,
      "num_tokens": 163832.0,
      "reward": 0.3531250059604645,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.3531250059604645,
      "step": 53
    },
    {
      "clip_ratio": 0.0002604968976811506,
      "epoch": 0.007373020207536865,
      "grad_norm": 0.028979483991861343,
      "kl": 0.0005701856089217472,
      "learning_rate": 4.978947368421053e-06,
      "loss": -0.0,
      "step": 54
    },
    {
      "clip_ratio": 0.00026939655072055757,
      "epoch": 0.007509557618787548,
      "grad_norm": 0.028597978875041008,
      "kl": 0.0005870076556675485,
      "learning_rate": 4.973684210526316e-06,
      "loss": 0.0001,
      "step": 55
    },
    {
      "clip_ratio": 0.0001426940580131486,
      "epoch": 0.007646095030038231,
      "grad_norm": 0.02883176878094673,
      "kl": 0.0005412003038145485,
      "learning_rate": 4.968421052631579e-06,
      "loss": -0.0002,
      "step": 56
    },
    {
      "clip_ratio": 0.000891457311809063,
      "completion_length": 150.71875,
      "epoch": 0.007782632441288913,
      "grad_norm": 0.029951218515634537,
      "kl": 0.0005701104482795927,
      "learning_rate": 4.9631578947368426e-06,
      "loss": 0.0001,
      "num_tokens": 176059.0,
      "reward": 0.578125,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.578125,
      "step": 57
    },
    {
      "clip_ratio": 0.0006488768121926114,
      "epoch": 0.007919169852539596,
      "grad_norm": 0.029265111312270164,
      "kl": 0.000576867132622283,
      "learning_rate": 4.957894736842106e-06,
      "loss": -0.0001,
      "step": 58
    },
    {
      "clip_ratio": 0.0006633551238337532,
      "epoch": 0.008055707263790278,
      "grad_norm": 0.029740851372480392,
      "kl": 0.0005770292154920753,
      "learning_rate": 4.952631578947369e-06,
      "loss": 0.0001,
      "step": 59
    },
    {
      "clip_ratio": 0.0007974752952577546,
      "epoch": 0.008192244675040962,
      "grad_norm": 0.029165882617235184,
      "kl": 0.0006021304664045601,
      "learning_rate": 4.947368421052632e-06,
      "loss": 0.0,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.15625,
      "epoch": 0.008328782086291644,
      "grad_norm": 0.02565835975110531,
      "kl": 0.0005988110287944437,
      "learning_rate": 4.942105263157895e-06,
      "loss": -0.0002,
      "num_tokens": 188028.0,
      "reward": 0.859375,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.859375,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.008465319497542327,
      "grad_norm": 0.024776969105005264,
      "kl": 0.0006643234455623315,
      "learning_rate": 4.936842105263158e-06,
      "loss": 0.0,
      "step": 62
    },
    {
      "clip_ratio": 0.0006443567835958675,
      "epoch": 0.00860185690879301,
      "grad_norm": 0.025867076590657234,
      "kl": 0.0006416208398150047,
      "learning_rate": 4.9315789473684215e-06,
      "loss": -0.0002,
      "step": 63
    },
    {
      "clip_ratio": 0.0002910231560235843,
      "epoch": 0.008738394320043691,
      "grad_norm": 0.026004023849964142,
      "kl": 0.0006436795183617505,
      "learning_rate": 4.926315789473685e-06,
      "loss": -0.0,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.90625,
      "epoch": 0.008874931731294375,
      "grad_norm": 0.019275352358818054,
      "kl": 0.0006075235532989609,
      "learning_rate": 4.921052631578948e-06,
      "loss": 0.0001,
      "num_tokens": 199657.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 65
    },
    {
      "clip_ratio": 0.00015862943837419152,
      "epoch": 0.009011469142545057,
      "grad_norm": 0.01945505477488041,
      "kl": 0.000544819579772593,
      "learning_rate": 4.915789473684211e-06,
      "loss": 0.0,
      "step": 66
    },
    {
      "clip_ratio": 0.0003850787179544568,
      "epoch": 0.00914800655379574,
      "grad_norm": 0.019484233111143112,
      "kl": 0.0005976865359116346,
      "learning_rate": 4.910526315789474e-06,
      "loss": -0.0,
      "step": 67
    },
    {
      "clip_ratio": 0.00015862943837419152,
      "epoch": 0.009284543965046422,
      "grad_norm": 0.019339406862854958,
      "kl": 0.0005465979302243795,
      "learning_rate": 4.905263157894737e-06,
      "loss": 0.0001,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.375,
      "epoch": 0.009421081376297106,
      "grad_norm": 0.02073509618639946,
      "kl": 0.0005910295212743222,
      "learning_rate": 4.9000000000000005e-06,
      "loss": 0.0001,
      "num_tokens": 210781.0,
      "reward": 0.887499988079071,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.887499988079071,
      "step": 69
    },
    {
      "clip_ratio": 0.0002500000118743628,
      "epoch": 0.009557618787547788,
      "grad_norm": 0.02097729779779911,
      "kl": 0.0006015199551256956,
      "learning_rate": 4.894736842105264e-06,
      "loss": 0.0001,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.009694156198798471,
      "grad_norm": 0.020941907539963722,
      "kl": 0.000576648867536278,
      "learning_rate": 4.889473684210527e-06,
      "loss": 0.0,
      "step": 71
    },
    {
      "clip_ratio": 0.00012860081915277988,
      "epoch": 0.009830693610049153,
      "grad_norm": 0.02064090594649315,
      "kl": 0.0005430334399534331,
      "learning_rate": 4.88421052631579e-06,
      "loss": 0.0,
      "step": 72
    },
    {
      "clip_ratio": 0.0012847917387261987,
      "completion_length": 169.78125,
      "epoch": 0.009967231021299837,
      "grad_norm": 0.023972708731889725,
      "kl": 0.0006444637119784602,
      "learning_rate": 4.878947368421053e-06,
      "loss": -0.0,
      "num_tokens": 223878.0,
      "reward": 0.578125,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.578125,
      "step": 73
    },
    {
      "clip_ratio": 0.00015624999650754035,
      "epoch": 0.010103768432550519,
      "grad_norm": 0.023986345157027245,
      "kl": 0.000609115405495686,
      "learning_rate": 4.873684210526316e-06,
      "loss": 0.0,
      "step": 74
    },
    {
      "clip_ratio": 0.00039849805762059987,
      "epoch": 0.0102403058438012,
      "grad_norm": 0.02389119192957878,
      "kl": 0.0005973214888399525,
      "learning_rate": 4.8684210526315795e-06,
      "loss": -0.0,
      "step": 75
    },
    {
      "clip_ratio": 0.0008600136352470145,
      "epoch": 0.010376843255051884,
      "grad_norm": 0.024189762771129608,
      "kl": 0.0005895726594644657,
      "learning_rate": 4.863157894736843e-06,
      "loss": -0.0001,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 144.09375,
      "epoch": 0.010513380666302566,
      "grad_norm": 0.017288263887166977,
      "kl": 0.0006489278166554868,
      "learning_rate": 4.857894736842106e-06,
      "loss": -0.0,
      "num_tokens": 235793.0,
      "reward": 0.746874988079071,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.746874988079071,
      "step": 77
    },
    {
      "clip_ratio": 0.000240384615608491,
      "epoch": 0.01064991807755325,
      "grad_norm": 0.01712276227772236,
      "kl": 0.0005898008203075733,
      "learning_rate": 4.852631578947369e-06,
      "loss": 0.0,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.010786455488803932,
      "grad_norm": 0.01740402728319168,
      "kl": 0.0005718218494621397,
      "learning_rate": 4.847368421052631e-06,
      "loss": -0.0,
      "step": 79
    },
    {
      "clip_ratio": 0.00012450199574232101,
      "epoch": 0.010922992900054615,
      "grad_norm": 0.017095359042286873,
      "kl": 0.0006164813421491999,
      "learning_rate": 4.842105263157895e-06,
      "loss": -0.0001,
      "step": 80
    },
    {
      "clip_ratio": 0.00046910810488043353,
      "completion_length": 131.34375,
      "epoch": 0.011059530311305297,
      "grad_norm": 0.02193499356508255,
      "kl": 0.0006088345667194517,
      "learning_rate": 4.8368421052631585e-06,
      "loss": 0.0,
      "num_tokens": 246760.0,
      "reward": 0.7718750238418579,
      "reward_std": 0.18124999105930328,
      "rewards/reward_fn": 0.7718750238418579,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01119606772255598,
      "grad_norm": 0.021939560770988464,
      "kl": 0.0006533018759000697,
      "learning_rate": 4.831578947368422e-06,
      "loss": 0.0,
      "step": 82
    },
    {
      "clip_ratio": 0.000101461038866546,
      "epoch": 0.011332605133806663,
      "grad_norm": 0.02191118523478508,
      "kl": 0.0006190983283431706,
      "learning_rate": 4.826315789473685e-06,
      "loss": 0.0,
      "step": 83
    },
    {
      "clip_ratio": 0.00047129648010013625,
      "epoch": 0.011469142545057346,
      "grad_norm": 0.021944301202893257,
      "kl": 0.000652034742415708,
      "learning_rate": 4.821052631578948e-06,
      "loss": 0.0001,
      "step": 84
    },
    {
      "clip_ratio": 0.0007275540265254676,
      "completion_length": 154.1875,
      "epoch": 0.011605679956308028,
      "grad_norm": 0.0288760494440794,
      "kl": 0.0006393879930328694,
      "learning_rate": 4.815789473684211e-06,
      "loss": 0.0001,
      "num_tokens": 258610.0,
      "reward": 0.8031249642372131,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.8031249642372131,
      "step": 85
    },
    {
      "clip_ratio": 0.0006221036019269377,
      "epoch": 0.011742217367558712,
      "grad_norm": 0.028509872034192085,
      "kl": 0.0006500086219602963,
      "learning_rate": 4.8105263157894735e-06,
      "loss": -0.0,
      "step": 86
    },
    {
      "clip_ratio": 0.001054934982676059,
      "epoch": 0.011878754778809394,
      "grad_norm": 0.029009725898504257,
      "kl": 0.0006744494376107468,
      "learning_rate": 4.8052631578947375e-06,
      "loss": -0.0001,
      "step": 87
    },
    {
      "clip_ratio": 0.0009002409788081422,
      "epoch": 0.012015292190060076,
      "grad_norm": 0.028925592079758644,
      "kl": 0.0006717115811625263,
      "learning_rate": 4.800000000000001e-06,
      "loss": 0.0001,
      "step": 88
    },
    {
      "clip_ratio": 0.0005800188446301036,
      "completion_length": 136.34375,
      "epoch": 0.01215182960131076,
      "grad_norm": 0.02483142353594303,
      "kl": 0.0006553460525537957,
      "learning_rate": 4.794736842105264e-06,
      "loss": -0.0001,
      "num_tokens": 270849.0,
      "reward": 0.6031249761581421,
      "reward_std": 0.2962018847465515,
      "rewards/reward_fn": 0.6031249761581421,
      "step": 89
    },
    {
      "clip_ratio": 0.0015143853743211366,
      "epoch": 0.012288367012561441,
      "grad_norm": 0.024743258953094482,
      "kl": 0.0006329920825010049,
      "learning_rate": 4.789473684210527e-06,
      "loss": 0.0,
      "step": 90
    },
    {
      "clip_ratio": 0.0005078124959254637,
      "epoch": 0.012424904423812125,
      "grad_norm": 0.025256063789129257,
      "kl": 0.0006229320388229098,
      "learning_rate": 4.78421052631579e-06,
      "loss": -0.0001,
      "step": 91
    },
    {
      "clip_ratio": 0.0005842391110491008,
      "epoch": 0.012561441835062807,
      "grad_norm": 0.025601226836442947,
      "kl": 0.0006528175799758174,
      "learning_rate": 4.778947368421053e-06,
      "loss": -0.0,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.3125,
      "epoch": 0.01269797924631349,
      "grad_norm": 0.03193929046392441,
      "kl": 0.0006130583351477981,
      "learning_rate": 4.773684210526316e-06,
      "loss": -0.0001,
      "num_tokens": 282379.0,
      "reward": 0.550000011920929,
      "reward_std": 0.3723076283931732,
      "rewards/reward_fn": 0.550000011920929,
      "step": 93
    },
    {
      "clip_ratio": 0.00044216986862011254,
      "epoch": 0.012834516657564172,
      "grad_norm": 0.0314643457531929,
      "kl": 0.0006293810620263685,
      "learning_rate": 4.76842105263158e-06,
      "loss": -0.0005,
      "step": 94
    },
    {
      "clip_ratio": 0.0005735749291488901,
      "epoch": 0.012971054068814856,
      "grad_norm": 0.03177262842655182,
      "kl": 0.0005880916755813814,
      "learning_rate": 4.763157894736842e-06,
      "loss": -0.0003,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.013107591480065538,
      "grad_norm": 0.03146577253937721,
      "kl": 0.0006502543369606428,
      "learning_rate": 4.757894736842106e-06,
      "loss": -0.0004,
      "step": 96
    },
    {
      "clip_ratio": 0.00025201612152159214,
      "completion_length": 134.71875,
      "epoch": 0.013244128891316221,
      "grad_norm": 0.019158681854605675,
      "kl": 0.0005447731527965516,
      "learning_rate": 4.752631578947369e-06,
      "loss": -0.0001,
      "num_tokens": 293978.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 97
    },
    {
      "clip_ratio": 0.00025201612152159214,
      "epoch": 0.013380666302566903,
      "grad_norm": 0.020707514137029648,
      "kl": 0.0005420907405095932,
      "learning_rate": 4.747368421052632e-06,
      "loss": 0.0002,
      "step": 98
    },
    {
      "clip_ratio": 0.0004881850036326796,
      "epoch": 0.013517203713817587,
      "grad_norm": 0.02058817818760872,
      "kl": 0.0005488166934810579,
      "learning_rate": 4.7421052631578954e-06,
      "loss": 0.0,
      "step": 99
    },
    {
      "clip_ratio": 0.0006924755871295929,
      "epoch": 0.013653741125068269,
      "grad_norm": 0.020786749199032784,
      "kl": 0.0005361115108826198,
      "learning_rate": 4.736842105263158e-06,
      "loss": 0.0001,
      "step": 100
    },
    {
      "clip_ratio": 0.00035785317595582455,
      "completion_length": 149.21875,
      "epoch": 0.01379027853631895,
      "grad_norm": 0.017742682248353958,
      "kl": 0.0006423360018743551,
      "learning_rate": 4.731578947368422e-06,
      "loss": -0.0002,
      "num_tokens": 306241.0,
      "reward": 0.6875,
      "reward_std": 0.17499999701976776,
      "rewards/reward_fn": 0.6875,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.013926815947569634,
      "grad_norm": 0.018075380474328995,
      "kl": 0.000610419586791977,
      "learning_rate": 4.726315789473684e-06,
      "loss": -0.0,
      "step": 102
    },
    {
      "clip_ratio": 0.0002297794126207009,
      "epoch": 0.014063353358820316,
      "grad_norm": 0.017695877701044083,
      "kl": 0.0006176056695039733,
      "learning_rate": 4.721052631578948e-06,
      "loss": -0.0,
      "step": 103
    },
    {
      "clip_ratio": 0.00027076782134827226,
      "epoch": 0.014199890770071,
      "grad_norm": 0.01795818656682968,
      "kl": 0.0006254155405258643,
      "learning_rate": 4.71578947368421e-06,
      "loss": -0.0001,
      "step": 104
    },
    {
      "clip_ratio": 0.0004153236513957381,
      "completion_length": 128.25,
      "epoch": 0.014336428181321682,
      "grad_norm": 0.022969288751482964,
      "kl": 0.0007000857394814375,
      "learning_rate": 4.710526315789474e-06,
      "loss": 0.0002,
      "num_tokens": 317705.0,
      "reward": 0.859375,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.859375,
      "step": 105
    },
    {
      "clip_ratio": 0.0004153236513957381,
      "epoch": 0.014472965592572365,
      "grad_norm": 0.023185281082987785,
      "kl": 0.0006248567869988619,
      "learning_rate": 4.705263157894738e-06,
      "loss": -0.0,
      "step": 106
    },
    {
      "clip_ratio": 0.0012351995101198554,
      "epoch": 0.014609503003823047,
      "grad_norm": 0.021740928292274475,
      "kl": 0.0006979771355872799,
      "learning_rate": 4.7e-06,
      "loss": 0.0001,
      "step": 107
    },
    {
      "clip_ratio": 0.0011130489874631166,
      "epoch": 0.01474604041507373,
      "grad_norm": 0.02287100814282894,
      "kl": 0.000650095523269556,
      "learning_rate": 4.694736842105264e-06,
      "loss": 0.0001,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.3125,
      "epoch": 0.014882577826324413,
      "grad_norm": 0.01759045571088791,
      "kl": 0.0006179187330417335,
      "learning_rate": 4.689473684210526e-06,
      "loss": -0.0001,
      "num_tokens": 329695.0,
      "reward": 0.8031249642372131,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.8031249642372131,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.015019115237575096,
      "grad_norm": 0.018014615401625633,
      "kl": 0.0006711829173582373,
      "learning_rate": 4.68421052631579e-06,
      "loss": 0.0001,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.015155652648825778,
      "grad_norm": 0.018217777833342552,
      "kl": 0.0006618361967412056,
      "learning_rate": 4.6789473684210525e-06,
      "loss": 0.0002,
      "step": 111
    },
    {
      "clip_ratio": 0.0002281021879753098,
      "epoch": 0.015292190060076462,
      "grad_norm": 0.01793798618018627,
      "kl": 0.0006607132654607994,
      "learning_rate": 4.6736842105263166e-06,
      "loss": 0.0001,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.34375,
      "epoch": 0.015428727471327144,
      "grad_norm": 0.010602413676679134,
      "kl": 0.0006683868668915238,
      "learning_rate": 4.668421052631579e-06,
      "loss": -0.0001,
      "num_tokens": 341294.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 113
    },
    {
      "clip_ratio": 0.0003324467979837209,
      "epoch": 0.015565264882577826,
      "grad_norm": 0.010346218943595886,
      "kl": 0.000636374299574527,
      "learning_rate": 4.663157894736842e-06,
      "loss": 0.0,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.015701802293828507,
      "grad_norm": 0.010380822233855724,
      "kl": 0.0005902485654587508,
      "learning_rate": 4.657894736842106e-06,
      "loss": -0.0,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.015838339705079193,
      "grad_norm": 0.010410645976662636,
      "kl": 0.0006477099495896255,
      "learning_rate": 4.652631578947368e-06,
      "loss": -0.0,
      "step": 116
    },
    {
      "clip_ratio": 0.0007784575573168695,
      "completion_length": 121.3125,
      "epoch": 0.015974877116329875,
      "grad_norm": 0.025988347828388214,
      "kl": 0.0006193161070768838,
      "learning_rate": 4.647368421052632e-06,
      "loss": 0.0002,
      "num_tokens": 352216.0,
      "reward": 0.46562498807907104,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.46562498807907104,
      "step": 117
    },
    {
      "clip_ratio": 0.0006022804300300777,
      "epoch": 0.016111414527580557,
      "grad_norm": 0.0258104857057333,
      "kl": 0.0006151870020403294,
      "learning_rate": 4.642105263157895e-06,
      "loss": 0.0002,
      "step": 118
    },
    {
      "clip_ratio": 0.0004505813994910568,
      "epoch": 0.01624795193883124,
      "grad_norm": 0.02582407183945179,
      "kl": 0.0006998937201387889,
      "learning_rate": 4.636842105263159e-06,
      "loss": 0.0,
      "step": 119
    },
    {
      "clip_ratio": 0.0004073779273312539,
      "epoch": 0.016384489350081924,
      "grad_norm": 0.02589990384876728,
      "kl": 0.0005731130522690364,
      "learning_rate": 4.631578947368421e-06,
      "loss": 0.0002,
      "step": 120
    },
    {
      "clip_ratio": 0.00043370435014367104,
      "completion_length": 146.9375,
      "epoch": 0.016521026761332606,
      "grad_norm": 0.020061973482370377,
      "kl": 0.0006172579032863723,
      "learning_rate": 4.626315789473684e-06,
      "loss": 0.0002,
      "num_tokens": 363906.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 121
    },
    {
      "clip_ratio": 0.0007088417623890564,
      "epoch": 0.016657564172583288,
      "grad_norm": 0.019345788285136223,
      "kl": 0.0006093144372698589,
      "learning_rate": 4.621052631578948e-06,
      "loss": 0.0002,
      "step": 122
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "epoch": 0.01679410158383397,
      "grad_norm": 0.01971699669957161,
      "kl": 0.000635543535281613,
      "learning_rate": 4.6157894736842105e-06,
      "loss": 0.0002,
      "step": 123
    },
    {
      "clip_ratio": 0.0009507163631496951,
      "epoch": 0.016930638995084655,
      "grad_norm": 0.019793082028627396,
      "kl": 0.0005919806335441535,
      "learning_rate": 4.6105263157894745e-06,
      "loss": 0.0001,
      "step": 124
    },
    {
      "clip_ratio": 0.0004717841511592269,
      "completion_length": 128.40625,
      "epoch": 0.017067176406335337,
      "grad_norm": 0.027957087382674217,
      "kl": 0.0006280076445364102,
      "learning_rate": 4.605263157894737e-06,
      "loss": -0.0002,
      "num_tokens": 375419.0,
      "reward": 0.46562501788139343,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.46562501788139343,
      "step": 125
    },
    {
      "clip_ratio": 0.00044455942406784743,
      "epoch": 0.01720371381758602,
      "grad_norm": 0.026606649160385132,
      "kl": 0.0005630467830997077,
      "learning_rate": 4.600000000000001e-06,
      "loss": -0.0001,
      "step": 126
    },
    {
      "clip_ratio": 0.0004891337011940777,
      "epoch": 0.0173402512288367,
      "grad_norm": 0.027467843145132065,
      "kl": 0.0006166973080325988,
      "learning_rate": 4.594736842105263e-06,
      "loss": -0.0002,
      "step": 127
    },
    {
      "clip_ratio": 0.0004891337011940777,
      "epoch": 0.017476788640087382,
      "grad_norm": 0.027233824133872986,
      "kl": 0.0006092134076425282,
      "learning_rate": 4.589473684210526e-06,
      "loss": -0.0002,
      "step": 128
    },
    {
      "clip_ratio": 0.00046258504153229296,
      "completion_length": 129.625,
      "epoch": 0.017613326051338068,
      "grad_norm": 0.018424691632390022,
      "kl": 0.0006046048201824306,
      "learning_rate": 4.5842105263157895e-06,
      "loss": -0.0001,
      "num_tokens": 386775.0,
      "reward": 0.550000011920929,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.550000011920929,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.01774986346258875,
      "grad_norm": 0.0189372468739748,
      "kl": 0.0006345385913846258,
      "learning_rate": 4.578947368421053e-06,
      "loss": -0.0001,
      "step": 130
    },
    {
      "clip_ratio": 0.00021258502965793014,
      "epoch": 0.01788640087383943,
      "grad_norm": 0.018727825954556465,
      "kl": 0.0006059091974748299,
      "learning_rate": 4.573684210526317e-06,
      "loss": -0.0001,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.018022938285090113,
      "grad_norm": 0.01906794309616089,
      "kl": 0.0005894042797081056,
      "learning_rate": 4.568421052631579e-06,
      "loss": -0.0002,
      "step": 132
    },
    {
      "clip_ratio": 0.0006876413826830685,
      "completion_length": 124.3125,
      "epoch": 0.0181594756963408,
      "grad_norm": 0.023546243086457253,
      "kl": 0.0004997413302589848,
      "learning_rate": 4.563157894736843e-06,
      "loss": -0.0001,
      "num_tokens": 397609.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 133
    },
    {
      "clip_ratio": 0.0005090699560241774,
      "epoch": 0.01829601310759148,
      "grad_norm": 0.02335953712463379,
      "kl": 0.0005571195488300873,
      "learning_rate": 4.557894736842105e-06,
      "loss": -0.0001,
      "step": 134
    },
    {
      "clip_ratio": 0.0006317299848888069,
      "epoch": 0.018432550518842163,
      "grad_norm": 0.02356523647904396,
      "kl": 0.0005225548211456044,
      "learning_rate": 4.552631578947369e-06,
      "loss": -0.0001,
      "step": 135
    },
    {
      "clip_ratio": 0.0005343041266314685,
      "epoch": 0.018569087930092845,
      "grad_norm": 0.023870345205068588,
      "kl": 0.0005025722261962073,
      "learning_rate": 4.547368421052632e-06,
      "loss": -0.0001,
      "step": 136
    },
    {
      "clip_ratio": 0.0001502403902122751,
      "completion_length": 153.5,
      "epoch": 0.01870562534134353,
      "grad_norm": 0.012812867760658264,
      "kl": 0.0006005578643453191,
      "learning_rate": 4.542105263157895e-06,
      "loss": 0.0001,
      "num_tokens": 409909.0,
      "reward": 0.9437500238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.9437500238418579,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.018842162752594212,
      "grad_norm": 0.012940590269863605,
      "kl": 0.0005563261474890169,
      "learning_rate": 4.536842105263158e-06,
      "loss": 0.0001,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.018978700163844894,
      "grad_norm": 0.012597217224538326,
      "kl": 0.0006045326954335906,
      "learning_rate": 4.531578947368421e-06,
      "loss": 0.0,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.019115237575095576,
      "grad_norm": 0.012925639748573303,
      "kl": 0.0005933965840085875,
      "learning_rate": 4.526315789473685e-06,
      "loss": -0.0001,
      "step": 140
    },
    {
      "clip_ratio": 0.0009048875363077968,
      "completion_length": 150.78125,
      "epoch": 0.019251774986346257,
      "grad_norm": 0.02253352664411068,
      "kl": 0.0005975085450700135,
      "learning_rate": 4.5210526315789475e-06,
      "loss": -0.0001,
      "num_tokens": 422358.0,
      "reward": 0.578125,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.578125,
      "step": 141
    },
    {
      "clip_ratio": 0.00021114865376148373,
      "epoch": 0.019388312397596943,
      "grad_norm": 0.021753521636128426,
      "kl": 0.0006178569246912957,
      "learning_rate": 4.5157894736842115e-06,
      "loss": -0.0004,
      "step": 142
    },
    {
      "clip_ratio": 0.00021114865376148373,
      "epoch": 0.019524849808847625,
      "grad_norm": 0.022238247096538544,
      "kl": 0.0005942240950389532,
      "learning_rate": 4.510526315789474e-06,
      "loss": -0.0002,
      "step": 143
    },
    {
      "clip_ratio": 0.00021114865376148373,
      "epoch": 0.019661387220098307,
      "grad_norm": 0.022495048120617867,
      "kl": 0.000565216106679145,
      "learning_rate": 4.505263157894737e-06,
      "loss": -0.0001,
      "step": 144
    },
    {
      "clip_ratio": 0.0006272522732615471,
      "completion_length": 121.375,
      "epoch": 0.01979792463134899,
      "grad_norm": 0.025133801624178886,
      "kl": 0.0006039909007995448,
      "learning_rate": 4.5e-06,
      "loss": 0.0,
      "num_tokens": 432838.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 145
    },
    {
      "clip_ratio": 0.0011298577737761661,
      "epoch": 0.019934462042599674,
      "grad_norm": 0.025172198191285133,
      "kl": 0.0005761020265708794,
      "learning_rate": 4.494736842105263e-06,
      "loss": -0.0001,
      "step": 146
    },
    {
      "clip_ratio": 0.0006077199795981869,
      "epoch": 0.020070999453850356,
      "grad_norm": 0.02526123821735382,
      "kl": 0.0006754654223186662,
      "learning_rate": 4.489473684210527e-06,
      "loss": -0.0,
      "step": 147
    },
    {
      "clip_ratio": 0.0004832089616684243,
      "epoch": 0.020207536865101038,
      "grad_norm": 0.02527388371527195,
      "kl": 0.0007189459374785656,
      "learning_rate": 4.48421052631579e-06,
      "loss": 0.0001,
      "step": 148
    },
    {
      "clip_ratio": 0.00026939655072055757,
      "completion_length": 153.34375,
      "epoch": 0.02034407427635172,
      "grad_norm": 0.02411748096346855,
      "kl": 0.0006902036120663979,
      "learning_rate": 4.478947368421054e-06,
      "loss": -0.0001,
      "num_tokens": 444481.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 149
    },
    {
      "clip_ratio": 0.0006806772944401018,
      "epoch": 0.0204806116876024,
      "grad_norm": 0.024839146062731743,
      "kl": 0.0006397511897375807,
      "learning_rate": 4.473684210526316e-06,
      "loss": -0.0004,
      "step": 150
    },
    {
      "clip_ratio": 0.0005650091770803556,
      "epoch": 0.020617149098853087,
      "grad_norm": 0.02479824796319008,
      "kl": 0.0006595764225494349,
      "learning_rate": 4.468421052631579e-06,
      "loss": -0.0001,
      "step": 151
    },
    {
      "clip_ratio": 0.0005590896907960996,
      "epoch": 0.02075368651010377,
      "grad_norm": 0.024913184344768524,
      "kl": 0.0006492447819255176,
      "learning_rate": 4.463157894736842e-06,
      "loss": -0.0003,
      "step": 152
    },
    {
      "clip_ratio": 0.0008531439525540918,
      "completion_length": 133.96875,
      "epoch": 0.02089022392135445,
      "grad_norm": 0.016449004411697388,
      "kl": 0.000623302158601291,
      "learning_rate": 4.4578947368421054e-06,
      "loss": -0.0001,
      "num_tokens": 455948.0,
      "reward": 0.4375,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.4375,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.021026761332605132,
      "grad_norm": 0.016440996900200844,
      "kl": 0.0006272347632148012,
      "learning_rate": 4.452631578947369e-06,
      "loss": -0.0,
      "step": 154
    },
    {
      "clip_ratio": 0.00013185653369873762,
      "epoch": 0.021163298743855818,
      "grad_norm": 0.016483375802636147,
      "kl": 0.0005931556556788564,
      "learning_rate": 4.447368421052632e-06,
      "loss": -0.0002,
      "step": 155
    },
    {
      "clip_ratio": 0.00013185653369873762,
      "epoch": 0.0212998361551065,
      "grad_norm": 0.016074543818831444,
      "kl": 0.0005772067979705753,
      "learning_rate": 4.442105263157896e-06,
      "loss": -0.0002,
      "step": 156
    },
    {
      "clip_ratio": 0.001116631436161697,
      "completion_length": 131.5625,
      "epoch": 0.02143637356635718,
      "grad_norm": 0.026868479326367378,
      "kl": 0.000641528177766304,
      "learning_rate": 4.436842105263158e-06,
      "loss": 0.0002,
      "num_tokens": 466882.0,
      "reward": 0.6031249761581421,
      "reward_std": 0.24478399753570557,
      "rewards/reward_fn": 0.6031249761581421,
      "step": 157
    },
    {
      "clip_ratio": 0.0008329408010467887,
      "epoch": 0.021572910977607863,
      "grad_norm": 0.02681853622198105,
      "kl": 0.0005750876798629179,
      "learning_rate": 4.431578947368421e-06,
      "loss": 0.0,
      "step": 158
    },
    {
      "clip_ratio": 0.0020322165510151535,
      "epoch": 0.02170944838885855,
      "grad_norm": 0.026626134291291237,
      "kl": 0.000633980415386759,
      "learning_rate": 4.426315789473684e-06,
      "loss": 0.0002,
      "step": 159
    },
    {
      "clip_ratio": 0.0011721871269401163,
      "epoch": 0.02184598580010923,
      "grad_norm": 0.026342855766415596,
      "kl": 0.000581728978886531,
      "learning_rate": 4.4210526315789476e-06,
      "loss": -0.0002,
      "step": 160
    },
    {
      "clip_ratio": 0.00034722223062999547,
      "completion_length": 142.96875,
      "epoch": 0.021982523211359913,
      "grad_norm": 0.03223702311515808,
      "kl": 0.000677699447805935,
      "learning_rate": 4.415789473684211e-06,
      "loss": -0.0001,
      "num_tokens": 478801.0,
      "reward": 0.746874988079071,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.746874988079071,
      "step": 161
    },
    {
      "clip_ratio": 0.00045375632180366665,
      "epoch": 0.022119060622610594,
      "grad_norm": 0.03263309225440025,
      "kl": 0.0006566386318809236,
      "learning_rate": 4.410526315789474e-06,
      "loss": -0.0,
      "step": 162
    },
    {
      "clip_ratio": 0.0004954312171321362,
      "epoch": 0.022255598033861276,
      "grad_norm": 0.03234660625457764,
      "kl": 0.0006995576613917365,
      "learning_rate": 4.405263157894737e-06,
      "loss": -0.0001,
      "step": 163
    },
    {
      "clip_ratio": 0.0003730945463757962,
      "epoch": 0.02239213544511196,
      "grad_norm": 0.03226694092154503,
      "kl": 0.000669665740133496,
      "learning_rate": 4.4e-06,
      "loss": 0.0001,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.90625,
      "epoch": 0.022528672856362644,
      "grad_norm": 0.02280597761273384,
      "kl": 0.000539449552888982,
      "learning_rate": 4.394736842105263e-06,
      "loss": -0.0001,
      "num_tokens": 491206.0,
      "reward": 0.7156250476837158,
      "reward_std": 0.23124998807907104,
      "rewards/reward_fn": 0.7156250476837158,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.022665210267613325,
      "grad_norm": 0.022639747709035873,
      "kl": 0.0005247992294243886,
      "learning_rate": 4.3894736842105266e-06,
      "loss": -0.0001,
      "step": 166
    },
    {
      "clip_ratio": 0.0003004807804245502,
      "epoch": 0.022801747678864007,
      "grad_norm": 0.023010019212961197,
      "kl": 0.0005579576090894989,
      "learning_rate": 4.38421052631579e-06,
      "loss": 0.0,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.022938285090114693,
      "grad_norm": 0.023213736712932587,
      "kl": 0.0005310979404384852,
      "learning_rate": 4.378947368421053e-06,
      "loss": 0.0001,
      "step": 168
    },
    {
      "clip_ratio": 0.000481564158690162,
      "completion_length": 162.25,
      "epoch": 0.023074822501365375,
      "grad_norm": 0.026983601972460747,
      "kl": 0.0005834002631672774,
      "learning_rate": 4.373684210526316e-06,
      "loss": 0.0,
      "num_tokens": 503338.0,
      "reward": 0.71875,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.71875,
      "step": 169
    },
    {
      "clip_ratio": 0.00032799315522424877,
      "epoch": 0.023211359912616056,
      "grad_norm": 0.02684599533677101,
      "kl": 0.0005691815322279581,
      "learning_rate": 4.368421052631579e-06,
      "loss": 0.0,
      "step": 170
    },
    {
      "clip_ratio": 0.0010038797772722319,
      "epoch": 0.02334789732386674,
      "grad_norm": 0.027076352387666702,
      "kl": 0.0005902027933188947,
      "learning_rate": 4.363157894736842e-06,
      "loss": 0.0002,
      "step": 171
    },
    {
      "clip_ratio": 0.0005283136706566438,
      "epoch": 0.023484434735117424,
      "grad_norm": 0.02659190259873867,
      "kl": 0.0005958642700534256,
      "learning_rate": 4.3578947368421055e-06,
      "loss": 0.0001,
      "step": 172
    },
    {
      "clip_ratio": 0.000557463092263788,
      "completion_length": 137.28125,
      "epoch": 0.023620972146368106,
      "grad_norm": 0.019615458324551582,
      "kl": 0.0006092825424275361,
      "learning_rate": 4.352631578947369e-06,
      "loss": 0.0002,
      "num_tokens": 514483.0,
      "reward": 0.9156249761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.9156249761581421,
      "step": 173
    },
    {
      "clip_ratio": 0.0007839123718440533,
      "epoch": 0.023757509557618788,
      "grad_norm": 0.019610155373811722,
      "kl": 0.0006007257366036356,
      "learning_rate": 4.347368421052632e-06,
      "loss": 0.0002,
      "step": 174
    },
    {
      "clip_ratio": 0.0003033980610780418,
      "epoch": 0.02389404696886947,
      "grad_norm": 0.01908615045249462,
      "kl": 0.0006212977809809672,
      "learning_rate": 4.342105263157895e-06,
      "loss": 0.0002,
      "step": 175
    },
    {
      "clip_ratio": 0.001106989337131381,
      "epoch": 0.02403058438012015,
      "grad_norm": 0.01965913362801075,
      "kl": 0.0006572387419510051,
      "learning_rate": 4.336842105263158e-06,
      "loss": 0.0001,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.5,
      "epoch": 0.024167121791370837,
      "grad_norm": 0.02616649493575096,
      "kl": 0.0006305564247668372,
      "learning_rate": 4.331578947368421e-06,
      "loss": -0.0001,
      "num_tokens": 526339.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 177
    },
    {
      "clip_ratio": 0.0005066010635346174,
      "epoch": 0.02430365920262152,
      "grad_norm": 0.02656235173344612,
      "kl": 0.000589995510836161,
      "learning_rate": 4.3263157894736845e-06,
      "loss": 0.0,
      "step": 178
    },
    {
      "clip_ratio": 0.0006540067261084914,
      "epoch": 0.0244401966138722,
      "grad_norm": 0.025882191956043243,
      "kl": 0.0006225678325790795,
      "learning_rate": 4.321052631578948e-06,
      "loss": -0.0001,
      "step": 179
    },
    {
      "clip_ratio": 0.000147405662573874,
      "epoch": 0.024576734025122882,
      "grad_norm": 0.026141585782170296,
      "kl": 0.0007103291836756398,
      "learning_rate": 4.315789473684211e-06,
      "loss": -0.0002,
      "step": 180
    },
    {
      "clip_ratio": 0.0008024979470064864,
      "completion_length": 136.84375,
      "epoch": 0.024713271436373568,
      "grad_norm": 0.02634594403207302,
      "kl": 0.0006556506277775043,
      "learning_rate": 4.310526315789474e-06,
      "loss": -0.0001,
      "num_tokens": 538294.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 181
    },
    {
      "clip_ratio": 0.0003042656317120418,
      "epoch": 0.02484980884762425,
      "grad_norm": 0.02641279436647892,
      "kl": 0.0005787361596958362,
      "learning_rate": 4.305263157894737e-06,
      "loss": -0.0001,
      "step": 182
    },
    {
      "clip_ratio": 0.0002822835522238165,
      "epoch": 0.02498634625887493,
      "grad_norm": 0.026922186836600304,
      "kl": 0.0006166264679450251,
      "learning_rate": 4.3e-06,
      "loss": -0.0002,
      "step": 183
    },
    {
      "clip_ratio": 0.00028801843291148543,
      "epoch": 0.025122883670125613,
      "grad_norm": 0.02697991020977497,
      "kl": 0.0006883774053676461,
      "learning_rate": 4.2947368421052635e-06,
      "loss": -0.0002,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.96875,
      "epoch": 0.0252594210813763,
      "grad_norm": 0.014164031483232975,
      "kl": 0.0006083018552089925,
      "learning_rate": 4.289473684210527e-06,
      "loss": 0.0,
      "num_tokens": 550441.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02539595849262698,
      "grad_norm": 0.014139293693006039,
      "kl": 0.0006685331882181345,
      "learning_rate": 4.28421052631579e-06,
      "loss": 0.0,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.025532495903877662,
      "grad_norm": 0.014367860741913319,
      "kl": 0.0006282098470364872,
      "learning_rate": 4.278947368421053e-06,
      "loss": 0.0001,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.025669033315128344,
      "grad_norm": 0.014389986172318459,
      "kl": 0.0006063026085030288,
      "learning_rate": 4.273684210526316e-06,
      "loss": 0.0,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.75,
      "epoch": 0.025805570726379026,
      "grad_norm": 0.016773829236626625,
      "kl": 0.0006331677345770004,
      "learning_rate": 4.268421052631579e-06,
      "loss": -0.0001,
      "num_tokens": 560797.0,
      "reward": 0.831250011920929,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.831250011920929,
      "step": 189
    },
    {
      "clip_ratio": 0.00026483050896786153,
      "epoch": 0.02594210813762971,
      "grad_norm": 0.016798226162791252,
      "kl": 0.000630740689302911,
      "learning_rate": 4.2631578947368425e-06,
      "loss": 0.0001,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.026078645548880394,
      "grad_norm": 0.01683247834444046,
      "kl": 0.0007194746331151691,
      "learning_rate": 4.257894736842106e-06,
      "loss": 0.0,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.026215182960131075,
      "grad_norm": 0.017007527872920036,
      "kl": 0.000644579552954383,
      "learning_rate": 4.252631578947369e-06,
      "loss": 0.0001,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.1875,
      "epoch": 0.026351720371381757,
      "grad_norm": 0.02215302176773548,
      "kl": 0.000574918890833942,
      "learning_rate": 4.247368421052632e-06,
      "loss": -0.0002,
      "num_tokens": 571887.0,
      "reward": 0.71875,
      "reward_std": 0.1948557198047638,
      "rewards/reward_fn": 0.71875,
      "step": 193
    },
    {
      "clip_ratio": 0.00019654087373055518,
      "epoch": 0.026488257782632443,
      "grad_norm": 0.021721050143241882,
      "kl": 0.0006944113565623411,
      "learning_rate": 4.242105263157895e-06,
      "loss": -0.0002,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.026624795193883125,
      "grad_norm": 0.02209274098277092,
      "kl": 0.0006284140536081395,
      "learning_rate": 4.236842105263158e-06,
      "loss": -0.0002,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.026761332605133806,
      "grad_norm": 0.022048382088541985,
      "kl": 0.000593657642184553,
      "learning_rate": 4.2315789473684215e-06,
      "loss": -0.0002,
      "step": 196
    },
    {
      "clip_ratio": 0.0005545529711525887,
      "completion_length": 124.375,
      "epoch": 0.02689787001638449,
      "grad_norm": 0.03627476841211319,
      "kl": 0.0006863787939437316,
      "learning_rate": 4.226315789473685e-06,
      "loss": 0.0003,
      "num_tokens": 582723.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.606249988079071,
      "step": 197
    },
    {
      "clip_ratio": 0.0011091082269558683,
      "epoch": 0.027034407427635174,
      "grad_norm": 0.03692282736301422,
      "kl": 0.0007320116324081027,
      "learning_rate": 4.221052631578948e-06,
      "loss": 0.0001,
      "step": 198
    },
    {
      "clip_ratio": 0.0007693048100918531,
      "epoch": 0.027170944838885856,
      "grad_norm": 0.03559679165482521,
      "kl": 0.0007174053198468755,
      "learning_rate": 4.215789473684211e-06,
      "loss": 0.0003,
      "step": 199
    },
    {
      "clip_ratio": 0.0009161705384030938,
      "epoch": 0.027307482250136537,
      "grad_norm": 0.036062467843294144,
      "kl": 0.0007688817172493145,
      "learning_rate": 4.210526315789474e-06,
      "loss": 0.0002,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.4375,
      "epoch": 0.02744401966138722,
      "grad_norm": 0.01810489036142826,
      "kl": 0.0006160379116408876,
      "learning_rate": 4.205263157894737e-06,
      "loss": -0.0001,
      "num_tokens": 594265.0,
      "reward": 0.606249988079071,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.606249988079071,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0275805570726379,
      "grad_norm": 0.017982803285121918,
      "kl": 0.0005821473964715551,
      "learning_rate": 4.2000000000000004e-06,
      "loss": -0.0001,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.027717094483888587,
      "grad_norm": 0.018066201359033585,
      "kl": 0.0005501055475178873,
      "learning_rate": 4.194736842105264e-06,
      "loss": -0.0001,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02785363189513927,
      "grad_norm": 0.018188124522566795,
      "kl": 0.0005876172319858597,
      "learning_rate": 4.189473684210527e-06,
      "loss": -0.0001,
      "step": 204
    },
    {
      "clip_ratio": 0.0003033980610780418,
      "completion_length": 129.4375,
      "epoch": 0.02799016930638995,
      "grad_norm": 0.01925286278128624,
      "kl": 0.0006736774898854492,
      "learning_rate": 4.18421052631579e-06,
      "loss": 0.0002,
      "num_tokens": 605391.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 205
    },
    {
      "clip_ratio": 0.0003033980610780418,
      "epoch": 0.028126706717640632,
      "grad_norm": 0.019308850169181824,
      "kl": 0.000748576178921212,
      "learning_rate": 4.178947368421053e-06,
      "loss": 0.0002,
      "step": 206
    },
    {
      "clip_ratio": 0.0004951158480253071,
      "epoch": 0.028263244128891318,
      "grad_norm": 0.01930277608335018,
      "kl": 0.0007205891324701952,
      "learning_rate": 4.173684210526316e-06,
      "loss": 0.0002,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.028399781540142,
      "grad_norm": 0.018969830125570297,
      "kl": 0.0007646826934433193,
      "learning_rate": 4.1684210526315794e-06,
      "loss": 0.0,
      "step": 208
    },
    {
      "clip_ratio": 0.0002367424312978983,
      "completion_length": 142.59375,
      "epoch": 0.02853631895139268,
      "grad_norm": 0.018274182453751564,
      "kl": 0.0006968205607336131,
      "learning_rate": 4.163157894736843e-06,
      "loss": 0.0002,
      "num_tokens": 617790.0,
      "reward": 0.578125,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.578125,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.028672856362643363,
      "grad_norm": 0.018192008137702942,
      "kl": 0.0006250791311686044,
      "learning_rate": 4.157894736842106e-06,
      "loss": 0.0001,
      "step": 210
    },
    {
      "clip_ratio": 0.0002604166802484542,
      "epoch": 0.02880939377389405,
      "grad_norm": 0.017575176432728767,
      "kl": 0.0006424866801353346,
      "learning_rate": 4.152631578947369e-06,
      "loss": 0.0001,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.02894593118514473,
      "grad_norm": 0.01796414516866207,
      "kl": 0.0006217293284862535,
      "learning_rate": 4.147368421052632e-06,
      "loss": 0.0,
      "step": 212
    },
    {
      "clip_ratio": 0.0001653439103392884,
      "completion_length": 144.0625,
      "epoch": 0.029082468596395412,
      "grad_norm": 0.01483822800219059,
      "kl": 0.0006835903727733239,
      "learning_rate": 4.142105263157895e-06,
      "loss": -0.0001,
      "num_tokens": 629920.0,
      "reward": 0.690625011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.690625011920929,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.029219006007646094,
      "grad_norm": 0.014619370922446251,
      "kl": 0.0006971015118324431,
      "learning_rate": 4.136842105263158e-06,
      "loss": -0.0001,
      "step": 214
    },
    {
      "clip_ratio": 0.0001653439103392884,
      "epoch": 0.029355543418896776,
      "grad_norm": 0.014896603301167488,
      "kl": 0.0007456434082087071,
      "learning_rate": 4.1315789473684216e-06,
      "loss": 0.0,
      "step": 215
    },
    {
      "clip_ratio": 0.0001653439103392884,
      "epoch": 0.02949208083014746,
      "grad_norm": 0.015029766596853733,
      "kl": 0.0007792614933350706,
      "learning_rate": 4.126315789473685e-06,
      "loss": -0.0,
      "step": 216
    },
    {
      "clip_ratio": 0.0007776991260470822,
      "completion_length": 149.4375,
      "epoch": 0.029628618241398143,
      "grad_norm": 0.024460721760988235,
      "kl": 0.0006598385712095478,
      "learning_rate": 4.121052631578948e-06,
      "loss": -0.0,
      "num_tokens": 642062.0,
      "reward": 0.6031250357627869,
      "reward_std": 0.23124998807907104,
      "rewards/reward_fn": 0.6031250357627869,
      "step": 217
    },
    {
      "clip_ratio": 0.0002281021879753098,
      "epoch": 0.029765155652648825,
      "grad_norm": 0.024433676153421402,
      "kl": 0.0006593063117179554,
      "learning_rate": 4.115789473684211e-06,
      "loss": -0.0,
      "step": 218
    },
    {
      "clip_ratio": 0.0013088609266560525,
      "epoch": 0.029901693063899507,
      "grad_norm": 0.024088183417916298,
      "kl": 0.0007106323237167089,
      "learning_rate": 4.110526315789474e-06,
      "loss": -0.0001,
      "step": 219
    },
    {
      "clip_ratio": 0.0007659851835342124,
      "epoch": 0.030038230475150193,
      "grad_norm": 0.02432289533317089,
      "kl": 0.0006990563970248331,
      "learning_rate": 4.105263157894737e-06,
      "loss": -0.0001,
      "step": 220
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "completion_length": 138.3125,
      "epoch": 0.030174767886400874,
      "grad_norm": 0.018672458827495575,
      "kl": 0.0006957703599255183,
      "learning_rate": 4.1e-06,
      "loss": 0.0,
      "num_tokens": 653788.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 221
    },
    {
      "clip_ratio": 0.000294811325147748,
      "epoch": 0.030311305297651556,
      "grad_norm": 0.018861589953303337,
      "kl": 0.0006577782787644537,
      "learning_rate": 4.094736842105264e-06,
      "loss": -0.0,
      "step": 222
    },
    {
      "clip_ratio": 0.0005017649673391134,
      "epoch": 0.030447842708902238,
      "grad_norm": 0.019544165581464767,
      "kl": 0.0006526250754177454,
      "learning_rate": 4.089473684210527e-06,
      "loss": -0.0001,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.030584380120152924,
      "grad_norm": 0.018741516396403313,
      "kl": 0.0006388758838511421,
      "learning_rate": 4.08421052631579e-06,
      "loss": -0.0,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.75,
      "epoch": 0.030720917531403606,
      "grad_norm": 0.01722196862101555,
      "kl": 0.0007343545703406562,
      "learning_rate": 4.078947368421053e-06,
      "loss": 0.0,
      "num_tokens": 665160.0,
      "reward": 0.7750000357627869,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7750000357627869,
      "step": 225
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "epoch": 0.030857454942654287,
      "grad_norm": 0.017116790637373924,
      "kl": 0.0007918194187368499,
      "learning_rate": 4.073684210526316e-06,
      "loss": -0.0,
      "step": 226
    },
    {
      "clip_ratio": 0.0005372232117224485,
      "epoch": 0.03099399235390497,
      "grad_norm": 0.017037423327565193,
      "kl": 0.0007311694735108176,
      "learning_rate": 4.0684210526315795e-06,
      "loss": -0.0,
      "step": 227
    },
    {
      "clip_ratio": 0.0006525368662551045,
      "epoch": 0.03113052976515565,
      "grad_norm": 0.01683720573782921,
      "kl": 0.0007448554715665523,
      "learning_rate": 4.063157894736842e-06,
      "loss": -0.0,
      "step": 228
    },
    {
      "clip_ratio": 0.000498970810440369,
      "completion_length": 142.75,
      "epoch": 0.03126706717640634,
      "grad_norm": 0.02255455031991005,
      "kl": 0.0006459420278588368,
      "learning_rate": 4.057894736842106e-06,
      "loss": 0.0001,
      "num_tokens": 677400.0,
      "reward": 0.46562501788139343,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.46562501788139343,
      "step": 229
    },
    {
      "clip_ratio": 0.0006992945709498599,
      "epoch": 0.031403604587657015,
      "grad_norm": 0.02236250601708889,
      "kl": 0.0006349996087919862,
      "learning_rate": 4.052631578947368e-06,
      "loss": 0.0,
      "step": 230
    },
    {
      "clip_ratio": 0.0001307531347265467,
      "epoch": 0.0315401419989077,
      "grad_norm": 0.02220042608678341,
      "kl": 0.0006470854705185047,
      "learning_rate": 4.047368421052632e-06,
      "loss": 0.0,
      "step": 231
    },
    {
      "clip_ratio": 0.0008540844573872164,
      "epoch": 0.031676679410158386,
      "grad_norm": 0.02246800623834133,
      "kl": 0.0006389342952388688,
      "learning_rate": 4.042105263157895e-06,
      "loss": 0.0001,
      "step": 232
    },
    {
      "clip_ratio": 0.00048048709868453443,
      "completion_length": 149.6875,
      "epoch": 0.031813216821409064,
      "grad_norm": 0.02225206233561039,
      "kl": 0.0006582574633284821,
      "learning_rate": 4.0368421052631585e-06,
      "loss": -0.0,
      "num_tokens": 689414.0,
      "reward": 0.690625011920929,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.690625011920929,
      "step": 233
    },
    {
      "clip_ratio": 0.0016851396212587133,
      "epoch": 0.03194975423265975,
      "grad_norm": 0.022733833640813828,
      "kl": 0.0006728457501594676,
      "learning_rate": 4.031578947368422e-06,
      "loss": 0.0001,
      "step": 234
    },
    {
      "clip_ratio": 0.000523468479514122,
      "epoch": 0.032086291643910435,
      "grad_norm": 0.022337261587381363,
      "kl": 0.0006649887154708267,
      "learning_rate": 4.026315789473684e-06,
      "loss": 0.0,
      "step": 235
    },
    {
      "clip_ratio": 0.00039966811891645193,
      "epoch": 0.03222282905516111,
      "grad_norm": 0.02236855961382389,
      "kl": 0.0006946580124349566,
      "learning_rate": 4.021052631578948e-06,
      "loss": 0.0001,
      "step": 236
    },
    {
      "clip_ratio": 0.0007370143721345812,
      "completion_length": 139.5,
      "epoch": 0.0323593664664118,
      "grad_norm": 0.022790394723415375,
      "kl": 0.0007473547284462256,
      "learning_rate": 4.01578947368421e-06,
      "loss": 0.0002,
      "num_tokens": 701014.0,
      "reward": 0.4937499761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.4937499761581421,
      "step": 237
    },
    {
      "clip_ratio": 0.0005428440490504727,
      "epoch": 0.03249590387766248,
      "grad_norm": 0.022693805396556854,
      "kl": 0.0006957066789254895,
      "learning_rate": 4.010526315789474e-06,
      "loss": 0.0001,
      "step": 238
    },
    {
      "clip_ratio": 0.0003408734919503331,
      "epoch": 0.03263244128891316,
      "grad_norm": 0.02278122492134571,
      "kl": 0.0007415731188302743,
      "learning_rate": 4.005263157894737e-06,
      "loss": 0.0002,
      "step": 239
    },
    {
      "clip_ratio": 0.00013528138515539467,
      "epoch": 0.03276897870016385,
      "grad_norm": 0.02245188318192959,
      "kl": 0.0007022500935818243,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0001,
      "step": 240
    },
    {
      "clip_ratio": 0.0005109125340823084,
      "completion_length": 141.28125,
      "epoch": 0.032905516111414526,
      "grad_norm": 0.017593976110219955,
      "kl": 0.000685278452692728,
      "learning_rate": 3.994736842105264e-06,
      "loss": 0.0001,
      "num_tokens": 712323.0,
      "reward": 0.550000011920929,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.550000011920929,
      "step": 241
    },
    {
      "clip_ratio": 0.00013130252773407847,
      "epoch": 0.03304205352266521,
      "grad_norm": 0.016243450343608856,
      "kl": 0.0007128663355615572,
      "learning_rate": 3.989473684210526e-06,
      "loss": 0.0001,
      "step": 242
    },
    {
      "clip_ratio": 0.0006084017659304664,
      "epoch": 0.03317859093391589,
      "grad_norm": 0.017042577266693115,
      "kl": 0.0007111189861461753,
      "learning_rate": 3.98421052631579e-06,
      "loss": 0.0,
      "step": 243
    },
    {
      "clip_ratio": 0.0006747301958967,
      "epoch": 0.033315128345166575,
      "grad_norm": 0.018135398626327515,
      "kl": 0.000669868842123833,
      "learning_rate": 3.9789473684210525e-06,
      "loss": 0.0,
      "step": 244
    },
    {
      "clip_ratio": 0.00022482014901470393,
      "completion_length": 156.09375,
      "epoch": 0.03345166575641726,
      "grad_norm": 0.018865320831537247,
      "kl": 0.0007178983569247066,
      "learning_rate": 3.9736842105263165e-06,
      "loss": -0.0,
      "num_tokens": 724074.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.03358820316766794,
      "grad_norm": 0.01888267695903778,
      "kl": 0.000737229005608242,
      "learning_rate": 3.968421052631579e-06,
      "loss": 0.0001,
      "step": 246
    },
    {
      "clip_ratio": 0.00029761905898340046,
      "epoch": 0.033724740578918624,
      "grad_norm": 0.018460268154740334,
      "kl": 0.0007195647185653797,
      "learning_rate": 3.963157894736843e-06,
      "loss": 0.0001,
      "step": 247
    },
    {
      "clip_ratio": 0.00040545598312746733,
      "epoch": 0.03386127799016931,
      "grad_norm": 0.01854436658322811,
      "kl": 0.000718526627679239,
      "learning_rate": 3.957894736842106e-06,
      "loss": -0.0,
      "step": 248
    },
    {
      "clip_ratio": 0.000364551626262255,
      "completion_length": 123.5625,
      "epoch": 0.03399781540141999,
      "grad_norm": 0.016607927158474922,
      "kl": 0.0006165570266603027,
      "learning_rate": 3.952631578947368e-06,
      "loss": 0.0,
      "num_tokens": 736048.0,
      "reward": 0.7750000357627869,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7750000357627869,
      "step": 249
    },
    {
      "clip_ratio": 0.0003149120893795043,
      "epoch": 0.034134352812670674,
      "grad_norm": 0.01636321283876896,
      "kl": 0.0006596833213734499,
      "learning_rate": 3.947368421052632e-06,
      "loss": -0.0,
      "step": 250
    },
    {
      "clip_ratio": 0.00033602150506339967,
      "epoch": 0.03427089022392135,
      "grad_norm": 0.016457047313451767,
      "kl": 0.000654415664030239,
      "learning_rate": 3.942105263157895e-06,
      "loss": 0.0001,
      "step": 251
    },
    {
      "clip_ratio": 0.00019654087373055518,
      "epoch": 0.03440742763517204,
      "grad_norm": 0.01638452522456646,
      "kl": 0.0006012939916217874,
      "learning_rate": 3.936842105263159e-06,
      "loss": 0.0001,
      "step": 252
    },
    {
      "clip_ratio": 0.0009389391634613276,
      "completion_length": 161.71875,
      "epoch": 0.03454396504642272,
      "grad_norm": 0.03075333870947361,
      "kl": 0.0006442447129302309,
      "learning_rate": 3.931578947368421e-06,
      "loss": 0.0002,
      "num_tokens": 748595.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 253
    },
    {
      "clip_ratio": 0.0006016489787725732,
      "epoch": 0.0346805024576734,
      "grad_norm": 0.030499013140797615,
      "kl": 0.0007103143916538102,
      "learning_rate": 3.926315789473685e-06,
      "loss": 0.0001,
      "step": 254
    },
    {
      "clip_ratio": 0.0010966659901896492,
      "epoch": 0.034817039868924086,
      "grad_norm": 0.030418358743190765,
      "kl": 0.0007258211571752327,
      "learning_rate": 3.921052631578947e-06,
      "loss": 0.0,
      "step": 255
    },
    {
      "clip_ratio": 0.0003524696658132598,
      "epoch": 0.034953577280174765,
      "grad_norm": 0.030616672709584236,
      "kl": 0.0007367220950982301,
      "learning_rate": 3.9157894736842104e-06,
      "loss": 0.0001,
      "step": 256
    },
    {
      "clip_ratio": 0.0010683327927836217,
      "completion_length": 153.0625,
      "epoch": 0.03509011469142545,
      "grad_norm": 0.02800179086625576,
      "kl": 0.0007471022909157909,
      "learning_rate": 3.9105263157894744e-06,
      "loss": 0.0002,
      "num_tokens": 760769.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 257
    },
    {
      "clip_ratio": 0.0007546676933998242,
      "epoch": 0.035226652102676136,
      "grad_norm": 0.028518524020910263,
      "kl": 0.000712448805643362,
      "learning_rate": 3.905263157894737e-06,
      "loss": 0.0002,
      "step": 258
    },
    {
      "clip_ratio": 0.000800817426352296,
      "epoch": 0.035363189513926814,
      "grad_norm": 0.02910081297159195,
      "kl": 0.0006784590418646985,
      "learning_rate": 3.900000000000001e-06,
      "loss": 0.0001,
      "step": 259
    },
    {
      "clip_ratio": 0.0006838897534180433,
      "epoch": 0.0354997269251775,
      "grad_norm": 0.027883443981409073,
      "kl": 0.0007144165310819517,
      "learning_rate": 3.894736842105263e-06,
      "loss": -0.0001,
      "step": 260
    },
    {
      "clip_ratio": 0.0007415218569803983,
      "completion_length": 139.59375,
      "epoch": 0.035636264336428185,
      "grad_norm": 0.025706559419631958,
      "kl": 0.0006818120314164844,
      "learning_rate": 3.889473684210527e-06,
      "loss": 0.0001,
      "num_tokens": 772592.0,
      "reward": 0.828125,
      "reward_std": 0.18370190262794495,
      "rewards/reward_fn": 0.828125,
      "step": 261
    },
    {
      "clip_ratio": 0.0008036253275349736,
      "epoch": 0.03577280174767886,
      "grad_norm": 0.02648027054965496,
      "kl": 0.000685886194332852,
      "learning_rate": 3.884210526315789e-06,
      "loss": 0.0,
      "step": 262
    },
    {
      "clip_ratio": 0.0006222750525921583,
      "epoch": 0.03590933915892955,
      "grad_norm": 0.025682132691144943,
      "kl": 0.0006569743923137139,
      "learning_rate": 3.878947368421053e-06,
      "loss": -0.0,
      "step": 263
    },
    {
      "clip_ratio": 0.0006222750525921583,
      "epoch": 0.03604587657018023,
      "grad_norm": 0.026056313887238503,
      "kl": 0.0006821834294896689,
      "learning_rate": 3.873684210526316e-06,
      "loss": 0.0,
      "step": 264
    },
    {
      "clip_ratio": 0.0006582448550034314,
      "completion_length": 155.9375,
      "epoch": 0.03618241398143091,
      "grad_norm": 0.030329573899507523,
      "kl": 0.0007156636456784327,
      "learning_rate": 3.868421052631579e-06,
      "loss": 0.0002,
      "num_tokens": 784742.0,
      "reward": 0.690625011920929,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.690625011920929,
      "step": 265
    },
    {
      "clip_ratio": 0.0011503307177918032,
      "epoch": 0.0363189513926816,
      "grad_norm": 0.031328026205301285,
      "kl": 0.0007437661415679031,
      "learning_rate": 3.863157894736843e-06,
      "loss": 0.0003,
      "step": 266
    },
    {
      "clip_ratio": 0.001414415441104211,
      "epoch": 0.036455488803932276,
      "grad_norm": 0.03110205940902233,
      "kl": 0.0007249777718243422,
      "learning_rate": 3.857894736842105e-06,
      "loss": 0.0002,
      "step": 267
    },
    {
      "clip_ratio": 0.0005313157889759168,
      "epoch": 0.03659202621518296,
      "grad_norm": 0.031048839911818504,
      "kl": 0.0006714642913721036,
      "learning_rate": 3.852631578947369e-06,
      "loss": 0.0002,
      "step": 268
    },
    {
      "clip_ratio": 0.0004928899579681456,
      "completion_length": 141.84375,
      "epoch": 0.03672856362643364,
      "grad_norm": 0.032659851014614105,
      "kl": 0.0007797479997861956,
      "learning_rate": 3.8473684210526316e-06,
      "loss": 0.0001,
      "num_tokens": 796197.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.25980761647224426,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 269
    },
    {
      "clip_ratio": 0.00028935185400769114,
      "epoch": 0.036865101037684325,
      "grad_norm": 0.03203575685620308,
      "kl": 0.0007469654974556761,
      "learning_rate": 3.842105263157895e-06,
      "loss": -0.0,
      "step": 270
    },
    {
      "clip_ratio": 0.0006925335328560323,
      "epoch": 0.03700163844893501,
      "grad_norm": 0.03224669024348259,
      "kl": 0.0008460631906928029,
      "learning_rate": 3.836842105263158e-06,
      "loss": 0.0,
      "step": 271
    },
    {
      "clip_ratio": 0.000613619398791343,
      "epoch": 0.03713817586018569,
      "grad_norm": 0.03178337216377258,
      "kl": 0.0008344126890733605,
      "learning_rate": 3.831578947368421e-06,
      "loss": 0.0,
      "step": 272
    },
    {
      "clip_ratio": 0.0007658389222342521,
      "completion_length": 148.75,
      "epoch": 0.037274713271436374,
      "grad_norm": 0.02685234323143959,
      "kl": 0.0008694035859662108,
      "learning_rate": 3.826315789473685e-06,
      "loss": -0.0001,
      "num_tokens": 808537.0,
      "reward": 0.859375,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.859375,
      "step": 273
    },
    {
      "clip_ratio": 0.00042204382771160454,
      "epoch": 0.03741125068268706,
      "grad_norm": 0.02722744457423687,
      "kl": 0.0008062587003223598,
      "learning_rate": 3.821052631578947e-06,
      "loss": -0.0001,
      "step": 274
    },
    {
      "clip_ratio": 0.0006064001499908045,
      "epoch": 0.03754778809393774,
      "grad_norm": 0.02703207917511463,
      "kl": 0.0008390755356231239,
      "learning_rate": 3.815789473684211e-06,
      "loss": -0.0002,
      "step": 275
    },
    {
      "clip_ratio": 0.0004554339684545994,
      "epoch": 0.037684325505188423,
      "grad_norm": 0.027210967615246773,
      "kl": 0.0007293312564797816,
      "learning_rate": 3.810526315789474e-06,
      "loss": -0.0001,
      "step": 276
    },
    {
      "clip_ratio": 0.001045014927512966,
      "completion_length": 137.09375,
      "epoch": 0.0378208629164391,
      "grad_norm": 0.03316161409020424,
      "kl": 0.0008686158525961218,
      "learning_rate": 3.805263157894737e-06,
      "loss": 0.0001,
      "num_tokens": 820732.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 277
    },
    {
      "clip_ratio": 0.0005361070216167718,
      "epoch": 0.03795740032768979,
      "grad_norm": 0.034403275698423386,
      "kl": 0.0009175811010209145,
      "learning_rate": 3.8000000000000005e-06,
      "loss": 0.0003,
      "step": 278
    },
    {
      "clip_ratio": 0.0006925088528078049,
      "epoch": 0.03809393773894047,
      "grad_norm": 0.0344231016933918,
      "kl": 0.0007938581566122593,
      "learning_rate": 3.794736842105263e-06,
      "loss": -0.0,
      "step": 279
    },
    {
      "clip_ratio": 0.0013625302235595882,
      "epoch": 0.03823047515019115,
      "grad_norm": 0.03481278568506241,
      "kl": 0.0009434933390366496,
      "learning_rate": 3.789473684210527e-06,
      "loss": 0.0001,
      "step": 280
    },
    {
      "clip_ratio": 0.00020424836839083582,
      "completion_length": 124.59375,
      "epoch": 0.038367012561441836,
      "grad_norm": 0.022259853780269623,
      "kl": 0.0007939861261547776,
      "learning_rate": 3.7842105263157895e-06,
      "loss": 0.0002,
      "num_tokens": 832071.0,
      "reward": 0.690625011920929,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.690625011920929,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.038503549972692515,
      "grad_norm": 0.022215358912944794,
      "kl": 0.0008975725677373703,
      "learning_rate": 3.778947368421053e-06,
      "loss": 0.0,
      "step": 282
    },
    {
      "clip_ratio": 0.0004173782654106617,
      "epoch": 0.0386400873839432,
      "grad_norm": 0.022612985223531723,
      "kl": 0.0008190222797566094,
      "learning_rate": 3.773684210526316e-06,
      "loss": 0.0001,
      "step": 283
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.038776624795193886,
      "grad_norm": 0.022016093134880066,
      "kl": 0.0008076918084043427,
      "learning_rate": 3.768421052631579e-06,
      "loss": 0.0001,
      "step": 284
    },
    {
      "clip_ratio": 0.000884739805769641,
      "completion_length": 144.40625,
      "epoch": 0.038913162206444564,
      "grad_norm": 0.03007262945175171,
      "kl": 0.0008287996988656232,
      "learning_rate": 3.7631578947368426e-06,
      "loss": 0.0,
      "num_tokens": 844080.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 285
    },
    {
      "clip_ratio": 0.0011433004328864627,
      "epoch": 0.03904969961769525,
      "grad_norm": 0.029923973605036736,
      "kl": 0.0008635769727334264,
      "learning_rate": 3.7578947368421053e-06,
      "loss": 0.0001,
      "step": 286
    },
    {
      "clip_ratio": 0.0006655217075604014,
      "epoch": 0.03918623702894593,
      "grad_norm": 0.029381604865193367,
      "kl": 0.0008121648797896341,
      "learning_rate": 3.752631578947369e-06,
      "loss": -0.0001,
      "step": 287
    },
    {
      "clip_ratio": 0.00125420526455855,
      "epoch": 0.03932277444019661,
      "grad_norm": 0.03023548424243927,
      "kl": 0.0008140054451359902,
      "learning_rate": 3.7473684210526317e-06,
      "loss": -0.0001,
      "step": 288
    },
    {
      "clip_ratio": 0.0003085214993916452,
      "completion_length": 144.75,
      "epoch": 0.0394593118514473,
      "grad_norm": 0.02250145748257637,
      "kl": 0.000704204682733689,
      "learning_rate": 3.7421052631578953e-06,
      "loss": -0.0001,
      "num_tokens": 855800.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 289
    },
    {
      "clip_ratio": 0.00027173911803402007,
      "epoch": 0.03959584926269798,
      "grad_norm": 0.022790009155869484,
      "kl": 0.0006996235561018693,
      "learning_rate": 3.736842105263158e-06,
      "loss": -0.0001,
      "step": 290
    },
    {
      "clip_ratio": 0.0008939800900407135,
      "epoch": 0.03973238667394866,
      "grad_norm": 0.022531475871801376,
      "kl": 0.0007605676910316106,
      "learning_rate": 3.7315789473684216e-06,
      "loss": -0.0002,
      "step": 291
    },
    {
      "clip_ratio": 0.000830608099931851,
      "epoch": 0.03986892408519935,
      "grad_norm": 0.022774893790483475,
      "kl": 0.0007272760003615986,
      "learning_rate": 3.7263157894736848e-06,
      "loss": 0.0,
      "step": 292
    },
    {
      "clip_ratio": 0.0004863886861130595,
      "completion_length": 144.34375,
      "epoch": 0.040005461496450026,
      "grad_norm": 0.025127986446022987,
      "kl": 0.0007745489801891381,
      "learning_rate": 3.7210526315789475e-06,
      "loss": 0.0001,
      "num_tokens": 867691.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337018847465515,
      "rewards/reward_fn": 0.746874988079071,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04014199890770071,
      "grad_norm": 0.024640265852212906,
      "kl": 0.0007439228484145133,
      "learning_rate": 3.715789473684211e-06,
      "loss": -0.0001,
      "step": 294
    },
    {
      "clip_ratio": 0.0004863886861130595,
      "epoch": 0.04027853631895139,
      "grad_norm": 0.02452578768134117,
      "kl": 0.0008032110617932631,
      "learning_rate": 3.710526315789474e-06,
      "loss": -0.0001,
      "step": 295
    },
    {
      "clip_ratio": 0.00024224806111305952,
      "epoch": 0.040415073730202075,
      "grad_norm": 0.02467331476509571,
      "kl": 0.0007492301938327728,
      "learning_rate": 3.7052631578947374e-06,
      "loss": -0.0002,
      "step": 296
    },
    {
      "clip_ratio": 0.000669916596962139,
      "completion_length": 140.65625,
      "epoch": 0.04055161114145276,
      "grad_norm": 0.020856456831097603,
      "kl": 0.0007392201123366249,
      "learning_rate": 3.7e-06,
      "loss": 0.0001,
      "num_tokens": 879700.0,
      "reward": 0.46562498807907104,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.46562498807907104,
      "step": 297
    },
    {
      "clip_ratio": 0.00017361111531499773,
      "epoch": 0.04068814855270344,
      "grad_norm": 0.02090269699692726,
      "kl": 0.0008008243175936514,
      "learning_rate": 3.6947368421052637e-06,
      "loss": 0.0001,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.040824685963954124,
      "grad_norm": 0.02103441022336483,
      "kl": 0.0007869898254284635,
      "learning_rate": 3.6894736842105265e-06,
      "loss": 0.0,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0409612233752048,
      "grad_norm": 0.020737936720252037,
      "kl": 0.0007604343190905638,
      "learning_rate": 3.6842105263157896e-06,
      "loss": -0.0,
      "step": 300
    },
    {
      "clip_ratio": 0.00020559210679493845,
      "completion_length": 144.25,
      "epoch": 0.04109776078645549,
      "grad_norm": 0.02263866737484932,
      "kl": 0.0008127225319185527,
      "learning_rate": 3.6789473684210532e-06,
      "loss": -0.0001,
      "num_tokens": 891956.0,
      "reward": 0.859375,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.859375,
      "step": 301
    },
    {
      "clip_ratio": 0.00020559210679493845,
      "epoch": 0.04123429819770617,
      "grad_norm": 0.022851375862956047,
      "kl": 0.0007813003094270243,
      "learning_rate": 3.673684210526316e-06,
      "loss": -0.0,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04137083560895685,
      "grad_norm": 0.02316156215965748,
      "kl": 0.0007418948507620371,
      "learning_rate": 3.6684210526315796e-06,
      "loss": 0.0001,
      "step": 303
    },
    {
      "clip_ratio": 0.00037425151094794273,
      "epoch": 0.04150737302020754,
      "grad_norm": 0.02298700250685215,
      "kl": 0.0008119569065456744,
      "learning_rate": 3.6631578947368423e-06,
      "loss": 0.0,
      "step": 304
    },
    {
      "clip_ratio": 0.0004397498705657199,
      "completion_length": 119.96875,
      "epoch": 0.04164391043145822,
      "grad_norm": 0.013564548455178738,
      "kl": 0.0009123221407207893,
      "learning_rate": 3.657894736842106e-06,
      "loss": 0.0001,
      "num_tokens": 902491.0,
      "reward": 0.971875011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.971875011920929,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0417804478427089,
      "grad_norm": 0.01371033862233162,
      "kl": 0.0009642941181482456,
      "learning_rate": 3.6526315789473686e-06,
      "loss": -0.0001,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.041916985253959586,
      "grad_norm": 0.013809485360980034,
      "kl": 0.0009494323851413355,
      "learning_rate": 3.6473684210526318e-06,
      "loss": 0.0001,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.042053522665210265,
      "grad_norm": 0.01366579532623291,
      "kl": 0.0009975432599276246,
      "learning_rate": 3.642105263157895e-06,
      "loss": -0.0,
      "step": 308
    },
    {
      "clip_ratio": 0.00017361111531499773,
      "completion_length": 151.6875,
      "epoch": 0.04219006007646095,
      "grad_norm": 0.029042799025774002,
      "kl": 0.0007244326034197002,
      "learning_rate": 3.636842105263158e-06,
      "loss": -0.0,
      "num_tokens": 914525.0,
      "reward": 0.746874988079071,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.746874988079071,
      "step": 309
    },
    {
      "clip_ratio": 0.00020559210679493845,
      "epoch": 0.042326597487711635,
      "grad_norm": 0.02877066843211651,
      "kl": 0.0007629632664247765,
      "learning_rate": 3.6315789473684217e-06,
      "loss": 0.0001,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.042463134898962314,
      "grad_norm": 0.028539150953292847,
      "kl": 0.0007963754860611516,
      "learning_rate": 3.6263157894736844e-06,
      "loss": 0.0001,
      "step": 311
    },
    {
      "clip_ratio": 0.000292056065518409,
      "epoch": 0.042599672310213,
      "grad_norm": 0.02861402928829193,
      "kl": 0.0007653530692550703,
      "learning_rate": 3.621052631578948e-06,
      "loss": 0.0001,
      "step": 312
    },
    {
      "clip_ratio": 0.00019171778694726527,
      "completion_length": 119.59375,
      "epoch": 0.04273620972146368,
      "grad_norm": 0.030657904222607613,
      "kl": 0.0009028592303366167,
      "learning_rate": 3.6157894736842108e-06,
      "loss": 0.0002,
      "num_tokens": 925264.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04287274713271436,
      "grad_norm": 0.030080867931246758,
      "kl": 0.0009077482118300395,
      "learning_rate": 3.610526315789474e-06,
      "loss": 0.0001,
      "step": 314
    },
    {
      "clip_ratio": 0.0003787878667935729,
      "epoch": 0.04300928454396505,
      "grad_norm": 0.031132614240050316,
      "kl": 0.0009562106060911901,
      "learning_rate": 3.605263157894737e-06,
      "loss": 0.0001,
      "step": 315
    },
    {
      "clip_ratio": 0.0005290678527671844,
      "epoch": 0.04314582195521573,
      "grad_norm": 0.03099122643470764,
      "kl": 0.0010130553719136515,
      "learning_rate": 3.6000000000000003e-06,
      "loss": 0.0002,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.65625,
      "epoch": 0.04328235936646641,
      "grad_norm": 0.02720688469707966,
      "kl": 0.0009564748033881187,
      "learning_rate": 3.5947368421052634e-06,
      "loss": 0.0001,
      "num_tokens": 937309.0,
      "reward": 0.859375,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.859375,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0434188967777171,
      "grad_norm": 0.027184858918190002,
      "kl": 0.0009709294772619614,
      "learning_rate": 3.5894736842105266e-06,
      "loss": 0.0001,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.043555434188967776,
      "grad_norm": 0.027308201417326927,
      "kl": 0.0009768028121470707,
      "learning_rate": 3.58421052631579e-06,
      "loss": -0.0001,
      "step": 319
    },
    {
      "clip_ratio": 0.00025826445198617876,
      "epoch": 0.04369197160021846,
      "grad_norm": 0.026938337832689285,
      "kl": 0.0008980743559732218,
      "learning_rate": 3.578947368421053e-06,
      "loss": -0.0001,
      "step": 320
    },
    {
      "clip_ratio": 0.0005560319623327814,
      "completion_length": 140.34375,
      "epoch": 0.04382850901146914,
      "grad_norm": 0.02959519624710083,
      "kl": 0.000880028532264987,
      "learning_rate": 3.5736842105263157e-06,
      "loss": 0.0001,
      "num_tokens": 949188.0,
      "reward": 0.690625011920929,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.690625011920929,
      "step": 321
    },
    {
      "clip_ratio": 0.0009927479404723272,
      "epoch": 0.043965046422719825,
      "grad_norm": 0.030023330822587013,
      "kl": 0.0009043698364621378,
      "learning_rate": 3.5684210526315792e-06,
      "loss": -0.0001,
      "step": 322
    },
    {
      "clip_ratio": 0.0003880719013977796,
      "epoch": 0.04410158383397051,
      "grad_norm": 0.028148531913757324,
      "kl": 0.0008921725293475902,
      "learning_rate": 3.5631578947368424e-06,
      "loss": 0.0002,
      "step": 323
    },
    {
      "clip_ratio": 0.0006740285753039643,
      "epoch": 0.04423812124522119,
      "grad_norm": 0.029892824590206146,
      "kl": 0.0008777467337495182,
      "learning_rate": 3.5578947368421056e-06,
      "loss": -0.0,
      "step": 324
    },
    {
      "clip_ratio": 0.00022644927958026528,
      "completion_length": 108.53125,
      "epoch": 0.044374658656471874,
      "grad_norm": 0.03082483820617199,
      "kl": 0.0009367997481604107,
      "learning_rate": 3.5526315789473687e-06,
      "loss": 0.0,
      "num_tokens": 959709.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 325
    },
    {
      "clip_ratio": 0.00022644927958026528,
      "epoch": 0.04451119606772255,
      "grad_norm": 0.031293999403715134,
      "kl": 0.0010181294901485671,
      "learning_rate": 3.5473684210526323e-06,
      "loss": -0.0,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04464773347897324,
      "grad_norm": 0.030545929446816444,
      "kl": 0.0009729322382554528,
      "learning_rate": 3.542105263157895e-06,
      "loss": 0.0001,
      "step": 327
    },
    {
      "clip_ratio": 0.00022644927958026528,
      "epoch": 0.04478427089022392,
      "grad_norm": 0.030758768320083618,
      "kl": 0.0009197474073516787,
      "learning_rate": 3.536842105263158e-06,
      "loss": 0.0,
      "step": 328
    },
    {
      "clip_ratio": 0.00018712575547397137,
      "completion_length": 127.875,
      "epoch": 0.0449208083014746,
      "grad_norm": 0.029162487015128136,
      "kl": 0.0010116572320839623,
      "learning_rate": 3.5315789473684214e-06,
      "loss": -0.0,
      "num_tokens": 970921.0,
      "reward": 0.690625011920929,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.690625011920929,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.04505734571272529,
      "grad_norm": 0.029205741360783577,
      "kl": 0.0009253943944713683,
      "learning_rate": 3.5263157894736846e-06,
      "loss": 0.0003,
      "step": 330
    },
    {
      "clip_ratio": 0.0004011668497696519,
      "epoch": 0.04519388312397597,
      "grad_norm": 0.028902918100357056,
      "kl": 0.0009834289649006678,
      "learning_rate": 3.5210526315789477e-06,
      "loss": -0.0,
      "step": 331
    },
    {
      "clip_ratio": 0.00021404109429568052,
      "epoch": 0.04533042053522665,
      "grad_norm": 0.02907485142350197,
      "kl": 0.0008809025821392424,
      "learning_rate": 3.515789473684211e-06,
      "loss": 0.0,
      "step": 332
    },
    {
      "clip_ratio": 0.0005798441779916175,
      "completion_length": 124.75,
      "epoch": 0.045466957946477336,
      "grad_norm": 0.03617256507277489,
      "kl": 0.000798491771092813,
      "learning_rate": 3.510526315789474e-06,
      "loss": 0.0001,
      "num_tokens": 981921.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 333
    },
    {
      "clip_ratio": 0.00011488970631035045,
      "epoch": 0.045603495357728015,
      "grad_norm": 0.03615871071815491,
      "kl": 0.0008176482906492311,
      "learning_rate": 3.505263157894737e-06,
      "loss": -0.0004,
      "step": 334
    },
    {
      "clip_ratio": 0.0014091167831793427,
      "epoch": 0.0457400327689787,
      "grad_norm": 0.03517930954694748,
      "kl": 0.0008173711526069383,
      "learning_rate": 3.5e-06,
      "loss": 0.0002,
      "step": 335
    },
    {
      "clip_ratio": 0.0006452707311837003,
      "epoch": 0.045876570180229385,
      "grad_norm": 0.03678533807396889,
      "kl": 0.0008238869195338339,
      "learning_rate": 3.4947368421052635e-06,
      "loss": -0.0001,
      "step": 336
    },
    {
      "clip_ratio": 0.0006435871036956087,
      "completion_length": 171.5625,
      "epoch": 0.046013107591480064,
      "grad_norm": 0.030732152983546257,
      "kl": 0.0008044209635045263,
      "learning_rate": 3.4894736842105263e-06,
      "loss": 0.0001,
      "num_tokens": 994447.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 337
    },
    {
      "clip_ratio": 0.0009438273482373916,
      "epoch": 0.04614964500273075,
      "grad_norm": 0.030625060200691223,
      "kl": 0.0007668728567296057,
      "learning_rate": 3.48421052631579e-06,
      "loss": -0.0002,
      "step": 338
    },
    {
      "clip_ratio": 0.0010408969974378124,
      "epoch": 0.04628618241398143,
      "grad_norm": 0.030618058517575264,
      "kl": 0.0008538224528820137,
      "learning_rate": 3.478947368421053e-06,
      "loss": -0.0001,
      "step": 339
    },
    {
      "clip_ratio": 0.0005348550475900993,
      "epoch": 0.04642271982523211,
      "grad_norm": 0.029850170016288757,
      "kl": 0.0007929099783723359,
      "learning_rate": 3.473684210526316e-06,
      "loss": -0.0001,
      "step": 340
    },
    {
      "clip_ratio": 0.0007453141151927412,
      "completion_length": 136.15625,
      "epoch": 0.0465592572364828,
      "grad_norm": 0.028187168762087822,
      "kl": 0.0007331060778597021,
      "learning_rate": 3.4684210526315794e-06,
      "loss": -0.0001,
      "num_tokens": 1006284.0,
      "reward": 0.4656250476837158,
      "reward_std": 0.25110572576522827,
      "rewards/reward_fn": 0.4656250476837158,
      "step": 341
    },
    {
      "clip_ratio": 0.0011472807818790898,
      "epoch": 0.04669579464773348,
      "grad_norm": 0.028124259784817696,
      "kl": 0.0007809556900610914,
      "learning_rate": 3.463157894736842e-06,
      "loss": 0.0001,
      "step": 342
    },
    {
      "clip_ratio": 0.000936449650907889,
      "epoch": 0.04683233205898416,
      "grad_norm": 0.028143279254436493,
      "kl": 0.0007603804115206003,
      "learning_rate": 3.4578947368421057e-06,
      "loss": -0.0,
      "step": 343
    },
    {
      "clip_ratio": 0.000640960643067956,
      "epoch": 0.04696886947023485,
      "grad_norm": 0.02959413267672062,
      "kl": 0.0007092921259754803,
      "learning_rate": 3.4526315789473684e-06,
      "loss": 0.0002,
      "step": 344
    },
    {
      "clip_ratio": 0.0005915082147112116,
      "completion_length": 141.1875,
      "epoch": 0.047105406881485526,
      "grad_norm": 0.027673540636897087,
      "kl": 0.0008731992320463178,
      "learning_rate": 3.447368421052632e-06,
      "loss": -0.0,
      "num_tokens": 1017466.0,
      "reward": 0.71875,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.71875,
      "step": 345
    },
    {
      "clip_ratio": 0.0004701640282291919,
      "epoch": 0.04724194429273621,
      "grad_norm": 0.028596054762601852,
      "kl": 0.0008803572027318296,
      "learning_rate": 3.4421052631578947e-06,
      "loss": -0.0001,
      "step": 346
    },
    {
      "clip_ratio": 0.0004612608900060877,
      "epoch": 0.04737848170398689,
      "grad_norm": 0.02798176184296608,
      "kl": 0.0008108743318189227,
      "learning_rate": 3.4368421052631583e-06,
      "loss": -0.0003,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.047515019115237575,
      "grad_norm": 0.028018614277243614,
      "kl": 0.0007496376688322925,
      "learning_rate": 3.4315789473684215e-06,
      "loss": -0.0003,
      "step": 348
    },
    {
      "clip_ratio": 0.00015470296784769744,
      "completion_length": 131.5,
      "epoch": 0.04765155652648826,
      "grad_norm": 0.013035676442086697,
      "kl": 0.0007108611252988339,
      "learning_rate": 3.4263157894736842e-06,
      "loss": -0.0001,
      "num_tokens": 1028970.0,
      "reward": 0.4375,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.4375,
      "step": 349
    },
    {
      "clip_ratio": 0.00015470296784769744,
      "epoch": 0.04778809393773894,
      "grad_norm": 0.012961960397660732,
      "kl": 0.0006872391941215028,
      "learning_rate": 3.421052631578948e-06,
      "loss": -0.0001,
      "step": 350
    },
    {
      "clip_ratio": 0.00015470296784769744,
      "epoch": 0.047924631348989624,
      "grad_norm": 0.013093845918774605,
      "kl": 0.0007618096824444365,
      "learning_rate": 3.4157894736842106e-06,
      "loss": -0.0001,
      "step": 351
    },
    {
      "clip_ratio": 0.0009627791150705889,
      "epoch": 0.0480611687602403,
      "grad_norm": 0.013092254288494587,
      "kl": 0.0007593772188556613,
      "learning_rate": 3.410526315789474e-06,
      "loss": 0.0001,
      "step": 352
    },
    {
      "clip_ratio": 0.0012184588267700747,
      "completion_length": 133.15625,
      "epoch": 0.04819770617149099,
      "grad_norm": 0.05045875906944275,
      "kl": 0.0010233524417344597,
      "learning_rate": 3.405263157894737e-06,
      "loss": 0.0002,
      "num_tokens": 1040631.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.4198557138442993,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 353
    },
    {
      "clip_ratio": 0.001344054937362671,
      "epoch": 0.04833424358274167,
      "grad_norm": 0.04969387128949165,
      "kl": 0.0010599575684864249,
      "learning_rate": 3.4000000000000005e-06,
      "loss": -0.0001,
      "step": 354
    },
    {
      "clip_ratio": 0.0011714279535226524,
      "epoch": 0.04847078099399235,
      "grad_norm": 0.04989272728562355,
      "kl": 0.0010831583349499851,
      "learning_rate": 3.3947368421052636e-06,
      "loss": 0.0002,
      "step": 355
    },
    {
      "clip_ratio": 0.0010523917735554278,
      "epoch": 0.04860731840524304,
      "grad_norm": 0.0498301163315773,
      "kl": 0.0011495640255816397,
      "learning_rate": 3.3894736842105264e-06,
      "loss": -0.0001,
      "step": 356
    },
    {
      "clip_ratio": 0.0001707650226308033,
      "completion_length": 134.96875,
      "epoch": 0.04874385581649372,
      "grad_norm": 0.015665635466575623,
      "kl": 0.0008176299570550327,
      "learning_rate": 3.38421052631579e-06,
      "loss": 0.0,
      "num_tokens": 1052394.0,
      "reward": 0.550000011920929,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.550000011920929,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0488803932277444,
      "grad_norm": 0.015186646021902561,
      "kl": 0.0008519297498423839,
      "learning_rate": 3.3789473684210527e-06,
      "loss": -0.0001,
      "step": 358
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.049016930638995086,
      "grad_norm": 0.015088633634150028,
      "kl": 0.0008564612980990205,
      "learning_rate": 3.3736842105263163e-06,
      "loss": 0.0001,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.049153468050245765,
      "grad_norm": 0.0157124362885952,
      "kl": 0.0008022732190511306,
      "learning_rate": 3.368421052631579e-06,
      "loss": -0.0,
      "step": 360
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "completion_length": 140.40625,
      "epoch": 0.04929000546149645,
      "grad_norm": 0.028024472296237946,
      "kl": 0.0008776496542850509,
      "learning_rate": 3.3631578947368426e-06,
      "loss": 0.0001,
      "num_tokens": 1064123.0,
      "reward": 0.8031249642372131,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.8031249642372131,
      "step": 361
    },
    {
      "clip_ratio": 0.0001849112450145185,
      "epoch": 0.049426542872747135,
      "grad_norm": 0.028005549684166908,
      "kl": 0.0008063845743890852,
      "learning_rate": 3.3578947368421054e-06,
      "loss": 0.0,
      "step": 362
    },
    {
      "clip_ratio": 0.0010166573629248887,
      "epoch": 0.049563080283997814,
      "grad_norm": 0.0277395136654377,
      "kl": 0.0008172210209522746,
      "learning_rate": 3.3526315789473685e-06,
      "loss": -0.0,
      "step": 363
    },
    {
      "clip_ratio": 0.0004943171807099134,
      "epoch": 0.0496996176952485,
      "grad_norm": 0.028183378279209137,
      "kl": 0.0008585352416048408,
      "learning_rate": 3.347368421052632e-06,
      "loss": 0.0001,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.21875,
      "epoch": 0.04983615510649918,
      "grad_norm": 0.01484043151140213,
      "kl": 0.0007672449992242036,
      "learning_rate": 3.342105263157895e-06,
      "loss": 0.0,
      "num_tokens": 1075514.0,
      "reward": 0.746874988079071,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.746874988079071,
      "step": 365
    },
    {
      "clip_ratio": 0.00029761905898340046,
      "epoch": 0.04997269251774986,
      "grad_norm": 0.015448699705302715,
      "kl": 0.0008690163053870492,
      "learning_rate": 3.3368421052631584e-06,
      "loss": 0.0001,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05010922992900055,
      "grad_norm": 0.014827921986579895,
      "kl": 0.0008811628867988475,
      "learning_rate": 3.331578947368421e-06,
      "loss": 0.0001,
      "step": 367
    },
    {
      "clip_ratio": 0.00029761905898340046,
      "epoch": 0.05024576734025123,
      "grad_norm": 0.015469317324459553,
      "kl": 0.0008598208462444745,
      "learning_rate": 3.3263157894736848e-06,
      "loss": 0.0001,
      "step": 368
    },
    {
      "clip_ratio": 0.0006039029685780406,
      "completion_length": 142.125,
      "epoch": 0.05038230475150191,
      "grad_norm": 0.05040661618113518,
      "kl": 0.0017897128709591925,
      "learning_rate": 3.3210526315789475e-06,
      "loss": 0.0,
      "num_tokens": 1087494.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.34620189666748047,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 369
    },
    {
      "clip_ratio": 0.001837524599977769,
      "epoch": 0.0505188421627526,
      "grad_norm": 0.04978993907570839,
      "kl": 0.0019511687733029248,
      "learning_rate": 3.3157894736842107e-06,
      "loss": -0.0001,
      "step": 370
    },
    {
      "clip_ratio": 0.0009692513267509639,
      "epoch": 0.050655379574003276,
      "grad_norm": 0.05005129054188728,
      "kl": 0.0018768518375509302,
      "learning_rate": 3.310526315789474e-06,
      "loss": -0.0002,
      "step": 371
    },
    {
      "clip_ratio": 0.0006632323638768867,
      "epoch": 0.05079191698525396,
      "grad_norm": 0.03911221772432327,
      "kl": 0.0015103784116945462,
      "learning_rate": 3.305263157894737e-06,
      "loss": -0.0002,
      "step": 372
    },
    {
      "clip_ratio": 0.0008642399334348738,
      "completion_length": 123.84375,
      "epoch": 0.05092845439650464,
      "grad_norm": 0.03477134928107262,
      "kl": 0.0009179235403280472,
      "learning_rate": 3.3000000000000006e-06,
      "loss": 0.0001,
      "num_tokens": 1098697.0,
      "reward": 0.550000011920929,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.550000011920929,
      "step": 373
    },
    {
      "clip_ratio": 0.0007545908156316727,
      "epoch": 0.051064991807755325,
      "grad_norm": 0.034040454775094986,
      "kl": 0.0009303928327426547,
      "learning_rate": 3.2947368421052633e-06,
      "loss": -0.0002,
      "step": 374
    },
    {
      "clip_ratio": 0.0010670908086467534,
      "epoch": 0.05120152921900601,
      "grad_norm": 0.03519129753112793,
      "kl": 0.0009441823649467551,
      "learning_rate": 3.289473684210527e-06,
      "loss": 0.0001,
      "step": 375
    },
    {
      "clip_ratio": 0.0007545908156316727,
      "epoch": 0.05133806663025669,
      "grad_norm": 0.03399660438299179,
      "kl": 0.0009638729961807257,
      "learning_rate": 3.2842105263157897e-06,
      "loss": 0.0001,
      "step": 376
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "completion_length": 150.34375,
      "epoch": 0.051474604041507374,
      "grad_norm": 0.02090802974998951,
      "kl": 0.0008481619370286353,
      "learning_rate": 3.278947368421053e-06,
      "loss": -0.0,
      "num_tokens": 1111616.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 377
    },
    {
      "clip_ratio": 0.00041578053787816316,
      "epoch": 0.05161114145275805,
      "grad_norm": 0.02162465825676918,
      "kl": 0.0009244795583072118,
      "learning_rate": 3.273684210526316e-06,
      "loss": 0.0,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05174767886400874,
      "grad_norm": 0.021669728681445122,
      "kl": 0.0007857876262278296,
      "learning_rate": 3.268421052631579e-06,
      "loss": -0.0,
      "step": 379
    },
    {
      "clip_ratio": 0.00026939655072055757,
      "epoch": 0.05188421627525942,
      "grad_norm": 0.02159099467098713,
      "kl": 0.0008994436720968224,
      "learning_rate": 3.2631578947368423e-06,
      "loss": -0.0,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 164.21875,
      "epoch": 0.0520207536865101,
      "grad_norm": 0.020035242661833763,
      "kl": 0.0010208635330855031,
      "learning_rate": 3.2578947368421055e-06,
      "loss": -0.0,
      "num_tokens": 1123775.0,
      "reward": 0.690625011920929,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.690625011920929,
      "step": 381
    },
    {
      "clip_ratio": 0.00016191709437407553,
      "epoch": 0.05215729109776079,
      "grad_norm": 0.021156245842576027,
      "kl": 0.0009584235158399679,
      "learning_rate": 3.252631578947369e-06,
      "loss": -0.0001,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05229382850901147,
      "grad_norm": 0.02144774980843067,
      "kl": 0.0009559597119732643,
      "learning_rate": 3.247368421052632e-06,
      "loss": -0.0,
      "step": 383
    },
    {
      "clip_ratio": 0.00016191709437407553,
      "epoch": 0.05243036592026215,
      "grad_norm": 0.021347761154174805,
      "kl": 0.0009219003768521361,
      "learning_rate": 3.2421052631578945e-06,
      "loss": 0.0,
      "step": 384
    },
    {
      "clip_ratio": 0.0002741228090599179,
      "completion_length": 125.1875,
      "epoch": 0.052566903331512836,
      "grad_norm": 0.02965986356139183,
      "kl": 0.0010802704318848555,
      "learning_rate": 3.236842105263158e-06,
      "loss": 0.0,
      "num_tokens": 1134441.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 385
    },
    {
      "clip_ratio": 0.0003844313760055229,
      "epoch": 0.052703440742763515,
      "grad_norm": 0.029097799211740494,
      "kl": 0.000968684209510684,
      "learning_rate": 3.2315789473684213e-06,
      "loss": 0.0001,
      "step": 386
    },
    {
      "clip_ratio": 0.000584462919505313,
      "epoch": 0.0528399781540142,
      "grad_norm": 0.02920384705066681,
      "kl": 0.0010418808997201268,
      "learning_rate": 3.2263157894736845e-06,
      "loss": -0.0001,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.052976515565264885,
      "grad_norm": 0.03033665381371975,
      "kl": 0.0010443700157338753,
      "learning_rate": 3.2210526315789476e-06,
      "loss": -0.0,
      "step": 388
    },
    {
      "clip_ratio": 0.0006488894578069448,
      "completion_length": 118.15625,
      "epoch": 0.053113052976515564,
      "grad_norm": 0.019158417358994484,
      "kl": 0.0010731482871051412,
      "learning_rate": 3.215789473684211e-06,
      "loss": 0.0002,
      "num_tokens": 1145598.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 389
    },
    {
      "clip_ratio": 0.0002604166802484542,
      "epoch": 0.05324959038776625,
      "grad_norm": 0.01972230151295662,
      "kl": 0.0011826686286440236,
      "learning_rate": 3.210526315789474e-06,
      "loss": 0.0001,
      "step": 390
    },
    {
      "clip_ratio": 0.00025826445198617876,
      "epoch": 0.05338612779901693,
      "grad_norm": 0.020151875913143158,
      "kl": 0.001059102758517838,
      "learning_rate": 3.2052631578947367e-06,
      "loss": -0.0001,
      "step": 391
    },
    {
      "clip_ratio": 0.00012159533071098849,
      "epoch": 0.05352266521026761,
      "grad_norm": 0.020285218954086304,
      "kl": 0.0010877625118155265,
      "learning_rate": 3.2000000000000003e-06,
      "loss": 0.0001,
      "step": 392
    },
    {
      "clip_ratio": 0.00028669723542407155,
      "completion_length": 124.875,
      "epoch": 0.0536592026215183,
      "grad_norm": 0.02960928902029991,
      "kl": 0.0010169045353904949,
      "learning_rate": 3.1947368421052634e-06,
      "loss": -0.0002,
      "num_tokens": 1156902.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05379574003276898,
      "grad_norm": 0.029819682240486145,
      "kl": 0.00100540832409024,
      "learning_rate": 3.1894736842105266e-06,
      "loss": -0.0004,
      "step": 394
    },
    {
      "clip_ratio": 0.00024224806111305952,
      "epoch": 0.05393227744401966,
      "grad_norm": 0.029693953692913055,
      "kl": 0.001026836252094654,
      "learning_rate": 3.1842105263157898e-06,
      "loss": -0.0002,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05406881485527035,
      "grad_norm": 0.029733866453170776,
      "kl": 0.001003824468170933,
      "learning_rate": 3.178947368421053e-06,
      "loss": -0.0001,
      "step": 396
    },
    {
      "clip_ratio": 0.0008818957139737904,
      "completion_length": 120.90625,
      "epoch": 0.054205352266521026,
      "grad_norm": 0.0288943350315094,
      "kl": 0.0010142269729840336,
      "learning_rate": 3.173684210526316e-06,
      "loss": 0.0001,
      "num_tokens": 1167923.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.25110572576522827,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 397
    },
    {
      "clip_ratio": 0.0011852937750518322,
      "epoch": 0.05434188967777171,
      "grad_norm": 0.028492823243141174,
      "kl": 0.001117868141591316,
      "learning_rate": 3.168421052631579e-06,
      "loss": 0.0002,
      "step": 398
    },
    {
      "clip_ratio": 0.0006109336100053042,
      "epoch": 0.05447842708902239,
      "grad_norm": 0.02876265160739422,
      "kl": 0.0010261732304570614,
      "learning_rate": 3.1631578947368424e-06,
      "loss": 0.0001,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.054614964500273075,
      "grad_norm": 0.0290256105363369,
      "kl": 0.0011352793753758306,
      "learning_rate": 3.157894736842105e-06,
      "loss": -0.0,
      "step": 400
    },
    {
      "clip_ratio": 0.0006023591413395479,
      "completion_length": 144.59375,
      "epoch": 0.05475150191152376,
      "grad_norm": 0.02488422952592373,
      "kl": 0.0010054338454210665,
      "learning_rate": 3.1526315789473688e-06,
      "loss": 0.0003,
      "num_tokens": 1179646.0,
      "reward": 0.578125,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.578125,
      "step": 401
    },
    {
      "clip_ratio": 0.0006849976925877854,
      "epoch": 0.05488803932277444,
      "grad_norm": 0.024585776031017303,
      "kl": 0.0009108762242249213,
      "learning_rate": 3.147368421052632e-06,
      "loss": 0.0002,
      "step": 402
    },
    {
      "clip_ratio": 0.0004349976807134226,
      "epoch": 0.055024576734025124,
      "grad_norm": 0.02463947981595993,
      "kl": 0.0009352307270091842,
      "learning_rate": 3.142105263157895e-06,
      "loss": 0.0,
      "step": 403
    },
    {
      "clip_ratio": 0.00047781202010810375,
      "epoch": 0.0551611141452758,
      "grad_norm": 0.02484274096786976,
      "kl": 0.000943178461966454,
      "learning_rate": 3.1368421052631582e-06,
      "loss": 0.0001,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.96875,
      "epoch": 0.05529765155652649,
      "grad_norm": 0.027189401909708977,
      "kl": 0.0009860549935183371,
      "learning_rate": 3.131578947368421e-06,
      "loss": -0.0,
      "num_tokens": 1191045.0,
      "reward": 0.578125,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.578125,
      "step": 405
    },
    {
      "clip_ratio": 0.00025201612152159214,
      "epoch": 0.05543418896777717,
      "grad_norm": 0.027663057669997215,
      "kl": 0.001040305780406925,
      "learning_rate": 3.1263157894736846e-06,
      "loss": -0.0,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05557072637902785,
      "grad_norm": 0.027513474225997925,
      "kl": 0.0009971423378374311,
      "learning_rate": 3.1210526315789473e-06,
      "loss": 0.0,
      "step": 407
    },
    {
      "clip_ratio": 0.0005750250711571425,
      "epoch": 0.05570726379027854,
      "grad_norm": 0.027500582858920097,
      "kl": 0.0009878080927592237,
      "learning_rate": 3.115789473684211e-06,
      "loss": 0.0001,
      "step": 408
    },
    {
      "clip_ratio": 0.00047506178088951856,
      "completion_length": 128.96875,
      "epoch": 0.05584380120152922,
      "grad_norm": 0.02907399833202362,
      "kl": 0.0009158288730759523,
      "learning_rate": 3.1105263157894736e-06,
      "loss": 0.0001,
      "num_tokens": 1202488.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.606249988079071,
      "step": 409
    },
    {
      "clip_ratio": 0.0005465437716338784,
      "epoch": 0.0559803386127799,
      "grad_norm": 0.029220519587397575,
      "kl": 0.0009589706305632717,
      "learning_rate": 3.1052631578947372e-06,
      "loss": -0.0001,
      "step": 410
    },
    {
      "clip_ratio": 0.00017458100046496838,
      "epoch": 0.056116876024030586,
      "grad_norm": 0.02900395542383194,
      "kl": 0.0009313819728049566,
      "learning_rate": 3.1000000000000004e-06,
      "loss": 0.0,
      "step": 411
    },
    {
      "clip_ratio": 0.00017458100046496838,
      "epoch": 0.056253413435281265,
      "grad_norm": 0.029268642887473106,
      "kl": 0.0009859869569481816,
      "learning_rate": 3.094736842105263e-06,
      "loss": -0.0003,
      "step": 412
    },
    {
      "clip_ratio": 0.0002604166802484542,
      "completion_length": 129.0,
      "epoch": 0.05638995084653195,
      "grad_norm": 0.02232382819056511,
      "kl": 0.0009413343414053088,
      "learning_rate": 3.0894736842105267e-06,
      "loss": -0.0,
      "num_tokens": 1213700.0,
      "reward": 0.550000011920929,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.550000011920929,
      "step": 413
    },
    {
      "clip_ratio": 0.0001755617995513603,
      "epoch": 0.056526488257782635,
      "grad_norm": 0.02243160828948021,
      "kl": 0.0009721652440930484,
      "learning_rate": 3.0842105263157895e-06,
      "loss": 0.0003,
      "step": 414
    },
    {
      "clip_ratio": 0.0002604166802484542,
      "epoch": 0.056663025669033314,
      "grad_norm": 0.022144054993987083,
      "kl": 0.000967281979683321,
      "learning_rate": 3.078947368421053e-06,
      "loss": -0.0,
      "step": 415
    },
    {
      "clip_ratio": 0.0002604166802484542,
      "epoch": 0.056799563080284,
      "grad_norm": 0.02247004024684429,
      "kl": 0.0010008905883296393,
      "learning_rate": 3.0736842105263158e-06,
      "loss": 0.0001,
      "step": 416
    },
    {
      "clip_ratio": 0.0012989779352210462,
      "completion_length": 126.15625,
      "epoch": 0.05693610049153468,
      "grad_norm": 0.03142450377345085,
      "kl": 0.0011929241682082647,
      "learning_rate": 3.0684210526315794e-06,
      "loss": 0.0,
      "num_tokens": 1225305.0,
      "reward": 0.578125,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.578125,
      "step": 417
    },
    {
      "clip_ratio": 0.0010220625554211438,
      "epoch": 0.05707263790278536,
      "grad_norm": 0.03162077069282532,
      "kl": 0.001126760105307767,
      "learning_rate": 3.0631578947368425e-06,
      "loss": -0.0,
      "step": 418
    },
    {
      "clip_ratio": 0.0013637234515044838,
      "epoch": 0.05720917531403605,
      "grad_norm": 0.0330648310482502,
      "kl": 0.0011995817449133028,
      "learning_rate": 3.0578947368421053e-06,
      "loss": 0.0003,
      "step": 419
    },
    {
      "clip_ratio": 0.0012198473705211654,
      "epoch": 0.05734571272528673,
      "grad_norm": 0.03194950148463249,
      "kl": 0.0011162917576257314,
      "learning_rate": 3.052631578947369e-06,
      "loss": 0.0001,
      "step": 420
    },
    {
      "clip_ratio": 0.00016891892300918698,
      "completion_length": 125.84375,
      "epoch": 0.05748225013653741,
      "grad_norm": 0.02752695232629776,
      "kl": 0.0010589144121695426,
      "learning_rate": 3.0473684210526316e-06,
      "loss": -0.0001,
      "num_tokens": 1236240.0,
      "reward": 0.550000011920929,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.550000011920929,
      "step": 421
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.0576187875477881,
      "grad_norm": 0.02796095237135887,
      "kl": 0.0010964719203911955,
      "learning_rate": 3.042105263157895e-06,
      "loss": -0.0001,
      "step": 422
    },
    {
      "clip_ratio": 0.0003004807804245502,
      "epoch": 0.057755324959038776,
      "grad_norm": 0.028477521613240242,
      "kl": 0.0011441280676081078,
      "learning_rate": 3.036842105263158e-06,
      "loss": -0.0001,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05789186237028946,
      "grad_norm": 0.027919191867113113,
      "kl": 0.000996430138911819,
      "learning_rate": 3.0315789473684215e-06,
      "loss": -0.0001,
      "step": 424
    },
    {
      "clip_ratio": 0.00034171076549682766,
      "completion_length": 123.78125,
      "epoch": 0.05802839978154014,
      "grad_norm": 0.025030149146914482,
      "kl": 0.0009267698869734886,
      "learning_rate": 3.0263157894736843e-06,
      "loss": -0.0,
      "num_tokens": 1247061.0,
      "reward": 0.578125,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.578125,
      "step": 425
    },
    {
      "clip_ratio": 0.00035305789788253605,
      "epoch": 0.058164937192790825,
      "grad_norm": 0.02742120996117592,
      "kl": 0.0009424066829524236,
      "learning_rate": 3.0210526315789474e-06,
      "loss": -0.0002,
      "step": 426
    },
    {
      "clip_ratio": 0.00014880952949170023,
      "epoch": 0.05830147460404151,
      "grad_norm": 0.024884475395083427,
      "kl": 0.0008936434551287675,
      "learning_rate": 3.015789473684211e-06,
      "loss": 0.0001,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05843801201529219,
      "grad_norm": 0.024734022095799446,
      "kl": 0.0008923630566641805,
      "learning_rate": 3.0105263157894737e-06,
      "loss": -0.0,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.0625,
      "epoch": 0.058574549426542874,
      "grad_norm": 0.015420363284647465,
      "kl": 0.0011353582085575908,
      "learning_rate": 3.0052631578947373e-06,
      "loss": -0.0002,
      "num_tokens": 1258491.0,
      "reward": 0.746874988079071,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.746874988079071,
      "step": 429
    },
    {
      "clip_ratio": 0.0003955696302000433,
      "epoch": 0.05871108683779355,
      "grad_norm": 0.015479897148907185,
      "kl": 0.0011547396525202203,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05884762424904424,
      "grad_norm": 0.015702426433563232,
      "kl": 0.0011680879488267237,
      "learning_rate": 2.9947368421052637e-06,
      "loss": -0.0,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.05898416166029492,
      "grad_norm": 0.015461385250091553,
      "kl": 0.0011976907471762388,
      "learning_rate": 2.9894736842105264e-06,
      "loss": -0.0001,
      "step": 432
    },
    {
      "clip_ratio": 0.0001755617995513603,
      "completion_length": 134.96875,
      "epoch": 0.0591206990715456,
      "grad_norm": 0.03721882402896881,
      "kl": 0.0010735591249613208,
      "learning_rate": 2.9842105263157896e-06,
      "loss": -0.0002,
      "num_tokens": 1269578.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 433
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.05925723648279629,
      "grad_norm": 0.03812366724014282,
      "kl": 0.0011335061217323528,
      "learning_rate": 2.9789473684210527e-06,
      "loss": 0.0001,
      "step": 434
    },
    {
      "clip_ratio": 0.0007790682866470888,
      "epoch": 0.05939377389404697,
      "grad_norm": 0.037977952510118484,
      "kl": 0.0010755884868558496,
      "learning_rate": 2.973684210526316e-06,
      "loss": -0.0002,
      "step": 435
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.05953031130529765,
      "grad_norm": 0.038097258657217026,
      "kl": 0.0011848643234770861,
      "learning_rate": 2.9684210526315795e-06,
      "loss": -0.0001,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.0625,
      "epoch": 0.059666848716548336,
      "grad_norm": 0.02606877312064171,
      "kl": 0.0008522843054379337,
      "learning_rate": 2.9631578947368422e-06,
      "loss": 0.0,
      "num_tokens": 1281764.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 437
    },
    {
      "clip_ratio": 0.0004951451264787465,
      "epoch": 0.059803386127799014,
      "grad_norm": 0.025950873270630836,
      "kl": 0.000901157999123825,
      "learning_rate": 2.957894736842106e-06,
      "loss": -0.0,
      "step": 438
    },
    {
      "clip_ratio": 0.00032438010384794325,
      "epoch": 0.0599399235390497,
      "grad_norm": 0.02585132047533989,
      "kl": 0.000955841273025726,
      "learning_rate": 2.9526315789473685e-06,
      "loss": -0.0,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.060076460950300385,
      "grad_norm": 0.026470227167010307,
      "kl": 0.0009173221933451714,
      "learning_rate": 2.9473684210526317e-06,
      "loss": -0.0,
      "step": 440
    },
    {
      "clip_ratio": 0.00022321428696159273,
      "completion_length": 123.15625,
      "epoch": 0.060212998361551064,
      "grad_norm": 0.031343188136816025,
      "kl": 0.0011035336447093869,
      "learning_rate": 2.942105263157895e-06,
      "loss": 0.0001,
      "num_tokens": 1292701.0,
      "reward": 0.690625011920929,
      "reward_std": 0.25110572576522827,
      "rewards/reward_fn": 0.690625011920929,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06034953577280175,
      "grad_norm": 0.031478047370910645,
      "kl": 0.0010842259507626295,
      "learning_rate": 2.936842105263158e-06,
      "loss": 0.0001,
      "step": 442
    },
    {
      "clip_ratio": 0.00018939393339678645,
      "epoch": 0.06048607318405243,
      "grad_norm": 0.03135223686695099,
      "kl": 0.0017851412476375117,
      "learning_rate": 2.931578947368421e-06,
      "loss": 0.0,
      "step": 443
    },
    {
      "clip_ratio": 0.00022321428696159273,
      "epoch": 0.06062261059530311,
      "grad_norm": 0.0316835418343544,
      "kl": 0.0014097988059802447,
      "learning_rate": 2.9263157894736844e-06,
      "loss": 0.0,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.5625,
      "epoch": 0.0607591480065538,
      "grad_norm": 0.0311806071549654,
      "kl": 0.0013436454155453248,
      "learning_rate": 2.921052631578948e-06,
      "loss": 0.0001,
      "num_tokens": 1304243.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.690625011920929,
      "step": 445
    },
    {
      "clip_ratio": 0.0007211912743514404,
      "epoch": 0.060895685417804477,
      "grad_norm": 0.03040701150894165,
      "kl": 0.0012179117611594847,
      "learning_rate": 2.9157894736842107e-06,
      "loss": -0.0002,
      "step": 446
    },
    {
      "clip_ratio": 0.001032393323839642,
      "epoch": 0.06103222282905516,
      "grad_norm": 0.030803782865405083,
      "kl": 0.001290351542593271,
      "learning_rate": 2.9105263157894743e-06,
      "loss": -0.0001,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06116876024030585,
      "grad_norm": 0.030800629407167435,
      "kl": 0.0013306584751262562,
      "learning_rate": 2.905263157894737e-06,
      "loss": -0.0001,
      "step": 448
    },
    {
      "clip_ratio": 0.00028935185400769114,
      "completion_length": 141.59375,
      "epoch": 0.061305297651556526,
      "grad_norm": 0.027338694781064987,
      "kl": 0.0014368064021255122,
      "learning_rate": 2.9e-06,
      "loss": 0.0002,
      "num_tokens": 1315890.0,
      "reward": 0.746874988079071,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.746874988079071,
      "step": 449
    },
    {
      "clip_ratio": 0.0004810696409549564,
      "epoch": 0.06144183506280721,
      "grad_norm": 0.02741483598947525,
      "kl": 0.0013909679764765315,
      "learning_rate": 2.8947368421052634e-06,
      "loss": 0.0001,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06157837247405789,
      "grad_norm": 0.027010517194867134,
      "kl": 0.001380715920276998,
      "learning_rate": 2.8894736842105265e-06,
      "loss": 0.0,
      "step": 451
    },
    {
      "clip_ratio": 0.0004810696409549564,
      "epoch": 0.061714909885308575,
      "grad_norm": 0.027187755331397057,
      "kl": 0.0014208953525667312,
      "learning_rate": 2.88421052631579e-06,
      "loss": -0.0,
      "step": 452
    },
    {
      "clip_ratio": 0.0008928571478463709,
      "completion_length": 145.4375,
      "epoch": 0.06185144729655926,
      "grad_norm": 0.027749359607696533,
      "kl": 0.001469686958444072,
      "learning_rate": 2.878947368421053e-06,
      "loss": 0.0001,
      "num_tokens": 1327856.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.06495190411806107,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06198798470780994,
      "grad_norm": 0.0267344918102026,
      "kl": 0.001519542402093066,
      "learning_rate": 2.8736842105263164e-06,
      "loss": 0.0,
      "step": 454
    },
    {
      "clip_ratio": 0.0003810975467786193,
      "epoch": 0.062124522119060624,
      "grad_norm": 0.027275780215859413,
      "kl": 0.001477261032050592,
      "learning_rate": 2.868421052631579e-06,
      "loss": 0.0001,
      "step": 455
    },
    {
      "clip_ratio": 0.00044642857392318547,
      "epoch": 0.0622610595303113,
      "grad_norm": 0.027177719399333,
      "kl": 0.0014757139615539927,
      "learning_rate": 2.8631578947368423e-06,
      "loss": 0.0,
      "step": 456
    },
    {
      "clip_ratio": 0.0011708111996995285,
      "completion_length": 145.4375,
      "epoch": 0.06239759694156199,
      "grad_norm": 0.03548799455165863,
      "kl": 0.0013813442674290854,
      "learning_rate": 2.8578947368421055e-06,
      "loss": 0.0002,
      "num_tokens": 1339522.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.30735570192337036,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 457
    },
    {
      "clip_ratio": 0.0011642339522950351,
      "epoch": 0.06253413435281267,
      "grad_norm": 0.03586866334080696,
      "kl": 0.0013935480292275315,
      "learning_rate": 2.8526315789473687e-06,
      "loss": 0.0002,
      "step": 458
    },
    {
      "clip_ratio": 0.0009245706896763295,
      "epoch": 0.06267067176406335,
      "grad_norm": 0.035171736031770706,
      "kl": 0.0013296840588736814,
      "learning_rate": 2.847368421052632e-06,
      "loss": -0.0001,
      "step": 459
    },
    {
      "clip_ratio": 0.0005984370218357071,
      "epoch": 0.06280720917531403,
      "grad_norm": 0.03586185351014137,
      "kl": 0.0013636875646625413,
      "learning_rate": 2.842105263157895e-06,
      "loss": 0.0001,
      "step": 460
    },
    {
      "clip_ratio": 0.00046950820251367986,
      "completion_length": 118.875,
      "epoch": 0.06294374658656472,
      "grad_norm": 0.03280496224761009,
      "kl": 0.0013567173700721469,
      "learning_rate": 2.8368421052631586e-06,
      "loss": 0.0002,
      "num_tokens": 1350862.0,
      "reward": 0.690625011920929,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.690625011920929,
      "step": 461
    },
    {
      "clip_ratio": 0.0004111842135898769,
      "epoch": 0.0630802839978154,
      "grad_norm": 0.03311141952872276,
      "kl": 0.0014376636845554458,
      "learning_rate": 2.8315789473684213e-06,
      "loss": 0.0003,
      "step": 462
    },
    {
      "clip_ratio": 0.00044646734022535384,
      "epoch": 0.06321682140906608,
      "grad_norm": 0.03446916490793228,
      "kl": 0.0015070214722072706,
      "learning_rate": 2.826315789473684e-06,
      "loss": 0.0003,
      "step": 463
    },
    {
      "clip_ratio": 0.0007496662874473259,
      "epoch": 0.06335335882031677,
      "grad_norm": 0.03369483724236488,
      "kl": 0.001399203034452512,
      "learning_rate": 2.8210526315789476e-06,
      "loss": 0.0001,
      "step": 464
    },
    {
      "clip_ratio": 0.0006772563501726836,
      "completion_length": 126.5625,
      "epoch": 0.06348989623156745,
      "grad_norm": 0.034894075244665146,
      "kl": 0.0011480197363198386,
      "learning_rate": 2.815789473684211e-06,
      "loss": 0.0001,
      "num_tokens": 1361844.0,
      "reward": 0.6875,
      "reward_std": 0.3049038052558899,
      "rewards/reward_fn": 0.6875,
      "step": 465
    },
    {
      "clip_ratio": 0.0009165278461296111,
      "epoch": 0.06362643364281813,
      "grad_norm": 0.03461375832557678,
      "kl": 0.0012377909497445216,
      "learning_rate": 2.810526315789474e-06,
      "loss": 0.0,
      "step": 466
    },
    {
      "clip_ratio": 0.0010300020658178255,
      "epoch": 0.06376297105406882,
      "grad_norm": 0.03713339567184448,
      "kl": 0.0013127127203915734,
      "learning_rate": 2.805263157894737e-06,
      "loss": 0.0001,
      "step": 467
    },
    {
      "clip_ratio": 0.00047134239866863936,
      "epoch": 0.0638995084653195,
      "grad_norm": 0.03711392730474472,
      "kl": 0.001211642022099113,
      "learning_rate": 2.8000000000000003e-06,
      "loss": 0.0002,
      "step": 468
    },
    {
      "clip_ratio": 0.0008621815213700756,
      "completion_length": 162.28125,
      "epoch": 0.06403604587657018,
      "grad_norm": 0.026019882410764694,
      "kl": 0.0012119517618884856,
      "learning_rate": 2.7947368421052635e-06,
      "loss": 0.0001,
      "num_tokens": 1374489.0,
      "reward": 0.550000011920929,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.550000011920929,
      "step": 469
    },
    {
      "clip_ratio": 0.00040302579873241484,
      "epoch": 0.06417258328782087,
      "grad_norm": 0.025591466575860977,
      "kl": 0.0012105955256629386,
      "learning_rate": 2.789473684210526e-06,
      "loss": 0.0,
      "step": 470
    },
    {
      "clip_ratio": 0.0014681293978355825,
      "epoch": 0.06430912069907155,
      "grad_norm": 0.025961099192500114,
      "kl": 0.001206063278914371,
      "learning_rate": 2.7842105263157898e-06,
      "loss": 0.0002,
      "step": 471
    },
    {
      "clip_ratio": 0.0008783431985648349,
      "epoch": 0.06444565811032223,
      "grad_norm": 0.026147864758968353,
      "kl": 0.0011393453842174495,
      "learning_rate": 2.7789473684210525e-06,
      "loss": 0.0001,
      "step": 472
    },
    {
      "clip_ratio": 0.00012755101488437504,
      "completion_length": 129.78125,
      "epoch": 0.0645821955215729,
      "grad_norm": 0.03525552526116371,
      "kl": 0.001519918667327147,
      "learning_rate": 2.773684210526316e-06,
      "loss": -0.0001,
      "num_tokens": 1385870.0,
      "reward": 0.746874988079071,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.746874988079071,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0647187329328236,
      "grad_norm": 0.03530130535364151,
      "kl": 0.0015549474192084745,
      "learning_rate": 2.7684210526315793e-06,
      "loss": 0.0002,
      "step": 474
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "epoch": 0.06485527034407428,
      "grad_norm": 0.03444594517350197,
      "kl": 0.0015299975057132542,
      "learning_rate": 2.7631578947368424e-06,
      "loss": 0.0,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06499180775532495,
      "grad_norm": 0.03540601208806038,
      "kl": 0.0014829309839115012,
      "learning_rate": 2.7578947368421056e-06,
      "loss": -0.0002,
      "step": 476
    },
    {
      "completion_length": 139.4375,
      "epoch": 0.06512834516657565,
      "grad_norm": 0.0,
      "learning_rate": 2.7526315789473683e-06,
      "loss": 0.0,
      "num_tokens": 1398096.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.0,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 477
    },
    {
      "epoch": 0.06526488257782632,
      "grad_norm": 0.0,
      "learning_rate": 2.747368421052632e-06,
      "loss": 0.0,
      "step": 478
    },
    {
      "epoch": 0.065401419989077,
      "grad_norm": 0.0,
      "learning_rate": 2.7421052631578947e-06,
      "loss": 0.0,
      "step": 479
    },
    {
      "epoch": 0.0655379574003277,
      "grad_norm": 0.0,
      "learning_rate": 2.7368421052631583e-06,
      "loss": 0.0,
      "step": 480
    },
    {
      "clip_ratio": 0.00048363096721004695,
      "completion_length": 139.28125,
      "epoch": 0.06567449481157837,
      "grad_norm": 0.03068307414650917,
      "kl": 0.0013596344333564048,
      "learning_rate": 2.7315789473684214e-06,
      "loss": 0.0001,
      "num_tokens": 1410073.0,
      "reward": 0.550000011920929,
      "reward_std": 0.289951890707016,
      "rewards/reward_fn": 0.550000011920929,
      "step": 481
    },
    {
      "clip_ratio": 0.0004263965238351375,
      "epoch": 0.06581103222282905,
      "grad_norm": 0.03083832748234272,
      "kl": 0.0012160772521383478,
      "learning_rate": 2.7263157894736846e-06,
      "loss": 0.0002,
      "step": 482
    },
    {
      "clip_ratio": 0.0007327690836973488,
      "epoch": 0.06594756963407974,
      "grad_norm": 0.03032069467008114,
      "kl": 0.0012884464822491282,
      "learning_rate": 2.7210526315789478e-06,
      "loss": 0.0,
      "step": 483
    },
    {
      "clip_ratio": 0.0004923844680888578,
      "epoch": 0.06608410704533042,
      "grad_norm": 0.030407611280679703,
      "kl": 0.0013384391750150826,
      "learning_rate": 2.7157894736842105e-06,
      "loss": -0.0002,
      "step": 484
    },
    {
      "clip_ratio": 0.0007657096430193633,
      "completion_length": 120.59375,
      "epoch": 0.0662206444565811,
      "grad_norm": 0.043436191976070404,
      "kl": 0.0014728355108672986,
      "learning_rate": 2.710526315789474e-06,
      "loss": 0.0001,
      "num_tokens": 1420660.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.30735570192337036,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 485
    },
    {
      "clip_ratio": 0.0010727702174335718,
      "epoch": 0.06635718186783178,
      "grad_norm": 0.044896580278873444,
      "kl": 0.0015333851679315558,
      "learning_rate": 2.705263157894737e-06,
      "loss": 0.0006,
      "step": 486
    },
    {
      "clip_ratio": 0.0008705731888767332,
      "epoch": 0.06649371927908247,
      "grad_norm": 0.04429871216416359,
      "kl": 0.0014375828959600767,
      "learning_rate": 2.7000000000000004e-06,
      "loss": 0.0003,
      "step": 487
    },
    {
      "clip_ratio": 0.0010845871875062585,
      "epoch": 0.06663025669033315,
      "grad_norm": 0.04436115548014641,
      "kl": 0.0015077010029926896,
      "learning_rate": 2.694736842105263e-06,
      "loss": 0.0001,
      "step": 488
    },
    {
      "clip_ratio": 0.00015703517419751734,
      "completion_length": 138.9375,
      "epoch": 0.06676679410158383,
      "grad_norm": 0.025293614715337753,
      "kl": 0.0016928213699429762,
      "learning_rate": 2.6894736842105267e-06,
      "loss": -0.0001,
      "num_tokens": 1431886.0,
      "reward": 0.578125,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.578125,
      "step": 489
    },
    {
      "clip_ratio": 0.00044642855937127024,
      "epoch": 0.06690333151283452,
      "grad_norm": 0.025101078674197197,
      "kl": 0.001892799466077122,
      "learning_rate": 2.68421052631579e-06,
      "loss": -0.0002,
      "step": 490
    },
    {
      "clip_ratio": 0.0003188775444868952,
      "epoch": 0.0670398689240852,
      "grad_norm": 0.025167712941765785,
      "kl": 0.001736639860610012,
      "learning_rate": 2.6789473684210526e-06,
      "loss": -0.0,
      "step": 491
    },
    {
      "clip_ratio": 0.0009300961974076927,
      "epoch": 0.06717640633533588,
      "grad_norm": 0.025257227942347527,
      "kl": 0.0017978880205191672,
      "learning_rate": 2.6736842105263162e-06,
      "loss": -0.0001,
      "step": 492
    },
    {
      "clip_ratio": 0.0003878545976476744,
      "completion_length": 147.4375,
      "epoch": 0.06731294374658657,
      "grad_norm": 0.022938350215554237,
      "kl": 0.0015158076530497056,
      "learning_rate": 2.668421052631579e-06,
      "loss": -0.0,
      "num_tokens": 1444020.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 493
    },
    {
      "clip_ratio": 0.0008949182229116559,
      "epoch": 0.06744948115783725,
      "grad_norm": 0.023230843245983124,
      "kl": 0.0014488295801129425,
      "learning_rate": 2.6631578947368426e-06,
      "loss": 0.0,
      "step": 494
    },
    {
      "clip_ratio": 0.0007672396022826433,
      "epoch": 0.06758601856908793,
      "grad_norm": 0.023685956373810768,
      "kl": 0.0014960741591494298,
      "learning_rate": 2.6578947368421053e-06,
      "loss": 0.0001,
      "step": 495
    },
    {
      "clip_ratio": 0.0004285848408471793,
      "epoch": 0.06772255598033862,
      "grad_norm": 0.022663943469524384,
      "kl": 0.0014857233782095136,
      "learning_rate": 2.652631578947369e-06,
      "loss": -0.0001,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.8125,
      "epoch": 0.0678590933915893,
      "grad_norm": 0.023737484589219093,
      "kl": 0.0015226335908664623,
      "learning_rate": 2.6473684210526316e-06,
      "loss": 0.0,
      "num_tokens": 1455314.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06799563080283998,
      "grad_norm": 0.023308604955673218,
      "kl": 0.0014484160819847602,
      "learning_rate": 2.6421052631578948e-06,
      "loss": 0.0001,
      "step": 498
    },
    {
      "clip_ratio": 0.00022163119865581393,
      "epoch": 0.06813216821409065,
      "grad_norm": 0.02275722846388817,
      "kl": 0.0015247040919348365,
      "learning_rate": 2.6368421052631584e-06,
      "loss": -0.0,
      "step": 499
    },
    {
      "clip_ratio": 0.00018939393339678645,
      "epoch": 0.06826870562534135,
      "grad_norm": 0.024789828807115555,
      "kl": 0.0015402598492073594,
      "learning_rate": 2.631578947368421e-06,
      "loss": 0.0001,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.34375,
      "epoch": 0.06840524303659203,
      "grad_norm": 0.029355965554714203,
      "kl": 0.0014410110106837237,
      "learning_rate": 2.6263157894736847e-06,
      "loss": -0.0001,
      "num_tokens": 1466205.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 501
    },
    {
      "clip_ratio": 0.0008779402705840766,
      "epoch": 0.0685417804478427,
      "grad_norm": 0.02950431779026985,
      "kl": 0.0014032465078344103,
      "learning_rate": 2.6210526315789474e-06,
      "loss": 0.0001,
      "step": 502
    },
    {
      "clip_ratio": 0.0004614599165506661,
      "epoch": 0.0686783178590934,
      "grad_norm": 0.02926722541451454,
      "kl": 0.0014136335121293087,
      "learning_rate": 2.615789473684211e-06,
      "loss": 0.0001,
      "step": 503
    },
    {
      "clip_ratio": 0.00046644278336316347,
      "epoch": 0.06881485527034407,
      "grad_norm": 0.02934475615620613,
      "kl": 0.0013926027413617703,
      "learning_rate": 2.6105263157894738e-06,
      "loss": -0.0001,
      "step": 504
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "completion_length": 117.78125,
      "epoch": 0.06895139268159475,
      "grad_norm": 0.0215784702450037,
      "kl": 0.0014604836833314039,
      "learning_rate": 2.605263157894737e-06,
      "loss": 0.0,
      "num_tokens": 1476486.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 505
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.06908793009284545,
      "grad_norm": 0.02178478240966797,
      "kl": 0.0016060788602771936,
      "learning_rate": 2.6e-06,
      "loss": -0.0001,
      "step": 506
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06922446750409612,
      "grad_norm": 0.021680708974599838,
      "kl": 0.0015948796844895696,
      "learning_rate": 2.5947368421052633e-06,
      "loss": 0.0001,
      "step": 507
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0693610049153468,
      "grad_norm": 0.021806875243782997,
      "kl": 0.001626272858629818,
      "learning_rate": 2.589473684210527e-06,
      "loss": -0.0001,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.96875,
      "epoch": 0.0694975423265975,
      "grad_norm": 0.025997139513492584,
      "kl": 0.0012444071562640602,
      "learning_rate": 2.5842105263157896e-06,
      "loss": -0.0001,
      "num_tokens": 1487469.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 509
    },
    {
      "clip_ratio": 0.000666738415020518,
      "epoch": 0.06963407973784817,
      "grad_norm": 0.026637002825737,
      "kl": 0.0013541714697566931,
      "learning_rate": 2.578947368421053e-06,
      "loss": -0.0001,
      "step": 510
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06977061714909885,
      "grad_norm": 0.026402544230222702,
      "kl": 0.0012964879524588468,
      "learning_rate": 2.573684210526316e-06,
      "loss": -0.0,
      "step": 511
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.06990715456034953,
      "grad_norm": 0.0267286766320467,
      "kl": 0.0014071960695218877,
      "learning_rate": 2.568421052631579e-06,
      "loss": -0.0,
      "step": 512
    },
    {
      "clip_ratio": 0.0003955851716455072,
      "completion_length": 118.46875,
      "epoch": 0.07004369197160022,
      "grad_norm": 0.03863956406712532,
      "kl": 0.001288709736400051,
      "learning_rate": 2.5631578947368422e-06,
      "loss": -0.0003,
      "num_tokens": 1498808.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.30735570192337036,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 513
    },
    {
      "clip_ratio": 0.00048096767568495125,
      "epoch": 0.0701802293828509,
      "grad_norm": 0.03899668529629707,
      "kl": 0.0013409877155936556,
      "learning_rate": 2.5578947368421054e-06,
      "loss": -0.0,
      "step": 514
    },
    {
      "clip_ratio": 0.0004269125493010506,
      "epoch": 0.07031676679410158,
      "grad_norm": 0.03946976363658905,
      "kl": 0.0014272014959715307,
      "learning_rate": 2.552631578947369e-06,
      "loss": -0.0001,
      "step": 515
    },
    {
      "clip_ratio": 0.0005278866447042674,
      "epoch": 0.07045330420535227,
      "grad_norm": 0.03886709362268448,
      "kl": 0.0013788384367217077,
      "learning_rate": 2.5473684210526317e-06,
      "loss": -0.0,
      "step": 516
    },
    {
      "clip_ratio": 0.0007051962893456221,
      "completion_length": 137.40625,
      "epoch": 0.07058984161660295,
      "grad_norm": 0.030156614258885384,
      "kl": 0.0012403783885019948,
      "learning_rate": 2.5421052631578953e-06,
      "loss": -0.0002,
      "num_tokens": 1510737.0,
      "reward": 0.4937500059604645,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.4937500059604645,
      "step": 517
    },
    {
      "clip_ratio": 0.0010378391889389604,
      "epoch": 0.07072637902785363,
      "grad_norm": 0.030437057837843895,
      "kl": 0.0012195480640002643,
      "learning_rate": 2.536842105263158e-06,
      "loss": 0.0001,
      "step": 518
    },
    {
      "clip_ratio": 0.0005122950533404946,
      "epoch": 0.07086291643910432,
      "grad_norm": 0.030496113002300262,
      "kl": 0.001190645452879835,
      "learning_rate": 2.5315789473684212e-06,
      "loss": -0.0002,
      "step": 519
    },
    {
      "clip_ratio": 0.0006950435927137733,
      "epoch": 0.070999453850355,
      "grad_norm": 0.02995339408516884,
      "kl": 0.0010328257139917696,
      "learning_rate": 2.5263157894736844e-06,
      "loss": -0.0002,
      "step": 520
    },
    {
      "clip_ratio": 0.00037165620597079396,
      "completion_length": 130.6875,
      "epoch": 0.07113599126160568,
      "grad_norm": 0.027491774410009384,
      "kl": 0.0015749909580335952,
      "learning_rate": 2.5210526315789475e-06,
      "loss": 0.0003,
      "num_tokens": 1521387.0,
      "reward": 0.550000011920929,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.550000011920929,
      "step": 521
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.07127252867285637,
      "grad_norm": 0.02750663459300995,
      "kl": 0.0013783774675175664,
      "learning_rate": 2.5157894736842107e-06,
      "loss": 0.0002,
      "step": 522
    },
    {
      "clip_ratio": 0.0005790924478787929,
      "epoch": 0.07140906608410705,
      "grad_norm": 0.027644384652376175,
      "kl": 0.0013931172179582063,
      "learning_rate": 2.510526315789474e-06,
      "loss": 0.0003,
      "step": 523
    },
    {
      "clip_ratio": 0.00037165620597079396,
      "epoch": 0.07154560349535773,
      "grad_norm": 0.02771710976958275,
      "kl": 0.001518670924269827,
      "learning_rate": 2.5052631578947375e-06,
      "loss": 0.0002,
      "step": 524
    },
    {
      "completion_length": 133.84375,
      "epoch": 0.0716821409066084,
      "grad_norm": 0.0,
      "learning_rate": 2.5e-06,
      "loss": 0.0,
      "num_tokens": 1532574.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.0,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 525
    },
    {
      "epoch": 0.0718186783178591,
      "grad_norm": 0.0,
      "learning_rate": 2.4947368421052634e-06,
      "loss": 0.0,
      "step": 526
    },
    {
      "epoch": 0.07195521572910978,
      "grad_norm": 0.0,
      "learning_rate": 2.4894736842105265e-06,
      "loss": 0.0,
      "step": 527
    },
    {
      "epoch": 0.07209175314036045,
      "grad_norm": 0.0,
      "learning_rate": 2.4842105263157897e-06,
      "loss": 0.0,
      "step": 528
    },
    {
      "clip_ratio": 0.0002982954465551302,
      "completion_length": 142.46875,
      "epoch": 0.07222829055161115,
      "grad_norm": 0.029288657009601593,
      "kl": 0.0009811991058086278,
      "learning_rate": 2.478947368421053e-06,
      "loss": 0.0003,
      "num_tokens": 1544593.0,
      "reward": 0.7718750238418579,
      "reward_std": 0.2962018847465515,
      "rewards/reward_fn": 0.7718750238418579,
      "step": 529
    },
    {
      "clip_ratio": 0.0008637555874884129,
      "epoch": 0.07236482796286182,
      "grad_norm": 0.029271140694618225,
      "kl": 0.0010114734732269426,
      "learning_rate": 2.473684210526316e-06,
      "loss": 0.0003,
      "step": 530
    },
    {
      "clip_ratio": 0.0008495510555803776,
      "epoch": 0.0725013653741125,
      "grad_norm": 0.028701355680823326,
      "kl": 0.0010413858981337398,
      "learning_rate": 2.468421052631579e-06,
      "loss": 0.0002,
      "step": 531
    },
    {
      "clip_ratio": 0.0005510496121132746,
      "epoch": 0.0726379027853632,
      "grad_norm": 0.02870851755142212,
      "kl": 0.0010655641935954918,
      "learning_rate": 2.4631578947368424e-06,
      "loss": 0.0004,
      "step": 532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.78125,
      "epoch": 0.07277444019661387,
      "grad_norm": 0.024263199418783188,
      "kl": 0.0011644043825072004,
      "learning_rate": 2.4578947368421055e-06,
      "loss": -0.0002,
      "num_tokens": 1556426.0,
      "reward": 0.578125,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.578125,
      "step": 533
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07291097760786455,
      "grad_norm": 0.023698801174759865,
      "kl": 0.001130858380747668,
      "learning_rate": 2.4526315789473687e-06,
      "loss": -0.0002,
      "step": 534
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "epoch": 0.07304751501911524,
      "grad_norm": 0.024516500532627106,
      "kl": 0.001214617892401293,
      "learning_rate": 2.447368421052632e-06,
      "loss": -0.0,
      "step": 535
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "epoch": 0.07318405243036592,
      "grad_norm": 0.0237928107380867,
      "kl": 0.0012256574964339961,
      "learning_rate": 2.442105263157895e-06,
      "loss": -0.0002,
      "step": 536
    },
    {
      "clip_ratio": 0.0012445505853975192,
      "completion_length": 140.96875,
      "epoch": 0.0733205898416166,
      "grad_norm": 0.034096427261829376,
      "kl": 0.001380213063384872,
      "learning_rate": 2.436842105263158e-06,
      "loss": -0.0001,
      "num_tokens": 1568913.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.35490381717681885,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 537
    },
    {
      "clip_ratio": 0.0008802999946055934,
      "epoch": 0.07345712725286728,
      "grad_norm": 0.033610839396715164,
      "kl": 0.0014004747145008878,
      "learning_rate": 2.4315789473684213e-06,
      "loss": -0.0001,
      "step": 538
    },
    {
      "clip_ratio": 0.00060980801936239,
      "epoch": 0.07359366466411797,
      "grad_norm": 0.03407885134220123,
      "kl": 0.0013439315152936615,
      "learning_rate": 2.4263157894736845e-06,
      "loss": -0.0001,
      "step": 539
    },
    {
      "clip_ratio": 0.0004953578027198091,
      "epoch": 0.07373020207536865,
      "grad_norm": 0.033993177115917206,
      "kl": 0.0013473886228894116,
      "learning_rate": 2.4210526315789477e-06,
      "loss": -0.0,
      "step": 540
    },
    {
      "clip_ratio": 0.001183721236884594,
      "completion_length": 155.21875,
      "epoch": 0.07386673948661933,
      "grad_norm": 0.0310914795845747,
      "kl": 0.0014184249976096908,
      "learning_rate": 2.415789473684211e-06,
      "loss": 0.0002,
      "num_tokens": 1581456.0,
      "reward": 0.71875,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.71875,
      "step": 541
    },
    {
      "clip_ratio": 0.0008445946150459349,
      "epoch": 0.07400327689787002,
      "grad_norm": 0.03024432435631752,
      "kl": 0.0014385326594492653,
      "learning_rate": 2.410526315789474e-06,
      "loss": 0.0,
      "step": 542
    },
    {
      "clip_ratio": 0.00036845837894361466,
      "epoch": 0.0741398143091207,
      "grad_norm": 0.030283359810709953,
      "kl": 0.0013663766912941355,
      "learning_rate": 2.4052631578947367e-06,
      "loss": -0.0001,
      "step": 543
    },
    {
      "clip_ratio": 0.00018601190822664648,
      "epoch": 0.07427635172037138,
      "grad_norm": 0.03067188523709774,
      "kl": 0.0013775449333479628,
      "learning_rate": 2.4000000000000003e-06,
      "loss": 0.0002,
      "step": 544
    },
    {
      "clip_ratio": 0.0005580357392318547,
      "completion_length": 119.5625,
      "epoch": 0.07441288913162207,
      "grad_norm": 0.02598925493657589,
      "kl": 0.001481451478866802,
      "learning_rate": 2.3947368421052635e-06,
      "loss": 0.0,
      "num_tokens": 1592442.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 545
    },
    {
      "clip_ratio": 0.0001653439103392884,
      "epoch": 0.07454942654287275,
      "grad_norm": 0.028425514698028564,
      "kl": 0.0016474172589369118,
      "learning_rate": 2.3894736842105266e-06,
      "loss": 0.0,
      "step": 546
    },
    {
      "clip_ratio": 0.00048688988317735493,
      "epoch": 0.07468596395412343,
      "grad_norm": 0.027035797014832497,
      "kl": 0.0014499265525955707,
      "learning_rate": 2.38421052631579e-06,
      "loss": -0.0,
      "step": 547
    },
    {
      "clip_ratio": 0.0005580357392318547,
      "epoch": 0.07482250136537412,
      "grad_norm": 0.027835296466946602,
      "kl": 0.0014104215515544638,
      "learning_rate": 2.378947368421053e-06,
      "loss": 0.0,
      "step": 548
    },
    {
      "clip_ratio": 0.001446785288862884,
      "completion_length": 141.5625,
      "epoch": 0.0749590387766248,
      "grad_norm": 0.02882438339293003,
      "kl": 0.0013727315872529289,
      "learning_rate": 2.373684210526316e-06,
      "loss": 0.0001,
      "num_tokens": 1603960.0,
      "reward": 0.4625000059604645,
      "reward_std": 0.23995190858840942,
      "rewards/reward_fn": 0.4625000059604645,
      "step": 549
    },
    {
      "clip_ratio": 0.0004970216541551054,
      "epoch": 0.07509557618787548,
      "grad_norm": 0.028407303616404533,
      "kl": 0.001324782728261198,
      "learning_rate": 2.368421052631579e-06,
      "loss": -0.0,
      "step": 550
    },
    {
      "clip_ratio": 0.0012807868115487508,
      "epoch": 0.07523211359912615,
      "grad_norm": 0.028264285996556282,
      "kl": 0.001328441574514727,
      "learning_rate": 2.363157894736842e-06,
      "loss": -0.0001,
      "step": 551
    },
    {
      "clip_ratio": 0.0008249489692389034,
      "epoch": 0.07536865101037685,
      "grad_norm": 0.029252443462610245,
      "kl": 0.0013075096630927874,
      "learning_rate": 2.357894736842105e-06,
      "loss": 0.0001,
      "step": 552
    },
    {
      "clip_ratio": 0.0006459349242504686,
      "completion_length": 131.78125,
      "epoch": 0.07550518842162753,
      "grad_norm": 0.030669942498207092,
      "kl": 0.0012164198715254315,
      "learning_rate": 2.352631578947369e-06,
      "loss": 0.0,
      "num_tokens": 1615629.0,
      "reward": 0.71875,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.71875,
      "step": 553
    },
    {
      "clip_ratio": 0.0011245697824051604,
      "epoch": 0.0756417258328782,
      "grad_norm": 0.030605750158429146,
      "kl": 0.00125411778481066,
      "learning_rate": 2.347368421052632e-06,
      "loss": -0.0001,
      "step": 554
    },
    {
      "clip_ratio": 0.00022163119865581393,
      "epoch": 0.0757782632441289,
      "grad_norm": 0.030652441084384918,
      "kl": 0.0013444009427985293,
      "learning_rate": 2.342105263157895e-06,
      "loss": -0.0001,
      "step": 555
    },
    {
      "clip_ratio": 0.0008194020629161969,
      "epoch": 0.07591480065537957,
      "grad_norm": 0.03041120246052742,
      "kl": 0.0011914784172404325,
      "learning_rate": 2.3368421052631583e-06,
      "loss": -0.0002,
      "step": 556
    },
    {
      "clip_ratio": 0.00019409938249737024,
      "completion_length": 153.25,
      "epoch": 0.07605133806663025,
      "grad_norm": 0.01942029595375061,
      "kl": 0.0013100661726639373,
      "learning_rate": 2.331578947368421e-06,
      "loss": 0.0002,
      "num_tokens": 1628237.0,
      "reward": 0.690625011920929,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.690625011920929,
      "step": 557
    },
    {
      "clip_ratio": 0.00019409938249737024,
      "epoch": 0.07618787547788095,
      "grad_norm": 0.019458364695310593,
      "kl": 0.0011170834286531317,
      "learning_rate": 2.326315789473684e-06,
      "loss": 0.0002,
      "step": 558
    },
    {
      "clip_ratio": 0.0006215237954165787,
      "epoch": 0.07632441288913162,
      "grad_norm": 0.020504020154476166,
      "kl": 0.0012800872691514087,
      "learning_rate": 2.3210526315789473e-06,
      "loss": 0.0002,
      "step": 559
    },
    {
      "clip_ratio": 0.00019409938249737024,
      "epoch": 0.0764609503003823,
      "grad_norm": 0.019755452871322632,
      "kl": 0.001140367172411061,
      "learning_rate": 2.3157894736842105e-06,
      "loss": 0.0002,
      "step": 560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.96875,
      "epoch": 0.07659748771163298,
      "grad_norm": 0.02310808189213276,
      "kl": 0.0012227943288962706,
      "learning_rate": 2.310526315789474e-06,
      "loss": -0.0001,
      "num_tokens": 1639516.0,
      "reward": 0.887499988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.887499988079071,
      "step": 561
    },
    {
      "clip_ratio": 0.0005299604235915467,
      "epoch": 0.07673402512288367,
      "grad_norm": 0.022301802411675453,
      "kl": 0.0012091731859982247,
      "learning_rate": 2.3052631578947373e-06,
      "loss": -0.0002,
      "step": 562
    },
    {
      "clip_ratio": 0.00021701389050576836,
      "epoch": 0.07687056253413435,
      "grad_norm": 0.02227598801255226,
      "kl": 0.0011962663684244035,
      "learning_rate": 2.3000000000000004e-06,
      "loss": -0.0001,
      "step": 563
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07700709994538503,
      "grad_norm": 0.02261110581457615,
      "kl": 0.0012483229429562925,
      "learning_rate": 2.294736842105263e-06,
      "loss": -0.0002,
      "step": 564
    },
    {
      "clip_ratio": 0.00038580247201025486,
      "completion_length": 111.53125,
      "epoch": 0.07714363735663572,
      "grad_norm": 0.03374376520514488,
      "kl": 0.0012109982199035585,
      "learning_rate": 2.2894736842105263e-06,
      "loss": 0.0,
      "num_tokens": 1649869.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.690625011920929,
      "step": 565
    },
    {
      "clip_ratio": 0.0007292090740520507,
      "epoch": 0.0772801747678864,
      "grad_norm": 0.03348137438297272,
      "kl": 0.001319482877079281,
      "learning_rate": 2.2842105263157895e-06,
      "loss": 0.0001,
      "step": 566
    },
    {
      "clip_ratio": 0.000866827875142917,
      "epoch": 0.07741671217913708,
      "grad_norm": 0.032686445862054825,
      "kl": 0.0012758291504724184,
      "learning_rate": 2.2789473684210527e-06,
      "loss": 0.0001,
      "step": 567
    },
    {
      "clip_ratio": 0.000631865463219583,
      "epoch": 0.07755324959038777,
      "grad_norm": 0.03336594998836517,
      "kl": 0.0013503911413863534,
      "learning_rate": 2.273684210526316e-06,
      "loss": 0.0001,
      "step": 568
    },
    {
      "clip_ratio": 0.0005172694654902443,
      "completion_length": 138.75,
      "epoch": 0.07768978700163845,
      "grad_norm": 0.0384264700114727,
      "kl": 0.0013226108276285231,
      "learning_rate": 2.268421052631579e-06,
      "loss": -0.0001,
      "num_tokens": 1661965.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 569
    },
    {
      "clip_ratio": 0.0009963809716282412,
      "epoch": 0.07782632441288913,
      "grad_norm": 0.03718084841966629,
      "kl": 0.0013709642798858113,
      "learning_rate": 2.2631578947368426e-06,
      "loss": 0.0001,
      "step": 570
    },
    {
      "clip_ratio": 0.0006437876872951165,
      "epoch": 0.07796286182413982,
      "grad_norm": 0.03695379197597504,
      "kl": 0.0014017439098097384,
      "learning_rate": 2.2578947368421057e-06,
      "loss": 0.0002,
      "step": 571
    },
    {
      "clip_ratio": 0.0006249233265407383,
      "epoch": 0.0780993992353905,
      "grad_norm": 0.03681650757789612,
      "kl": 0.0013024672862229636,
      "learning_rate": 2.2526315789473685e-06,
      "loss": -0.0001,
      "step": 572
    },
    {
      "clip_ratio": 0.0007458344407496043,
      "completion_length": 131.125,
      "epoch": 0.07823593664664118,
      "grad_norm": 0.033540453761816025,
      "kl": 0.0015056260308483616,
      "learning_rate": 2.2473684210526316e-06,
      "loss": 0.0001,
      "num_tokens": 1673245.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.289951890707016,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 573
    },
    {
      "clip_ratio": 0.0011247631337027997,
      "epoch": 0.07837247405789186,
      "grad_norm": 0.034303613007068634,
      "kl": 0.0013787490188406082,
      "learning_rate": 2.242105263157895e-06,
      "loss": 0.0001,
      "step": 574
    },
    {
      "clip_ratio": 0.0005023798657930456,
      "epoch": 0.07850901146914255,
      "grad_norm": 0.03354345262050629,
      "kl": 0.0014687197344755987,
      "learning_rate": 2.236842105263158e-06,
      "loss": 0.0002,
      "step": 575
    },
    {
      "clip_ratio": 0.0006665511391474865,
      "epoch": 0.07864554888039323,
      "grad_norm": 0.034005530178546906,
      "kl": 0.0014215231167327147,
      "learning_rate": 2.231578947368421e-06,
      "loss": -0.0,
      "step": 576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.21875,
      "epoch": 0.0787820862916439,
      "grad_norm": 0.028434976935386658,
      "kl": 0.0013502443598554237,
      "learning_rate": 2.2263157894736843e-06,
      "loss": 0.0001,
      "num_tokens": 1684708.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 577
    },
    {
      "clip_ratio": 0.0005482456181198359,
      "epoch": 0.0789186237028946,
      "grad_norm": 0.028591668233275414,
      "kl": 0.0012434296741048456,
      "learning_rate": 2.221052631578948e-06,
      "loss": -0.0,
      "step": 578
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.07905516111414528,
      "grad_norm": 0.028464889153838158,
      "kl": 0.0013504233384082909,
      "learning_rate": 2.2157894736842106e-06,
      "loss": -0.0002,
      "step": 579
    },
    {
      "clip_ratio": 0.0005482456181198359,
      "epoch": 0.07919169852539595,
      "grad_norm": 0.0283973291516304,
      "kl": 0.0013455405560307554,
      "learning_rate": 2.2105263157894738e-06,
      "loss": -0.0001,
      "step": 580
    },
    {
      "clip_ratio": 0.0003379267000127584,
      "completion_length": 144.90625,
      "epoch": 0.07932823593664665,
      "grad_norm": 0.01949334517121315,
      "kl": 0.0012513340170698939,
      "learning_rate": 2.205263157894737e-06,
      "loss": 0.0001,
      "num_tokens": 1697297.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 581
    },
    {
      "clip_ratio": 0.0006014183891238645,
      "epoch": 0.07946477334789732,
      "grad_norm": 0.01936924457550049,
      "kl": 0.0012721861930913292,
      "learning_rate": 2.2e-06,
      "loss": 0.0,
      "step": 582
    },
    {
      "clip_ratio": 0.0005715087900171056,
      "epoch": 0.079601310759148,
      "grad_norm": 0.018492933362722397,
      "kl": 0.0012525502661446808,
      "learning_rate": 2.1947368421052633e-06,
      "loss": 0.0001,
      "step": 583
    },
    {
      "clip_ratio": 0.00016622339899186045,
      "epoch": 0.0797378481703987,
      "grad_norm": 0.017930587753653526,
      "kl": 0.0011951930664508836,
      "learning_rate": 2.1894736842105264e-06,
      "loss": -0.0,
      "step": 584
    },
    {
      "clip_ratio": 0.0004695706011261791,
      "completion_length": 139.1875,
      "epoch": 0.07987438558164937,
      "grad_norm": 0.028043018653988838,
      "kl": 0.0011986813478870317,
      "learning_rate": 2.1842105263157896e-06,
      "loss": 0.0,
      "num_tokens": 1708779.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 585
    },
    {
      "clip_ratio": 0.000575878111703787,
      "epoch": 0.08001092299290005,
      "grad_norm": 0.027894066646695137,
      "kl": 0.0012194329829071648,
      "learning_rate": 2.1789473684210528e-06,
      "loss": 0.0002,
      "step": 586
    },
    {
      "clip_ratio": 0.00023320894979406148,
      "epoch": 0.08014746040415073,
      "grad_norm": 0.02750025875866413,
      "kl": 0.0012120107594455476,
      "learning_rate": 2.173684210526316e-06,
      "loss": -0.0001,
      "step": 587
    },
    {
      "clip_ratio": 0.0008329564152518287,
      "epoch": 0.08028399781540142,
      "grad_norm": 0.028404943645000458,
      "kl": 0.0013067316140222829,
      "learning_rate": 2.168421052631579e-06,
      "loss": 0.0,
      "step": 588
    },
    {
      "clip_ratio": 0.0001849112450145185,
      "completion_length": 112.21875,
      "epoch": 0.0804205352266521,
      "grad_norm": 0.025281289592385292,
      "kl": 0.0012155929061918869,
      "learning_rate": 2.1631578947368423e-06,
      "loss": -0.0001,
      "num_tokens": 1720098.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 589
    },
    {
      "clip_ratio": 0.0004163927223999053,
      "epoch": 0.08055707263790278,
      "grad_norm": 0.025050563737750053,
      "kl": 0.0011125751880172174,
      "learning_rate": 2.1578947368421054e-06,
      "loss": -0.0002,
      "step": 590
    },
    {
      "clip_ratio": 0.0004163927223999053,
      "epoch": 0.08069361004915347,
      "grad_norm": 0.025595294311642647,
      "kl": 0.001212971978020505,
      "learning_rate": 2.1526315789473686e-06,
      "loss": -0.0001,
      "step": 591
    },
    {
      "clip_ratio": 0.0007168735028244555,
      "epoch": 0.08083014746040415,
      "grad_norm": 0.024896765127778053,
      "kl": 0.0012561739672491967,
      "learning_rate": 2.1473684210526317e-06,
      "loss": -0.0001,
      "step": 592
    },
    {
      "clip_ratio": 0.0005804953689221293,
      "completion_length": 133.1875,
      "epoch": 0.08096668487165483,
      "grad_norm": 0.03213340416550636,
      "kl": 0.0012622286103578517,
      "learning_rate": 2.142105263157895e-06,
      "loss": -0.0001,
      "num_tokens": 1731732.0,
      "reward": 0.6031249761581421,
      "reward_std": 0.2962018847465515,
      "rewards/reward_fn": 0.6031249761581421,
      "step": 593
    },
    {
      "clip_ratio": 0.00047927717969287187,
      "epoch": 0.08110322228290552,
      "grad_norm": 0.031801752746105194,
      "kl": 0.0011793114590545883,
      "learning_rate": 2.136842105263158e-06,
      "loss": -0.0001,
      "step": 594
    },
    {
      "clip_ratio": 0.0005405196279753,
      "epoch": 0.0812397596941562,
      "grad_norm": 0.03253817930817604,
      "kl": 0.0011914531751244795,
      "learning_rate": 2.1315789473684212e-06,
      "loss": -0.0001,
      "step": 595
    },
    {
      "clip_ratio": 0.0007670347695238888,
      "epoch": 0.08137629710540688,
      "grad_norm": 0.03316066786646843,
      "kl": 0.0012645062852243427,
      "learning_rate": 2.1263157894736844e-06,
      "loss": -0.0,
      "step": 596
    },
    {
      "clip_ratio": 0.0003789131442317739,
      "completion_length": 144.28125,
      "epoch": 0.08151283451665757,
      "grad_norm": 0.028041541576385498,
      "kl": 0.001233616606441501,
      "learning_rate": 2.1210526315789476e-06,
      "loss": -0.0,
      "num_tokens": 1744297.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 597
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08164937192790825,
      "grad_norm": 0.028079692274332047,
      "kl": 0.0011803826682807994,
      "learning_rate": 2.1157894736842107e-06,
      "loss": 0.0,
      "step": 598
    },
    {
      "clip_ratio": 0.0004720039287349209,
      "epoch": 0.08178590933915893,
      "grad_norm": 0.02811983786523342,
      "kl": 0.0012397360787872458,
      "learning_rate": 2.110526315789474e-06,
      "loss": -0.0002,
      "step": 599
    },
    {
      "clip_ratio": 0.0004097465134691447,
      "epoch": 0.0819224467504096,
      "grad_norm": 0.028073711320757866,
      "kl": 0.0012096571817892254,
      "learning_rate": 2.105263157894737e-06,
      "loss": -0.0,
      "step": 600
    },
    {
      "clip_ratio": 0.0010307422780897468,
      "completion_length": 129.46875,
      "epoch": 0.0820589841616603,
      "grad_norm": 0.03012477047741413,
      "kl": 0.001331004872554331,
      "learning_rate": 2.1000000000000002e-06,
      "loss": 0.0002,
      "num_tokens": 1755712.0,
      "reward": 0.746874988079071,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.746874988079071,
      "step": 601
    },
    {
      "clip_ratio": 0.0010868488752748817,
      "epoch": 0.08219552157291098,
      "grad_norm": 0.027972938492894173,
      "kl": 0.0012423287271303707,
      "learning_rate": 2.0947368421052634e-06,
      "loss": 0.0001,
      "step": 602
    },
    {
      "clip_ratio": 0.0017305946748820134,
      "epoch": 0.08233205898416165,
      "grad_norm": 0.027943221852183342,
      "kl": 0.0013830821371811908,
      "learning_rate": 2.0894736842105266e-06,
      "loss": 0.0002,
      "step": 603
    },
    {
      "clip_ratio": 0.0005490178009495139,
      "epoch": 0.08246859639541235,
      "grad_norm": 0.028850063681602478,
      "kl": 0.0013167489851184655,
      "learning_rate": 2.0842105263157897e-06,
      "loss": -0.0,
      "step": 604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.90625,
      "epoch": 0.08260513380666303,
      "grad_norm": 0.024748941883444786,
      "kl": 0.0014295073833636707,
      "learning_rate": 2.078947368421053e-06,
      "loss": 0.0,
      "num_tokens": 1766085.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 605
    },
    {
      "clip_ratio": 0.00019654087373055518,
      "epoch": 0.0827416712179137,
      "grad_norm": 0.02504793554544449,
      "kl": 0.001298841527386685,
      "learning_rate": 2.073684210526316e-06,
      "loss": -0.0,
      "step": 606
    },
    {
      "clip_ratio": 0.00020559210679493845,
      "epoch": 0.0828782086291644,
      "grad_norm": 0.02473580837249756,
      "kl": 0.0013759256125922548,
      "learning_rate": 2.068421052631579e-06,
      "loss": 0.0,
      "step": 607
    },
    {
      "clip_ratio": 0.00020559210679493845,
      "epoch": 0.08301474604041507,
      "grad_norm": 0.024983009323477745,
      "kl": 0.0013146149885869818,
      "learning_rate": 2.0631578947368424e-06,
      "loss": -0.0001,
      "step": 608
    },
    {
      "clip_ratio": 0.00013950893480796367,
      "completion_length": 159.40625,
      "epoch": 0.08315128345166575,
      "grad_norm": 0.02585506998002529,
      "kl": 0.0011791864890255965,
      "learning_rate": 2.0578947368421055e-06,
      "loss": -0.0002,
      "num_tokens": 1778654.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 609
    },
    {
      "clip_ratio": 0.000621384650003165,
      "epoch": 0.08328782086291645,
      "grad_norm": 0.026020504534244537,
      "kl": 0.0011703061281878036,
      "learning_rate": 2.0526315789473687e-06,
      "loss": -0.0001,
      "step": 610
    },
    {
      "clip_ratio": 0.0009574061550665647,
      "epoch": 0.08342435827416712,
      "grad_norm": 0.026134071871638298,
      "kl": 0.0012730637799904798,
      "learning_rate": 2.047368421052632e-06,
      "loss": -0.0001,
      "step": 611
    },
    {
      "clip_ratio": 0.00047553043987136334,
      "epoch": 0.0835608956854178,
      "grad_norm": 0.02622632496058941,
      "kl": 0.0011777592935686698,
      "learning_rate": 2.042105263157895e-06,
      "loss": -0.0001,
      "step": 612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.28125,
      "epoch": 0.08369743309666848,
      "grad_norm": 0.029103118926286697,
      "kl": 0.0013813998102705227,
      "learning_rate": 2.036842105263158e-06,
      "loss": -0.0,
      "num_tokens": 1789519.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.2511056959629059,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 613
    },
    {
      "clip_ratio": 0.0003215289398212917,
      "epoch": 0.08383397050791917,
      "grad_norm": 0.028894973918795586,
      "kl": 0.0013992627809784608,
      "learning_rate": 2.031578947368421e-06,
      "loss": -0.0001,
      "step": 614
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08397050791916985,
      "grad_norm": 0.02881738543510437,
      "kl": 0.0014436542860494228,
      "learning_rate": 2.026315789473684e-06,
      "loss": -0.0003,
      "step": 615
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08410704533042053,
      "grad_norm": 0.029226887971162796,
      "kl": 0.0015083678954397328,
      "learning_rate": 2.0210526315789477e-06,
      "loss": -0.0002,
      "step": 616
    },
    {
      "clip_ratio": 0.0003229463181924075,
      "completion_length": 168.1875,
      "epoch": 0.08424358274167122,
      "grad_norm": 0.024407584220170975,
      "kl": 0.0014167944955261191,
      "learning_rate": 2.015789473684211e-06,
      "loss": -0.0,
      "num_tokens": 1802857.0,
      "reward": 0.4625000059604645,
      "reward_std": 0.17499999701976776,
      "rewards/reward_fn": 0.4625000059604645,
      "step": 617
    },
    {
      "clip_ratio": 0.00030754510225960985,
      "epoch": 0.0843801201529219,
      "grad_norm": 0.025380659848451614,
      "kl": 0.0012805712885892717,
      "learning_rate": 2.010526315789474e-06,
      "loss": 0.0001,
      "step": 618
    },
    {
      "clip_ratio": 0.00031566374673275277,
      "epoch": 0.08451665756417258,
      "grad_norm": 0.025637686252593994,
      "kl": 0.0012550040901260218,
      "learning_rate": 2.005263157894737e-06,
      "loss": 0.0001,
      "step": 619
    },
    {
      "clip_ratio": 0.00017959770048037171,
      "epoch": 0.08465319497542327,
      "grad_norm": 0.02544740028679371,
      "kl": 0.0013416227866400732,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0001,
      "step": 620
    },
    {
      "clip_ratio": 0.000400834578613285,
      "completion_length": 143.625,
      "epoch": 0.08478973238667395,
      "grad_norm": 0.025843657553195953,
      "kl": 0.001131466372498835,
      "learning_rate": 1.994736842105263e-06,
      "loss": 0.0,
      "num_tokens": 1815005.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.690625011920929,
      "step": 621
    },
    {
      "clip_ratio": 0.0006174136869958602,
      "epoch": 0.08492626979792463,
      "grad_norm": 0.02638258785009384,
      "kl": 0.0010728317356551997,
      "learning_rate": 1.9894736842105262e-06,
      "loss": 0.0,
      "step": 622
    },
    {
      "clip_ratio": 0.0002513130530132912,
      "epoch": 0.08506280720917532,
      "grad_norm": 0.025244932621717453,
      "kl": 0.0011344658514644834,
      "learning_rate": 1.9842105263157894e-06,
      "loss": -0.0001,
      "step": 623
    },
    {
      "clip_ratio": 0.00045424259587889537,
      "epoch": 0.085199344620426,
      "grad_norm": 0.025821218267083168,
      "kl": 0.0011071752214775188,
      "learning_rate": 1.978947368421053e-06,
      "loss": 0.0,
      "step": 624
    },
    {
      "clip_ratio": 0.00040070006798487157,
      "completion_length": 138.96875,
      "epoch": 0.08533588203167668,
      "grad_norm": 0.020593425258994102,
      "kl": 0.0014952101555536501,
      "learning_rate": 1.973684210526316e-06,
      "loss": 0.0001,
      "num_tokens": 1827092.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 625
    },
    {
      "clip_ratio": 0.0007846917069400661,
      "epoch": 0.08547241944292736,
      "grad_norm": 0.01970861665904522,
      "kl": 0.0015636316857126076,
      "learning_rate": 1.9684210526315793e-06,
      "loss": 0.0002,
      "step": 626
    },
    {
      "clip_ratio": 0.00026483050896786153,
      "epoch": 0.08560895685417805,
      "grad_norm": 0.020389128476381302,
      "kl": 0.0015640047313354444,
      "learning_rate": 1.9631578947368425e-06,
      "loss": 0.0001,
      "step": 627
    },
    {
      "clip_ratio": 0.0005353983433451504,
      "epoch": 0.08574549426542873,
      "grad_norm": 0.020528005436062813,
      "kl": 0.001569879857925116,
      "learning_rate": 1.9578947368421052e-06,
      "loss": 0.0003,
      "step": 628
    },
    {
      "clip_ratio": 0.00027173911803402007,
      "completion_length": 106.28125,
      "epoch": 0.0858820316766794,
      "grad_norm": 0.025182850658893585,
      "kl": 0.0017427417806175072,
      "learning_rate": 1.9526315789473684e-06,
      "loss": 0.0001,
      "num_tokens": 1838041.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 629
    },
    {
      "clip_ratio": 0.00015394088404718786,
      "epoch": 0.0860185690879301,
      "grad_norm": 0.025246020406484604,
      "kl": 0.0017945856025107787,
      "learning_rate": 1.9473684210526315e-06,
      "loss": -0.0,
      "step": 630
    },
    {
      "clip_ratio": 0.00027173911803402007,
      "epoch": 0.08615510649918078,
      "grad_norm": 0.02506876550614834,
      "kl": 0.0017630000711506,
      "learning_rate": 1.9421052631578947e-06,
      "loss": 0.0,
      "step": 631
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.08629164391043145,
      "grad_norm": 0.02513444982469082,
      "kl": 0.0018713257522904314,
      "learning_rate": 1.936842105263158e-06,
      "loss": -0.0,
      "step": 632
    },
    {
      "clip_ratio": 0.0001806358341127634,
      "completion_length": 133.9375,
      "epoch": 0.08642818132168215,
      "grad_norm": 0.030734168365597725,
      "kl": 0.0014602373357774923,
      "learning_rate": 1.9315789473684215e-06,
      "loss": -0.0,
      "num_tokens": 1849627.0,
      "reward": 0.71875,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.71875,
      "step": 633
    },
    {
      "clip_ratio": 0.0008967998146545142,
      "epoch": 0.08656471873293282,
      "grad_norm": 0.031097473576664925,
      "kl": 0.001530306273707538,
      "learning_rate": 1.9263157894736846e-06,
      "loss": -0.0001,
      "step": 634
    },
    {
      "clip_ratio": 0.0002281021879753098,
      "epoch": 0.0867012561441835,
      "grad_norm": 0.031329941004514694,
      "kl": 0.0013291898703755578,
      "learning_rate": 1.9210526315789474e-06,
      "loss": -0.0001,
      "step": 635
    },
    {
      "clip_ratio": 0.0004247764591127634,
      "epoch": 0.0868377935554342,
      "grad_norm": 0.030992189422249794,
      "kl": 0.0014359870256157592,
      "learning_rate": 1.9157894736842105e-06,
      "loss": -0.0004,
      "step": 636
    },
    {
      "clip_ratio": 0.0003124999930150807,
      "completion_length": 129.71875,
      "epoch": 0.08697433096668487,
      "grad_norm": 0.02779513970017433,
      "kl": 0.0014640076806244906,
      "learning_rate": 1.9105263157894737e-06,
      "loss": 0.0001,
      "num_tokens": 1861258.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 637
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08711086837793555,
      "grad_norm": 0.02775580994784832,
      "kl": 0.001482383013353683,
      "learning_rate": 1.905263157894737e-06,
      "loss": 0.0001,
      "step": 638
    },
    {
      "clip_ratio": 0.0001453488366678357,
      "epoch": 0.08724740578918623,
      "grad_norm": 0.027669286355376244,
      "kl": 0.0014783034021093044,
      "learning_rate": 1.9000000000000002e-06,
      "loss": 0.0001,
      "step": 639
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08738394320043692,
      "grad_norm": 0.02769199199974537,
      "kl": 0.0014845102041363134,
      "learning_rate": 1.8947368421052634e-06,
      "loss": 0.0001,
      "step": 640
    },
    {
      "clip_ratio": 0.0009185846138279885,
      "completion_length": 128.46875,
      "epoch": 0.0875204806116876,
      "grad_norm": 0.033943865448236465,
      "kl": 0.0014185677864588797,
      "learning_rate": 1.8894736842105266e-06,
      "loss": 0.0,
      "num_tokens": 1872497.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.606249988079071,
      "step": 641
    },
    {
      "clip_ratio": 0.0007410277903545648,
      "epoch": 0.08765701802293828,
      "grad_norm": 0.033117737621068954,
      "kl": 0.0015516698476858437,
      "learning_rate": 1.8842105263157895e-06,
      "loss": -0.0,
      "step": 642
    },
    {
      "clip_ratio": 0.0007410277903545648,
      "epoch": 0.08779355543418897,
      "grad_norm": 0.032690513879060745,
      "kl": 0.0014765476635147934,
      "learning_rate": 1.8789473684210527e-06,
      "loss": -0.0002,
      "step": 643
    },
    {
      "clip_ratio": 0.0007410277903545648,
      "epoch": 0.08793009284543965,
      "grad_norm": 0.033511463552713394,
      "kl": 0.0015954046502884012,
      "learning_rate": 1.8736842105263158e-06,
      "loss": -0.0002,
      "step": 644
    },
    {
      "clip_ratio": 0.0007349282532231882,
      "completion_length": 117.1875,
      "epoch": 0.08806663025669033,
      "grad_norm": 0.04200991615653038,
      "kl": 0.0018133009270968614,
      "learning_rate": 1.868421052631579e-06,
      "loss": 0.0002,
      "num_tokens": 1883011.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2337018847465515,
      "rewards/reward_fn": 0.690625011920929,
      "step": 645
    },
    {
      "clip_ratio": 0.0011147170735057443,
      "epoch": 0.08820316766794102,
      "grad_norm": 0.03769839555025101,
      "kl": 0.0018559084401204018,
      "learning_rate": 1.8631578947368424e-06,
      "loss": -0.0001,
      "step": 646
    },
    {
      "clip_ratio": 0.0013395372225204483,
      "epoch": 0.0883397050791917,
      "grad_norm": 0.03914198279380798,
      "kl": 0.0018133489347746945,
      "learning_rate": 1.8578947368421055e-06,
      "loss": 0.0001,
      "step": 647
    },
    {
      "clip_ratio": 0.001044334188918583,
      "epoch": 0.08847624249044238,
      "grad_norm": 0.03948010131716728,
      "kl": 0.0017424372199457139,
      "learning_rate": 1.8526315789473687e-06,
      "loss": 0.0001,
      "step": 648
    },
    {
      "clip_ratio": 0.0009479038853896782,
      "completion_length": 147.8125,
      "epoch": 0.08861277990169307,
      "grad_norm": 0.034093838185071945,
      "kl": 0.0017491482576588169,
      "learning_rate": 1.8473684210526319e-06,
      "loss": 0.0002,
      "num_tokens": 1895205.0,
      "reward": 0.606249988079071,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.606249988079071,
      "step": 649
    },
    {
      "clip_ratio": 0.0013839395687682554,
      "epoch": 0.08874931731294375,
      "grad_norm": 0.034466780722141266,
      "kl": 0.0017224287621502299,
      "learning_rate": 1.8421052631578948e-06,
      "loss": 0.0003,
      "step": 650
    },
    {
      "clip_ratio": 0.0007027992105577141,
      "epoch": 0.08888585472419443,
      "grad_norm": 0.034083131700754166,
      "kl": 0.0017858712380984798,
      "learning_rate": 1.836842105263158e-06,
      "loss": 0.0001,
      "step": 651
    },
    {
      "clip_ratio": 0.00064598981407471,
      "epoch": 0.0890223921354451,
      "grad_norm": 0.03420592471957207,
      "kl": 0.0017675116159807658,
      "learning_rate": 1.8315789473684211e-06,
      "loss": 0.0,
      "step": 652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.78125,
      "epoch": 0.0891589295466958,
      "grad_norm": 0.025346998125314713,
      "kl": 0.001804993613404804,
      "learning_rate": 1.8263157894736843e-06,
      "loss": -0.0001,
      "num_tokens": 1906734.0,
      "reward": 0.71875,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.71875,
      "step": 653
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08929546695794648,
      "grad_norm": 0.025825245305895805,
      "kl": 0.001727437149384059,
      "learning_rate": 1.8210526315789475e-06,
      "loss": -0.0001,
      "step": 654
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.08943200436919715,
      "grad_norm": 0.025717921555042267,
      "kl": 0.0016938845965341898,
      "learning_rate": 1.8157894736842109e-06,
      "loss": 0.0,
      "step": 655
    },
    {
      "clip_ratio": 0.00037425151094794273,
      "epoch": 0.08956854178044785,
      "grad_norm": 0.02527354285120964,
      "kl": 0.0016179548201762373,
      "learning_rate": 1.810526315789474e-06,
      "loss": -0.0001,
      "step": 656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.1875,
      "epoch": 0.08970507919169853,
      "grad_norm": 0.03043227270245552,
      "kl": 0.0015594559863529867,
      "learning_rate": 1.805263157894737e-06,
      "loss": -0.0,
      "num_tokens": 1918476.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 657
    },
    {
      "clip_ratio": 0.0001806358341127634,
      "epoch": 0.0898416166029492,
      "grad_norm": 0.03057057410478592,
      "kl": 0.0016266269267362077,
      "learning_rate": 1.8000000000000001e-06,
      "loss": -0.0001,
      "step": 658
    },
    {
      "clip_ratio": 0.0009754410712048411,
      "epoch": 0.0899781540141999,
      "grad_norm": 0.031286321580410004,
      "kl": 0.0016341756581823574,
      "learning_rate": 1.7947368421052633e-06,
      "loss": -0.0,
      "step": 659
    },
    {
      "clip_ratio": 0.0006547989178216085,
      "epoch": 0.09011469142545057,
      "grad_norm": 0.030410924926400185,
      "kl": 0.0016616588927718112,
      "learning_rate": 1.7894736842105265e-06,
      "loss": -0.0,
      "step": 660
    },
    {
      "clip_ratio": 0.0005230949609540403,
      "completion_length": 138.0,
      "epoch": 0.09025122883670125,
      "grad_norm": 0.034563641995191574,
      "kl": 0.0014913913328200579,
      "learning_rate": 1.7842105263157896e-06,
      "loss": 0.0001,
      "num_tokens": 1930076.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 661
    },
    {
      "clip_ratio": 0.000984658210654743,
      "epoch": 0.09038776624795195,
      "grad_norm": 0.03449584171175957,
      "kl": 0.0014851825108053163,
      "learning_rate": 1.7789473684210528e-06,
      "loss": 0.0003,
      "step": 662
    },
    {
      "clip_ratio": 0.000619944155914709,
      "epoch": 0.09052430365920262,
      "grad_norm": 0.03485969081521034,
      "kl": 0.0014608390283683548,
      "learning_rate": 1.7736842105263162e-06,
      "loss": 0.0,
      "step": 663
    },
    {
      "clip_ratio": 0.0005230949609540403,
      "epoch": 0.0906608410704533,
      "grad_norm": 0.03468193858861923,
      "kl": 0.0013985099394631106,
      "learning_rate": 1.768421052631579e-06,
      "loss": -0.0001,
      "step": 664
    },
    {
      "clip_ratio": 0.0016197122313315049,
      "completion_length": 126.0625,
      "epoch": 0.09079737848170398,
      "grad_norm": 0.048120178282260895,
      "kl": 0.0019150414373143576,
      "learning_rate": 1.7631578947368423e-06,
      "loss": 0.0002,
      "num_tokens": 1941278.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.4372595250606537,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 665
    },
    {
      "clip_ratio": 0.0028011579561280087,
      "epoch": 0.09093391589295467,
      "grad_norm": 0.04828806221485138,
      "kl": 0.001905280525534181,
      "learning_rate": 1.7578947368421054e-06,
      "loss": 0.0003,
      "step": 666
    },
    {
      "clip_ratio": 0.0024009775515878573,
      "epoch": 0.09107045330420535,
      "grad_norm": 0.050769418478012085,
      "kl": 0.0019341932602401357,
      "learning_rate": 1.7526315789473686e-06,
      "loss": 0.0003,
      "step": 667
    },
    {
      "clip_ratio": 0.0010695689998101443,
      "epoch": 0.09120699071545603,
      "grad_norm": 0.04936697706580162,
      "kl": 0.0019743770517379744,
      "learning_rate": 1.7473684210526318e-06,
      "loss": -0.0001,
      "step": 668
    },
    {
      "clip_ratio": 0.0007503781234845519,
      "completion_length": 140.65625,
      "epoch": 0.09134352812670672,
      "grad_norm": 0.03651706874370575,
      "kl": 0.002162061420676764,
      "learning_rate": 1.742105263157895e-06,
      "loss": 0.0003,
      "num_tokens": 1953231.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.25110572576522827,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 669
    },
    {
      "clip_ratio": 0.000699896685546264,
      "epoch": 0.0914800655379574,
      "grad_norm": 0.03636300936341286,
      "kl": 0.0022203527332749218,
      "learning_rate": 1.736842105263158e-06,
      "loss": 0.0002,
      "step": 670
    },
    {
      "clip_ratio": 0.00034614611649885774,
      "epoch": 0.09161660294920808,
      "grad_norm": 0.037476662546396255,
      "kl": 0.002253743024994037,
      "learning_rate": 1.731578947368421e-06,
      "loss": 0.0003,
      "step": 671
    },
    {
      "clip_ratio": 0.0007263705774676055,
      "epoch": 0.09175314036045877,
      "grad_norm": 0.03752395883202553,
      "kl": 0.0022044695178919937,
      "learning_rate": 1.7263157894736842e-06,
      "loss": 0.0002,
      "step": 672
    },
    {
      "clip_ratio": 0.0023742812336422503,
      "completion_length": 128.5,
      "epoch": 0.09188967777170945,
      "grad_norm": 0.04179362207651138,
      "kl": 0.0015345551819336833,
      "learning_rate": 1.7210526315789474e-06,
      "loss": 0.0001,
      "num_tokens": 1964579.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.35490381717681885,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 673
    },
    {
      "clip_ratio": 0.0012075189006282017,
      "epoch": 0.09202621518296013,
      "grad_norm": 0.04128982126712799,
      "kl": 0.0015574862100038445,
      "learning_rate": 1.7157894736842107e-06,
      "loss": 0.0,
      "step": 674
    },
    {
      "clip_ratio": 0.0009616574534447864,
      "epoch": 0.09216275259421082,
      "grad_norm": 0.04147768020629883,
      "kl": 0.0015054902996780584,
      "learning_rate": 1.710526315789474e-06,
      "loss": -0.0,
      "step": 675
    },
    {
      "clip_ratio": 0.0014374384481925517,
      "epoch": 0.0922992900054615,
      "grad_norm": 0.041720401495695114,
      "kl": 0.0014951720877434127,
      "learning_rate": 1.705263157894737e-06,
      "loss": 0.0001,
      "step": 676
    },
    {
      "clip_ratio": 0.0004928899579681456,
      "completion_length": 128.59375,
      "epoch": 0.09243582741671218,
      "grad_norm": 0.02729075774550438,
      "kl": 0.0019795900498138508,
      "learning_rate": 1.7000000000000002e-06,
      "loss": 0.0,
      "num_tokens": 1975790.0,
      "reward": 0.746874988079071,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.746874988079071,
      "step": 677
    },
    {
      "clip_ratio": 0.0010389106319053099,
      "epoch": 0.09257236482796286,
      "grad_norm": 0.027434304356575012,
      "kl": 0.001802716407837579,
      "learning_rate": 1.6947368421052632e-06,
      "loss": 0.0003,
      "step": 678
    },
    {
      "clip_ratio": 0.0002561475266702473,
      "epoch": 0.09270890223921355,
      "grad_norm": 0.02690211683511734,
      "kl": 0.0018389189117442584,
      "learning_rate": 1.6894736842105264e-06,
      "loss": -0.0,
      "step": 679
    },
    {
      "clip_ratio": 0.0002561475266702473,
      "epoch": 0.09284543965046423,
      "grad_norm": 0.027215635403990746,
      "kl": 0.001933387746248627,
      "learning_rate": 1.6842105263157895e-06,
      "loss": 0.0,
      "step": 680
    },
    {
      "clip_ratio": 0.000294811325147748,
      "completion_length": 121.09375,
      "epoch": 0.0929819770617149,
      "grad_norm": 0.029706580564379692,
      "kl": 0.002037748434304376,
      "learning_rate": 1.6789473684210527e-06,
      "loss": 0.0,
      "num_tokens": 1986749.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 681
    },
    {
      "clip_ratio": 0.000294811325147748,
      "epoch": 0.0931185144729656,
      "grad_norm": 0.02914939634501934,
      "kl": 0.0020570547858369537,
      "learning_rate": 1.673684210526316e-06,
      "loss": 0.0001,
      "step": 682
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09325505188421628,
      "grad_norm": 0.02946673519909382,
      "kl": 0.002120211442161235,
      "learning_rate": 1.6684210526315792e-06,
      "loss": 0.0,
      "step": 683
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09339158929546695,
      "grad_norm": 0.0294260922819376,
      "kl": 0.0021350198639993323,
      "learning_rate": 1.6631578947368424e-06,
      "loss": -0.0001,
      "step": 684
    },
    {
      "clip_ratio": 0.00019409938249737024,
      "completion_length": 149.40625,
      "epoch": 0.09352812670671765,
      "grad_norm": 0.024748150259256363,
      "kl": 0.001692789483058732,
      "learning_rate": 1.6578947368421053e-06,
      "loss": -0.0,
      "num_tokens": 1998122.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 685
    },
    {
      "clip_ratio": 0.0007354209810728207,
      "epoch": 0.09366466411796832,
      "grad_norm": 0.025075767189264297,
      "kl": 0.0016773628740338609,
      "learning_rate": 1.6526315789473685e-06,
      "loss": 0.0,
      "step": 686
    },
    {
      "clip_ratio": 0.0005357511981856078,
      "epoch": 0.093801201529219,
      "grad_norm": 0.024616461247205734,
      "kl": 0.001673653707257472,
      "learning_rate": 1.6473684210526317e-06,
      "loss": -0.0,
      "step": 687
    },
    {
      "clip_ratio": 0.00012400794366840273,
      "epoch": 0.0939377389404697,
      "grad_norm": 0.025905825197696686,
      "kl": 0.0016978087533061625,
      "learning_rate": 1.6421052631578948e-06,
      "loss": 0.0,
      "step": 688
    },
    {
      "clip_ratio": 0.0005619380681309849,
      "completion_length": 134.65625,
      "epoch": 0.09407427635172037,
      "grad_norm": 0.031999699771404266,
      "kl": 0.0016089139571704436,
      "learning_rate": 1.636842105263158e-06,
      "loss": -0.0001,
      "num_tokens": 2010007.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 689
    },
    {
      "clip_ratio": 0.00034340660204179585,
      "epoch": 0.09421081376297105,
      "grad_norm": 0.03210426867008209,
      "kl": 0.0015337139411712997,
      "learning_rate": 1.6315789473684212e-06,
      "loss": 0.0001,
      "step": 690
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09434735117422173,
      "grad_norm": 0.03191930055618286,
      "kl": 0.001649985551921418,
      "learning_rate": 1.6263157894736845e-06,
      "loss": -0.0002,
      "step": 691
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09448388858547242,
      "grad_norm": 0.0321015864610672,
      "kl": 0.0016289945742755663,
      "learning_rate": 1.6210526315789473e-06,
      "loss": 0.0001,
      "step": 692
    },
    {
      "clip_ratio": 0.0004166830622125417,
      "completion_length": 119.09375,
      "epoch": 0.0946204259967231,
      "grad_norm": 0.027585109695792198,
      "kl": 0.0017405743255949346,
      "learning_rate": 1.6157894736842106e-06,
      "loss": 0.0,
      "num_tokens": 2021430.0,
      "reward": 0.71875,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.71875,
      "step": 693
    },
    {
      "clip_ratio": 0.0006837770924903452,
      "epoch": 0.09475696340797378,
      "grad_norm": 0.02802807278931141,
      "kl": 0.0017661270230746595,
      "learning_rate": 1.6105263157894738e-06,
      "loss": 0.0002,
      "step": 694
    },
    {
      "clip_ratio": 0.0004166830622125417,
      "epoch": 0.09489350081922447,
      "grad_norm": 0.02761889435350895,
      "kl": 0.0017125707254308509,
      "learning_rate": 1.605263157894737e-06,
      "loss": -0.0,
      "step": 695
    },
    {
      "clip_ratio": 0.0008214422850869596,
      "epoch": 0.09503003823047515,
      "grad_norm": 0.027634145691990852,
      "kl": 0.0018781941034831107,
      "learning_rate": 1.6000000000000001e-06,
      "loss": -0.0002,
      "step": 696
    },
    {
      "clip_ratio": 0.0011033393384423107,
      "completion_length": 138.78125,
      "epoch": 0.09516657564172583,
      "grad_norm": 0.03645732253789902,
      "kl": 0.0015163161588134244,
      "learning_rate": 1.5947368421052633e-06,
      "loss": 0.0,
      "num_tokens": 2033031.0,
      "reward": 0.746874988079071,
      "reward_std": 0.298653781414032,
      "rewards/reward_fn": 0.746874988079071,
      "step": 697
    },
    {
      "clip_ratio": 0.00014076576917432249,
      "epoch": 0.09530311305297652,
      "grad_norm": 0.035988595336675644,
      "kl": 0.0015767957502248464,
      "learning_rate": 1.5894736842105265e-06,
      "loss": -0.0001,
      "step": 698
    },
    {
      "clip_ratio": 0.0010361238601035438,
      "epoch": 0.0954396504642272,
      "grad_norm": 0.03622249513864517,
      "kl": 0.0015686019669374218,
      "learning_rate": 1.5842105263157894e-06,
      "loss": -0.0001,
      "step": 699
    },
    {
      "clip_ratio": 0.00037931538827251643,
      "epoch": 0.09557618787547788,
      "grad_norm": 0.03611570969223976,
      "kl": 0.0015960140808601864,
      "learning_rate": 1.5789473684210526e-06,
      "loss": -0.0003,
      "step": 700
    },
    {
      "clip_ratio": 0.0005520998747670092,
      "completion_length": 161.96875,
      "epoch": 0.09571272528672857,
      "grad_norm": 0.03334690257906914,
      "kl": 0.0014301782102847937,
      "learning_rate": 1.573684210526316e-06,
      "loss": -0.0001,
      "num_tokens": 2046202.0,
      "reward": 0.40625,
      "reward_std": 0.3049038052558899,
      "rewards/reward_fn": 0.40625,
      "step": 701
    },
    {
      "clip_ratio": 0.00041565170977264643,
      "epoch": 0.09584926269797925,
      "grad_norm": 0.03214196488261223,
      "kl": 0.001410563058925618,
      "learning_rate": 1.5684210526315791e-06,
      "loss": -0.0002,
      "step": 702
    },
    {
      "clip_ratio": 0.000545183866051957,
      "epoch": 0.09598580010922993,
      "grad_norm": 0.033150359988212585,
      "kl": 0.0013807993827867904,
      "learning_rate": 1.5631578947368423e-06,
      "loss": 0.0001,
      "step": 703
    },
    {
      "clip_ratio": 0.00011160714348079637,
      "epoch": 0.0961223375204806,
      "grad_norm": 0.032138846814632416,
      "kl": 0.0013679018711627577,
      "learning_rate": 1.5578947368421054e-06,
      "loss": -0.0,
      "step": 704
    },
    {
      "clip_ratio": 0.0004139072843827307,
      "completion_length": 129.375,
      "epoch": 0.0962588749317313,
      "grad_norm": 0.03673640638589859,
      "kl": 0.0017190880898851901,
      "learning_rate": 1.5526315789473686e-06,
      "loss": 0.0001,
      "num_tokens": 2057270.0,
      "reward": 0.690625011920929,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.690625011920929,
      "step": 705
    },
    {
      "clip_ratio": 0.00036971406370867044,
      "epoch": 0.09639541234298198,
      "grad_norm": 0.034445621073246,
      "kl": 0.001723575060168514,
      "learning_rate": 1.5473684210526316e-06,
      "loss": 0.0001,
      "step": 706
    },
    {
      "clip_ratio": 0.0008860736415954307,
      "epoch": 0.09653194975423265,
      "grad_norm": 0.036579687148332596,
      "kl": 0.0016895698117878055,
      "learning_rate": 1.5421052631578947e-06,
      "loss": -0.0002,
      "step": 707
    },
    {
      "clip_ratio": 0.00046615848259534687,
      "epoch": 0.09666848716548335,
      "grad_norm": 0.036325111985206604,
      "kl": 0.0016936773936322425,
      "learning_rate": 1.5368421052631579e-06,
      "loss": -0.0002,
      "step": 708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.28125,
      "epoch": 0.09680502457673403,
      "grad_norm": 0.02210831828415394,
      "kl": 0.0017170392002299195,
      "learning_rate": 1.5315789473684213e-06,
      "loss": -0.0001,
      "num_tokens": 2068215.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 709
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0969415619879847,
      "grad_norm": 0.02184915728867054,
      "kl": 0.0016970243068499258,
      "learning_rate": 1.5263157894736844e-06,
      "loss": -0.0001,
      "step": 710
    },
    {
      "clip_ratio": 0.0001594387722434476,
      "epoch": 0.0970780993992354,
      "grad_norm": 0.022008102387189865,
      "kl": 0.001666097336965322,
      "learning_rate": 1.5210526315789476e-06,
      "loss": -0.0,
      "step": 711
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.09721463681048607,
      "grad_norm": 0.02160630375146866,
      "kl": 0.0016568601295148255,
      "learning_rate": 1.5157894736842108e-06,
      "loss": -0.0,
      "step": 712
    },
    {
      "clip_ratio": 0.0008411372982664034,
      "completion_length": 149.9375,
      "epoch": 0.09735117422173675,
      "grad_norm": 0.03211577236652374,
      "kl": 0.0015268612896761624,
      "learning_rate": 1.5105263157894737e-06,
      "loss": -0.0003,
      "num_tokens": 2080137.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2511056959629059,
      "rewards/reward_fn": 0.746874988079071,
      "step": 713
    },
    {
      "clip_ratio": 0.0008598162676207721,
      "epoch": 0.09748771163298744,
      "grad_norm": 0.03194946050643921,
      "kl": 0.0014656807561550522,
      "learning_rate": 1.5052631578947369e-06,
      "loss": -0.0001,
      "step": 714
    },
    {
      "clip_ratio": 0.0011155456595588475,
      "epoch": 0.09762424904423812,
      "grad_norm": 0.03199795261025429,
      "kl": 0.0015198987184703583,
      "learning_rate": 1.5e-06,
      "loss": -0.0001,
      "step": 715
    },
    {
      "clip_ratio": 0.0004340070008765906,
      "epoch": 0.0977607864554888,
      "grad_norm": 0.03166012093424797,
      "kl": 0.0015317311590479221,
      "learning_rate": 1.4947368421052632e-06,
      "loss": -0.0002,
      "step": 716
    },
    {
      "clip_ratio": 0.0010012474449467845,
      "completion_length": 150.5625,
      "epoch": 0.09789732386673948,
      "grad_norm": 0.033033497631549835,
      "kl": 0.0016807273759695818,
      "learning_rate": 1.4894736842105264e-06,
      "loss": -0.0002,
      "num_tokens": 2092099.0,
      "reward": 0.859375,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.859375,
      "step": 717
    },
    {
      "clip_ratio": 0.000548348885786254,
      "epoch": 0.09803386127799017,
      "grad_norm": 0.033237796276807785,
      "kl": 0.0017241216792172054,
      "learning_rate": 1.4842105263157897e-06,
      "loss": -0.0,
      "step": 718
    },
    {
      "clip_ratio": 0.00046202958037611097,
      "epoch": 0.09817039868924085,
      "grad_norm": 0.03423339128494263,
      "kl": 0.0016696170132490806,
      "learning_rate": 1.478947368421053e-06,
      "loss": -0.0001,
      "step": 719
    },
    {
      "clip_ratio": 0.0008547664401703514,
      "epoch": 0.09830693610049153,
      "grad_norm": 0.03277941793203354,
      "kl": 0.0016932836006162688,
      "learning_rate": 1.4736842105263159e-06,
      "loss": -0.0001,
      "step": 720
    },
    {
      "clip_ratio": 0.0007789192459313199,
      "completion_length": 126.40625,
      "epoch": 0.09844347351174222,
      "grad_norm": 0.02932615391910076,
      "kl": 0.0018991009465025854,
      "learning_rate": 1.468421052631579e-06,
      "loss": -0.0,
      "num_tokens": 2103080.0,
      "reward": 0.550000011920929,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.550000011920929,
      "step": 721
    },
    {
      "clip_ratio": 0.00021701389050576836,
      "epoch": 0.0985800109229929,
      "grad_norm": 0.02889619581401348,
      "kl": 0.0018387689356131887,
      "learning_rate": 1.4631578947368422e-06,
      "loss": 0.0001,
      "step": 722
    },
    {
      "clip_ratio": 0.0009659523202572018,
      "epoch": 0.09871654833424358,
      "grad_norm": 0.029005665332078934,
      "kl": 0.0018426774186082184,
      "learning_rate": 1.4578947368421053e-06,
      "loss": 0.0001,
      "step": 723
    },
    {
      "clip_ratio": 0.0007789192459313199,
      "epoch": 0.09885308574549427,
      "grad_norm": 0.029167210683226585,
      "kl": 0.001885860742731893,
      "learning_rate": 1.4526315789473685e-06,
      "loss": -0.0001,
      "step": 724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 108.65625,
      "epoch": 0.09898962315674495,
      "grad_norm": 0.036384548991918564,
      "kl": 0.0014372121031556162,
      "learning_rate": 1.4473684210526317e-06,
      "loss": 0.0001,
      "num_tokens": 2114013.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.2511056959629059,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 725
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "epoch": 0.09912616056799563,
      "grad_norm": 0.036850765347480774,
      "kl": 0.0016032819266911247,
      "learning_rate": 1.442105263157895e-06,
      "loss": -0.0,
      "step": 726
    },
    {
      "clip_ratio": 0.0003396739193703979,
      "epoch": 0.09926269797924632,
      "grad_norm": 0.0372999869287014,
      "kl": 0.0015850669196879608,
      "learning_rate": 1.4368421052631582e-06,
      "loss": 0.0001,
      "step": 727
    },
    {
      "clip_ratio": 0.00021853146608918905,
      "epoch": 0.099399235390497,
      "grad_norm": 0.03686143085360527,
      "kl": 0.001440588292098255,
      "learning_rate": 1.4315789473684212e-06,
      "loss": 0.0,
      "step": 728
    },
    {
      "clip_ratio": 0.00048737886390881613,
      "completion_length": 144.4375,
      "epoch": 0.09953577280174768,
      "grad_norm": 0.03900820016860962,
      "kl": 0.0018115455322913476,
      "learning_rate": 1.4263157894736843e-06,
      "loss": -0.0002,
      "num_tokens": 2125843.0,
      "reward": 0.71875,
      "reward_std": 0.35490378737449646,
      "rewards/reward_fn": 0.71875,
      "step": 729
    },
    {
      "clip_ratio": 0.0004231298589729704,
      "epoch": 0.09967231021299836,
      "grad_norm": 0.03907107934355736,
      "kl": 0.0018483502553863218,
      "learning_rate": 1.4210526315789475e-06,
      "loss": -0.0002,
      "step": 730
    },
    {
      "clip_ratio": 0.0006401437494787388,
      "epoch": 0.09980884762424905,
      "grad_norm": 0.03969133272767067,
      "kl": 0.001942682627486647,
      "learning_rate": 1.4157894736842107e-06,
      "loss": -0.0,
      "step": 731
    },
    {
      "clip_ratio": 0.0005634278713841923,
      "epoch": 0.09994538503549973,
      "grad_norm": 0.03949359059333801,
      "kl": 0.0019668521690618945,
      "learning_rate": 1.4105263157894738e-06,
      "loss": -0.0002,
      "step": 732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.09375,
      "epoch": 0.1000819224467504,
      "grad_norm": 0.012566876597702503,
      "kl": 0.0020119995751883835,
      "learning_rate": 1.405263157894737e-06,
      "loss": -0.0,
      "num_tokens": 2137698.0,
      "reward": 0.859375,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.859375,
      "step": 733
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1002184598580011,
      "grad_norm": 0.017249109223484993,
      "kl": 0.0017891458755912026,
      "learning_rate": 1.4000000000000001e-06,
      "loss": -0.0001,
      "step": 734
    },
    {
      "clip_ratio": 0.0001849112450145185,
      "epoch": 0.10035499726925177,
      "grad_norm": 0.017133330926299095,
      "kl": 0.001893964958981087,
      "learning_rate": 1.394736842105263e-06,
      "loss": -0.0,
      "step": 735
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10049153468050245,
      "grad_norm": 0.017315877601504326,
      "kl": 0.0018965235340147046,
      "learning_rate": 1.3894736842105263e-06,
      "loss": -0.0001,
      "step": 736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 106.625,
      "epoch": 0.10062807209175315,
      "grad_norm": 0.02570240944623947,
      "kl": 0.002060361744952388,
      "learning_rate": 1.3842105263157896e-06,
      "loss": 0.0,
      "num_tokens": 2149018.0,
      "reward": 0.859375,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.859375,
      "step": 737
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.10076460950300382,
      "grad_norm": 0.026232900097966194,
      "kl": 0.0021926913996139774,
      "learning_rate": 1.3789473684210528e-06,
      "loss": -0.0,
      "step": 738
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.1009011469142545,
      "grad_norm": 0.025098463520407677,
      "kl": 0.002106565338181099,
      "learning_rate": 1.373684210526316e-06,
      "loss": -0.0002,
      "step": 739
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.1010376843255052,
      "grad_norm": 0.025355098769068718,
      "kl": 0.0020467557496886,
      "learning_rate": 1.3684210526315791e-06,
      "loss": 0.0002,
      "step": 740
    },
    {
      "clip_ratio": 0.00098407655605115,
      "completion_length": 132.4375,
      "epoch": 0.10117422173675587,
      "grad_norm": 0.036574024707078934,
      "kl": 0.0018209284698968986,
      "learning_rate": 1.3631578947368423e-06,
      "loss": 0.0,
      "num_tokens": 2160252.0,
      "reward": 0.4937499761581421,
      "reward_std": 0.289951890707016,
      "rewards/reward_fn": 0.4937499761581421,
      "step": 741
    },
    {
      "clip_ratio": 0.0007863230566726997,
      "epoch": 0.10131075914800655,
      "grad_norm": 0.03622036427259445,
      "kl": 0.0018895867688115686,
      "learning_rate": 1.3578947368421052e-06,
      "loss": -0.0001,
      "step": 742
    },
    {
      "clip_ratio": 0.0010683000582503155,
      "epoch": 0.10144729655925723,
      "grad_norm": 0.03665731102228165,
      "kl": 0.0018325582204852253,
      "learning_rate": 1.3526315789473684e-06,
      "loss": -0.0001,
      "step": 743
    },
    {
      "clip_ratio": 0.0008244863129220903,
      "epoch": 0.10158383397050792,
      "grad_norm": 0.036230310797691345,
      "kl": 0.0018342306175327394,
      "learning_rate": 1.3473684210526316e-06,
      "loss": -0.0002,
      "step": 744
    },
    {
      "clip_ratio": 0.00041680221329443157,
      "completion_length": 141.625,
      "epoch": 0.1017203713817586,
      "grad_norm": 0.03272077441215515,
      "kl": 0.0017692799974611262,
      "learning_rate": 1.342105263157895e-06,
      "loss": -0.0001,
      "num_tokens": 2171828.0,
      "reward": 0.4937500059604645,
      "reward_std": 0.289951890707016,
      "rewards/reward_fn": 0.4937500059604645,
      "step": 745
    },
    {
      "clip_ratio": 0.000294811325147748,
      "epoch": 0.10185690879300928,
      "grad_norm": 0.03340604528784752,
      "kl": 0.0016943258451647125,
      "learning_rate": 1.3368421052631581e-06,
      "loss": 0.0001,
      "step": 746
    },
    {
      "clip_ratio": 0.0011052459594793618,
      "epoch": 0.10199344620425997,
      "grad_norm": 0.03306342288851738,
      "kl": 0.0018010039475484518,
      "learning_rate": 1.3315789473684213e-06,
      "loss": 0.0,
      "step": 747
    },
    {
      "clip_ratio": 0.00026939655072055757,
      "epoch": 0.10212998361551065,
      "grad_norm": 0.032833609730005264,
      "kl": 0.0018719100971793523,
      "learning_rate": 1.3263157894736844e-06,
      "loss": 0.0003,
      "step": 748
    },
    {
      "clip_ratio": 0.0009049124200828373,
      "completion_length": 122.84375,
      "epoch": 0.10226652102676133,
      "grad_norm": 0.027507351711392403,
      "kl": 0.0014249746527639218,
      "learning_rate": 1.3210526315789474e-06,
      "loss": -0.0001,
      "num_tokens": 2183135.0,
      "reward": 0.690625011920929,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.690625011920929,
      "step": 749
    },
    {
      "clip_ratio": 0.0008354411402251571,
      "epoch": 0.10240305843801202,
      "grad_norm": 0.02685280330479145,
      "kl": 0.0016018145142879803,
      "learning_rate": 1.3157894736842106e-06,
      "loss": 0.0001,
      "step": 750
    },
    {
      "clip_ratio": 0.0004125279519939795,
      "epoch": 0.1025395958492627,
      "grad_norm": 0.02706029638648033,
      "kl": 0.0015976888844306814,
      "learning_rate": 1.3105263157894737e-06,
      "loss": -0.0,
      "step": 751
    },
    {
      "clip_ratio": 0.0006423073646146804,
      "epoch": 0.10267613326051338,
      "grad_norm": 0.027195792645215988,
      "kl": 0.001565335212944774,
      "learning_rate": 1.3052631578947369e-06,
      "loss": -0.0001,
      "step": 752
    },
    {
      "clip_ratio": 0.00047186609299387783,
      "completion_length": 125.5,
      "epoch": 0.10281267067176407,
      "grad_norm": 0.04647589102387428,
      "kl": 0.0022257442942645866,
      "learning_rate": 1.3e-06,
      "loss": -0.0003,
      "num_tokens": 2194823.0,
      "reward": 0.49375003576278687,
      "reward_std": 0.37230759859085083,
      "rewards/reward_fn": 0.49375003576278687,
      "step": 753
    },
    {
      "clip_ratio": 0.00031565656536258757,
      "epoch": 0.10294920808301475,
      "grad_norm": 0.046547677367925644,
      "kl": 0.00213234518741956,
      "learning_rate": 1.2947368421052634e-06,
      "loss": 0.0,
      "step": 754
    },
    {
      "clip_ratio": 0.0009570495167281479,
      "epoch": 0.10308574549426543,
      "grad_norm": 0.046105701476335526,
      "kl": 0.002177114802179858,
      "learning_rate": 1.2894736842105266e-06,
      "loss": -0.0001,
      "step": 755
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1032222829055161,
      "grad_norm": 0.04649042338132858,
      "kl": 0.002199585574999219,
      "learning_rate": 1.2842105263157895e-06,
      "loss": -0.0004,
      "step": 756
    },
    {
      "clip_ratio": 0.00028935185400769114,
      "completion_length": 131.09375,
      "epoch": 0.1033588203167668,
      "grad_norm": 0.032286182045936584,
      "kl": 0.0018966078678204212,
      "learning_rate": 1.2789473684210527e-06,
      "loss": -0.0001,
      "num_tokens": 2206574.0,
      "reward": 0.746874988079071,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.746874988079071,
      "step": 757
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10349535772801748,
      "grad_norm": 0.031949903815984726,
      "kl": 0.001969371132872766,
      "learning_rate": 1.2736842105263159e-06,
      "loss": -0.0002,
      "step": 758
    },
    {
      "clip_ratio": 0.00028935185400769114,
      "epoch": 0.10363189513926815,
      "grad_norm": 0.03214770555496216,
      "kl": 0.0018590922591101844,
      "learning_rate": 1.268421052631579e-06,
      "loss": 0.0002,
      "step": 759
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10376843255051885,
      "grad_norm": 0.041023969650268555,
      "kl": 0.0018697814284678316,
      "learning_rate": 1.2631578947368422e-06,
      "loss": 0.0,
      "step": 760
    },
    {
      "clip_ratio": 0.00035995818325318396,
      "completion_length": 137.375,
      "epoch": 0.10390496996176952,
      "grad_norm": 0.03233378753066063,
      "kl": 0.0022407320539059583,
      "learning_rate": 1.2578947368421054e-06,
      "loss": -0.0,
      "num_tokens": 2218178.0,
      "reward": 0.578125,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.578125,
      "step": 761
    },
    {
      "clip_ratio": 0.00014880952949170023,
      "epoch": 0.1040415073730202,
      "grad_norm": 0.03180888295173645,
      "kl": 0.0023300516895687906,
      "learning_rate": 1.2526315789473687e-06,
      "loss": -0.0001,
      "step": 762
    },
    {
      "clip_ratio": 0.00021114865376148373,
      "epoch": 0.1041780447842709,
      "grad_norm": 0.03168882802128792,
      "kl": 0.0021711552963097347,
      "learning_rate": 1.2473684210526317e-06,
      "loss": -0.0002,
      "step": 763
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10431458219552157,
      "grad_norm": 0.03188063204288483,
      "kl": 0.002269363276354852,
      "learning_rate": 1.2421052631578948e-06,
      "loss": -0.0001,
      "step": 764
    },
    {
      "clip_ratio": 0.0005279374017845839,
      "completion_length": 134.09375,
      "epoch": 0.10445111960677225,
      "grad_norm": 0.025516286492347717,
      "kl": 0.0019899348426406505,
      "learning_rate": 1.236842105263158e-06,
      "loss": 0.0001,
      "num_tokens": 2229037.0,
      "reward": 0.9437500238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.9437500238418579,
      "step": 765
    },
    {
      "clip_ratio": 0.00016622339899186045,
      "epoch": 0.10458765701802294,
      "grad_norm": 0.025222698226571083,
      "kl": 0.001958257113074069,
      "learning_rate": 1.2315789473684212e-06,
      "loss": -0.0,
      "step": 766
    },
    {
      "clip_ratio": 0.0006586723029613495,
      "epoch": 0.10472419442927362,
      "grad_norm": 0.02710876613855362,
      "kl": 0.0018938343164336402,
      "learning_rate": 1.2263157894736843e-06,
      "loss": 0.0001,
      "step": 767
    },
    {
      "clip_ratio": 0.00021853146608918905,
      "epoch": 0.1048607318405243,
      "grad_norm": 0.025685269385576248,
      "kl": 0.0020639228823711164,
      "learning_rate": 1.2210526315789475e-06,
      "loss": -0.0,
      "step": 768
    },
    {
      "clip_ratio": 0.0011217906430829316,
      "completion_length": 133.96875,
      "epoch": 0.10499726925177498,
      "grad_norm": 0.0428641252219677,
      "kl": 0.0020228277189744404,
      "learning_rate": 1.2157894736842107e-06,
      "loss": 0.0003,
      "num_tokens": 2240432.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 769
    },
    {
      "clip_ratio": 0.0015004310989752412,
      "epoch": 0.10513380666302567,
      "grad_norm": 0.04304884001612663,
      "kl": 0.0020226746928528883,
      "learning_rate": 1.2105263157894738e-06,
      "loss": -0.0001,
      "step": 770
    },
    {
      "clip_ratio": 0.0010420014732517302,
      "epoch": 0.10527034407427635,
      "grad_norm": 0.04133768379688263,
      "kl": 0.001905648708998342,
      "learning_rate": 1.205263157894737e-06,
      "loss": 0.0001,
      "step": 771
    },
    {
      "clip_ratio": 0.0007046800164971501,
      "epoch": 0.10540688148552703,
      "grad_norm": 0.04461239278316498,
      "kl": 0.0019738287810469046,
      "learning_rate": 1.2000000000000002e-06,
      "loss": 0.0002,
      "step": 772
    },
    {
      "clip_ratio": 0.00048007245641201735,
      "completion_length": 123.4375,
      "epoch": 0.10554341889677772,
      "grad_norm": 0.03209725022315979,
      "kl": 0.002003139950829791,
      "learning_rate": 1.1947368421052633e-06,
      "loss": -0.0,
      "num_tokens": 2251734.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.18615379929542542,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 773
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1056799563080284,
      "grad_norm": 0.032378919422626495,
      "kl": 0.0019621488609118387,
      "learning_rate": 1.1894736842105265e-06,
      "loss": 0.0,
      "step": 774
    },
    {
      "clip_ratio": 0.000294811325147748,
      "epoch": 0.10581649371927908,
      "grad_norm": 0.032106876373291016,
      "kl": 0.001987692747206893,
      "learning_rate": 1.1842105263157894e-06,
      "loss": -0.0001,
      "step": 775
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10595303113052977,
      "grad_norm": 0.031922806054353714,
      "kl": 0.0019523974406183697,
      "learning_rate": 1.1789473684210526e-06,
      "loss": -0.0001,
      "step": 776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.5,
      "epoch": 0.10608956854178045,
      "grad_norm": 0.02806154638528824,
      "kl": 0.0022280882021732396,
      "learning_rate": 1.173684210526316e-06,
      "loss": -0.0002,
      "num_tokens": 2262646.0,
      "reward": 0.578125,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.578125,
      "step": 777
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10622610595303113,
      "grad_norm": 0.027715560048818588,
      "kl": 0.0022063598844397347,
      "learning_rate": 1.1684210526315791e-06,
      "loss": -0.0001,
      "step": 778
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10636264336428182,
      "grad_norm": 0.027679534628987312,
      "kl": 0.002072686085739406,
      "learning_rate": 1.163157894736842e-06,
      "loss": -0.0001,
      "step": 779
    },
    {
      "clip_ratio": 0.000240384615608491,
      "epoch": 0.1064991807755325,
      "grad_norm": 0.026711298152804375,
      "kl": 0.002109782257321058,
      "learning_rate": 1.1578947368421053e-06,
      "loss": -0.0,
      "step": 780
    },
    {
      "clip_ratio": 0.0003037487913388759,
      "completion_length": 150.71875,
      "epoch": 0.10663571818678318,
      "grad_norm": 0.03064105659723282,
      "kl": 0.001606904103027773,
      "learning_rate": 1.1526315789473686e-06,
      "loss": -0.0001,
      "num_tokens": 2275025.0,
      "reward": 0.550000011920929,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.550000011920929,
      "step": 781
    },
    {
      "clip_ratio": 0.0008279759786091745,
      "epoch": 0.10677225559803386,
      "grad_norm": 0.03059159778058529,
      "kl": 0.0015235491227940656,
      "learning_rate": 1.1473684210526316e-06,
      "loss": -0.0001,
      "step": 782
    },
    {
      "clip_ratio": 0.0013057087053311989,
      "epoch": 0.10690879300928455,
      "grad_norm": 0.029724495485424995,
      "kl": 0.0015655803872505203,
      "learning_rate": 1.1421052631578947e-06,
      "loss": -0.0002,
      "step": 783
    },
    {
      "clip_ratio": 0.0009672492742538452,
      "epoch": 0.10704533042053523,
      "grad_norm": 0.030576549470424652,
      "kl": 0.001495385740781785,
      "learning_rate": 1.136842105263158e-06,
      "loss": 0.0001,
      "step": 784
    },
    {
      "clip_ratio": 0.0011856760102091357,
      "completion_length": 128.90625,
      "epoch": 0.1071818678317859,
      "grad_norm": 0.04623905569314957,
      "kl": 0.0020615452212950913,
      "learning_rate": 1.1315789473684213e-06,
      "loss": 0.0001,
      "num_tokens": 2286446.0,
      "reward": 0.690625011920929,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.690625011920929,
      "step": 785
    },
    {
      "clip_ratio": 0.000777867971919477,
      "epoch": 0.1073184052430366,
      "grad_norm": 0.046840786933898926,
      "kl": 0.0020759970630024327,
      "learning_rate": 1.1263157894736842e-06,
      "loss": -0.0001,
      "step": 786
    },
    {
      "clip_ratio": 0.0015322749968618155,
      "epoch": 0.10745494265428727,
      "grad_norm": 0.04625287652015686,
      "kl": 0.002053738779068226,
      "learning_rate": 1.1210526315789474e-06,
      "loss": 0.0002,
      "step": 787
    },
    {
      "clip_ratio": 0.0009743925911607221,
      "epoch": 0.10759148006553795,
      "grad_norm": 0.04577214643359184,
      "kl": 0.001995397404243704,
      "learning_rate": 1.1157894736842106e-06,
      "loss": 0.0001,
      "step": 788
    },
    {
      "clip_ratio": 0.000294811325147748,
      "completion_length": 123.875,
      "epoch": 0.10772801747678865,
      "grad_norm": 0.02258908562362194,
      "kl": 0.002167003383874544,
      "learning_rate": 1.110526315789474e-06,
      "loss": 0.0,
      "num_tokens": 2296770.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 789
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10786455488803932,
      "grad_norm": 0.023535488173365593,
      "kl": 0.001987856176128844,
      "learning_rate": 1.1052631578947369e-06,
      "loss": 0.0001,
      "step": 790
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10800109229929,
      "grad_norm": 0.023333881050348282,
      "kl": 0.002119716620654799,
      "learning_rate": 1.1e-06,
      "loss": 0.0,
      "step": 791
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1081376297105407,
      "grad_norm": 0.024103818461298943,
      "kl": 0.00215182734427799,
      "learning_rate": 1.0947368421052632e-06,
      "loss": 0.0001,
      "step": 792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.96875,
      "epoch": 0.10827416712179137,
      "grad_norm": 0.02682146430015564,
      "kl": 0.0017019015285768546,
      "learning_rate": 1.0894736842105264e-06,
      "loss": 0.0001,
      "num_tokens": 2308301.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 793
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10841070453304205,
      "grad_norm": 0.0268485676497221,
      "kl": 0.0017056646174751222,
      "learning_rate": 1.0842105263157895e-06,
      "loss": 0.0001,
      "step": 794
    },
    {
      "clip_ratio": 0.00034722223062999547,
      "epoch": 0.10854724194429273,
      "grad_norm": 0.027060694992542267,
      "kl": 0.0017693005065666512,
      "learning_rate": 1.0789473684210527e-06,
      "loss": -0.0002,
      "step": 795
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10868377935554342,
      "grad_norm": 0.026868777349591255,
      "kl": 0.0017992367465922143,
      "learning_rate": 1.0736842105263159e-06,
      "loss": -0.0,
      "step": 796
    },
    {
      "clip_ratio": 0.00020032051543239504,
      "completion_length": 129.21875,
      "epoch": 0.1088203167667941,
      "grad_norm": 0.031718045473098755,
      "kl": 0.0016662773796269903,
      "learning_rate": 1.068421052631579e-06,
      "loss": -0.0002,
      "num_tokens": 2320572.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 797
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.10895685417804478,
      "grad_norm": 0.03147350996732712,
      "kl": 0.0020512565124590765,
      "learning_rate": 1.0631578947368422e-06,
      "loss": -0.0001,
      "step": 798
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.10909339158929547,
      "grad_norm": 0.030978526920080185,
      "kl": 0.0017799611032387475,
      "learning_rate": 1.0578947368421054e-06,
      "loss": -0.0001,
      "step": 799
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.10922992900054615,
      "grad_norm": 0.03200105205178261,
      "kl": 0.0019068860565312207,
      "learning_rate": 1.0526315789473685e-06,
      "loss": -0.0001,
      "step": 800
    },
    {
      "clip_ratio": 0.00021701389050576836,
      "completion_length": 133.96875,
      "epoch": 0.10936646641179683,
      "grad_norm": 0.030265985056757927,
      "kl": 0.0018098962100339122,
      "learning_rate": 1.0473684210526317e-06,
      "loss": 0.0001,
      "num_tokens": 2331767.0,
      "reward": 0.578125,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.578125,
      "step": 801
    },
    {
      "clip_ratio": 0.00021701389050576836,
      "epoch": 0.10950300382304752,
      "grad_norm": 0.030390843749046326,
      "kl": 0.0019942210747103672,
      "learning_rate": 1.0421052631578949e-06,
      "loss": 0.0002,
      "step": 802
    },
    {
      "clip_ratio": 0.0004973889590473846,
      "epoch": 0.1096395412342982,
      "grad_norm": 0.03022087551653385,
      "kl": 0.0018650772599357879,
      "learning_rate": 1.036842105263158e-06,
      "loss": 0.0002,
      "step": 803
    },
    {
      "clip_ratio": 0.000717032453394495,
      "epoch": 0.10977607864554888,
      "grad_norm": 0.03012370690703392,
      "kl": 0.0018872718937927857,
      "learning_rate": 1.0315789473684212e-06,
      "loss": 0.0002,
      "step": 804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.4375,
      "epoch": 0.10991261605679957,
      "grad_norm": 0.03756886348128319,
      "kl": 0.0018428273033350706,
      "learning_rate": 1.0263157894736843e-06,
      "loss": -0.0001,
      "num_tokens": 2343521.0,
      "reward": 0.831250011920929,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.831250011920929,
      "step": 805
    },
    {
      "clip_ratio": 0.0007708884513704106,
      "epoch": 0.11004915346805025,
      "grad_norm": 0.03728393465280533,
      "kl": 0.0018604374545247993,
      "learning_rate": 1.0210526315789475e-06,
      "loss": -0.0001,
      "step": 806
    },
    {
      "clip_ratio": 0.00040751635970082134,
      "epoch": 0.11018569087930093,
      "grad_norm": 0.037585336714982986,
      "kl": 0.0019263962039985927,
      "learning_rate": 1.0157894736842105e-06,
      "loss": -0.0002,
      "step": 807
    },
    {
      "clip_ratio": 0.0006620027706958354,
      "epoch": 0.1103222282905516,
      "grad_norm": 0.037390291690826416,
      "kl": 0.0018384076520305825,
      "learning_rate": 1.0105263157894738e-06,
      "loss": 0.0003,
      "step": 808
    },
    {
      "clip_ratio": 0.001664920237089973,
      "completion_length": 135.28125,
      "epoch": 0.1104587657018023,
      "grad_norm": 0.0413917638361454,
      "kl": 0.002032072839938337,
      "learning_rate": 1.005263157894737e-06,
      "loss": -0.0001,
      "num_tokens": 2354734.0,
      "reward": 0.606249988079071,
      "reward_std": 0.35490381717681885,
      "rewards/reward_fn": 0.606249988079071,
      "step": 809
    },
    {
      "clip_ratio": 0.0011410745937610045,
      "epoch": 0.11059530311305298,
      "grad_norm": 0.04129190742969513,
      "kl": 0.002106687028572196,
      "learning_rate": 1.0000000000000002e-06,
      "loss": -0.0003,
      "step": 810
    },
    {
      "clip_ratio": 0.0010704625310609117,
      "epoch": 0.11073184052430365,
      "grad_norm": 0.041056614369153976,
      "kl": 0.0018973293499584543,
      "learning_rate": 9.947368421052631e-07,
      "loss": -0.0002,
      "step": 811
    },
    {
      "clip_ratio": 0.001158156941528432,
      "epoch": 0.11086837793555435,
      "grad_norm": 0.04233892634510994,
      "kl": 0.0020257557353033917,
      "learning_rate": 9.894736842105265e-07,
      "loss": -0.0002,
      "step": 812
    },
    {
      "clip_ratio": 0.001496172248153016,
      "completion_length": 122.28125,
      "epoch": 0.11100491534680502,
      "grad_norm": 0.040789976716041565,
      "kl": 0.002083957220747834,
      "learning_rate": 9.842105263157897e-07,
      "loss": -0.0001,
      "num_tokens": 2365983.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 813
    },
    {
      "clip_ratio": 0.0008011132304091007,
      "epoch": 0.1111414527580557,
      "grad_norm": 0.041051674634218216,
      "kl": 0.0022389385812857654,
      "learning_rate": 9.789473684210526e-07,
      "loss": -0.0001,
      "step": 814
    },
    {
      "clip_ratio": 0.0021354319324018434,
      "epoch": 0.1112779901693064,
      "grad_norm": 0.040859155356884,
      "kl": 0.0022529203379235696,
      "learning_rate": 9.736842105263158e-07,
      "loss": 0.0001,
      "step": 815
    },
    {
      "clip_ratio": 0.0009132857667282224,
      "epoch": 0.11141452758055707,
      "grad_norm": 0.04067659378051758,
      "kl": 0.0021882897126488388,
      "learning_rate": 9.68421052631579e-07,
      "loss": -0.0002,
      "step": 816
    },
    {
      "clip_ratio": 0.0008015833009267226,
      "completion_length": 130.875,
      "epoch": 0.11155106499180775,
      "grad_norm": 0.0404217429459095,
      "kl": 0.00207597758344491,
      "learning_rate": 9.631578947368423e-07,
      "loss": 0.0001,
      "num_tokens": 2377083.0,
      "reward": 0.550000011920929,
      "reward_std": 0.2424038052558899,
      "rewards/reward_fn": 0.550000011920929,
      "step": 817
    },
    {
      "clip_ratio": 0.00045197630242910236,
      "epoch": 0.11168760240305844,
      "grad_norm": 0.04167267680168152,
      "kl": 0.001948569952219259,
      "learning_rate": 9.578947368421053e-07,
      "loss": 0.0003,
      "step": 818
    },
    {
      "clip_ratio": 0.0008133314258884639,
      "epoch": 0.11182413981430912,
      "grad_norm": 0.04034573212265968,
      "kl": 0.00205451292640646,
      "learning_rate": 9.526315789473685e-07,
      "loss": 0.0004,
      "step": 819
    },
    {
      "clip_ratio": 0.0005783690139651299,
      "epoch": 0.1119606772255598,
      "grad_norm": 0.0407131165266037,
      "kl": 0.002034339318925049,
      "learning_rate": 9.473684210526317e-07,
      "loss": 0.0002,
      "step": 820
    },
    {
      "clip_ratio": 0.0006272622267715633,
      "completion_length": 140.40625,
      "epoch": 0.11209721463681048,
      "grad_norm": 0.022182300686836243,
      "kl": 0.0017465550354245352,
      "learning_rate": 9.421052631578948e-07,
      "loss": -0.0002,
      "num_tokens": 2388600.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 821
    },
    {
      "clip_ratio": 0.0006651895891991444,
      "epoch": 0.11223375204806117,
      "grad_norm": 0.02211550623178482,
      "kl": 0.001654662546570762,
      "learning_rate": 9.368421052631579e-07,
      "loss": -0.0002,
      "step": 822
    },
    {
      "clip_ratio": 0.00039533356903120875,
      "epoch": 0.11237028945931185,
      "grad_norm": 0.021990373730659485,
      "kl": 0.0017915865701070288,
      "learning_rate": 9.315789473684212e-07,
      "loss": -0.0001,
      "step": 823
    },
    {
      "clip_ratio": 0.0004717895935755223,
      "epoch": 0.11250682687056253,
      "grad_norm": 0.022272175177931786,
      "kl": 0.001733036981022451,
      "learning_rate": 9.263157894736844e-07,
      "loss": 0.0,
      "step": 824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.8125,
      "epoch": 0.11264336428181322,
      "grad_norm": 0.023923585191369057,
      "kl": 0.002135575396096101,
      "learning_rate": 9.210526315789474e-07,
      "loss": 0.0002,
      "num_tokens": 2400042.0,
      "reward": 0.9156249761581421,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.9156249761581421,
      "step": 825
    },
    {
      "clip_ratio": 0.00015318627993110567,
      "epoch": 0.1127799016930639,
      "grad_norm": 0.024121804162859917,
      "kl": 0.0022043623648642097,
      "learning_rate": 9.157894736842106e-07,
      "loss": 0.0001,
      "step": 826
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11291643910431458,
      "grad_norm": 0.02391844429075718,
      "kl": 0.002101333857353893,
      "learning_rate": 9.105263157894737e-07,
      "loss": 0.0003,
      "step": 827
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11305297651556527,
      "grad_norm": 0.023999251425266266,
      "kl": 0.00220367505971808,
      "learning_rate": 9.05263157894737e-07,
      "loss": 0.0003,
      "step": 828
    },
    {
      "clip_ratio": 0.0005337733018677682,
      "completion_length": 142.71875,
      "epoch": 0.11318951392681595,
      "grad_norm": 0.030275406315922737,
      "kl": 0.002487844758434221,
      "learning_rate": 9.000000000000001e-07,
      "loss": 0.0,
      "num_tokens": 2411269.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 829
    },
    {
      "clip_ratio": 0.0008539288392057642,
      "epoch": 0.11332605133806663,
      "grad_norm": 0.03197161853313446,
      "kl": 0.002642420648044208,
      "learning_rate": 8.947368421052632e-07,
      "loss": -0.0001,
      "step": 830
    },
    {
      "clip_ratio": 0.0016706544120097533,
      "epoch": 0.11346258874931732,
      "grad_norm": 0.0321902334690094,
      "kl": 0.002622455087475828,
      "learning_rate": 8.894736842105264e-07,
      "loss": -0.0001,
      "step": 831
    },
    {
      "clip_ratio": 0.0005035283829784021,
      "epoch": 0.113599126160568,
      "grad_norm": 0.031006773933768272,
      "kl": 0.0025140510661003646,
      "learning_rate": 8.842105263157895e-07,
      "loss": 0.0,
      "step": 832
    },
    {
      "clip_ratio": 0.0008417521603405476,
      "completion_length": 143.28125,
      "epoch": 0.11373566357181868,
      "grad_norm": 0.03075164556503296,
      "kl": 0.0016546154947718605,
      "learning_rate": 8.789473684210527e-07,
      "loss": 0.0003,
      "num_tokens": 2423254.0,
      "reward": 0.550000011920929,
      "reward_std": 0.289951890707016,
      "rewards/reward_fn": 0.550000011920929,
      "step": 833
    },
    {
      "clip_ratio": 0.0014790181594435126,
      "epoch": 0.11387220098306935,
      "grad_norm": 0.03217291459441185,
      "kl": 0.0017813782724260818,
      "learning_rate": 8.736842105263159e-07,
      "loss": 0.0001,
      "step": 834
    },
    {
      "clip_ratio": 0.0005646957288263366,
      "epoch": 0.11400873839432005,
      "grad_norm": 0.03188389539718628,
      "kl": 0.0018060972142848186,
      "learning_rate": 8.68421052631579e-07,
      "loss": 0.0001,
      "step": 835
    },
    {
      "clip_ratio": 0.00032028641726356,
      "epoch": 0.11414527580557073,
      "grad_norm": 0.031433261930942535,
      "kl": 0.0017021511393977562,
      "learning_rate": 8.631578947368421e-07,
      "loss": 0.0,
      "step": 836
    },
    {
      "clip_ratio": 0.00037020017043687403,
      "completion_length": 131.1875,
      "epoch": 0.1142818132168214,
      "grad_norm": 0.026657752692699432,
      "kl": 0.0015303445052268216,
      "learning_rate": 8.578947368421054e-07,
      "loss": -0.0001,
      "num_tokens": 2434568.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 837
    },
    {
      "clip_ratio": 0.00010888501856243238,
      "epoch": 0.1144183506280721,
      "grad_norm": 0.026700396090745926,
      "kl": 0.0015061656304169446,
      "learning_rate": 8.526315789473685e-07,
      "loss": -0.0003,
      "step": 838
    },
    {
      "clip_ratio": 0.0006371819763444364,
      "epoch": 0.11455488803932277,
      "grad_norm": 0.026319589465856552,
      "kl": 0.001553235877508996,
      "learning_rate": 8.473684210526316e-07,
      "loss": -0.0,
      "step": 839
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11469142545057345,
      "grad_norm": 0.027091626077890396,
      "kl": 0.0014733337520738132,
      "learning_rate": 8.421052631578948e-07,
      "loss": -0.0,
      "step": 840
    },
    {
      "clip_ratio": 0.00023854961909819394,
      "completion_length": 122.71875,
      "epoch": 0.11482796286182415,
      "grad_norm": 0.04396679624915123,
      "kl": 0.0016556861010030843,
      "learning_rate": 8.36842105263158e-07,
      "loss": 0.0002,
      "num_tokens": 2445595.0,
      "reward": 0.690625011920929,
      "reward_std": 0.29865381121635437,
      "rewards/reward_fn": 0.690625011920929,
      "step": 841
    },
    {
      "clip_ratio": 0.000645464620902203,
      "epoch": 0.11496450027307482,
      "grad_norm": 0.04361685737967491,
      "kl": 0.0017527456711832201,
      "learning_rate": 8.315789473684212e-07,
      "loss": 0.0002,
      "step": 842
    },
    {
      "clip_ratio": 0.0008407137356698513,
      "epoch": 0.1151010376843255,
      "grad_norm": 0.044291649013757706,
      "kl": 0.0017438465802115388,
      "learning_rate": 8.263157894736843e-07,
      "loss": 0.0,
      "step": 843
    },
    {
      "clip_ratio": 0.0006041670858394355,
      "epoch": 0.1152375750955762,
      "grad_norm": 0.04404996335506439,
      "kl": 0.0018427322065690532,
      "learning_rate": 8.210526315789474e-07,
      "loss": 0.0001,
      "step": 844
    },
    {
      "clip_ratio": 0.0008153812377713621,
      "completion_length": 140.40625,
      "epoch": 0.11537411250682687,
      "grad_norm": 0.03839636966586113,
      "kl": 0.0015892905485088704,
      "learning_rate": 8.157894736842106e-07,
      "loss": -0.0001,
      "num_tokens": 2457952.0,
      "reward": 0.5499999523162842,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.5499999523162842,
      "step": 845
    },
    {
      "clip_ratio": 0.0012618098116945475,
      "epoch": 0.11551064991807755,
      "grad_norm": 0.037849295884370804,
      "kl": 0.0015891550483502215,
      "learning_rate": 8.105263157894736e-07,
      "loss": -0.0002,
      "step": 846
    },
    {
      "clip_ratio": 0.0005228656809777021,
      "epoch": 0.11564718732932823,
      "grad_norm": 0.035613447427749634,
      "kl": 0.0016328740239259787,
      "learning_rate": 8.052631578947369e-07,
      "loss": -0.0001,
      "step": 847
    },
    {
      "clip_ratio": 0.001042295160004869,
      "epoch": 0.11578372474057892,
      "grad_norm": 0.035603221505880356,
      "kl": 0.0016090690733108204,
      "learning_rate": 8.000000000000001e-07,
      "loss": -0.0002,
      "step": 848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.9375,
      "epoch": 0.1159202621518296,
      "grad_norm": 0.0288266371935606,
      "kl": 0.002268145781272324,
      "learning_rate": 7.947368421052632e-07,
      "loss": 0.0001,
      "num_tokens": 2468550.0,
      "reward": 0.550000011920929,
      "reward_std": 0.12990380823612213,
      "rewards/reward_fn": 0.550000011920929,
      "step": 849
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "epoch": 0.11605679956308028,
      "grad_norm": 0.028440002351999283,
      "kl": 0.0022938305392017355,
      "learning_rate": 7.894736842105263e-07,
      "loss": -0.0,
      "step": 850
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.11619333697433097,
      "grad_norm": 0.0285995714366436,
      "kl": 0.002212948449596297,
      "learning_rate": 7.842105263157896e-07,
      "loss": -0.0001,
      "step": 851
    },
    {
      "clip_ratio": 0.0002840909000951797,
      "epoch": 0.11632987438558165,
      "grad_norm": 0.02846153825521469,
      "kl": 0.0023701565587543882,
      "learning_rate": 7.789473684210527e-07,
      "loss": -0.0,
      "step": 852
    },
    {
      "clip_ratio": 0.0001230314956046641,
      "completion_length": 128.46875,
      "epoch": 0.11646641179683233,
      "grad_norm": 0.01918521709740162,
      "kl": 0.002273245157994097,
      "learning_rate": 7.736842105263158e-07,
      "loss": -0.0,
      "num_tokens": 2479605.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 853
    },
    {
      "clip_ratio": 0.0001230314956046641,
      "epoch": 0.11660294920808302,
      "grad_norm": 0.019171753898262978,
      "kl": 0.0023291846446227282,
      "learning_rate": 7.684210526315789e-07,
      "loss": -0.0001,
      "step": 854
    },
    {
      "clip_ratio": 0.0003671721206046641,
      "epoch": 0.1167394866193337,
      "grad_norm": 0.019308310002088547,
      "kl": 0.0023352275111392373,
      "learning_rate": 7.631578947368422e-07,
      "loss": 0.0,
      "step": 855
    },
    {
      "clip_ratio": 0.0006592281861230731,
      "epoch": 0.11687602403058438,
      "grad_norm": 0.01928950659930706,
      "kl": 0.0023274405393749475,
      "learning_rate": 7.578947368421054e-07,
      "loss": 0.0,
      "step": 856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 159.09375,
      "epoch": 0.11701256144183507,
      "grad_norm": 0.0285057183355093,
      "kl": 0.0015260744385159342,
      "learning_rate": 7.526315789473684e-07,
      "loss": -0.0001,
      "num_tokens": 2492020.0,
      "reward": 0.690625011920929,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.690625011920929,
      "step": 857
    },
    {
      "clip_ratio": 0.00015703517419751734,
      "epoch": 0.11714909885308575,
      "grad_norm": 0.02803329937160015,
      "kl": 0.001516848760729772,
      "learning_rate": 7.473684210526316e-07,
      "loss": -0.0,
      "step": 858
    },
    {
      "clip_ratio": 0.00015703517419751734,
      "epoch": 0.11728563626433643,
      "grad_norm": 0.02815289795398712,
      "kl": 0.0014452427276410162,
      "learning_rate": 7.421052631578949e-07,
      "loss": -0.0001,
      "step": 859
    },
    {
      "clip_ratio": 0.0002603958855615929,
      "epoch": 0.1174221736755871,
      "grad_norm": 0.02827727235853672,
      "kl": 0.0014738957925146678,
      "learning_rate": 7.368421052631579e-07,
      "loss": -0.0001,
      "step": 860
    },
    {
      "clip_ratio": 0.00027901786961592734,
      "completion_length": 133.28125,
      "epoch": 0.1175587110868378,
      "grad_norm": 0.029622992500662804,
      "kl": 0.0017857067632576218,
      "learning_rate": 7.315789473684211e-07,
      "loss": 0.0001,
      "num_tokens": 2503657.0,
      "reward": 0.5218750238418579,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.5218750238418579,
      "step": 861
    },
    {
      "clip_ratio": 0.000202922077733092,
      "epoch": 0.11769524849808848,
      "grad_norm": 0.029818588867783546,
      "kl": 0.0018871719075832516,
      "learning_rate": 7.263157894736843e-07,
      "loss": -0.0001,
      "step": 862
    },
    {
      "clip_ratio": 0.0004327014903537929,
      "epoch": 0.11783178590933915,
      "grad_norm": 0.03024519979953766,
      "kl": 0.0019561543740564957,
      "learning_rate": 7.210526315789475e-07,
      "loss": 0.0001,
      "step": 863
    },
    {
      "clip_ratio": 0.0002297794126207009,
      "epoch": 0.11796832332058985,
      "grad_norm": 0.029245659708976746,
      "kl": 0.0018293752600584412,
      "learning_rate": 7.157894736842106e-07,
      "loss": 0.0,
      "step": 864
    },
    {
      "clip_ratio": 0.0004111669841222465,
      "completion_length": 135.375,
      "epoch": 0.11810486073184052,
      "grad_norm": 0.0331193283200264,
      "kl": 0.002089089653964038,
      "learning_rate": 7.105263157894737e-07,
      "loss": -0.0002,
      "num_tokens": 2515349.0,
      "reward": 0.46562501788139343,
      "reward_std": 0.2337019145488739,
      "rewards/reward_fn": 0.46562501788139343,
      "step": 865
    },
    {
      "clip_ratio": 0.0005714233993785456,
      "epoch": 0.1182413981430912,
      "grad_norm": 0.033026281744241714,
      "kl": 0.00206513051125512,
      "learning_rate": 7.052631578947369e-07,
      "loss": -0.0,
      "step": 866
    },
    {
      "clip_ratio": 0.0005714233993785456,
      "epoch": 0.1183779355543419,
      "grad_norm": 0.03322478011250496,
      "kl": 0.002053415919363033,
      "learning_rate": 7.000000000000001e-07,
      "loss": 0.0,
      "step": 867
    },
    {
      "clip_ratio": 0.0007658765680389479,
      "epoch": 0.11851447296559257,
      "grad_norm": 0.03364533185958862,
      "kl": 0.002117756648658542,
      "learning_rate": 6.947368421052631e-07,
      "loss": -0.0,
      "step": 868
    },
    {
      "clip_ratio": 0.0005630630766972899,
      "completion_length": 121.21875,
      "epoch": 0.11865101037684325,
      "grad_norm": 0.024038543924689293,
      "kl": 0.0018689564458327368,
      "learning_rate": 6.894736842105264e-07,
      "loss": 0.0001,
      "num_tokens": 2525896.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 869
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.11878754778809394,
      "grad_norm": 0.023961666971445084,
      "kl": 0.001825891291446169,
      "learning_rate": 6.842105263157896e-07,
      "loss": -0.0,
      "step": 870
    },
    {
      "clip_ratio": 0.00028153153834864497,
      "epoch": 0.11892408519934462,
      "grad_norm": 0.024005061015486717,
      "kl": 0.0018299317471246468,
      "learning_rate": 6.789473684210526e-07,
      "loss": -0.0,
      "step": 871
    },
    {
      "clip_ratio": 0.0006981982151046395,
      "epoch": 0.1190606226105953,
      "grad_norm": 0.024153009057044983,
      "kl": 0.0018959494791488396,
      "learning_rate": 6.736842105263158e-07,
      "loss": 0.0001,
      "step": 872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.9375,
      "epoch": 0.11919716002184598,
      "grad_norm": 0.020175298675894737,
      "kl": 0.0021377066386776278,
      "learning_rate": 6.684210526315791e-07,
      "loss": 0.0,
      "num_tokens": 2537458.0,
      "reward": 0.690625011920929,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.690625011920929,
      "step": 873
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.11933369743309667,
      "grad_norm": 0.020295897498726845,
      "kl": 0.002052186724540661,
      "learning_rate": 6.631578947368422e-07,
      "loss": 0.0001,
      "step": 874
    },
    {
      "clip_ratio": 0.0001707650226308033,
      "epoch": 0.11947023484434735,
      "grad_norm": 0.02031334675848484,
      "kl": 0.0021257563785184175,
      "learning_rate": 6.578947368421053e-07,
      "loss": 0.0001,
      "step": 875
    },
    {
      "clip_ratio": 0.000234962411923334,
      "epoch": 0.11960677225559803,
      "grad_norm": 0.020466400310397148,
      "kl": 0.002226112668722635,
      "learning_rate": 6.526315789473684e-07,
      "loss": 0.0001,
      "step": 876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.3125,
      "epoch": 0.11974330966684872,
      "grad_norm": 0.017409300431609154,
      "kl": 0.002158536493880092,
      "learning_rate": 6.473684210526317e-07,
      "loss": 0.0001,
      "num_tokens": 2548268.0,
      "reward": 0.746874988079071,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.746874988079071,
      "step": 877
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1198798470780994,
      "grad_norm": 0.017360569909214973,
      "kl": 0.002025924322879291,
      "learning_rate": 6.421052631578948e-07,
      "loss": 0.0001,
      "step": 878
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12001638448935008,
      "grad_norm": 0.017513062804937363,
      "kl": 0.0020348432663013227,
      "learning_rate": 6.368421052631579e-07,
      "loss": 0.0001,
      "step": 879
    },
    {
      "clip_ratio": 0.00025201612152159214,
      "epoch": 0.12015292190060077,
      "grad_norm": 0.01726604998111725,
      "kl": 0.0020535808343993267,
      "learning_rate": 6.315789473684211e-07,
      "loss": 0.0001,
      "step": 880
    },
    {
      "clip_ratio": 0.0006926970963831991,
      "completion_length": 142.28125,
      "epoch": 0.12028945931185145,
      "grad_norm": 0.036571867763996124,
      "kl": 0.002262445339511032,
      "learning_rate": 6.263157894736844e-07,
      "loss": 0.0001,
      "num_tokens": 2560229.0,
      "reward": 0.606249988079071,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.606249988079071,
      "step": 881
    },
    {
      "clip_ratio": 0.0008497322705807164,
      "epoch": 0.12042599672310213,
      "grad_norm": 0.037133604288101196,
      "kl": 0.0022962079056014773,
      "learning_rate": 6.210526315789474e-07,
      "loss": 0.0002,
      "step": 882
    },
    {
      "clip_ratio": 0.0005721866909880191,
      "epoch": 0.12056253413435282,
      "grad_norm": 0.03698795288801193,
      "kl": 0.002319942112080753,
      "learning_rate": 6.157894736842106e-07,
      "loss": 0.0001,
      "step": 883
    },
    {
      "clip_ratio": 0.00020032051543239504,
      "epoch": 0.1206990715456035,
      "grad_norm": 0.036612141877412796,
      "kl": 0.0023310945653065573,
      "learning_rate": 6.105263157894738e-07,
      "loss": 0.0001,
      "step": 884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.5625,
      "epoch": 0.12083560895685418,
      "grad_norm": 0.028590509667992592,
      "kl": 0.0021473601445904933,
      "learning_rate": 6.052631578947369e-07,
      "loss": -0.0001,
      "num_tokens": 2571031.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 885
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12097214636810485,
      "grad_norm": 0.028788069263100624,
      "kl": 0.002038355499280442,
      "learning_rate": 6.000000000000001e-07,
      "loss": -0.0,
      "step": 886
    },
    {
      "clip_ratio": 0.00035511364694684744,
      "epoch": 0.12110868377935555,
      "grad_norm": 0.02862849459052086,
      "kl": 0.001932536375534255,
      "learning_rate": 5.947368421052632e-07,
      "loss": -0.0002,
      "step": 887
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12124522119060623,
      "grad_norm": 0.02891075611114502,
      "kl": 0.0018507974418753292,
      "learning_rate": 5.894736842105263e-07,
      "loss": -0.0001,
      "step": 888
    },
    {
      "clip_ratio": 0.0002540650311857462,
      "completion_length": 117.28125,
      "epoch": 0.1213817586018569,
      "grad_norm": 0.019024843350052834,
      "kl": 0.002249214661787846,
      "learning_rate": 5.842105263157896e-07,
      "loss": -0.0001,
      "num_tokens": 2582120.0,
      "reward": 0.6031249761581421,
      "reward_std": 0.07120190560817719,
      "rewards/reward_fn": 0.6031249761581421,
      "step": 889
    },
    {
      "clip_ratio": 0.00021404109429568052,
      "epoch": 0.1215182960131076,
      "grad_norm": 0.019089598208665848,
      "kl": 0.0020849486736551626,
      "learning_rate": 5.789473684210526e-07,
      "loss": -0.0002,
      "step": 890
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12165483342435827,
      "grad_norm": 0.018902519717812538,
      "kl": 0.0022411892423406243,
      "learning_rate": 5.736842105263158e-07,
      "loss": -0.0001,
      "step": 891
    },
    {
      "clip_ratio": 0.0002540650311857462,
      "epoch": 0.12179137083560895,
      "grad_norm": 0.018905550241470337,
      "kl": 0.002317437955753121,
      "learning_rate": 5.68421052631579e-07,
      "loss": -0.0,
      "step": 892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.96875,
      "epoch": 0.12192790824685965,
      "grad_norm": 0.035427920520305634,
      "kl": 0.0019287737968625152,
      "learning_rate": 5.631578947368421e-07,
      "loss": -0.0,
      "num_tokens": 2592995.0,
      "reward": 0.32499998807907104,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.32499998807907104,
      "step": 893
    },
    {
      "clip_ratio": 0.0008413766045123339,
      "epoch": 0.12206444565811032,
      "grad_norm": 0.03540027514100075,
      "kl": 0.0019906538364011794,
      "learning_rate": 5.578947368421053e-07,
      "loss": -0.0002,
      "step": 894
    },
    {
      "clip_ratio": 0.0010899427579715848,
      "epoch": 0.122200983069361,
      "grad_norm": 0.03542100265622139,
      "kl": 0.002009846612054389,
      "learning_rate": 5.526315789473684e-07,
      "loss": 0.0001,
      "step": 895
    },
    {
      "clip_ratio": 0.0004976481723133475,
      "epoch": 0.1223375204806117,
      "grad_norm": 0.03547709807753563,
      "kl": 0.0020012245713587618,
      "learning_rate": 5.473684210526316e-07,
      "loss": -0.0,
      "step": 896
    },
    {
      "clip_ratio": 0.0003513729461701587,
      "completion_length": 144.25,
      "epoch": 0.12247405789186237,
      "grad_norm": 0.0332733653485775,
      "kl": 0.002094986011798028,
      "learning_rate": 5.421052631578948e-07,
      "loss": 0.0,
      "num_tokens": 2604815.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.25110572576522827,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 897
    },
    {
      "clip_ratio": 0.0010349433432566002,
      "epoch": 0.12261059530311305,
      "grad_norm": 0.03256555646657944,
      "kl": 0.002050184742984129,
      "learning_rate": 5.368421052631579e-07,
      "loss": 0.0002,
      "step": 898
    },
    {
      "clip_ratio": 0.0005613642715616152,
      "epoch": 0.12274713271436373,
      "grad_norm": 0.033078376203775406,
      "kl": 0.0021014586727687856,
      "learning_rate": 5.315789473684211e-07,
      "loss": 0.0001,
      "step": 899
    },
    {
      "clip_ratio": 0.00041538292134646326,
      "epoch": 0.12288367012561442,
      "grad_norm": 0.03323504701256752,
      "kl": 0.0020402646969159832,
      "learning_rate": 5.263157894736843e-07,
      "loss": -0.0001,
      "step": 900
    },
    {
      "clip_ratio": 0.0003972457634517923,
      "completion_length": 131.0,
      "epoch": 0.1230202075368651,
      "grad_norm": 0.024151014164090157,
      "kl": 0.0020411131608852884,
      "learning_rate": 5.210526315789474e-07,
      "loss": -0.0001,
      "num_tokens": 2616651.0,
      "reward": 0.550000011920929,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.550000011920929,
      "step": 901
    },
    {
      "clip_ratio": 0.00026483050896786153,
      "epoch": 0.12315674494811578,
      "grad_norm": 0.024675916880369186,
      "kl": 0.0021187742222537054,
      "learning_rate": 5.157894736842106e-07,
      "loss": -0.0001,
      "step": 902
    },
    {
      "clip_ratio": 0.0009654743334976956,
      "epoch": 0.12329328235936647,
      "grad_norm": 0.024687334895133972,
      "kl": 0.0020260024430172052,
      "learning_rate": 5.105263157894738e-07,
      "loss": -0.0001,
      "step": 903
    },
    {
      "clip_ratio": 0.0009654743334976956,
      "epoch": 0.12342981977061715,
      "grad_norm": 0.023990610614418983,
      "kl": 0.001970850536963553,
      "learning_rate": 5.052631578947369e-07,
      "loss": -0.0,
      "step": 904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.59375,
      "epoch": 0.12356635718186783,
      "grad_norm": 0.02948358654975891,
      "kl": 0.0017617333833186422,
      "learning_rate": 5.000000000000001e-07,
      "loss": -0.0001,
      "num_tokens": 2627562.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 905
    },
    {
      "clip_ratio": 0.0002500000118743628,
      "epoch": 0.12370289459311852,
      "grad_norm": 0.029498081654310226,
      "kl": 0.0018035039292954025,
      "learning_rate": 4.947368421052632e-07,
      "loss": -0.0001,
      "step": 906
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1238394320043692,
      "grad_norm": 0.029065372422337532,
      "kl": 0.001829242224630434,
      "learning_rate": 4.894736842105263e-07,
      "loss": -0.0001,
      "step": 907
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12397596941561988,
      "grad_norm": 0.02955593168735504,
      "kl": 0.001747171319948393,
      "learning_rate": 4.842105263157895e-07,
      "loss": 0.0,
      "step": 908
    },
    {
      "clip_ratio": 0.000662284146528691,
      "completion_length": 118.5,
      "epoch": 0.12411250682687057,
      "grad_norm": 0.03342261537909508,
      "kl": 0.0021508525096578524,
      "learning_rate": 4.789473684210526e-07,
      "loss": -0.0,
      "num_tokens": 2638254.0,
      "reward": 0.8031249642372131,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.8031249642372131,
      "step": 909
    },
    {
      "clip_ratio": 0.00034340660204179585,
      "epoch": 0.12424904423812125,
      "grad_norm": 0.03362087532877922,
      "kl": 0.0020725269005197333,
      "learning_rate": 4.7368421052631585e-07,
      "loss": 0.0002,
      "step": 910
    },
    {
      "clip_ratio": 0.00034340660204179585,
      "epoch": 0.12438558164937193,
      "grad_norm": 0.03338223695755005,
      "kl": 0.002021927133682766,
      "learning_rate": 4.6842105263157896e-07,
      "loss": 0.0001,
      "step": 911
    },
    {
      "clip_ratio": 0.00034340660204179585,
      "epoch": 0.1245221190606226,
      "grad_norm": 0.03381529077887535,
      "kl": 0.002014444373344304,
      "learning_rate": 4.631578947368422e-07,
      "loss": -0.0,
      "step": 912
    },
    {
      "clip_ratio": 0.0005952381179668009,
      "completion_length": 130.03125,
      "epoch": 0.1246586564718733,
      "grad_norm": 0.022914061322808266,
      "kl": 0.0017131423273895052,
      "learning_rate": 4.578947368421053e-07,
      "loss": 0.0003,
      "num_tokens": 2649651.0,
      "reward": 0.6625000238418579,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.6625000238418579,
      "step": 913
    },
    {
      "clip_ratio": 0.0006686990382149816,
      "epoch": 0.12479519388312398,
      "grad_norm": 0.023066677153110504,
      "kl": 0.0018404278443995281,
      "learning_rate": 4.526315789473685e-07,
      "loss": 0.0004,
      "step": 914
    },
    {
      "clip_ratio": 0.00018168604583479464,
      "epoch": 0.12493173129437465,
      "grad_norm": 0.02291272021830082,
      "kl": 0.001800586937861226,
      "learning_rate": 4.473684210526316e-07,
      "loss": 0.0003,
      "step": 915
    },
    {
      "clip_ratio": 0.00018168604583479464,
      "epoch": 0.12506826870562535,
      "grad_norm": 0.022816216573119164,
      "kl": 0.0018408020005153958,
      "learning_rate": 4.421052631578947e-07,
      "loss": 0.0003,
      "step": 916
    },
    {
      "clip_ratio": 0.0004030029522255063,
      "completion_length": 132.15625,
      "epoch": 0.125204806116876,
      "grad_norm": 0.024068795144557953,
      "kl": 0.002098910128552234,
      "learning_rate": 4.3684210526315794e-07,
      "loss": -0.0001,
      "num_tokens": 2661240.0,
      "reward": 0.578125,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.578125,
      "step": 917
    },
    {
      "clip_ratio": 0.0004030029522255063,
      "epoch": 0.1253413435281267,
      "grad_norm": 0.023403236642479897,
      "kl": 0.002160426525733783,
      "learning_rate": 4.3157894736842105e-07,
      "loss": 0.0001,
      "step": 918
    },
    {
      "clip_ratio": 0.0005795566248707473,
      "epoch": 0.1254778809393774,
      "grad_norm": 0.023773547261953354,
      "kl": 0.0020677255215559853,
      "learning_rate": 4.2631578947368427e-07,
      "loss": 0.0,
      "step": 919
    },
    {
      "clip_ratio": 0.00017655367264524102,
      "epoch": 0.12561441835062806,
      "grad_norm": 0.02403651364147663,
      "kl": 0.0021508349182113307,
      "learning_rate": 4.210526315789474e-07,
      "loss": 0.0001,
      "step": 920
    },
    {
      "clip_ratio": 0.00024801588733680546,
      "completion_length": 128.65625,
      "epoch": 0.12575095576187875,
      "grad_norm": 0.02821282111108303,
      "kl": 0.001836812205510796,
      "learning_rate": 4.157894736842106e-07,
      "loss": -0.0003,
      "num_tokens": 2672441.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 921
    },
    {
      "clip_ratio": 0.00024801588733680546,
      "epoch": 0.12588749317312944,
      "grad_norm": 0.028158273547887802,
      "kl": 0.0019186180161341326,
      "learning_rate": 4.105263157894737e-07,
      "loss": -0.0002,
      "step": 922
    },
    {
      "clip_ratio": 0.0007261889695655555,
      "epoch": 0.1260240305843801,
      "grad_norm": 0.028605299070477486,
      "kl": 0.0018298066697752802,
      "learning_rate": 4.052631578947368e-07,
      "loss": 0.0001,
      "step": 923
    },
    {
      "clip_ratio": 0.0001516990305390209,
      "epoch": 0.1261605679956308,
      "grad_norm": 0.027679719030857086,
      "kl": 0.001927206470099918,
      "learning_rate": 4.0000000000000003e-07,
      "loss": -0.0001,
      "step": 924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.96875,
      "epoch": 0.1262971054068815,
      "grad_norm": 0.013626189902424812,
      "kl": 0.001803158293114393,
      "learning_rate": 3.9473684210526315e-07,
      "loss": 0.0,
      "num_tokens": 2684516.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.05624999850988388,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 925
    },
    {
      "clip_ratio": 0.0001270325155928731,
      "epoch": 0.12643364281813216,
      "grad_norm": 0.013184582814574242,
      "kl": 0.0018444102661305806,
      "learning_rate": 3.8947368421052636e-07,
      "loss": -0.0001,
      "step": 926
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12657018022938285,
      "grad_norm": 0.013402871787548065,
      "kl": 0.0017522034195280867,
      "learning_rate": 3.8421052631578947e-07,
      "loss": -0.0,
      "step": 927
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12670671764063354,
      "grad_norm": 0.013282261788845062,
      "kl": 0.0018042439696728252,
      "learning_rate": 3.789473684210527e-07,
      "loss": -0.0001,
      "step": 928
    },
    {
      "clip_ratio": 0.0005196314741624519,
      "completion_length": 141.625,
      "epoch": 0.1268432550518842,
      "grad_norm": 0.034112170338630676,
      "kl": 0.0018444620891386876,
      "learning_rate": 3.736842105263158e-07,
      "loss": -0.0001,
      "num_tokens": 2696288.0,
      "reward": 0.550000011920929,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.550000011920929,
      "step": 929
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1269797924631349,
      "grad_norm": 0.034253526479005814,
      "kl": 0.0017955962957785232,
      "learning_rate": 3.6842105263157896e-07,
      "loss": -0.0003,
      "step": 930
    },
    {
      "clip_ratio": 0.0003722258115885779,
      "epoch": 0.1271163298743856,
      "grad_norm": 0.03406032919883728,
      "kl": 0.0016890436236280948,
      "learning_rate": 3.6315789473684213e-07,
      "loss": -0.0002,
      "step": 931
    },
    {
      "clip_ratio": 0.0006515669811051339,
      "epoch": 0.12725286728563626,
      "grad_norm": 0.0342014916241169,
      "kl": 0.0017644154213485308,
      "learning_rate": 3.578947368421053e-07,
      "loss": -0.0001,
      "step": 932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.5625,
      "epoch": 0.12738940469688695,
      "grad_norm": 0.028642408549785614,
      "kl": 0.0017243105503439438,
      "learning_rate": 3.5263157894736846e-07,
      "loss": -0.0001,
      "num_tokens": 2707946.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.8031250238418579,
      "step": 933
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.12752594210813764,
      "grad_norm": 0.02829623967409134,
      "kl": 0.0017418616225768346,
      "learning_rate": 3.4736842105263157e-07,
      "loss": -0.0001,
      "step": 934
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1276624795193883,
      "grad_norm": 0.028566766530275345,
      "kl": 0.0018181849009124562,
      "learning_rate": 3.421052631578948e-07,
      "loss": 0.0,
      "step": 935
    },
    {
      "clip_ratio": 0.00016447369125671685,
      "epoch": 0.127799016930639,
      "grad_norm": 0.028366414830088615,
      "kl": 0.0016921994192671264,
      "learning_rate": 3.368421052631579e-07,
      "loss": -0.0001,
      "step": 936
    },
    {
      "clip_ratio": 0.0006108862871769816,
      "completion_length": 155.28125,
      "epoch": 0.1279355543418897,
      "grad_norm": 0.0333460234105587,
      "kl": 0.0017041650116880191,
      "learning_rate": 3.315789473684211e-07,
      "loss": -0.0003,
      "num_tokens": 2720175.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.28125,
      "rewards/reward_fn": 0.6343749761581421,
      "step": 937
    },
    {
      "clip_ratio": 0.00046177000331226736,
      "epoch": 0.12807209175314035,
      "grad_norm": 0.03320127725601196,
      "kl": 0.0017484442232671427,
      "learning_rate": 3.263157894736842e-07,
      "loss": -0.0001,
      "step": 938
    },
    {
      "clip_ratio": 0.00014140272105578333,
      "epoch": 0.12820862916439105,
      "grad_norm": 0.03337475657463074,
      "kl": 0.0017337162571493536,
      "learning_rate": 3.210526315789474e-07,
      "loss": -0.0001,
      "step": 939
    },
    {
      "clip_ratio": 0.0008734305156394839,
      "epoch": 0.12834516657564174,
      "grad_norm": 0.03332708403468132,
      "kl": 0.001759997263434343,
      "learning_rate": 3.1578947368421055e-07,
      "loss": -0.0,
      "step": 940
    },
    {
      "clip_ratio": 0.0003595184243749827,
      "completion_length": 128.875,
      "epoch": 0.1284817039868924,
      "grad_norm": 0.04191797599196434,
      "kl": 0.0022904958823346533,
      "learning_rate": 3.105263157894737e-07,
      "loss": -0.0002,
      "num_tokens": 2731411.0,
      "reward": 0.71875,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.71875,
      "step": 941
    },
    {
      "clip_ratio": 0.0005447269359137863,
      "epoch": 0.1286182413981431,
      "grad_norm": 0.04139786958694458,
      "kl": 0.002369653229834512,
      "learning_rate": 3.052631578947369e-07,
      "loss": -0.0,
      "step": 942
    },
    {
      "clip_ratio": 0.0006348488095682114,
      "epoch": 0.12875477880939376,
      "grad_norm": 0.041293784976005554,
      "kl": 0.002344006352359429,
      "learning_rate": 3.0000000000000004e-07,
      "loss": -0.0002,
      "step": 943
    },
    {
      "clip_ratio": 0.00013469827536027879,
      "epoch": 0.12889131622064445,
      "grad_norm": 0.041487667709589005,
      "kl": 0.0024770467080088565,
      "learning_rate": 2.9473684210526315e-07,
      "loss": -0.0001,
      "step": 944
    },
    {
      "clip_ratio": 0.0001594387722434476,
      "completion_length": 123.25,
      "epoch": 0.12902785363189515,
      "grad_norm": 0.031169265508651733,
      "kl": 0.0020621584189939313,
      "learning_rate": 2.894736842105263e-07,
      "loss": -0.0004,
      "num_tokens": 2742219.0,
      "reward": 0.4906250238418579,
      "reward_std": 0.18370190262794495,
      "rewards/reward_fn": 0.4906250238418579,
      "step": 945
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1291643910431458,
      "grad_norm": 0.03082355111837387,
      "kl": 0.0021173268905840814,
      "learning_rate": 2.842105263157895e-07,
      "loss": -0.0002,
      "step": 946
    },
    {
      "clip_ratio": 0.0001594387722434476,
      "epoch": 0.1293009284543965,
      "grad_norm": 0.03010324388742447,
      "kl": 0.0021255461579130497,
      "learning_rate": 2.7894736842105264e-07,
      "loss": -0.0001,
      "step": 947
    },
    {
      "clip_ratio": 0.0001460280327592045,
      "epoch": 0.1294374658656472,
      "grad_norm": 0.031324584037065506,
      "kl": 0.001988987545701093,
      "learning_rate": 2.736842105263158e-07,
      "loss": -0.0003,
      "step": 948
    },
    {
      "clip_ratio": 0.000619448401266709,
      "completion_length": 152.0,
      "epoch": 0.12957400327689786,
      "grad_norm": 0.02701011672616005,
      "kl": 0.0018674684397410601,
      "learning_rate": 2.6842105263157897e-07,
      "loss": 0.0,
      "num_tokens": 2754427.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.22499999403953552,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 949
    },
    {
      "clip_ratio": 0.00023086981673259288,
      "epoch": 0.12971054068814855,
      "grad_norm": 0.027033735066652298,
      "kl": 0.0018265853186676395,
      "learning_rate": 2.6315789473684213e-07,
      "loss": 0.0001,
      "step": 950
    },
    {
      "clip_ratio": 0.00040395389805780724,
      "epoch": 0.12984707809939924,
      "grad_norm": 0.027212731540203094,
      "kl": 0.0019295160582259996,
      "learning_rate": 2.578947368421053e-07,
      "loss": -0.0001,
      "step": 951
    },
    {
      "clip_ratio": 0.0006968785892240703,
      "epoch": 0.1299836155106499,
      "grad_norm": 0.02716096304357052,
      "kl": 0.0018647334300112561,
      "learning_rate": 2.5263157894736846e-07,
      "loss": -0.0001,
      "step": 952
    },
    {
      "clip_ratio": 0.0007757260173093528,
      "completion_length": 119.3125,
      "epoch": 0.1301201529219006,
      "grad_norm": 0.04574109613895416,
      "kl": 0.0019425052614678862,
      "learning_rate": 2.473684210526316e-07,
      "loss": -0.0001,
      "num_tokens": 2765617.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.36360570788383484,
      "rewards/reward_fn": 0.6906249523162842,
      "step": 953
    },
    {
      "clip_ratio": 0.000234962411923334,
      "epoch": 0.1302566903331513,
      "grad_norm": 0.04522733762860298,
      "kl": 0.0018160702220484382,
      "learning_rate": 2.4210526315789473e-07,
      "loss": -0.0004,
      "step": 954
    },
    {
      "clip_ratio": 0.0008333333535119891,
      "epoch": 0.13039322774440196,
      "grad_norm": 0.04570286348462105,
      "kl": 0.0019073822350037517,
      "learning_rate": 2.3684210526315792e-07,
      "loss": -0.0002,
      "step": 955
    },
    {
      "clip_ratio": 0.0006666666886303574,
      "epoch": 0.13052976515565265,
      "grad_norm": 0.04525072127580643,
      "kl": 0.001965075010957662,
      "learning_rate": 2.315789473684211e-07,
      "loss": -0.0002,
      "step": 956
    },
    {
      "clip_ratio": 0.0005073317588539794,
      "completion_length": 124.0,
      "epoch": 0.13066630256690334,
      "grad_norm": 0.03176407143473625,
      "kl": 0.002387185357292765,
      "learning_rate": 2.2631578947368425e-07,
      "loss": 0.0,
      "num_tokens": 2776405.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 957
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.130802839978154,
      "grad_norm": 0.0319964699447155,
      "kl": 0.0022256208067119587,
      "learning_rate": 2.2105263157894736e-07,
      "loss": -0.0002,
      "step": 958
    },
    {
      "clip_ratio": 0.00025826445198617876,
      "epoch": 0.1309393773894047,
      "grad_norm": 0.03232432156801224,
      "kl": 0.0025238104517484317,
      "learning_rate": 2.1578947368421053e-07,
      "loss": 0.0001,
      "step": 959
    },
    {
      "clip_ratio": 0.0007217811944428831,
      "epoch": 0.1310759148006554,
      "grad_norm": 0.031727712601423264,
      "kl": 0.0023681106831645593,
      "learning_rate": 2.105263157894737e-07,
      "loss": -0.0002,
      "step": 960
    },
    {
      "clip_ratio": 0.0003830051573459059,
      "completion_length": 118.53125,
      "epoch": 0.13121245221190606,
      "grad_norm": 0.02657039277255535,
      "kl": 0.002205699001933681,
      "learning_rate": 2.0526315789473685e-07,
      "loss": 0.0,
      "num_tokens": 2787282.0,
      "reward": 0.887499988079071,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.887499988079071,
      "step": 961
    },
    {
      "clip_ratio": 0.00016447369125671685,
      "epoch": 0.13134898962315675,
      "grad_norm": 0.026693396270275116,
      "kl": 0.00212965470564086,
      "learning_rate": 2.0000000000000002e-07,
      "loss": 0.0,
      "step": 962
    },
    {
      "clip_ratio": 0.0004708462511189282,
      "epoch": 0.13148552703440744,
      "grad_norm": 0.026747936382889748,
      "kl": 0.0022483005668618716,
      "learning_rate": 1.9473684210526318e-07,
      "loss": -0.0001,
      "step": 963
    },
    {
      "clip_ratio": 0.000635319942375645,
      "epoch": 0.1316220644456581,
      "grad_norm": 0.02631334215402603,
      "kl": 0.002198568679887103,
      "learning_rate": 1.8947368421052634e-07,
      "loss": -0.0,
      "step": 964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.65625,
      "epoch": 0.1317586018569088,
      "grad_norm": 0.03489239513874054,
      "kl": 0.0026326864990551258,
      "learning_rate": 1.8421052631578948e-07,
      "loss": -0.0001,
      "num_tokens": 2798519.0,
      "reward": 0.6062500476837158,
      "reward_std": 0.2899519205093384,
      "rewards/reward_fn": 0.6062500476837158,
      "step": 965
    },
    {
      "clip_ratio": 0.0013032681017648429,
      "epoch": 0.1318951392681595,
      "grad_norm": 0.03495628386735916,
      "kl": 0.0025636351347202435,
      "learning_rate": 1.7894736842105265e-07,
      "loss": -0.0003,
      "step": 966
    },
    {
      "clip_ratio": 0.0010461228230269626,
      "epoch": 0.13203167667941015,
      "grad_norm": 0.03498396277427673,
      "kl": 0.002557776528192335,
      "learning_rate": 1.7368421052631578e-07,
      "loss": -0.0001,
      "step": 967
    },
    {
      "clip_ratio": 0.0011772600410040468,
      "epoch": 0.13216821409066085,
      "grad_norm": 0.035107169300317764,
      "kl": 0.0026038188407255802,
      "learning_rate": 1.6842105263157895e-07,
      "loss": 0.0001,
      "step": 968
    },
    {
      "clip_ratio": 0.00022321428696159273,
      "completion_length": 115.90625,
      "epoch": 0.1323047515019115,
      "grad_norm": 0.029477836564183235,
      "kl": 0.002139107233233517,
      "learning_rate": 1.631578947368421e-07,
      "loss": -0.0001,
      "num_tokens": 2809540.0,
      "reward": 0.7468750476837158,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.7468750476837158,
      "step": 969
    },
    {
      "clip_ratio": 0.0007052621658658609,
      "epoch": 0.1324412889131622,
      "grad_norm": 0.02851179987192154,
      "kl": 0.0021293932059052167,
      "learning_rate": 1.5789473684210527e-07,
      "loss": -0.0,
      "step": 970
    },
    {
      "clip_ratio": 0.0002604166802484542,
      "epoch": 0.1325778263244129,
      "grad_norm": 0.02857079543173313,
      "kl": 0.0020065300195710734,
      "learning_rate": 1.5263157894736844e-07,
      "loss": 0.0001,
      "step": 971
    },
    {
      "clip_ratio": 0.00044642857392318547,
      "epoch": 0.13271436373566356,
      "grad_norm": 0.029482480138540268,
      "kl": 0.002193307056586491,
      "learning_rate": 1.4736842105263158e-07,
      "loss": 0.0002,
      "step": 972
    },
    {
      "clip_ratio": 0.000703705329215154,
      "completion_length": 136.15625,
      "epoch": 0.13285090114691425,
      "grad_norm": 0.03487672656774521,
      "kl": 0.0018428106286592083,
      "learning_rate": 1.4210526315789474e-07,
      "loss": 0.0001,
      "num_tokens": 2820793.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.746874988079071,
      "step": 973
    },
    {
      "clip_ratio": 0.000788291115895845,
      "epoch": 0.13298743855816494,
      "grad_norm": 0.0352611243724823,
      "kl": 0.0017364407913191826,
      "learning_rate": 1.368421052631579e-07,
      "loss": 0.0,
      "step": 974
    },
    {
      "clip_ratio": 0.0010632339253788814,
      "epoch": 0.1331239759694156,
      "grad_norm": 0.03562775254249573,
      "kl": 0.0017056907054211479,
      "learning_rate": 1.3157894736842107e-07,
      "loss": -0.0002,
      "step": 975
    },
    {
      "clip_ratio": 0.0002540650311857462,
      "epoch": 0.1332605133806663,
      "grad_norm": 0.03521440923213959,
      "kl": 0.0017664205297478475,
      "learning_rate": 1.2631578947368423e-07,
      "loss": 0.0001,
      "step": 976
    },
    {
      "clip_ratio": 0.0006411774083971977,
      "completion_length": 125.9375,
      "epoch": 0.133397050791917,
      "grad_norm": 0.04052438214421272,
      "kl": 0.002075563157632132,
      "learning_rate": 1.2105263157894737e-07,
      "loss": 0.0001,
      "num_tokens": 2831843.0,
      "reward": 0.746874988079071,
      "reward_std": 0.2337018996477127,
      "rewards/reward_fn": 0.746874988079071,
      "step": 977
    },
    {
      "clip_ratio": 0.0007323232421185821,
      "epoch": 0.13353358820316766,
      "grad_norm": 0.041622359305620193,
      "kl": 0.0020363604762678733,
      "learning_rate": 1.1578947368421054e-07,
      "loss": 0.0002,
      "step": 978
    },
    {
      "clip_ratio": 0.00031565656536258757,
      "epoch": 0.13367012561441835,
      "grad_norm": 0.04189898818731308,
      "kl": 0.0020175932950223796,
      "learning_rate": 1.1052631578947368e-07,
      "loss": 0.0001,
      "step": 979
    },
    {
      "clip_ratio": 0.0007421875197906047,
      "epoch": 0.13380666302566904,
      "grad_norm": 0.04237787052989006,
      "kl": 0.001999835752940271,
      "learning_rate": 1.0526315789473685e-07,
      "loss": 0.0003,
      "step": 980
    },
    {
      "clip_ratio": 0.0004293356032576412,
      "completion_length": 123.09375,
      "epoch": 0.1339432004369197,
      "grad_norm": 0.044372156262397766,
      "kl": 0.0019276039001852041,
      "learning_rate": 1.0000000000000001e-07,
      "loss": 0.0,
      "num_tokens": 2842758.0,
      "reward": 0.7749999761581421,
      "reward_std": 0.35490381717681885,
      "rewards/reward_fn": 0.7749999761581421,
      "step": 981
    },
    {
      "clip_ratio": 0.000518681132234633,
      "epoch": 0.1340797378481704,
      "grad_norm": 0.04381715878844261,
      "kl": 0.0021304004676494515,
      "learning_rate": 9.473684210526317e-08,
      "loss": -0.0002,
      "step": 982
    },
    {
      "clip_ratio": 0.00043168605770915747,
      "epoch": 0.1342162752594211,
      "grad_norm": 0.04373766854405403,
      "kl": 0.001978013932784961,
      "learning_rate": 8.947368421052632e-08,
      "loss": -0.0001,
      "step": 983
    },
    {
      "clip_ratio": 0.0011094502406194806,
      "epoch": 0.13435281267067176,
      "grad_norm": 0.04434705153107643,
      "kl": 0.0019842102301481646,
      "learning_rate": 8.421052631578947e-08,
      "loss": -0.0001,
      "step": 984
    },
    {
      "clip_ratio": 0.0008912696212064475,
      "completion_length": 125.6875,
      "epoch": 0.13448935008192245,
      "grad_norm": 0.035390034317970276,
      "kl": 0.0026560259429970756,
      "learning_rate": 7.894736842105264e-08,
      "loss": 0.0001,
      "num_tokens": 2853588.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.17745190858840942,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 985
    },
    {
      "clip_ratio": 0.0007543278188677505,
      "epoch": 0.13462588749317314,
      "grad_norm": 0.0348115973174572,
      "kl": 0.0027009638433810323,
      "learning_rate": 7.368421052631579e-08,
      "loss": 0.0002,
      "step": 986
    },
    {
      "clip_ratio": 0.0007222114945761859,
      "epoch": 0.1347624249044238,
      "grad_norm": 0.03600397706031799,
      "kl": 0.002620842893520603,
      "learning_rate": 6.842105263157895e-08,
      "loss": 0.0002,
      "step": 987
    },
    {
      "clip_ratio": 0.0007801339961588383,
      "epoch": 0.1348989623156745,
      "grad_norm": 0.034362636506557465,
      "kl": 0.0025556167329341406,
      "learning_rate": 6.315789473684211e-08,
      "loss": 0.0002,
      "step": 988
    },
    {
      "clip_ratio": 0.0004814189160242677,
      "completion_length": 133.28125,
      "epoch": 0.1350354997269252,
      "grad_norm": 0.020515473559498787,
      "kl": 0.001966405860002851,
      "learning_rate": 5.789473684210527e-08,
      "loss": 0.0001,
      "num_tokens": 2865025.0,
      "reward": 0.6343750357627869,
      "reward_std": 0.12120190262794495,
      "rewards/reward_fn": 0.6343750357627869,
      "step": 989
    },
    {
      "clip_ratio": 0.0006052327662473544,
      "epoch": 0.13517203713817585,
      "grad_norm": 0.020682375878095627,
      "kl": 0.001828348807975999,
      "learning_rate": 5.263157894736842e-08,
      "loss": -0.0,
      "step": 990
    },
    {
      "clip_ratio": 0.00054094273946248,
      "epoch": 0.13530857454942655,
      "grad_norm": 0.020333580672740936,
      "kl": 0.0018641406913957326,
      "learning_rate": 4.7368421052631586e-08,
      "loss": -0.0001,
      "step": 991
    },
    {
      "clip_ratio": 0.0003124999930150807,
      "epoch": 0.13544511196067724,
      "grad_norm": 0.020512109622359276,
      "kl": 0.0017612487336009508,
      "learning_rate": 4.2105263157894737e-08,
      "loss": -0.0001,
      "step": 992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.15625,
      "epoch": 0.1355816493719279,
      "grad_norm": 0.021672414615750313,
      "kl": 0.001844227123001474,
      "learning_rate": 3.6842105263157894e-08,
      "loss": 0.0002,
      "num_tokens": 2876602.0,
      "reward": 0.8312499523162842,
      "reward_std": 0.11249999701976776,
      "rewards/reward_fn": 0.8312499523162842,
      "step": 993
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1357181867831786,
      "grad_norm": 0.02169492281973362,
      "kl": 0.001965407580428291,
      "learning_rate": 3.157894736842106e-08,
      "loss": 0.0001,
      "step": 994
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.13585472419442926,
      "grad_norm": 0.021661533042788506,
      "kl": 0.002062978146568639,
      "learning_rate": 2.631578947368421e-08,
      "loss": 0.0001,
      "step": 995
    },
    {
      "clip_ratio": 0.0004692772781709209,
      "epoch": 0.13599126160567995,
      "grad_norm": 0.021499216556549072,
      "kl": 0.0019374883922864683,
      "learning_rate": 2.1052631578947368e-08,
      "loss": 0.0001,
      "step": 996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.59375,
      "epoch": 0.13612779901693065,
      "grad_norm": 0.026189453899860382,
      "kl": 0.0020121177003602497,
      "learning_rate": 1.578947368421053e-08,
      "loss": 0.0,
      "num_tokens": 2887857.0,
      "reward": 0.690625011920929,
      "reward_std": 0.16874998807907104,
      "rewards/reward_fn": 0.690625011920929,
      "step": 997
    },
    {
      "clip_ratio": 0.00010629251482896507,
      "epoch": 0.1362643364281813,
      "grad_norm": 0.02555716410279274,
      "kl": 0.0019450822092039743,
      "learning_rate": 1.0526315789473684e-08,
      "loss": 0.0,
      "step": 998
    },
    {
      "clip_ratio": 0.00043367347097955644,
      "epoch": 0.136400873839432,
      "grad_norm": 0.025302477180957794,
      "kl": 0.0021157398405193817,
      "learning_rate": 5.263157894736842e-09,
      "loss": -0.0002,
      "step": 999
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.1365374112506827,
      "grad_norm": 0.025643914937973022,
      "kl": 0.0020015074787806952,
      "learning_rate": 0.0,
      "loss": -0.0,
      "step": 1000
    }
  ],
  "logging_steps": 1,
  "max_steps": 1000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 250,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 32,
  "trial_name": null,
  "trial_params": null
}