{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.23232851251669862,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0002323285125166986,
      "grad_norm": 26.544933319091797,
      "learning_rate": 1e-05,
      "logits/chosen": 2.489198923110962,
      "logits/rejected": 2.346872329711914,
      "logps/chosen": -82.36200714111328,
      "logps/rejected": -78.39857482910156,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0004646570250333972,
      "grad_norm": 25.423452377319336,
      "learning_rate": 9.99767711962834e-06,
      "logits/chosen": 2.47176456451416,
      "logits/rejected": 2.4068381786346436,
      "logps/chosen": -83.61863708496094,
      "logps/rejected": -72.68998718261719,
      "loss": 0.7031,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.012627649120986462,
      "rewards/margins": -0.019588422030210495,
      "rewards/rejected": 0.006960772909224033,
      "step": 2
    },
    {
      "epoch": 0.0006969855375500958,
      "grad_norm": 25.243507385253906,
      "learning_rate": 9.995354239256679e-06,
      "logits/chosen": 2.228482961654663,
      "logits/rejected": 2.150202512741089,
      "logps/chosen": -71.58912658691406,
      "logps/rejected": -83.01657104492188,
      "loss": 0.6954,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 7.674687367398292e-05,
      "rewards/margins": -0.00426566693931818,
      "rewards/rejected": 0.004342413507401943,
      "step": 3
    },
    {
      "epoch": 0.0009293140500667944,
      "grad_norm": 23.932086944580078,
      "learning_rate": 9.993031358885018e-06,
      "logits/chosen": 2.2671163082122803,
      "logits/rejected": 2.3315846920013428,
      "logps/chosen": -78.51152038574219,
      "logps/rejected": -75.3253173828125,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0018994330894201994,
      "rewards/margins": -0.0011305809020996094,
      "rewards/rejected": -0.00076885218732059,
      "step": 4
    },
    {
      "epoch": 0.001161642562583493,
      "grad_norm": 28.281946182250977,
      "learning_rate": 9.990708478513358e-06,
      "logits/chosen": 2.6626229286193848,
      "logits/rejected": 2.7411580085754395,
      "logps/chosen": -99.05874633789062,
      "logps/rejected": -87.10566711425781,
      "loss": 0.6926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007532119285315275,
      "rewards/margins": 0.0016193138435482979,
      "rewards/rejected": 0.005912806373089552,
      "step": 5
    },
    {
      "epoch": 0.0013939710751001916,
      "grad_norm": 25.21985626220703,
      "learning_rate": 9.988385598141697e-06,
      "logits/chosen": 2.1242856979370117,
      "logits/rejected": 2.2432708740234375,
      "logps/chosen": -81.8587646484375,
      "logps/rejected": -81.01544189453125,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006950853276066482,
      "rewards/margins": 0.00572471646592021,
      "rewards/rejected": -0.00641980255022645,
      "step": 6
    },
    {
      "epoch": 0.0016262995876168904,
      "grad_norm": 25.034494400024414,
      "learning_rate": 9.986062717770036e-06,
      "logits/chosen": 2.6563827991485596,
      "logits/rejected": 2.6968705654144287,
      "logps/chosen": -74.47640991210938,
      "logps/rejected": -79.97054290771484,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013524294830858707,
      "rewards/margins": 0.007795404642820358,
      "rewards/rejected": 0.005728888791054487,
      "step": 7
    },
    {
      "epoch": 0.001858628100133589,
      "grad_norm": 25.208173751831055,
      "learning_rate": 9.983739837398375e-06,
      "logits/chosen": 2.1611547470092773,
      "logits/rejected": 2.190526247024536,
      "logps/chosen": -71.9569091796875,
      "logps/rejected": -83.9079818725586,
      "loss": 0.7112,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.000784396892413497,
      "rewards/margins": -0.03520684316754341,
      "rewards/rejected": 0.03599124029278755,
      "step": 8
    },
    {
      "epoch": 0.0020909566126502874,
      "grad_norm": 24.812728881835938,
      "learning_rate": 9.981416957026714e-06,
      "logits/chosen": 1.981597661972046,
      "logits/rejected": 2.0364420413970947,
      "logps/chosen": -73.43708801269531,
      "logps/rejected": -72.46263122558594,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003046679776161909,
      "rewards/margins": 0.0036836147774010897,
      "rewards/rejected": -0.006730294320732355,
      "step": 9
    },
    {
      "epoch": 0.002323285125166986,
      "grad_norm": 27.564239501953125,
      "learning_rate": 9.979094076655053e-06,
      "logits/chosen": 2.5006470680236816,
      "logits/rejected": 2.4719185829162598,
      "logps/chosen": -80.63986206054688,
      "logps/rejected": -101.32374572753906,
      "loss": 0.7108,
      "rewards/accuracies": 0.125,
      "rewards/chosen": 0.0008775456808507442,
      "rewards/margins": -0.034381840378046036,
      "rewards/rejected": 0.035259392112493515,
      "step": 10
    },
    {
      "epoch": 0.002555613637683685,
      "grad_norm": 23.87603187561035,
      "learning_rate": 9.976771196283392e-06,
      "logits/chosen": 2.2564938068389893,
      "logits/rejected": 2.5674681663513184,
      "logps/chosen": -79.31163024902344,
      "logps/rejected": -73.00125885009766,
      "loss": 0.706,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007615161128342152,
      "rewards/margins": -0.02476191520690918,
      "rewards/rejected": 0.03237707540392876,
      "step": 11
    },
    {
      "epoch": 0.0027879421502003832,
      "grad_norm": 25.130205154418945,
      "learning_rate": 9.97444831591173e-06,
      "logits/chosen": 1.9622834920883179,
      "logits/rejected": 1.822800636291504,
      "logps/chosen": -78.5544662475586,
      "logps/rejected": -79.82791137695312,
      "loss": 0.6824,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.039655137807130814,
      "rewards/margins": 0.022574162110686302,
      "rewards/rejected": 0.01708097569644451,
      "step": 12
    },
    {
      "epoch": 0.003020270662717082,
      "grad_norm": 26.172447204589844,
      "learning_rate": 9.972125435540071e-06,
      "logits/chosen": 2.648510456085205,
      "logits/rejected": 2.6628782749176025,
      "logps/chosen": -76.5418930053711,
      "logps/rejected": -93.00799560546875,
      "loss": 0.6653,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04645419493317604,
      "rewards/margins": 0.05786591023206711,
      "rewards/rejected": -0.011411715298891068,
      "step": 13
    },
    {
      "epoch": 0.0032525991752337807,
      "grad_norm": 24.2358341217041,
      "learning_rate": 9.96980255516841e-06,
      "logits/chosen": 1.7121721506118774,
      "logits/rejected": 1.855048418045044,
      "logps/chosen": -81.92431640625,
      "logps/rejected": -79.02764892578125,
      "loss": 0.6815,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05940699577331543,
      "rewards/margins": 0.02543337456882,
      "rewards/rejected": 0.03397362306714058,
      "step": 14
    },
    {
      "epoch": 0.003484927687750479,
      "grad_norm": 22.968090057373047,
      "learning_rate": 9.967479674796748e-06,
      "logits/chosen": 2.1601884365081787,
      "logits/rejected": 2.246208906173706,
      "logps/chosen": -73.30953979492188,
      "logps/rejected": -62.957237243652344,
      "loss": 0.6907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012835003435611725,
      "rewards/margins": 0.005627276375889778,
      "rewards/rejected": 0.007207726128399372,
      "step": 15
    },
    {
      "epoch": 0.003717256200267178,
      "grad_norm": 23.415523529052734,
      "learning_rate": 9.965156794425088e-06,
      "logits/chosen": 2.2577569484710693,
      "logits/rejected": 2.4123809337615967,
      "logps/chosen": -71.63866424560547,
      "logps/rejected": -64.214111328125,
      "loss": 0.6859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02216820791363716,
      "rewards/margins": 0.016334807500243187,
      "rewards/rejected": 0.005833399016410112,
      "step": 16
    },
    {
      "epoch": 0.003949584712783876,
      "grad_norm": 25.63241195678711,
      "learning_rate": 9.962833914053427e-06,
      "logits/chosen": 2.1968536376953125,
      "logits/rejected": 2.1120736598968506,
      "logps/chosen": -88.56842803955078,
      "logps/rejected": -78.5439224243164,
      "loss": 0.6887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.061372753232717514,
      "rewards/margins": 0.010415052995085716,
      "rewards/rejected": 0.050957705825567245,
      "step": 17
    },
    {
      "epoch": 0.004181913225300575,
      "grad_norm": 22.836362838745117,
      "learning_rate": 9.960511033681766e-06,
      "logits/chosen": 2.6677451133728027,
      "logits/rejected": 2.6506311893463135,
      "logps/chosen": -78.59965515136719,
      "logps/rejected": -78.5658950805664,
      "loss": 0.6689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03840620815753937,
      "rewards/margins": 0.050463031977415085,
      "rewards/rejected": -0.012056825682520866,
      "step": 18
    },
    {
      "epoch": 0.004414241737817274,
      "grad_norm": 22.588682174682617,
      "learning_rate": 9.958188153310105e-06,
      "logits/chosen": 1.7466487884521484,
      "logits/rejected": 1.715010643005371,
      "logps/chosen": -72.5875244140625,
      "logps/rejected": -76.71061706542969,
      "loss": 0.6926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023352695629000664,
      "rewards/margins": 0.0019424175843596458,
      "rewards/rejected": 0.021410275250673294,
      "step": 19
    },
    {
      "epoch": 0.004646570250333972,
      "grad_norm": 25.90950584411621,
      "learning_rate": 9.955865272938444e-06,
      "logits/chosen": 2.01682710647583,
      "logits/rejected": 2.2097835540771484,
      "logps/chosen": -83.37155151367188,
      "logps/rejected": -88.21090698242188,
      "loss": 0.6783,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0657111406326294,
      "rewards/margins": 0.0358596071600914,
      "rewards/rejected": 0.029851533472537994,
      "step": 20
    },
    {
      "epoch": 0.004878898762850671,
      "grad_norm": 23.781816482543945,
      "learning_rate": 9.953542392566785e-06,
      "logits/chosen": 1.761909008026123,
      "logits/rejected": 1.8345574140548706,
      "logps/chosen": -83.01974487304688,
      "logps/rejected": -71.64412689208984,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.024792063981294632,
      "rewards/margins": 0.0019376268610358238,
      "rewards/rejected": 0.022854436188936234,
      "step": 21
    },
    {
      "epoch": 0.00511122727536737,
      "grad_norm": 26.64802360534668,
      "learning_rate": 9.951219512195124e-06,
      "logits/chosen": 1.6238797903060913,
      "logits/rejected": 1.5423059463500977,
      "logps/chosen": -87.01276397705078,
      "logps/rejected": -74.9041519165039,
      "loss": 0.7119,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017537618055939674,
      "rewards/margins": -0.036546967923641205,
      "rewards/rejected": 0.01900935173034668,
      "step": 22
    },
    {
      "epoch": 0.005343555787884068,
      "grad_norm": 25.708192825317383,
      "learning_rate": 9.94889663182346e-06,
      "logits/chosen": 2.189649820327759,
      "logits/rejected": 2.1577956676483154,
      "logps/chosen": -76.0335693359375,
      "logps/rejected": -75.82823181152344,
      "loss": 0.7139,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.018225861713290215,
      "rewards/margins": -0.03962578997015953,
      "rewards/rejected": 0.0578516460955143,
      "step": 23
    },
    {
      "epoch": 0.0055758843004007665,
      "grad_norm": 26.050640106201172,
      "learning_rate": 9.946573751451801e-06,
      "logits/chosen": 1.8416625261306763,
      "logits/rejected": 1.75844144821167,
      "logps/chosen": -76.32035827636719,
      "logps/rejected": -76.03118133544922,
      "loss": 0.7186,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015236424282193184,
      "rewards/margins": -0.04777280613780022,
      "rewards/rejected": 0.06300923228263855,
      "step": 24
    },
    {
      "epoch": 0.005808212812917465,
      "grad_norm": 23.632600784301758,
      "learning_rate": 9.94425087108014e-06,
      "logits/chosen": 1.6662321090698242,
      "logits/rejected": 1.8364561796188354,
      "logps/chosen": -70.7703628540039,
      "logps/rejected": -83.5075912475586,
      "loss": 0.6816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03744876757264137,
      "rewards/margins": 0.0244155153632164,
      "rewards/rejected": 0.013033246621489525,
      "step": 25
    },
    {
      "epoch": 0.006040541325434164,
      "grad_norm": 26.79390525817871,
      "learning_rate": 9.94192799070848e-06,
      "logits/chosen": 1.7419593334197998,
      "logits/rejected": 1.7070484161376953,
      "logps/chosen": -87.23074340820312,
      "logps/rejected": -69.88967895507812,
      "loss": 0.7097,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006749964319169521,
      "rewards/margins": -0.029734179377555847,
      "rewards/rejected": 0.0229842197149992,
      "step": 26
    },
    {
      "epoch": 0.006272869837950863,
      "grad_norm": 23.454269409179688,
      "learning_rate": 9.939605110336818e-06,
      "logits/chosen": 1.9998044967651367,
      "logits/rejected": 2.00679349899292,
      "logps/chosen": -72.79754638671875,
      "logps/rejected": -67.37321472167969,
      "loss": 0.7086,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.03303203731775284,
      "rewards/margins": -0.027854587882757187,
      "rewards/rejected": 0.06088662147521973,
      "step": 27
    },
    {
      "epoch": 0.0065051983504675615,
      "grad_norm": 26.14141845703125,
      "learning_rate": 9.937282229965157e-06,
      "logits/chosen": 1.9277416467666626,
      "logits/rejected": 1.8402342796325684,
      "logps/chosen": -84.95695495605469,
      "logps/rejected": -72.75863647460938,
      "loss": 0.7121,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012473201379179955,
      "rewards/margins": -0.035994578152894974,
      "rewards/rejected": 0.02352137863636017,
      "step": 28
    },
    {
      "epoch": 0.006737526862984259,
      "grad_norm": 24.120149612426758,
      "learning_rate": 9.934959349593498e-06,
      "logits/chosen": 2.363116979598999,
      "logits/rejected": 2.3114633560180664,
      "logps/chosen": -74.10717010498047,
      "logps/rejected": -81.46306610107422,
      "loss": 0.6978,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0362170934677124,
      "rewards/margins": -0.006656435318291187,
      "rewards/rejected": 0.042873527854681015,
      "step": 29
    },
    {
      "epoch": 0.006969855375500958,
      "grad_norm": 23.367250442504883,
      "learning_rate": 9.932636469221835e-06,
      "logits/chosen": 2.072114944458008,
      "logits/rejected": 1.9454174041748047,
      "logps/chosen": -75.50045013427734,
      "logps/rejected": -72.88445281982422,
      "loss": 0.6964,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04860131815075874,
      "rewards/margins": -0.0038162730634212494,
      "rewards/rejected": 0.052417587488889694,
      "step": 30
    },
    {
      "epoch": 0.007202183888017657,
      "grad_norm": 26.80875587463379,
      "learning_rate": 9.930313588850174e-06,
      "logits/chosen": 2.0430378913879395,
      "logits/rejected": 1.8732497692108154,
      "logps/chosen": -81.90925598144531,
      "logps/rejected": -87.255126953125,
      "loss": 0.6813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07823934406042099,
      "rewards/margins": 0.024950195103883743,
      "rewards/rejected": 0.053289152681827545,
      "step": 31
    },
    {
      "epoch": 0.007434512400534356,
      "grad_norm": 25.338926315307617,
      "learning_rate": 9.927990708478515e-06,
      "logits/chosen": 1.9104430675506592,
      "logits/rejected": 1.9207375049591064,
      "logps/chosen": -83.48724365234375,
      "logps/rejected": -75.99659729003906,
      "loss": 0.6937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06634726375341415,
      "rewards/margins": 0.0007224567234516144,
      "rewards/rejected": 0.06562481820583344,
      "step": 32
    },
    {
      "epoch": 0.007666840913051054,
      "grad_norm": 25.798728942871094,
      "learning_rate": 9.925667828106853e-06,
      "logits/chosen": 1.6129276752471924,
      "logits/rejected": 1.7860119342803955,
      "logps/chosen": -78.94570922851562,
      "logps/rejected": -74.75910186767578,
      "loss": 0.7092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06511005759239197,
      "rewards/margins": -0.028507117182016373,
      "rewards/rejected": 0.09361718595027924,
      "step": 33
    },
    {
      "epoch": 0.007899169425567752,
      "grad_norm": 25.018274307250977,
      "learning_rate": 9.923344947735192e-06,
      "logits/chosen": 2.3370282649993896,
      "logits/rejected": 2.283297538757324,
      "logps/chosen": -63.34268569946289,
      "logps/rejected": -83.58838653564453,
      "loss": 0.7037,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06861378997564316,
      "rewards/margins": -0.020372439175844193,
      "rewards/rejected": 0.08898623287677765,
      "step": 34
    },
    {
      "epoch": 0.008131497938084452,
      "grad_norm": 27.241104125976562,
      "learning_rate": 9.921022067363531e-06,
      "logits/chosen": 3.0263028144836426,
      "logits/rejected": 3.104552984237671,
      "logps/chosen": -78.11624145507812,
      "logps/rejected": -102.92598724365234,
      "loss": 0.701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0666528046131134,
      "rewards/margins": -0.012287972494959831,
      "rewards/rejected": 0.07894077152013779,
      "step": 35
    },
    {
      "epoch": 0.00836382645060115,
      "grad_norm": 26.56825828552246,
      "learning_rate": 9.91869918699187e-06,
      "logits/chosen": 2.3515210151672363,
      "logits/rejected": 2.367500066757202,
      "logps/chosen": -85.02680206298828,
      "logps/rejected": -88.89542388916016,
      "loss": 0.6774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0800166130065918,
      "rewards/margins": 0.033708762377500534,
      "rewards/rejected": 0.046307846903800964,
      "step": 36
    },
    {
      "epoch": 0.00859615496311785,
      "grad_norm": 26.560543060302734,
      "learning_rate": 9.916376306620211e-06,
      "logits/chosen": 1.9316604137420654,
      "logits/rejected": 1.9338077306747437,
      "logps/chosen": -75.01915740966797,
      "logps/rejected": -74.08940887451172,
      "loss": 0.7319,
      "rewards/accuracies": 0.125,
      "rewards/chosen": 0.028211187571287155,
      "rewards/margins": -0.07484707981348038,
      "rewards/rejected": 0.10305827856063843,
      "step": 37
    },
    {
      "epoch": 0.008828483475634547,
      "grad_norm": 27.436309814453125,
      "learning_rate": 9.914053426248548e-06,
      "logits/chosen": 1.9309556484222412,
      "logits/rejected": 1.9130499362945557,
      "logps/chosen": -80.54497528076172,
      "logps/rejected": -94.24501037597656,
      "loss": 0.7393,
      "rewards/accuracies": 0.125,
      "rewards/chosen": 0.043723221868276596,
      "rewards/margins": -0.08802710473537445,
      "rewards/rejected": 0.13175031542778015,
      "step": 38
    },
    {
      "epoch": 0.009060811988151245,
      "grad_norm": 26.39231300354004,
      "learning_rate": 9.911730545876889e-06,
      "logits/chosen": 2.1016974449157715,
      "logits/rejected": 1.9595110416412354,
      "logps/chosen": -76.59342956542969,
      "logps/rejected": -76.72994232177734,
      "loss": 0.6998,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.08930272608995438,
      "rewards/margins": -0.010815193876624107,
      "rewards/rejected": 0.10011792182922363,
      "step": 39
    },
    {
      "epoch": 0.009293140500667945,
      "grad_norm": 23.721036911010742,
      "learning_rate": 9.909407665505228e-06,
      "logits/chosen": 1.7134631872177124,
      "logits/rejected": 1.7667949199676514,
      "logps/chosen": -79.33737182617188,
      "logps/rejected": -83.88177490234375,
      "loss": 0.6969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06931237876415253,
      "rewards/margins": -0.004717661999166012,
      "rewards/rejected": 0.07403004914522171,
      "step": 40
    },
    {
      "epoch": 0.009525469013184643,
      "grad_norm": 23.577003479003906,
      "learning_rate": 9.907084785133567e-06,
      "logits/chosen": 2.3501439094543457,
      "logits/rejected": 2.0931906700134277,
      "logps/chosen": -82.49964141845703,
      "logps/rejected": -78.04341125488281,
      "loss": 0.628,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1803060919046402,
      "rewards/margins": 0.22086814045906067,
      "rewards/rejected": -0.04056205600500107,
      "step": 41
    },
    {
      "epoch": 0.009757797525701342,
      "grad_norm": 25.63322639465332,
      "learning_rate": 9.904761904761906e-06,
      "logits/chosen": 2.156503200531006,
      "logits/rejected": 2.153296709060669,
      "logps/chosen": -88.50535583496094,
      "logps/rejected": -80.06227111816406,
      "loss": 0.7,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.03836202621459961,
      "rewards/margins": -0.012214899994432926,
      "rewards/rejected": 0.05057692527770996,
      "step": 42
    },
    {
      "epoch": 0.00999012603821804,
      "grad_norm": 22.583648681640625,
      "learning_rate": 9.902439024390245e-06,
      "logits/chosen": 1.5218547582626343,
      "logits/rejected": 1.666164517402649,
      "logps/chosen": -77.22662353515625,
      "logps/rejected": -70.24125671386719,
      "loss": 0.6873,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.06380771845579147,
      "rewards/margins": 0.02862047776579857,
      "rewards/rejected": 0.0351872444152832,
      "step": 43
    },
    {
      "epoch": 0.01022245455073474,
      "grad_norm": 25.545495986938477,
      "learning_rate": 9.900116144018583e-06,
      "logits/chosen": 1.6735045909881592,
      "logits/rejected": 1.6950719356536865,
      "logps/chosen": -75.29996490478516,
      "logps/rejected": -83.03594970703125,
      "loss": 0.6726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1299327164888382,
      "rewards/margins": 0.04630177468061447,
      "rewards/rejected": 0.08363094180822372,
      "step": 44
    },
    {
      "epoch": 0.010454783063251438,
      "grad_norm": 24.771657943725586,
      "learning_rate": 9.897793263646922e-06,
      "logits/chosen": 2.223294734954834,
      "logits/rejected": 2.3361949920654297,
      "logps/chosen": -85.27457427978516,
      "logps/rejected": -70.1801528930664,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07427213340997696,
      "rewards/margins": 0.01737535186111927,
      "rewards/rejected": 0.05689678341150284,
      "step": 45
    },
    {
      "epoch": 0.010687111575768135,
      "grad_norm": 24.368621826171875,
      "learning_rate": 9.895470383275261e-06,
      "logits/chosen": 1.674993872642517,
      "logits/rejected": 1.922520637512207,
      "logps/chosen": -80.94437408447266,
      "logps/rejected": -80.69613647460938,
      "loss": 0.6792,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0937371551990509,
      "rewards/margins": 0.03016524203121662,
      "rewards/rejected": 0.06357190757989883,
      "step": 46
    },
    {
      "epoch": 0.010919440088284835,
      "grad_norm": 23.917110443115234,
      "learning_rate": 9.893147502903602e-06,
      "logits/chosen": 1.4872246980667114,
      "logits/rejected": 1.7845369577407837,
      "logps/chosen": -79.3690185546875,
      "logps/rejected": -91.04258728027344,
      "loss": 0.6597,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06799228489398956,
      "rewards/margins": 0.07255524396896362,
      "rewards/rejected": -0.004562950227409601,
      "step": 47
    },
    {
      "epoch": 0.011151768600801533,
      "grad_norm": 25.309917449951172,
      "learning_rate": 9.890824622531941e-06,
      "logits/chosen": 1.6744539737701416,
      "logits/rejected": 1.600510835647583,
      "logps/chosen": -84.9424819946289,
      "logps/rejected": -73.61338806152344,
      "loss": 0.6889,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.12050977349281311,
      "rewards/margins": 0.010859325528144836,
      "rewards/rejected": 0.10965044051408768,
      "step": 48
    },
    {
      "epoch": 0.011384097113318233,
      "grad_norm": 27.1778507232666,
      "learning_rate": 9.88850174216028e-06,
      "logits/chosen": 2.0491511821746826,
      "logits/rejected": 1.9164681434631348,
      "logps/chosen": -81.17369079589844,
      "logps/rejected": -78.52923583984375,
      "loss": 0.7007,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06764011830091476,
      "rewards/margins": -0.013568542897701263,
      "rewards/rejected": 0.08120866119861603,
      "step": 49
    },
    {
      "epoch": 0.01161642562583493,
      "grad_norm": 29.29009437561035,
      "learning_rate": 9.886178861788619e-06,
      "logits/chosen": 1.7468621730804443,
      "logits/rejected": 1.7312324047088623,
      "logps/chosen": -98.3561782836914,
      "logps/rejected": -73.74363708496094,
      "loss": 0.7311,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.07353110611438751,
      "rewards/margins": -0.06932863593101501,
      "rewards/rejected": 0.14285974204540253,
      "step": 50
    },
    {
      "epoch": 0.01184875413835163,
      "grad_norm": 24.076007843017578,
      "learning_rate": 9.883855981416958e-06,
      "logits/chosen": 1.6535121202468872,
      "logits/rejected": 1.8407156467437744,
      "logps/chosen": -76.2763671875,
      "logps/rejected": -78.16549682617188,
      "loss": 0.718,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07884471118450165,
      "rewards/margins": -0.0450916513800621,
      "rewards/rejected": 0.12393637001514435,
      "step": 51
    },
    {
      "epoch": 0.012081082650868328,
      "grad_norm": 24.63661003112793,
      "learning_rate": 9.881533101045298e-06,
      "logits/chosen": 2.3171191215515137,
      "logits/rejected": 2.2840631008148193,
      "logps/chosen": -74.67701721191406,
      "logps/rejected": -83.5268783569336,
      "loss": 0.695,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.05177018418908119,
      "rewards/margins": -0.0014177579432725906,
      "rewards/rejected": 0.05318794399499893,
      "step": 52
    },
    {
      "epoch": 0.012313411163385026,
      "grad_norm": 23.193069458007812,
      "learning_rate": 9.879210220673636e-06,
      "logits/chosen": 2.4298601150512695,
      "logits/rejected": 2.252713918685913,
      "logps/chosen": -81.4945297241211,
      "logps/rejected": -60.496315002441406,
      "loss": 0.706,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09277703613042831,
      "rewards/margins": -0.02292318269610405,
      "rewards/rejected": 0.11570023000240326,
      "step": 53
    },
    {
      "epoch": 0.012545739675901725,
      "grad_norm": 27.1844425201416,
      "learning_rate": 9.876887340301975e-06,
      "logits/chosen": 2.5585250854492188,
      "logits/rejected": 2.7496328353881836,
      "logps/chosen": -86.01087951660156,
      "logps/rejected": -90.96095275878906,
      "loss": 0.6789,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1046748161315918,
      "rewards/margins": 0.03144533932209015,
      "rewards/rejected": 0.07322947680950165,
      "step": 54
    },
    {
      "epoch": 0.012778068188418423,
      "grad_norm": 25.9202880859375,
      "learning_rate": 9.874564459930315e-06,
      "logits/chosen": 2.112987518310547,
      "logits/rejected": 2.307497024536133,
      "logps/chosen": -89.74347686767578,
      "logps/rejected": -78.48823547363281,
      "loss": 0.6723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07724513858556747,
      "rewards/margins": 0.04334824159741402,
      "rewards/rejected": 0.033896900713443756,
      "step": 55
    },
    {
      "epoch": 0.013010396700935123,
      "grad_norm": 24.676788330078125,
      "learning_rate": 9.872241579558654e-06,
      "logits/chosen": 2.5764334201812744,
      "logits/rejected": 2.529407262802124,
      "logps/chosen": -71.40621948242188,
      "logps/rejected": -72.14805603027344,
      "loss": 0.7256,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.04570601135492325,
      "rewards/margins": -0.06266713887453079,
      "rewards/rejected": 0.10837314277887344,
      "step": 56
    },
    {
      "epoch": 0.01324272521345182,
      "grad_norm": 23.639705657958984,
      "learning_rate": 9.869918699186993e-06,
      "logits/chosen": 1.627376914024353,
      "logits/rejected": 1.5962104797363281,
      "logps/chosen": -78.68608093261719,
      "logps/rejected": -81.17896270751953,
      "loss": 0.7032,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.10436435043811798,
      "rewards/margins": -0.018074989318847656,
      "rewards/rejected": 0.12243934720754623,
      "step": 57
    },
    {
      "epoch": 0.013475053725968519,
      "grad_norm": 24.150094985961914,
      "learning_rate": 9.867595818815332e-06,
      "logits/chosen": 2.4876465797424316,
      "logits/rejected": 2.5634500980377197,
      "logps/chosen": -76.20947265625,
      "logps/rejected": -85.55035400390625,
      "loss": 0.6599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11965687572956085,
      "rewards/margins": 0.07920224964618683,
      "rewards/rejected": 0.04045462608337402,
      "step": 58
    },
    {
      "epoch": 0.013707382238485218,
      "grad_norm": 25.417823791503906,
      "learning_rate": 9.865272938443671e-06,
      "logits/chosen": 2.5532174110412598,
      "logits/rejected": 2.5578882694244385,
      "logps/chosen": -84.26395416259766,
      "logps/rejected": -73.08594512939453,
      "loss": 0.7044,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.051102928817272186,
      "rewards/margins": -0.019775938242673874,
      "rewards/rejected": 0.07087886333465576,
      "step": 59
    },
    {
      "epoch": 0.013939710751001916,
      "grad_norm": 25.123048782348633,
      "learning_rate": 9.86295005807201e-06,
      "logits/chosen": 1.923710584640503,
      "logits/rejected": 1.9758179187774658,
      "logps/chosen": -83.03857421875,
      "logps/rejected": -82.28742218017578,
      "loss": 0.6811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10221691429615021,
      "rewards/margins": 0.025673720985651016,
      "rewards/rejected": 0.07654318958520889,
      "step": 60
    },
    {
      "epoch": 0.014172039263518616,
      "grad_norm": 26.46537971496582,
      "learning_rate": 9.860627177700349e-06,
      "logits/chosen": 1.9620473384857178,
      "logits/rejected": 1.9895777702331543,
      "logps/chosen": -77.49363708496094,
      "logps/rejected": -76.91546630859375,
      "loss": 0.6912,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05003867298364639,
      "rewards/margins": 0.00703961867839098,
      "rewards/rejected": 0.04299905523657799,
      "step": 61
    },
    {
      "epoch": 0.014404367776035314,
      "grad_norm": 26.772850036621094,
      "learning_rate": 9.858304297328688e-06,
      "logits/chosen": 2.446756362915039,
      "logits/rejected": 2.456329107284546,
      "logps/chosen": -91.13645935058594,
      "logps/rejected": -97.60475158691406,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.047224998474121094,
      "rewards/margins": 0.0032025817781686783,
      "rewards/rejected": 0.044022418558597565,
      "step": 62
    },
    {
      "epoch": 0.014636696288552013,
      "grad_norm": 26.236709594726562,
      "learning_rate": 9.855981416957028e-06,
      "logits/chosen": 1.9222549200057983,
      "logits/rejected": 1.8407011032104492,
      "logps/chosen": -81.70240783691406,
      "logps/rejected": -81.59886169433594,
      "loss": 0.7463,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.007420565001666546,
      "rewards/margins": -0.09724569320678711,
      "rewards/rejected": 0.08982513099908829,
      "step": 63
    },
    {
      "epoch": 0.014869024801068711,
      "grad_norm": 25.77079963684082,
      "learning_rate": 9.853658536585367e-06,
      "logits/chosen": 2.8614184856414795,
      "logits/rejected": 2.9851694107055664,
      "logps/chosen": -73.4080581665039,
      "logps/rejected": -71.8211669921875,
      "loss": 0.6761,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09877953678369522,
      "rewards/margins": 0.035445258021354675,
      "rewards/rejected": 0.06333427876234055,
      "step": 64
    },
    {
      "epoch": 0.015101353313585409,
      "grad_norm": 27.664365768432617,
      "learning_rate": 9.851335656213704e-06,
      "logits/chosen": 2.620373249053955,
      "logits/rejected": 2.651179313659668,
      "logps/chosen": -87.67101287841797,
      "logps/rejected": -89.05938720703125,
      "loss": 0.6843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10441002994775772,
      "rewards/margins": 0.020449090749025345,
      "rewards/rejected": 0.08396093547344208,
      "step": 65
    },
    {
      "epoch": 0.015333681826102109,
      "grad_norm": 24.129974365234375,
      "learning_rate": 9.849012775842045e-06,
      "logits/chosen": 1.919310212135315,
      "logits/rejected": 1.9644291400909424,
      "logps/chosen": -69.11741638183594,
      "logps/rejected": -80.59484100341797,
      "loss": 0.6849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08919209986925125,
      "rewards/margins": 0.022063134238123894,
      "rewards/rejected": 0.06712896376848221,
      "step": 66
    },
    {
      "epoch": 0.015566010338618807,
      "grad_norm": 25.23989486694336,
      "learning_rate": 9.846689895470384e-06,
      "logits/chosen": 1.6064504384994507,
      "logits/rejected": 1.797716736793518,
      "logps/chosen": -79.51156616210938,
      "logps/rejected": -82.53575897216797,
      "loss": 0.6781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10089290142059326,
      "rewards/margins": 0.03414376080036163,
      "rewards/rejected": 0.06674914807081223,
      "step": 67
    },
    {
      "epoch": 0.015798338851135504,
      "grad_norm": 58.85096740722656,
      "learning_rate": 9.844367015098723e-06,
      "logits/chosen": 1.9074525833129883,
      "logits/rejected": 2.0409088134765625,
      "logps/chosen": -79.28838348388672,
      "logps/rejected": -78.9749526977539,
      "loss": 0.7264,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.005871390923857689,
      "rewards/margins": -0.0637439489364624,
      "rewards/rejected": 0.06961534172296524,
      "step": 68
    },
    {
      "epoch": 0.016030667363652204,
      "grad_norm": 24.11976432800293,
      "learning_rate": 9.842044134727062e-06,
      "logits/chosen": 1.8751786947250366,
      "logits/rejected": 1.9385555982589722,
      "logps/chosen": -79.04988861083984,
      "logps/rejected": -84.36827087402344,
      "loss": 0.6998,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011375858448445797,
      "rewards/margins": -0.011868285946547985,
      "rewards/rejected": 0.0004924284294247627,
      "step": 69
    },
    {
      "epoch": 0.016262995876168904,
      "grad_norm": 24.130413055419922,
      "learning_rate": 9.839721254355401e-06,
      "logits/chosen": 1.8247448205947876,
      "logits/rejected": 1.6972384452819824,
      "logps/chosen": -66.8177490234375,
      "logps/rejected": -75.09021759033203,
      "loss": 0.6783,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08765456825494766,
      "rewards/margins": 0.03657737001776695,
      "rewards/rejected": 0.05107720196247101,
      "step": 70
    },
    {
      "epoch": 0.0164953243886856,
      "grad_norm": 24.80650520324707,
      "learning_rate": 9.837398373983741e-06,
      "logits/chosen": 1.7210553884506226,
      "logits/rejected": 1.897660255432129,
      "logps/chosen": -81.87832641601562,
      "logps/rejected": -76.72029876708984,
      "loss": 0.6779,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.058904219418764114,
      "rewards/margins": 0.03424220159649849,
      "rewards/rejected": 0.024662017822265625,
      "step": 71
    },
    {
      "epoch": 0.0167276529012023,
      "grad_norm": 27.961864471435547,
      "learning_rate": 9.83507549361208e-06,
      "logits/chosen": 2.437018632888794,
      "logits/rejected": 2.2464234828948975,
      "logps/chosen": -84.42726135253906,
      "logps/rejected": -97.61540222167969,
      "loss": 0.6347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07640127837657928,
      "rewards/margins": 0.12302045524120331,
      "rewards/rejected": -0.04661917686462402,
      "step": 72
    },
    {
      "epoch": 0.016959981413719,
      "grad_norm": 24.35243034362793,
      "learning_rate": 9.83275261324042e-06,
      "logits/chosen": 1.4718319177627563,
      "logits/rejected": 1.5746363401412964,
      "logps/chosen": -72.75634002685547,
      "logps/rejected": -80.85359191894531,
      "loss": 0.6954,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06799433380365372,
      "rewards/margins": -0.00029962509870529175,
      "rewards/rejected": 0.06829395890235901,
      "step": 73
    },
    {
      "epoch": 0.0171923099262357,
      "grad_norm": 26.85625648498535,
      "learning_rate": 9.830429732868758e-06,
      "logits/chosen": 2.1132824420928955,
      "logits/rejected": 2.3064656257629395,
      "logps/chosen": -77.10936737060547,
      "logps/rejected": -78.73450469970703,
      "loss": 0.6937,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0007575256749987602,
      "rewards/margins": 0.0020042667165398598,
      "rewards/rejected": -0.0012467391788959503,
      "step": 74
    },
    {
      "epoch": 0.017424638438752395,
      "grad_norm": 25.052215576171875,
      "learning_rate": 9.828106852497097e-06,
      "logits/chosen": 2.023804187774658,
      "logits/rejected": 1.9421014785766602,
      "logps/chosen": -75.79916381835938,
      "logps/rejected": -76.78378295898438,
      "loss": 0.6773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1165846437215805,
      "rewards/margins": 0.03906099498271942,
      "rewards/rejected": 0.07752363383769989,
      "step": 75
    },
    {
      "epoch": 0.017656966951269094,
      "grad_norm": 23.70705795288086,
      "learning_rate": 9.825783972125436e-06,
      "logits/chosen": 1.8521742820739746,
      "logits/rejected": 1.8376868963241577,
      "logps/chosen": -68.0479736328125,
      "logps/rejected": -77.34935760498047,
      "loss": 0.6736,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.064688540995121,
      "rewards/margins": 0.039977602660655975,
      "rewards/rejected": 0.024710942059755325,
      "step": 76
    },
    {
      "epoch": 0.017889295463785794,
      "grad_norm": 24.258459091186523,
      "learning_rate": 9.823461091753775e-06,
      "logits/chosen": 1.8073270320892334,
      "logits/rejected": 1.9061001539230347,
      "logps/chosen": -80.93087768554688,
      "logps/rejected": -74.43304443359375,
      "loss": 0.6685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03962979465723038,
      "rewards/margins": 0.05214669555425644,
      "rewards/rejected": -0.012516905553638935,
      "step": 77
    },
    {
      "epoch": 0.01812162397630249,
      "grad_norm": 25.513917922973633,
      "learning_rate": 9.821138211382114e-06,
      "logits/chosen": 2.6621265411376953,
      "logits/rejected": 2.3249011039733887,
      "logps/chosen": -95.5090560913086,
      "logps/rejected": -64.27548217773438,
      "loss": 0.7274,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01939702033996582,
      "rewards/margins": -0.06307997554540634,
      "rewards/rejected": 0.04368296265602112,
      "step": 78
    },
    {
      "epoch": 0.01835395248881919,
      "grad_norm": 24.684385299682617,
      "learning_rate": 9.818815331010455e-06,
      "logits/chosen": 1.8767122030258179,
      "logits/rejected": 2.0713295936584473,
      "logps/chosen": -80.49508666992188,
      "logps/rejected": -70.62307739257812,
      "loss": 0.7333,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.02649994008243084,
      "rewards/margins": -0.07757081836462021,
      "rewards/rejected": 0.05107088014483452,
      "step": 79
    },
    {
      "epoch": 0.01858628100133589,
      "grad_norm": 21.998275756835938,
      "learning_rate": 9.816492450638792e-06,
      "logits/chosen": 1.4419735670089722,
      "logits/rejected": 1.5253500938415527,
      "logps/chosen": -79.55047607421875,
      "logps/rejected": -68.56495666503906,
      "loss": 0.6261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16743972897529602,
      "rewards/margins": 0.15437473356723785,
      "rewards/rejected": 0.013064992614090443,
      "step": 80
    },
    {
      "epoch": 0.01881860951385259,
      "grad_norm": 101.38760375976562,
      "learning_rate": 9.814169570267133e-06,
      "logits/chosen": 1.7964839935302734,
      "logits/rejected": 1.7768237590789795,
      "logps/chosen": -83.44844818115234,
      "logps/rejected": -74.59136962890625,
      "loss": 0.7236,
      "rewards/accuracies": 0.125,
      "rewards/chosen": 0.0367831215262413,
      "rewards/margins": -0.05722935125231743,
      "rewards/rejected": 0.09401248395442963,
      "step": 81
    },
    {
      "epoch": 0.019050938026369285,
      "grad_norm": 22.373836517333984,
      "learning_rate": 9.811846689895471e-06,
      "logits/chosen": 2.1996755599975586,
      "logits/rejected": 2.3457305431365967,
      "logps/chosen": -84.91327667236328,
      "logps/rejected": -78.07355499267578,
      "loss": 0.6633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.057767562568187714,
      "rewards/margins": 0.06439128518104553,
      "rewards/rejected": -0.006623722612857819,
      "step": 82
    },
    {
      "epoch": 0.019283266538885985,
      "grad_norm": 24.90047264099121,
      "learning_rate": 9.80952380952381e-06,
      "logits/chosen": 1.7361314296722412,
      "logits/rejected": 1.50971519947052,
      "logps/chosen": -84.3535385131836,
      "logps/rejected": -80.10987091064453,
      "loss": 0.6561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12052430957555771,
      "rewards/margins": 0.07879315316677094,
      "rewards/rejected": 0.04173116758465767,
      "step": 83
    },
    {
      "epoch": 0.019515595051402684,
      "grad_norm": 24.865108489990234,
      "learning_rate": 9.80720092915215e-06,
      "logits/chosen": 1.5763808488845825,
      "logits/rejected": 1.6495041847229004,
      "logps/chosen": -82.87889862060547,
      "logps/rejected": -75.51065063476562,
      "loss": 0.7,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004367826972156763,
      "rewards/margins": -0.011592531576752663,
      "rewards/rejected": 0.01596035808324814,
      "step": 84
    },
    {
      "epoch": 0.01974792356391938,
      "grad_norm": 24.123653411865234,
      "learning_rate": 9.804878048780488e-06,
      "logits/chosen": 1.679161787033081,
      "logits/rejected": 1.4935640096664429,
      "logps/chosen": -79.99159240722656,
      "logps/rejected": -66.52352142333984,
      "loss": 0.7438,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.05960530787706375,
      "rewards/margins": -0.09520193934440613,
      "rewards/rejected": 0.03559662774205208,
      "step": 85
    },
    {
      "epoch": 0.01998025207643608,
      "grad_norm": 24.048891067504883,
      "learning_rate": 9.802555168408829e-06,
      "logits/chosen": 1.5362915992736816,
      "logits/rejected": 1.658151626586914,
      "logps/chosen": -78.06932830810547,
      "logps/rejected": -72.51118469238281,
      "loss": 0.691,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.032294489443302155,
      "rewards/margins": 0.005856825038790703,
      "rewards/rejected": 0.0264376662671566,
      "step": 86
    },
    {
      "epoch": 0.02021258058895278,
      "grad_norm": 23.401119232177734,
      "learning_rate": 9.800232288037166e-06,
      "logits/chosen": 1.6821589469909668,
      "logits/rejected": 1.5869208574295044,
      "logps/chosen": -74.18900299072266,
      "logps/rejected": -79.1020278930664,
      "loss": 0.7171,
      "rewards/accuracies": 0.125,
      "rewards/chosen": 0.0004026414826512337,
      "rewards/margins": -0.04565449059009552,
      "rewards/rejected": 0.04605713114142418,
      "step": 87
    },
    {
      "epoch": 0.02044490910146948,
      "grad_norm": 25.13729476928711,
      "learning_rate": 9.797909407665505e-06,
      "logits/chosen": 1.615554928779602,
      "logits/rejected": 1.6884888410568237,
      "logps/chosen": -81.1583480834961,
      "logps/rejected": -79.55589294433594,
      "loss": 0.7064,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04984720051288605,
      "rewards/margins": -0.0221281535923481,
      "rewards/rejected": -0.02771904692053795,
      "step": 88
    },
    {
      "epoch": 0.020677237613986175,
      "grad_norm": 28.607189178466797,
      "learning_rate": 9.795586527293846e-06,
      "logits/chosen": 2.1622965335845947,
      "logits/rejected": 2.1760082244873047,
      "logps/chosen": -80.73334503173828,
      "logps/rejected": -81.92765808105469,
      "loss": 0.6634,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.10154452919960022,
      "rewards/margins": 0.0668458417057991,
      "rewards/rejected": 0.03469867259263992,
      "step": 89
    },
    {
      "epoch": 0.020909566126502875,
      "grad_norm": 29.330778121948242,
      "learning_rate": 9.793263646922185e-06,
      "logits/chosen": 3.0493593215942383,
      "logits/rejected": 2.7285170555114746,
      "logps/chosen": -88.40428924560547,
      "logps/rejected": -91.06063842773438,
      "loss": 0.6881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01602778024971485,
      "rewards/margins": 0.0176236592233181,
      "rewards/rejected": -0.0015958775766193867,
      "step": 90
    },
    {
      "epoch": 0.021141894639019575,
      "grad_norm": 25.83343505859375,
      "learning_rate": 9.790940766550524e-06,
      "logits/chosen": 1.5784181356430054,
      "logits/rejected": 1.7860231399536133,
      "logps/chosen": -84.33978271484375,
      "logps/rejected": -75.49471282958984,
      "loss": 0.6523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06263656914234161,
      "rewards/margins": 0.08720848709344864,
      "rewards/rejected": -0.024571921676397324,
      "step": 91
    },
    {
      "epoch": 0.02137422315153627,
      "grad_norm": 22.382783889770508,
      "learning_rate": 9.788617886178862e-06,
      "logits/chosen": 2.105940103530884,
      "logits/rejected": 2.06386137008667,
      "logps/chosen": -69.99339294433594,
      "logps/rejected": -76.8082504272461,
      "loss": 0.6676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04404342174530029,
      "rewards/margins": 0.05893457680940628,
      "rewards/rejected": -0.014891158789396286,
      "step": 92
    },
    {
      "epoch": 0.02160655166405297,
      "grad_norm": 26.053569793701172,
      "learning_rate": 9.786295005807201e-06,
      "logits/chosen": 1.684584140777588,
      "logits/rejected": 1.8862419128417969,
      "logps/chosen": -76.8394546508789,
      "logps/rejected": -81.55705261230469,
      "loss": 0.7007,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03772728517651558,
      "rewards/margins": -0.012339998967945576,
      "rewards/rejected": -0.025387287139892578,
      "step": 93
    },
    {
      "epoch": 0.02183888017656967,
      "grad_norm": 25.149398803710938,
      "learning_rate": 9.783972125435542e-06,
      "logits/chosen": 1.6767865419387817,
      "logits/rejected": 1.5772606134414673,
      "logps/chosen": -80.51941680908203,
      "logps/rejected": -78.90348815917969,
      "loss": 0.664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03773882985115051,
      "rewards/margins": 0.061766862869262695,
      "rewards/rejected": -0.02402804046869278,
      "step": 94
    },
    {
      "epoch": 0.02207120868908637,
      "grad_norm": 29.473079681396484,
      "learning_rate": 9.78164924506388e-06,
      "logits/chosen": 2.3061985969543457,
      "logits/rejected": 2.2962324619293213,
      "logps/chosen": -86.93255615234375,
      "logps/rejected": -80.72125244140625,
      "loss": 0.6917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03776422142982483,
      "rewards/margins": 0.016720369458198547,
      "rewards/rejected": 0.021043846383690834,
      "step": 95
    },
    {
      "epoch": 0.022303537201603066,
      "grad_norm": 22.719894409179688,
      "learning_rate": 9.779326364692218e-06,
      "logits/chosen": 1.6481540203094482,
      "logits/rejected": 1.394147276878357,
      "logps/chosen": -72.44271850585938,
      "logps/rejected": -74.70650482177734,
      "loss": 0.6689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04925873503088951,
      "rewards/margins": 0.05192830413579941,
      "rewards/rejected": -0.0026695728302001953,
      "step": 96
    },
    {
      "epoch": 0.022535865714119765,
      "grad_norm": 24.471372604370117,
      "learning_rate": 9.777003484320559e-06,
      "logits/chosen": 1.658986210823059,
      "logits/rejected": 1.6556494235992432,
      "logps/chosen": -74.11808776855469,
      "logps/rejected": -83.99127960205078,
      "loss": 0.6948,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.06438872963190079,
      "rewards/margins": 0.003132658079266548,
      "rewards/rejected": 0.06125607714056969,
      "step": 97
    },
    {
      "epoch": 0.022768194226636465,
      "grad_norm": 25.796268463134766,
      "learning_rate": 9.774680603948898e-06,
      "logits/chosen": 2.1815946102142334,
      "logits/rejected": 2.3106911182403564,
      "logps/chosen": -81.73091888427734,
      "logps/rejected": -81.36689758300781,
      "loss": 0.6406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.031129933893680573,
      "rewards/margins": 0.10954905301332474,
      "rewards/rejected": -0.07841911911964417,
      "step": 98
    },
    {
      "epoch": 0.02300052273915316,
      "grad_norm": 25.108381271362305,
      "learning_rate": 9.772357723577237e-06,
      "logits/chosen": 1.6949337720870972,
      "logits/rejected": 1.807483196258545,
      "logps/chosen": -80.49740600585938,
      "logps/rejected": -88.18728637695312,
      "loss": 0.6674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06719913333654404,
      "rewards/margins": 0.05966556817293167,
      "rewards/rejected": 0.007533573545515537,
      "step": 99
    },
    {
      "epoch": 0.02323285125166986,
      "grad_norm": 25.498998641967773,
      "learning_rate": 9.770034843205576e-06,
      "logits/chosen": 1.5122909545898438,
      "logits/rejected": 1.5292168855667114,
      "logps/chosen": -89.87741088867188,
      "logps/rejected": -84.98407745361328,
      "loss": 0.6652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05555038899183273,
      "rewards/margins": 0.05987496301531792,
      "rewards/rejected": -0.004324577748775482,
      "step": 100
    },
    {
      "epoch": 0.02346517976418656,
      "grad_norm": 21.71100616455078,
      "learning_rate": 9.767711962833915e-06,
      "logits/chosen": 1.8574323654174805,
      "logits/rejected": 2.075505256652832,
      "logps/chosen": -72.7991943359375,
      "logps/rejected": -70.93531036376953,
      "loss": 0.6479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06609399616718292,
      "rewards/margins": 0.09509226679801941,
      "rewards/rejected": -0.028998281806707382,
      "step": 101
    },
    {
      "epoch": 0.02369750827670326,
      "grad_norm": 27.07535743713379,
      "learning_rate": 9.765389082462254e-06,
      "logits/chosen": 1.4758707284927368,
      "logits/rejected": 1.6379084587097168,
      "logps/chosen": -83.02943420410156,
      "logps/rejected": -77.36613464355469,
      "loss": 0.6479,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0780515968799591,
      "rewards/margins": 0.119206003844738,
      "rewards/rejected": -0.0411544069647789,
      "step": 102
    },
    {
      "epoch": 0.023929836789219956,
      "grad_norm": 23.961532592773438,
      "learning_rate": 9.763066202090592e-06,
      "logits/chosen": 1.252142310142517,
      "logits/rejected": 1.2799500226974487,
      "logps/chosen": -77.11042785644531,
      "logps/rejected": -87.48603057861328,
      "loss": 0.6823,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.010325813665986061,
      "rewards/margins": 0.02402322366833687,
      "rewards/rejected": -0.03434903174638748,
      "step": 103
    },
    {
      "epoch": 0.024162165301736656,
      "grad_norm": 27.035995483398438,
      "learning_rate": 9.760743321718931e-06,
      "logits/chosen": 1.7154314517974854,
      "logits/rejected": 1.7886874675750732,
      "logps/chosen": -77.42863464355469,
      "logps/rejected": -93.68787384033203,
      "loss": 0.6663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07736267149448395,
      "rewards/margins": 0.0581146739423275,
      "rewards/rejected": 0.019248008728027344,
      "step": 104
    },
    {
      "epoch": 0.024394493814253355,
      "grad_norm": 25.17705535888672,
      "learning_rate": 9.758420441347272e-06,
      "logits/chosen": 1.6538807153701782,
      "logits/rejected": 1.5920181274414062,
      "logps/chosen": -81.7781982421875,
      "logps/rejected": -86.6353759765625,
      "loss": 0.653,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06357365101575851,
      "rewards/margins": 0.08581877499818802,
      "rewards/rejected": -0.022245123982429504,
      "step": 105
    },
    {
      "epoch": 0.02462682232677005,
      "grad_norm": 24.071313858032227,
      "learning_rate": 9.756097560975611e-06,
      "logits/chosen": 1.3487094640731812,
      "logits/rejected": 1.4865374565124512,
      "logps/chosen": -81.08248138427734,
      "logps/rejected": -73.72357177734375,
      "loss": 0.642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07907728850841522,
      "rewards/margins": 0.11009693890810013,
      "rewards/rejected": -0.03101964108645916,
      "step": 106
    },
    {
      "epoch": 0.02485915083928675,
      "grad_norm": 23.406444549560547,
      "learning_rate": 9.75377468060395e-06,
      "logits/chosen": 2.165569305419922,
      "logits/rejected": 2.0899436473846436,
      "logps/chosen": -73.32127380371094,
      "logps/rejected": -80.1220932006836,
      "loss": 0.6914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.034476205706596375,
      "rewards/margins": 0.01704404503107071,
      "rewards/rejected": 0.017432164400815964,
      "step": 107
    },
    {
      "epoch": 0.02509147935180345,
      "grad_norm": 24.063966751098633,
      "learning_rate": 9.751451800232289e-06,
      "logits/chosen": 1.7365080118179321,
      "logits/rejected": 1.8159117698669434,
      "logps/chosen": -75.74324035644531,
      "logps/rejected": -81.88143157958984,
      "loss": 0.6656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11494691669940948,
      "rewards/margins": 0.06658921390771866,
      "rewards/rejected": 0.04835769906640053,
      "step": 108
    },
    {
      "epoch": 0.025323807864320147,
      "grad_norm": 25.45781707763672,
      "learning_rate": 9.749128919860628e-06,
      "logits/chosen": 1.7996327877044678,
      "logits/rejected": 1.6671695709228516,
      "logps/chosen": -90.62440490722656,
      "logps/rejected": -66.36534118652344,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027943992987275124,
      "rewards/margins": -0.0010875221341848373,
      "rewards/rejected": 0.02903151512145996,
      "step": 109
    },
    {
      "epoch": 0.025556136376836847,
      "grad_norm": 24.88028907775879,
      "learning_rate": 9.746806039488967e-06,
      "logits/chosen": 1.7234630584716797,
      "logits/rejected": 1.7237104177474976,
      "logps/chosen": -73.85713958740234,
      "logps/rejected": -68.37217712402344,
      "loss": 0.7029,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.06238389015197754,
      "rewards/margins": -0.011479423381388187,
      "rewards/rejected": 0.07386332005262375,
      "step": 110
    },
    {
      "epoch": 0.025788464889353546,
      "grad_norm": 22.461929321289062,
      "learning_rate": 9.744483159117306e-06,
      "logits/chosen": 1.0226595401763916,
      "logits/rejected": 1.2100639343261719,
      "logps/chosen": -66.87179565429688,
      "logps/rejected": -68.86766052246094,
      "loss": 0.6827,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.042656973004341125,
      "rewards/margins": 0.03138089179992676,
      "rewards/rejected": 0.01127607747912407,
      "step": 111
    },
    {
      "epoch": 0.026020793401870246,
      "grad_norm": 27.204612731933594,
      "learning_rate": 9.742160278745645e-06,
      "logits/chosen": 1.4265903234481812,
      "logits/rejected": 1.4105942249298096,
      "logps/chosen": -76.76040649414062,
      "logps/rejected": -76.5082778930664,
      "loss": 0.7214,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.037148021161556244,
      "rewards/margins": -0.04279467836022377,
      "rewards/rejected": 0.005646658595651388,
      "step": 112
    },
    {
      "epoch": 0.026253121914386942,
      "grad_norm": 24.869287490844727,
      "learning_rate": 9.739837398373985e-06,
      "logits/chosen": 1.2271597385406494,
      "logits/rejected": 1.2083015441894531,
      "logps/chosen": -74.720703125,
      "logps/rejected": -82.41539764404297,
      "loss": 0.6633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0795014351606369,
      "rewards/margins": 0.09004393219947815,
      "rewards/rejected": -0.010542487725615501,
      "step": 113
    },
    {
      "epoch": 0.02648545042690364,
      "grad_norm": 24.15592384338379,
      "learning_rate": 9.737514518002324e-06,
      "logits/chosen": 0.7626086473464966,
      "logits/rejected": 0.9320838451385498,
      "logps/chosen": -77.52369689941406,
      "logps/rejected": -89.1755599975586,
      "loss": 0.685,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0014255521818995476,
      "rewards/margins": 0.019864918664097786,
      "rewards/rejected": -0.018439367413520813,
      "step": 114
    },
    {
      "epoch": 0.02671777893942034,
      "grad_norm": 22.304698944091797,
      "learning_rate": 9.735191637630663e-06,
      "logits/chosen": 2.054605484008789,
      "logits/rejected": 1.9895591735839844,
      "logps/chosen": -77.14823913574219,
      "logps/rejected": -63.1622200012207,
      "loss": 0.6861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0470518060028553,
      "rewards/margins": 0.026663942262530327,
      "rewards/rejected": 0.020387863740324974,
      "step": 115
    },
    {
      "epoch": 0.026950107451937037,
      "grad_norm": 25.65361213684082,
      "learning_rate": 9.732868757259002e-06,
      "logits/chosen": 0.863017201423645,
      "logits/rejected": 1.0035254955291748,
      "logps/chosen": -73.42313385009766,
      "logps/rejected": -81.6391372680664,
      "loss": 0.7049,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.027466751635074615,
      "rewards/margins": -0.012210682965815067,
      "rewards/rejected": 0.03967743366956711,
      "step": 116
    },
    {
      "epoch": 0.027182435964453737,
      "grad_norm": 22.997716903686523,
      "learning_rate": 9.730545876887341e-06,
      "logits/chosen": 1.393533706665039,
      "logits/rejected": 1.4625047445297241,
      "logps/chosen": -71.73375701904297,
      "logps/rejected": -74.02236938476562,
      "loss": 0.6827,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00979616492986679,
      "rewards/margins": 0.026103425770998,
      "rewards/rejected": -0.01630726084113121,
      "step": 117
    },
    {
      "epoch": 0.027414764476970437,
      "grad_norm": 22.688247680664062,
      "learning_rate": 9.72822299651568e-06,
      "logits/chosen": 1.843857765197754,
      "logits/rejected": 1.9069762229919434,
      "logps/chosen": -72.25578308105469,
      "logps/rejected": -69.68343353271484,
      "loss": 0.6793,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0808035358786583,
      "rewards/margins": 0.030942920595407486,
      "rewards/rejected": 0.049860622733831406,
      "step": 118
    },
    {
      "epoch": 0.027647092989487136,
      "grad_norm": 21.481569290161133,
      "learning_rate": 9.725900116144019e-06,
      "logits/chosen": 1.110047459602356,
      "logits/rejected": 1.046691656112671,
      "logps/chosen": -73.03572082519531,
      "logps/rejected": -81.59480285644531,
      "loss": 0.6445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0601714625954628,
      "rewards/margins": 0.10709219425916672,
      "rewards/rejected": -0.04692072421312332,
      "step": 119
    },
    {
      "epoch": 0.027879421502003832,
      "grad_norm": 25.963714599609375,
      "learning_rate": 9.72357723577236e-06,
      "logits/chosen": 1.1851603984832764,
      "logits/rejected": 1.2960668802261353,
      "logps/chosen": -76.0704345703125,
      "logps/rejected": -96.57462310791016,
      "loss": 0.7086,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.036742497235536575,
      "rewards/margins": -0.025018122047185898,
      "rewards/rejected": 0.061760611832141876,
      "step": 120
    },
    {
      "epoch": 0.028111750014520532,
      "grad_norm": 22.69502067565918,
      "learning_rate": 9.721254355400698e-06,
      "logits/chosen": 1.1909441947937012,
      "logits/rejected": 1.0538312196731567,
      "logps/chosen": -79.4405288696289,
      "logps/rejected": -80.79310607910156,
      "loss": 0.6531,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.029710862785577774,
      "rewards/margins": 0.08547158539295197,
      "rewards/rejected": -0.05576071888208389,
      "step": 121
    },
    {
      "epoch": 0.02834407852703723,
      "grad_norm": 28.017593383789062,
      "learning_rate": 9.718931475029036e-06,
      "logits/chosen": 0.7847111821174622,
      "logits/rejected": 0.9066658616065979,
      "logps/chosen": -87.8373794555664,
      "logps/rejected": -79.62187194824219,
      "loss": 0.7114,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.05482766777276993,
      "rewards/margins": -0.021097324788570404,
      "rewards/rejected": 0.07592499256134033,
      "step": 122
    },
    {
      "epoch": 0.028576407039553928,
      "grad_norm": 25.330312728881836,
      "learning_rate": 9.716608594657376e-06,
      "logits/chosen": 1.0675437450408936,
      "logits/rejected": 1.1942085027694702,
      "logps/chosen": -79.87265014648438,
      "logps/rejected": -95.25157928466797,
      "loss": 0.6519,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009310174733400345,
      "rewards/margins": 0.0964374840259552,
      "rewards/rejected": -0.08712730556726456,
      "step": 123
    },
    {
      "epoch": 0.028808735552070627,
      "grad_norm": 23.249961853027344,
      "learning_rate": 9.714285714285715e-06,
      "logits/chosen": 1.5048774480819702,
      "logits/rejected": 1.3851749897003174,
      "logps/chosen": -77.59626007080078,
      "logps/rejected": -75.91844940185547,
      "loss": 0.6731,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.026271101087331772,
      "rewards/margins": 0.04827455058693886,
      "rewards/rejected": -0.02200343832373619,
      "step": 124
    },
    {
      "epoch": 0.029041064064587327,
      "grad_norm": 26.36027717590332,
      "learning_rate": 9.711962833914054e-06,
      "logits/chosen": 0.8619117736816406,
      "logits/rejected": 0.9224223494529724,
      "logps/chosen": -74.47737884521484,
      "logps/rejected": -91.89643859863281,
      "loss": 0.6849,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.053105734288692474,
      "rewards/margins": 0.028877731412649155,
      "rewards/rejected": 0.024227997288107872,
      "step": 125
    },
    {
      "epoch": 0.029273392577104027,
      "grad_norm": 26.84822654724121,
      "learning_rate": 9.709639953542393e-06,
      "logits/chosen": 1.1549164056777954,
      "logits/rejected": 1.0195082426071167,
      "logps/chosen": -77.82099914550781,
      "logps/rejected": -76.01766967773438,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0406905934214592,
      "rewards/margins": 0.005040287971496582,
      "rewards/rejected": 0.035650305449962616,
      "step": 126
    },
    {
      "epoch": 0.029505721089620723,
      "grad_norm": 24.208023071289062,
      "learning_rate": 9.707317073170732e-06,
      "logits/chosen": 0.7252591252326965,
      "logits/rejected": 0.7087407112121582,
      "logps/chosen": -85.24772644042969,
      "logps/rejected": -69.93513488769531,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0075637828558683395,
      "rewards/margins": 0.018346186727285385,
      "rewards/rejected": -0.025909975171089172,
      "step": 127
    },
    {
      "epoch": 0.029738049602137422,
      "grad_norm": 25.470611572265625,
      "learning_rate": 9.704994192799073e-06,
      "logits/chosen": 1.6545016765594482,
      "logits/rejected": 1.6989911794662476,
      "logps/chosen": -82.15815734863281,
      "logps/rejected": -100.19397735595703,
      "loss": 0.6841,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.033689308911561966,
      "rewards/margins": 0.027858180925250053,
      "rewards/rejected": 0.0058311233296990395,
      "step": 128
    },
    {
      "epoch": 0.029970378114654122,
      "grad_norm": 19.62473487854004,
      "learning_rate": 9.702671312427412e-06,
      "logits/chosen": 0.6996293067932129,
      "logits/rejected": 0.8045562505722046,
      "logps/chosen": -77.6377182006836,
      "logps/rejected": -71.63854217529297,
      "loss": 0.5609,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1928681880235672,
      "rewards/margins": 0.3701432943344116,
      "rewards/rejected": -0.17727509140968323,
      "step": 129
    },
    {
      "epoch": 0.030202706627170818,
      "grad_norm": 23.87986183166504,
      "learning_rate": 9.700348432055749e-06,
      "logits/chosen": 1.0507431030273438,
      "logits/rejected": 0.8717767596244812,
      "logps/chosen": -75.0171890258789,
      "logps/rejected": -71.42559051513672,
      "loss": 0.6882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02100958675146103,
      "rewards/margins": 0.030065149068832397,
      "rewards/rejected": -0.009055564180016518,
      "step": 130
    },
    {
      "epoch": 0.030435035139687518,
      "grad_norm": 25.1107234954834,
      "learning_rate": 9.69802555168409e-06,
      "logits/chosen": 0.9183138608932495,
      "logits/rejected": 0.866865336894989,
      "logps/chosen": -87.5278549194336,
      "logps/rejected": -88.8864517211914,
      "loss": 0.7173,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03750017285346985,
      "rewards/margins": -0.039888933300971985,
      "rewards/rejected": 0.0023887641727924347,
      "step": 131
    },
    {
      "epoch": 0.030667363652204217,
      "grad_norm": 26.91111946105957,
      "learning_rate": 9.695702671312428e-06,
      "logits/chosen": 1.236215591430664,
      "logits/rejected": 1.2571947574615479,
      "logps/chosen": -80.36104583740234,
      "logps/rejected": -81.34573364257812,
      "loss": 0.7043,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.026833154261112213,
      "rewards/margins": -0.009217692539095879,
      "rewards/rejected": -0.017615463584661484,
      "step": 132
    },
    {
      "epoch": 0.030899692164720917,
      "grad_norm": 26.606536865234375,
      "learning_rate": 9.693379790940767e-06,
      "logits/chosen": 0.9858619570732117,
      "logits/rejected": 0.9235528111457825,
      "logps/chosen": -82.92285919189453,
      "logps/rejected": -73.2775650024414,
      "loss": 0.7448,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.026828523725271225,
      "rewards/margins": -0.08375802636146545,
      "rewards/rejected": 0.05692949891090393,
      "step": 133
    },
    {
      "epoch": 0.031132020677237613,
      "grad_norm": 27.200794219970703,
      "learning_rate": 9.691056910569106e-06,
      "logits/chosen": 1.190006136894226,
      "logits/rejected": 1.171931505203247,
      "logps/chosen": -90.56068420410156,
      "logps/rejected": -86.20176696777344,
      "loss": 0.7261,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0718030110001564,
      "rewards/margins": -0.05974910408258438,
      "rewards/rejected": -0.012053921818733215,
      "step": 134
    },
    {
      "epoch": 0.03136434918975431,
      "grad_norm": 26.36789321899414,
      "learning_rate": 9.688734030197445e-06,
      "logits/chosen": 0.7657046318054199,
      "logits/rejected": 0.8715552091598511,
      "logps/chosen": -81.07868194580078,
      "logps/rejected": -91.34465789794922,
      "loss": 0.6939,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0389862097799778,
      "rewards/margins": 0.003806780092418194,
      "rewards/rejected": -0.04279297962784767,
      "step": 135
    },
    {
      "epoch": 0.03159667770227101,
      "grad_norm": 27.81682014465332,
      "learning_rate": 9.686411149825786e-06,
      "logits/chosen": 0.8535057902336121,
      "logits/rejected": 0.9208365678787231,
      "logps/chosen": -93.580810546875,
      "logps/rejected": -92.31231689453125,
      "loss": 0.7038,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08433561027050018,
      "rewards/margins": -0.002650330774486065,
      "rewards/rejected": -0.08168528228998184,
      "step": 136
    },
    {
      "epoch": 0.03182900621478771,
      "grad_norm": 32.67242431640625,
      "learning_rate": 9.684088269454123e-06,
      "logits/chosen": 0.7974899411201477,
      "logits/rejected": 0.8422968983650208,
      "logps/chosen": -75.86585235595703,
      "logps/rejected": -79.54329681396484,
      "loss": 0.7038,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0043999627232551575,
      "rewards/margins": -0.009209103882312775,
      "rewards/rejected": 0.004809146746993065,
      "step": 137
    },
    {
      "epoch": 0.03206133472730441,
      "grad_norm": 22.540508270263672,
      "learning_rate": 9.681765389082462e-06,
      "logits/chosen": 0.9388677477836609,
      "logits/rejected": 1.0635464191436768,
      "logps/chosen": -89.39547729492188,
      "logps/rejected": -74.82655334472656,
      "loss": 0.6388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03728647157549858,
      "rewards/margins": 0.13039088249206543,
      "rewards/rejected": -0.1676773726940155,
      "step": 138
    },
    {
      "epoch": 0.03229366323982111,
      "grad_norm": 24.58083724975586,
      "learning_rate": 9.679442508710803e-06,
      "logits/chosen": 0.7958865165710449,
      "logits/rejected": 0.6199870705604553,
      "logps/chosen": -79.97803497314453,
      "logps/rejected": -86.03216552734375,
      "loss": 0.7259,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.11075848340988159,
      "rewards/margins": -0.060355350375175476,
      "rewards/rejected": -0.05040311813354492,
      "step": 139
    },
    {
      "epoch": 0.03252599175233781,
      "grad_norm": 23.942935943603516,
      "learning_rate": 9.677119628339142e-06,
      "logits/chosen": 2.4944605827331543,
      "logits/rejected": 2.38624906539917,
      "logps/chosen": -87.36884307861328,
      "logps/rejected": -81.0146713256836,
      "loss": 0.6418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.043311718851327896,
      "rewards/margins": 0.16201439499855042,
      "rewards/rejected": -0.11870267987251282,
      "step": 140
    },
    {
      "epoch": 0.03275832026485451,
      "grad_norm": 21.73566436767578,
      "learning_rate": 9.67479674796748e-06,
      "logits/chosen": 0.47082051634788513,
      "logits/rejected": 0.6741467118263245,
      "logps/chosen": -77.72731018066406,
      "logps/rejected": -73.4183120727539,
      "loss": 0.6309,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005721166729927063,
      "rewards/margins": 0.14412197470664978,
      "rewards/rejected": -0.1384008228778839,
      "step": 141
    },
    {
      "epoch": 0.0329906487773712,
      "grad_norm": 24.917295455932617,
      "learning_rate": 9.67247386759582e-06,
      "logits/chosen": 0.9368895888328552,
      "logits/rejected": 1.2217334508895874,
      "logps/chosen": -98.49968719482422,
      "logps/rejected": -76.61304473876953,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06494739651679993,
      "rewards/margins": -0.0022036265581846237,
      "rewards/rejected": -0.06274378299713135,
      "step": 142
    },
    {
      "epoch": 0.0332229772898879,
      "grad_norm": 23.257936477661133,
      "learning_rate": 9.670150987224158e-06,
      "logits/chosen": 0.8182318210601807,
      "logits/rejected": 0.9186137914657593,
      "logps/chosen": -75.82257080078125,
      "logps/rejected": -71.07379913330078,
      "loss": 0.6883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07362480461597443,
      "rewards/margins": 0.01579270511865616,
      "rewards/rejected": -0.08941750228404999,
      "step": 143
    },
    {
      "epoch": 0.0334553058024046,
      "grad_norm": 24.80918312072754,
      "learning_rate": 9.667828106852499e-06,
      "logits/chosen": 1.217673897743225,
      "logits/rejected": 0.869089663028717,
      "logps/chosen": -82.96888732910156,
      "logps/rejected": -83.36164855957031,
      "loss": 0.7329,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.010229445062577724,
      "rewards/margins": -0.07509203255176544,
      "rewards/rejected": 0.06486257910728455,
      "step": 144
    },
    {
      "epoch": 0.0336876343149213,
      "grad_norm": 26.246437072753906,
      "learning_rate": 9.665505226480836e-06,
      "logits/chosen": 1.5015833377838135,
      "logits/rejected": 1.5893099308013916,
      "logps/chosen": -91.93476867675781,
      "logps/rejected": -97.5189437866211,
      "loss": 0.7134,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.08258689939975739,
      "rewards/margins": -0.030202794820070267,
      "rewards/rejected": -0.052384112030267715,
      "step": 145
    },
    {
      "epoch": 0.033919962827438,
      "grad_norm": 24.71675682067871,
      "learning_rate": 9.663182346109177e-06,
      "logits/chosen": 0.6912932395935059,
      "logits/rejected": 0.8796858191490173,
      "logps/chosen": -70.5829086303711,
      "logps/rejected": -77.00018310546875,
      "loss": 0.6975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03875011205673218,
      "rewards/margins": 0.009975738823413849,
      "rewards/rejected": -0.04872584715485573,
      "step": 146
    },
    {
      "epoch": 0.0341522913399547,
      "grad_norm": 21.906028747558594,
      "learning_rate": 9.660859465737516e-06,
      "logits/chosen": 0.8913214206695557,
      "logits/rejected": 0.9305406808853149,
      "logps/chosen": -72.95903778076172,
      "logps/rejected": -65.98860168457031,
      "loss": 0.695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02702171541750431,
      "rewards/margins": 0.005129186902195215,
      "rewards/rejected": 0.021892527118325233,
      "step": 147
    },
    {
      "epoch": 0.0343846198524714,
      "grad_norm": 22.754444122314453,
      "learning_rate": 9.658536585365855e-06,
      "logits/chosen": 0.6079922318458557,
      "logits/rejected": 0.646517276763916,
      "logps/chosen": -75.02823638916016,
      "logps/rejected": -87.9752197265625,
      "loss": 0.6329,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013214348815381527,
      "rewards/margins": 0.13523083925247192,
      "rewards/rejected": -0.12201648205518723,
      "step": 148
    },
    {
      "epoch": 0.03461694836498809,
      "grad_norm": 25.25969123840332,
      "learning_rate": 9.656213704994194e-06,
      "logits/chosen": 0.7703552842140198,
      "logits/rejected": 0.6918190717697144,
      "logps/chosen": -83.9351806640625,
      "logps/rejected": -84.70355224609375,
      "loss": 0.7118,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03359396010637283,
      "rewards/margins": -0.034001659601926804,
      "rewards/rejected": 0.0004076967015862465,
      "step": 149
    },
    {
      "epoch": 0.03484927687750479,
      "grad_norm": 25.762052536010742,
      "learning_rate": 9.653890824622533e-06,
      "logits/chosen": 0.8892149925231934,
      "logits/rejected": 0.82904052734375,
      "logps/chosen": -78.97528839111328,
      "logps/rejected": -78.16500854492188,
      "loss": 0.7241,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08483923226594925,
      "rewards/margins": -0.05046680569648743,
      "rewards/rejected": -0.034372422844171524,
      "step": 150
    },
    {
      "epoch": 0.03508160539002149,
      "grad_norm": 25.963598251342773,
      "learning_rate": 9.651567944250871e-06,
      "logits/chosen": 0.7601613402366638,
      "logits/rejected": 0.9236962199211121,
      "logps/chosen": -76.4361572265625,
      "logps/rejected": -74.8977279663086,
      "loss": 0.6805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011766362003982067,
      "rewards/margins": 0.035644978284835815,
      "rewards/rejected": -0.023878619074821472,
      "step": 151
    },
    {
      "epoch": 0.03531393390253819,
      "grad_norm": 23.919998168945312,
      "learning_rate": 9.64924506387921e-06,
      "logits/chosen": 1.1502480506896973,
      "logits/rejected": 1.0901892185211182,
      "logps/chosen": -76.6703872680664,
      "logps/rejected": -96.2566146850586,
      "loss": 0.6402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.027442000806331635,
      "rewards/margins": 0.11506982147693634,
      "rewards/rejected": -0.14251182973384857,
      "step": 152
    },
    {
      "epoch": 0.03554626241505489,
      "grad_norm": 25.16694450378418,
      "learning_rate": 9.64692218350755e-06,
      "logits/chosen": 1.4753659963607788,
      "logits/rejected": 1.5824772119522095,
      "logps/chosen": -88.6231689453125,
      "logps/rejected": -89.57084655761719,
      "loss": 0.6432,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.04347161948680878,
      "rewards/margins": 0.14114293456077576,
      "rewards/rejected": -0.09767132997512817,
      "step": 153
    },
    {
      "epoch": 0.03577859092757159,
      "grad_norm": 23.647266387939453,
      "learning_rate": 9.64459930313589e-06,
      "logits/chosen": 1.156267523765564,
      "logits/rejected": 1.055008053779602,
      "logps/chosen": -72.036376953125,
      "logps/rejected": -79.46315002441406,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.059677932411432266,
      "rewards/margins": 0.017813801765441895,
      "rewards/rejected": -0.07749173790216446,
      "step": 154
    },
    {
      "epoch": 0.03601091944008829,
      "grad_norm": 24.356725692749023,
      "learning_rate": 9.642276422764229e-06,
      "logits/chosen": 1.0836918354034424,
      "logits/rejected": 0.8899031281471252,
      "logps/chosen": -85.39784240722656,
      "logps/rejected": -89.4796142578125,
      "loss": 0.6562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01596546359360218,
      "rewards/margins": 0.08610065281391144,
      "rewards/rejected": -0.10206611454486847,
      "step": 155
    },
    {
      "epoch": 0.03624324795260498,
      "grad_norm": 25.357450485229492,
      "learning_rate": 9.639953542392568e-06,
      "logits/chosen": 1.4896888732910156,
      "logits/rejected": 1.4365402460098267,
      "logps/chosen": -71.19708251953125,
      "logps/rejected": -91.87501525878906,
      "loss": 0.685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06497883796691895,
      "rewards/margins": 0.018375039100646973,
      "rewards/rejected": -0.08335387706756592,
      "step": 156
    },
    {
      "epoch": 0.03647557646512168,
      "grad_norm": 25.677522659301758,
      "learning_rate": 9.637630662020907e-06,
      "logits/chosen": 0.6010698080062866,
      "logits/rejected": 0.6183891296386719,
      "logps/chosen": -73.30345153808594,
      "logps/rejected": -78.95282745361328,
      "loss": 0.7052,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1014658510684967,
      "rewards/margins": -0.012188438326120377,
      "rewards/rejected": -0.08927740901708603,
      "step": 157
    },
    {
      "epoch": 0.03670790497763838,
      "grad_norm": 25.561960220336914,
      "learning_rate": 9.635307781649246e-06,
      "logits/chosen": 0.7301754355430603,
      "logits/rejected": 0.5582457780838013,
      "logps/chosen": -83.70674133300781,
      "logps/rejected": -75.43791961669922,
      "loss": 0.7629,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1517687886953354,
      "rewards/margins": -0.11894108355045319,
      "rewards/rejected": -0.0328277125954628,
      "step": 158
    },
    {
      "epoch": 0.03694023349015508,
      "grad_norm": 25.31032371520996,
      "learning_rate": 9.632984901277585e-06,
      "logits/chosen": 0.616348147392273,
      "logits/rejected": 0.878142237663269,
      "logps/chosen": -77.8045654296875,
      "logps/rejected": -88.8984375,
      "loss": 0.6541,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09570598602294922,
      "rewards/margins": 0.09221577644348145,
      "rewards/rejected": -0.18792176246643066,
      "step": 159
    },
    {
      "epoch": 0.03717256200267178,
      "grad_norm": 25.02229881286621,
      "learning_rate": 9.630662020905924e-06,
      "logits/chosen": 0.9756011962890625,
      "logits/rejected": 1.2848674058914185,
      "logps/chosen": -76.04415130615234,
      "logps/rejected": -81.3241195678711,
      "loss": 0.6845,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06352944672107697,
      "rewards/margins": 0.022676391527056694,
      "rewards/rejected": -0.08620583266019821,
      "step": 160
    },
    {
      "epoch": 0.03740489051518848,
      "grad_norm": 23.11696434020996,
      "learning_rate": 9.628339140534263e-06,
      "logits/chosen": 1.0564229488372803,
      "logits/rejected": 0.9220237731933594,
      "logps/chosen": -83.78263092041016,
      "logps/rejected": -83.23342895507812,
      "loss": 0.7014,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0516064390540123,
      "rewards/margins": 0.011201692745089531,
      "rewards/rejected": -0.06280814111232758,
      "step": 161
    },
    {
      "epoch": 0.03763721902770518,
      "grad_norm": 52.63197326660156,
      "learning_rate": 9.626016260162603e-06,
      "logits/chosen": 0.9179050326347351,
      "logits/rejected": 0.9746651649475098,
      "logps/chosen": -90.00971984863281,
      "logps/rejected": -81.41433715820312,
      "loss": 0.6714,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07298532128334045,
      "rewards/margins": 0.059744976460933685,
      "rewards/rejected": -0.13273030519485474,
      "step": 162
    },
    {
      "epoch": 0.03786954754022187,
      "grad_norm": 26.52187728881836,
      "learning_rate": 9.623693379790942e-06,
      "logits/chosen": 0.5978593826293945,
      "logits/rejected": 0.6327693462371826,
      "logps/chosen": -91.24750518798828,
      "logps/rejected": -98.72026062011719,
      "loss": 0.6411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03467149659991264,
      "rewards/margins": 0.12018885463476181,
      "rewards/rejected": -0.08551736176013947,
      "step": 163
    },
    {
      "epoch": 0.03810187605273857,
      "grad_norm": 27.444822311401367,
      "learning_rate": 9.621370499419281e-06,
      "logits/chosen": 0.5819846391677856,
      "logits/rejected": 0.5619577765464783,
      "logps/chosen": -83.36019134521484,
      "logps/rejected": -86.63079833984375,
      "loss": 0.7086,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.12360559403896332,
      "rewards/margins": -0.01832636073231697,
      "rewards/rejected": -0.10527923703193665,
      "step": 164
    },
    {
      "epoch": 0.03833420456525527,
      "grad_norm": 24.69837188720703,
      "learning_rate": 9.61904761904762e-06,
      "logits/chosen": 1.649139165878296,
      "logits/rejected": 1.710658311843872,
      "logps/chosen": -89.96896362304688,
      "logps/rejected": -93.73404693603516,
      "loss": 0.6456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09374471008777618,
      "rewards/margins": 0.11190159618854523,
      "rewards/rejected": -0.2056463062763214,
      "step": 165
    },
    {
      "epoch": 0.03856653307777197,
      "grad_norm": 23.825031280517578,
      "learning_rate": 9.616724738675959e-06,
      "logits/chosen": 0.8162941336631775,
      "logits/rejected": 0.866635262966156,
      "logps/chosen": -73.37603759765625,
      "logps/rejected": -88.7437973022461,
      "loss": 0.6029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.025563430041074753,
      "rewards/margins": 0.20350812375545502,
      "rewards/rejected": -0.17794468998908997,
      "step": 166
    },
    {
      "epoch": 0.03879886159028867,
      "grad_norm": 22.64305305480957,
      "learning_rate": 9.614401858304298e-06,
      "logits/chosen": 0.9881279468536377,
      "logits/rejected": 0.8642110824584961,
      "logps/chosen": -83.57404327392578,
      "logps/rejected": -72.68785858154297,
      "loss": 0.6438,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003731992095708847,
      "rewards/margins": 0.14462216198444366,
      "rewards/rejected": -0.1408901810646057,
      "step": 167
    },
    {
      "epoch": 0.03903119010280537,
      "grad_norm": 27.634489059448242,
      "learning_rate": 9.612078977932637e-06,
      "logits/chosen": 0.789940595626831,
      "logits/rejected": 0.6786967515945435,
      "logps/chosen": -82.15557861328125,
      "logps/rejected": -75.16957092285156,
      "loss": 0.7249,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04150848463177681,
      "rewards/margins": -0.046905770897865295,
      "rewards/rejected": 0.005397298373281956,
      "step": 168
    },
    {
      "epoch": 0.03926351861532207,
      "grad_norm": 23.019744873046875,
      "learning_rate": 9.609756097560976e-06,
      "logits/chosen": 0.7420105934143066,
      "logits/rejected": 0.6561073064804077,
      "logps/chosen": -82.24797821044922,
      "logps/rejected": -81.51502990722656,
      "loss": 0.6743,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.045328669250011444,
      "rewards/margins": 0.04786372184753418,
      "rewards/rejected": -0.09319238364696503,
      "step": 169
    },
    {
      "epoch": 0.03949584712783876,
      "grad_norm": 24.913414001464844,
      "learning_rate": 9.607433217189316e-06,
      "logits/chosen": 0.8789207935333252,
      "logits/rejected": 0.7808835506439209,
      "logps/chosen": -85.18133544921875,
      "logps/rejected": -86.51840209960938,
      "loss": 0.6251,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09639911353588104,
      "rewards/margins": 0.16184383630752563,
      "rewards/rejected": -0.25824296474456787,
      "step": 170
    },
    {
      "epoch": 0.03972817564035546,
      "grad_norm": 24.281892776489258,
      "learning_rate": 9.605110336817655e-06,
      "logits/chosen": 0.8700835704803467,
      "logits/rejected": 1.105660319328308,
      "logps/chosen": -80.8272476196289,
      "logps/rejected": -77.28353881835938,
      "loss": 0.7066,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13883355259895325,
      "rewards/margins": -0.018394997343420982,
      "rewards/rejected": -0.12043855339288712,
      "step": 171
    },
    {
      "epoch": 0.03996050415287216,
      "grad_norm": 23.377979278564453,
      "learning_rate": 9.602787456445993e-06,
      "logits/chosen": 1.1905995607376099,
      "logits/rejected": 1.302850365638733,
      "logps/chosen": -79.65522766113281,
      "logps/rejected": -79.85957336425781,
      "loss": 0.6647,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08393304795026779,
      "rewards/margins": 0.07165922969579697,
      "rewards/rejected": -0.15559227764606476,
      "step": 172
    },
    {
      "epoch": 0.04019283266538886,
      "grad_norm": 25.6408748626709,
      "learning_rate": 9.600464576074333e-06,
      "logits/chosen": 0.801346480846405,
      "logits/rejected": 0.6207796335220337,
      "logps/chosen": -82.48949432373047,
      "logps/rejected": -77.9223403930664,
      "loss": 0.6714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00852665863931179,
      "rewards/margins": 0.051790978759527206,
      "rewards/rejected": -0.04326431825757027,
      "step": 173
    },
    {
      "epoch": 0.04042516117790556,
      "grad_norm": 26.220539093017578,
      "learning_rate": 9.598141695702672e-06,
      "logits/chosen": 0.3460066318511963,
      "logits/rejected": 0.6080366969108582,
      "logps/chosen": -84.21121978759766,
      "logps/rejected": -78.95067596435547,
      "loss": 0.7585,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.22833383083343506,
      "rewards/margins": -0.12164497375488281,
      "rewards/rejected": -0.10668887197971344,
      "step": 174
    },
    {
      "epoch": 0.04065748969042226,
      "grad_norm": 38.469173431396484,
      "learning_rate": 9.595818815331011e-06,
      "logits/chosen": 0.9019977450370789,
      "logits/rejected": 0.729228138923645,
      "logps/chosen": -80.8230209350586,
      "logps/rejected": -76.061279296875,
      "loss": 0.7215,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1661253422498703,
      "rewards/margins": -0.035698361694812775,
      "rewards/rejected": -0.13042698800563812,
      "step": 175
    },
    {
      "epoch": 0.04088981820293896,
      "grad_norm": 24.68029022216797,
      "learning_rate": 9.59349593495935e-06,
      "logits/chosen": 1.0769799947738647,
      "logits/rejected": 0.9985893964767456,
      "logps/chosen": -90.17050170898438,
      "logps/rejected": -108.2805404663086,
      "loss": 0.5903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013095138594508171,
      "rewards/margins": 0.24680376052856445,
      "rewards/rejected": -0.23370863497257233,
      "step": 176
    },
    {
      "epoch": 0.04112214671545565,
      "grad_norm": 22.210182189941406,
      "learning_rate": 9.591173054587689e-06,
      "logits/chosen": 1.122673511505127,
      "logits/rejected": 1.3760311603546143,
      "logps/chosen": -83.86248016357422,
      "logps/rejected": -86.11193084716797,
      "loss": 0.6354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06086144968867302,
      "rewards/margins": 0.13492879271507263,
      "rewards/rejected": -0.19579024612903595,
      "step": 177
    },
    {
      "epoch": 0.04135447522797235,
      "grad_norm": 23.432720184326172,
      "learning_rate": 9.58885017421603e-06,
      "logits/chosen": 0.5170559883117676,
      "logits/rejected": 0.5322235226631165,
      "logps/chosen": -88.53645324707031,
      "logps/rejected": -86.81134796142578,
      "loss": 0.679,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16106106340885162,
      "rewards/margins": 0.049766965210437775,
      "rewards/rejected": -0.2108280211687088,
      "step": 178
    },
    {
      "epoch": 0.04158680374048905,
      "grad_norm": 25.338985443115234,
      "learning_rate": 9.586527293844368e-06,
      "logits/chosen": 0.5887912511825562,
      "logits/rejected": 0.3337961435317993,
      "logps/chosen": -87.5685806274414,
      "logps/rejected": -83.93437957763672,
      "loss": 0.7004,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.18717116117477417,
      "rewards/margins": 0.006549593061208725,
      "rewards/rejected": -0.1937207579612732,
      "step": 179
    },
    {
      "epoch": 0.04181913225300575,
      "grad_norm": 24.043415069580078,
      "learning_rate": 9.584204413472707e-06,
      "logits/chosen": 0.932618260383606,
      "logits/rejected": 0.8223248720169067,
      "logps/chosen": -80.76325988769531,
      "logps/rejected": -78.12005615234375,
      "loss": 0.6846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03523974120616913,
      "rewards/margins": 0.027580738067626953,
      "rewards/rejected": -0.06282048672437668,
      "step": 180
    },
    {
      "epoch": 0.04205146076552245,
      "grad_norm": 26.01968765258789,
      "learning_rate": 9.581881533101046e-06,
      "logits/chosen": 0.7482004165649414,
      "logits/rejected": 0.9481625556945801,
      "logps/chosen": -94.6074447631836,
      "logps/rejected": -89.83027648925781,
      "loss": 0.6923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23213204741477966,
      "rewards/margins": 0.02935132570564747,
      "rewards/rejected": -0.26148340106010437,
      "step": 181
    },
    {
      "epoch": 0.04228378927803915,
      "grad_norm": 23.208662033081055,
      "learning_rate": 9.579558652729385e-06,
      "logits/chosen": 0.9314935803413391,
      "logits/rejected": 0.9562511444091797,
      "logps/chosen": -78.97242736816406,
      "logps/rejected": -82.98225402832031,
      "loss": 0.6069,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.033845946192741394,
      "rewards/margins": 0.22310292720794678,
      "rewards/rejected": -0.1892569661140442,
      "step": 182
    },
    {
      "epoch": 0.04251611779055585,
      "grad_norm": 23.18341064453125,
      "learning_rate": 9.577235772357724e-06,
      "logits/chosen": 1.1921300888061523,
      "logits/rejected": 1.5245792865753174,
      "logps/chosen": -79.762939453125,
      "logps/rejected": -77.60730743408203,
      "loss": 0.6439,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05731511116027832,
      "rewards/margins": 0.1687927544116974,
      "rewards/rejected": -0.22610783576965332,
      "step": 183
    },
    {
      "epoch": 0.04274844630307254,
      "grad_norm": 28.481788635253906,
      "learning_rate": 9.574912891986063e-06,
      "logits/chosen": 1.49360990524292,
      "logits/rejected": 1.3969236612319946,
      "logps/chosen": -84.1294174194336,
      "logps/rejected": -85.93048095703125,
      "loss": 0.7721,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18237246572971344,
      "rewards/margins": -0.11297689378261566,
      "rewards/rejected": -0.06939559429883957,
      "step": 184
    },
    {
      "epoch": 0.04298077481558924,
      "grad_norm": 20.821380615234375,
      "learning_rate": 9.572590011614402e-06,
      "logits/chosen": 0.5743886232376099,
      "logits/rejected": 0.6558592915534973,
      "logps/chosen": -76.67902374267578,
      "logps/rejected": -84.19308471679688,
      "loss": 0.5459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14474807679653168,
      "rewards/margins": 0.3547048568725586,
      "rewards/rejected": -0.2099567949771881,
      "step": 185
    },
    {
      "epoch": 0.04321310332810594,
      "grad_norm": 34.835044860839844,
      "learning_rate": 9.570267131242743e-06,
      "logits/chosen": 1.942851185798645,
      "logits/rejected": 2.037898302078247,
      "logps/chosen": -90.30465698242188,
      "logps/rejected": -91.85946655273438,
      "loss": 0.701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13341791927814484,
      "rewards/margins": 0.031705550849437714,
      "rewards/rejected": -0.16512346267700195,
      "step": 186
    },
    {
      "epoch": 0.04344543184062264,
      "grad_norm": 26.158443450927734,
      "learning_rate": 9.56794425087108e-06,
      "logits/chosen": 0.6394087076187134,
      "logits/rejected": 0.6770368218421936,
      "logps/chosen": -91.15007019042969,
      "logps/rejected": -81.2726821899414,
      "loss": 0.7282,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20979642868041992,
      "rewards/margins": -0.062087319791316986,
      "rewards/rejected": -0.14770910143852234,
      "step": 187
    },
    {
      "epoch": 0.04367776035313934,
      "grad_norm": 25.855775833129883,
      "learning_rate": 9.56562137049942e-06,
      "logits/chosen": 1.7836347818374634,
      "logits/rejected": 1.8073960542678833,
      "logps/chosen": -96.72058868408203,
      "logps/rejected": -75.6865234375,
      "loss": 0.6885,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05247664451599121,
      "rewards/margins": 0.043247200548648834,
      "rewards/rejected": -0.09572384506464005,
      "step": 188
    },
    {
      "epoch": 0.04391008886565604,
      "grad_norm": 25.324739456176758,
      "learning_rate": 9.56329849012776e-06,
      "logits/chosen": 0.9527526497840881,
      "logits/rejected": 0.7819272875785828,
      "logps/chosen": -83.4541015625,
      "logps/rejected": -81.66917419433594,
      "loss": 0.7045,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1453428417444229,
      "rewards/margins": 0.07800273597240448,
      "rewards/rejected": -0.2233455926179886,
      "step": 189
    },
    {
      "epoch": 0.04414241737817274,
      "grad_norm": 24.482505798339844,
      "learning_rate": 9.560975609756098e-06,
      "logits/chosen": 0.779019296169281,
      "logits/rejected": 0.9800407290458679,
      "logps/chosen": -92.69786834716797,
      "logps/rejected": -68.34039306640625,
      "loss": 0.6246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07926034927368164,
      "rewards/margins": 0.1597546935081482,
      "rewards/rejected": -0.08049435913562775,
      "step": 190
    },
    {
      "epoch": 0.04437474589068943,
      "grad_norm": 25.976215362548828,
      "learning_rate": 9.558652729384437e-06,
      "logits/chosen": 0.9001207947731018,
      "logits/rejected": 0.8603881001472473,
      "logps/chosen": -82.6992416381836,
      "logps/rejected": -94.33602142333984,
      "loss": 0.6844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11915641278028488,
      "rewards/margins": 0.04024672508239746,
      "rewards/rejected": -0.15940314531326294,
      "step": 191
    },
    {
      "epoch": 0.04460707440320613,
      "grad_norm": 24.97604751586914,
      "learning_rate": 9.556329849012776e-06,
      "logits/chosen": 0.8038893938064575,
      "logits/rejected": 0.7792326211929321,
      "logps/chosen": -78.59691619873047,
      "logps/rejected": -83.5892562866211,
      "loss": 0.7435,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.24631312489509583,
      "rewards/margins": -0.0942593663930893,
      "rewards/rejected": -0.15205375850200653,
      "step": 192
    },
    {
      "epoch": 0.04483940291572283,
      "grad_norm": 25.38528823852539,
      "learning_rate": 9.554006968641115e-06,
      "logits/chosen": 0.3639446198940277,
      "logits/rejected": 0.5673648118972778,
      "logps/chosen": -82.59968566894531,
      "logps/rejected": -83.03562927246094,
      "loss": 0.6805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12597382068634033,
      "rewards/margins": 0.03330863267183304,
      "rewards/rejected": -0.15928244590759277,
      "step": 193
    },
    {
      "epoch": 0.04507173142823953,
      "grad_norm": 24.062732696533203,
      "learning_rate": 9.551684088269456e-06,
      "logits/chosen": 0.7799516320228577,
      "logits/rejected": 1.0170520544052124,
      "logps/chosen": -78.17079162597656,
      "logps/rejected": -104.1685791015625,
      "loss": 0.5935,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07671985030174255,
      "rewards/margins": 0.23819765448570251,
      "rewards/rejected": -0.31491750478744507,
      "step": 194
    },
    {
      "epoch": 0.04530405994075623,
      "grad_norm": 28.163639068603516,
      "learning_rate": 9.549361207897793e-06,
      "logits/chosen": 0.7950453162193298,
      "logits/rejected": 0.5940882563591003,
      "logps/chosen": -91.15036010742188,
      "logps/rejected": -84.74078369140625,
      "loss": 0.7792,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34206289052963257,
      "rewards/margins": -0.15103954076766968,
      "rewards/rejected": -0.1910233348608017,
      "step": 195
    },
    {
      "epoch": 0.04553638845327293,
      "grad_norm": 23.52483367919922,
      "learning_rate": 9.547038327526134e-06,
      "logits/chosen": 0.921002984046936,
      "logits/rejected": 0.8743106126785278,
      "logps/chosen": -74.60057067871094,
      "logps/rejected": -75.40564727783203,
      "loss": 0.7197,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.18451224267482758,
      "rewards/margins": -0.04760880395770073,
      "rewards/rejected": -0.13690343499183655,
      "step": 196
    },
    {
      "epoch": 0.04576871696578963,
      "grad_norm": 21.66035270690918,
      "learning_rate": 9.544715447154473e-06,
      "logits/chosen": 0.7122595906257629,
      "logits/rejected": 0.9245290160179138,
      "logps/chosen": -77.13469696044922,
      "logps/rejected": -86.87715911865234,
      "loss": 0.5544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02021944336593151,
      "rewards/margins": 0.32544180750846863,
      "rewards/rejected": -0.3456612527370453,
      "step": 197
    },
    {
      "epoch": 0.04600104547830632,
      "grad_norm": 24.59174919128418,
      "learning_rate": 9.542392566782812e-06,
      "logits/chosen": 0.44183671474456787,
      "logits/rejected": 0.5853345394134521,
      "logps/chosen": -89.69120788574219,
      "logps/rejected": -99.71695709228516,
      "loss": 0.6232,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19464464485645294,
      "rewards/margins": 0.22112798690795898,
      "rewards/rejected": -0.4157726466655731,
      "step": 198
    },
    {
      "epoch": 0.04623337399082302,
      "grad_norm": 27.349668502807617,
      "learning_rate": 9.54006968641115e-06,
      "logits/chosen": 1.628969669342041,
      "logits/rejected": 1.6922025680541992,
      "logps/chosen": -93.99549102783203,
      "logps/rejected": -101.67591857910156,
      "loss": 0.6649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11732275038957596,
      "rewards/margins": 0.10433518886566162,
      "rewards/rejected": -0.22165794670581818,
      "step": 199
    },
    {
      "epoch": 0.04646570250333972,
      "grad_norm": 21.54608917236328,
      "learning_rate": 9.53774680603949e-06,
      "logits/chosen": 1.1112756729125977,
      "logits/rejected": 1.0260870456695557,
      "logps/chosen": -75.53955078125,
      "logps/rejected": -82.27347564697266,
      "loss": 0.6282,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09597907215356827,
      "rewards/margins": 0.1938326060771942,
      "rewards/rejected": -0.2898116707801819,
      "step": 200
    },
    {
      "epoch": 0.04669803101585642,
      "grad_norm": 24.006656646728516,
      "learning_rate": 9.53542392566783e-06,
      "logits/chosen": 0.6280245184898376,
      "logits/rejected": 0.71958327293396,
      "logps/chosen": -81.09029388427734,
      "logps/rejected": -81.64341735839844,
      "loss": 0.7296,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1801968663930893,
      "rewards/margins": -0.04514436423778534,
      "rewards/rejected": -0.13505248725414276,
      "step": 201
    },
    {
      "epoch": 0.04693035952837312,
      "grad_norm": 25.550783157348633,
      "learning_rate": 9.533101045296167e-06,
      "logits/chosen": 0.6883896589279175,
      "logits/rejected": 0.6527040004730225,
      "logps/chosen": -83.14958953857422,
      "logps/rejected": -85.29740142822266,
      "loss": 0.6735,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18662473559379578,
      "rewards/margins": 0.04387976974248886,
      "rewards/rejected": -0.23050452768802643,
      "step": 202
    },
    {
      "epoch": 0.04716268804088982,
      "grad_norm": 21.3087100982666,
      "learning_rate": 9.530778164924506e-06,
      "logits/chosen": 1.3519216775894165,
      "logits/rejected": 1.3141582012176514,
      "logps/chosen": -78.48006439208984,
      "logps/rejected": -96.09600830078125,
      "loss": 0.5777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.024489376693964005,
      "rewards/margins": 0.25224506855010986,
      "rewards/rejected": -0.27673444151878357,
      "step": 203
    },
    {
      "epoch": 0.04739501655340652,
      "grad_norm": 23.849302291870117,
      "learning_rate": 9.528455284552847e-06,
      "logits/chosen": 0.6480240821838379,
      "logits/rejected": 0.6562947630882263,
      "logps/chosen": -78.85226440429688,
      "logps/rejected": -74.72954559326172,
      "loss": 0.673,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2097853273153305,
      "rewards/margins": 0.0668829083442688,
      "rewards/rejected": -0.2766682505607605,
      "step": 204
    },
    {
      "epoch": 0.04762734506592321,
      "grad_norm": 24.65671730041504,
      "learning_rate": 9.526132404181186e-06,
      "logits/chosen": 0.9490463137626648,
      "logits/rejected": 0.9440734386444092,
      "logps/chosen": -80.49921417236328,
      "logps/rejected": -103.0182113647461,
      "loss": 0.6153,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11659792810678482,
      "rewards/margins": 0.2098953276872635,
      "rewards/rejected": -0.3264932632446289,
      "step": 205
    },
    {
      "epoch": 0.04785967357843991,
      "grad_norm": 26.0067138671875,
      "learning_rate": 9.523809523809525e-06,
      "logits/chosen": 0.8453131914138794,
      "logits/rejected": 0.8406972289085388,
      "logps/chosen": -85.34453582763672,
      "logps/rejected": -66.71489715576172,
      "loss": 0.7878,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2639509439468384,
      "rewards/margins": -0.1555192470550537,
      "rewards/rejected": -0.10843171179294586,
      "step": 206
    },
    {
      "epoch": 0.04809200209095661,
      "grad_norm": 25.13216209411621,
      "learning_rate": 9.521486643437864e-06,
      "logits/chosen": 1.1840423345565796,
      "logits/rejected": 1.1542432308197021,
      "logps/chosen": -81.85821533203125,
      "logps/rejected": -90.05056762695312,
      "loss": 0.6829,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.21208006143569946,
      "rewards/margins": 0.06129186600446701,
      "rewards/rejected": -0.2733719050884247,
      "step": 207
    },
    {
      "epoch": 0.04832433060347331,
      "grad_norm": 24.443567276000977,
      "learning_rate": 9.519163763066203e-06,
      "logits/chosen": 0.623783528804779,
      "logits/rejected": 0.8758875727653503,
      "logps/chosen": -83.2656478881836,
      "logps/rejected": -88.24087524414062,
      "loss": 0.6152,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1001816987991333,
      "rewards/margins": 0.2066730409860611,
      "rewards/rejected": -0.3068547546863556,
      "step": 208
    },
    {
      "epoch": 0.04855665911599001,
      "grad_norm": 23.610536575317383,
      "learning_rate": 9.516840882694543e-06,
      "logits/chosen": 0.630200982093811,
      "logits/rejected": 0.7346654534339905,
      "logps/chosen": -92.14993286132812,
      "logps/rejected": -79.349365234375,
      "loss": 0.6089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.060889098793268204,
      "rewards/margins": 0.20746535062789917,
      "rewards/rejected": -0.26835447549819946,
      "step": 209
    },
    {
      "epoch": 0.04878898762850671,
      "grad_norm": 29.6751708984375,
      "learning_rate": 9.51451800232288e-06,
      "logits/chosen": 1.3564164638519287,
      "logits/rejected": 1.2262893915176392,
      "logps/chosen": -96.78337097167969,
      "logps/rejected": -88.72765350341797,
      "loss": 0.8565,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.3366764783859253,
      "rewards/margins": -0.29102811217308044,
      "rewards/rejected": -0.04564835876226425,
      "step": 210
    },
    {
      "epoch": 0.04902131614102341,
      "grad_norm": 25.131568908691406,
      "learning_rate": 9.51219512195122e-06,
      "logits/chosen": 0.9517356157302856,
      "logits/rejected": 1.0066896677017212,
      "logps/chosen": -77.97272491455078,
      "logps/rejected": -78.64808654785156,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20897045731544495,
      "rewards/margins": 0.00877980887889862,
      "rewards/rejected": -0.21775026619434357,
      "step": 211
    },
    {
      "epoch": 0.0492536446535401,
      "grad_norm": 25.985334396362305,
      "learning_rate": 9.50987224157956e-06,
      "logits/chosen": 0.6175941228866577,
      "logits/rejected": 0.5941746234893799,
      "logps/chosen": -86.25373840332031,
      "logps/rejected": -88.81474304199219,
      "loss": 0.6877,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2419128119945526,
      "rewards/margins": 0.01964937150478363,
      "rewards/rejected": -0.26156216859817505,
      "step": 212
    },
    {
      "epoch": 0.0494859731660568,
      "grad_norm": 31.10892105102539,
      "learning_rate": 9.507549361207899e-06,
      "logits/chosen": 1.081666111946106,
      "logits/rejected": 1.0217894315719604,
      "logps/chosen": -103.37283325195312,
      "logps/rejected": -89.26905059814453,
      "loss": 0.7307,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32980090379714966,
      "rewards/margins": -0.06596161425113678,
      "rewards/rejected": -0.2638393044471741,
      "step": 213
    },
    {
      "epoch": 0.0497183016785735,
      "grad_norm": 25.076208114624023,
      "learning_rate": 9.505226480836238e-06,
      "logits/chosen": 0.991582453250885,
      "logits/rejected": 0.940561830997467,
      "logps/chosen": -80.62861633300781,
      "logps/rejected": -78.90901947021484,
      "loss": 0.7115,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15771079063415527,
      "rewards/margins": -0.026337573304772377,
      "rewards/rejected": -0.13137321174144745,
      "step": 214
    },
    {
      "epoch": 0.0499506301910902,
      "grad_norm": 28.435401916503906,
      "learning_rate": 9.502903600464577e-06,
      "logits/chosen": 1.288082242012024,
      "logits/rejected": 1.0713335275650024,
      "logps/chosen": -92.73802185058594,
      "logps/rejected": -72.17249298095703,
      "loss": 0.7323,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1974586546421051,
      "rewards/margins": -0.047986991703510284,
      "rewards/rejected": -0.14947164058685303,
      "step": 215
    },
    {
      "epoch": 0.0501829587036069,
      "grad_norm": 22.377595901489258,
      "learning_rate": 9.500580720092916e-06,
      "logits/chosen": 1.285351276397705,
      "logits/rejected": 1.3568681478500366,
      "logps/chosen": -99.1650390625,
      "logps/rejected": -103.68816375732422,
      "loss": 0.5298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0026804450899362564,
      "rewards/margins": 0.41726332902908325,
      "rewards/rejected": -0.41994377970695496,
      "step": 216
    },
    {
      "epoch": 0.0504152872161236,
      "grad_norm": 25.159852981567383,
      "learning_rate": 9.498257839721255e-06,
      "logits/chosen": 0.915169894695282,
      "logits/rejected": 1.0977768898010254,
      "logps/chosen": -97.24781799316406,
      "logps/rejected": -95.92755126953125,
      "loss": 0.6358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21563345193862915,
      "rewards/margins": 0.20002567768096924,
      "rewards/rejected": -0.4156591296195984,
      "step": 217
    },
    {
      "epoch": 0.050647615728640294,
      "grad_norm": 24.437490463256836,
      "learning_rate": 9.495934959349594e-06,
      "logits/chosen": 0.9077317118644714,
      "logits/rejected": 1.0738000869750977,
      "logps/chosen": -83.65917205810547,
      "logps/rejected": -76.18121337890625,
      "loss": 0.7094,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21276262402534485,
      "rewards/margins": -0.015402242541313171,
      "rewards/rejected": -0.19736036658287048,
      "step": 218
    },
    {
      "epoch": 0.050879944241156994,
      "grad_norm": 22.51068687438965,
      "learning_rate": 9.493612078977933e-06,
      "logits/chosen": 1.2256091833114624,
      "logits/rejected": 1.2311805486679077,
      "logps/chosen": -78.90677642822266,
      "logps/rejected": -82.08403015136719,
      "loss": 0.7013,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22245410084724426,
      "rewards/margins": -0.00599205307662487,
      "rewards/rejected": -0.21646203100681305,
      "step": 219
    },
    {
      "epoch": 0.05111227275367369,
      "grad_norm": 22.90703010559082,
      "learning_rate": 9.491289198606273e-06,
      "logits/chosen": 0.7386928796768188,
      "logits/rejected": 0.8563076853752136,
      "logps/chosen": -89.8940658569336,
      "logps/rejected": -82.94794464111328,
      "loss": 0.6438,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20790429413318634,
      "rewards/margins": 0.1584940105676651,
      "rewards/rejected": -0.36639833450317383,
      "step": 220
    },
    {
      "epoch": 0.05134460126619039,
      "grad_norm": 24.280672073364258,
      "learning_rate": 9.488966318234612e-06,
      "logits/chosen": 1.1168150901794434,
      "logits/rejected": 1.001579999923706,
      "logps/chosen": -81.94800567626953,
      "logps/rejected": -76.64828491210938,
      "loss": 0.6987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20591965317726135,
      "rewards/margins": 0.021196484565734863,
      "rewards/rejected": -0.22711613774299622,
      "step": 221
    },
    {
      "epoch": 0.05157692977870709,
      "grad_norm": 22.840314865112305,
      "learning_rate": 9.486643437862951e-06,
      "logits/chosen": 1.2425003051757812,
      "logits/rejected": 1.3737773895263672,
      "logps/chosen": -79.92900085449219,
      "logps/rejected": -84.80998229980469,
      "loss": 0.6213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.046738557517528534,
      "rewards/margins": 0.16848358511924744,
      "rewards/rejected": -0.21522215008735657,
      "step": 222
    },
    {
      "epoch": 0.05180925829122379,
      "grad_norm": 26.917728424072266,
      "learning_rate": 9.48432055749129e-06,
      "logits/chosen": 1.2572247982025146,
      "logits/rejected": 1.3949685096740723,
      "logps/chosen": -80.06588745117188,
      "logps/rejected": -91.55889892578125,
      "loss": 0.6755,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19516249001026154,
      "rewards/margins": 0.060037143528461456,
      "rewards/rejected": -0.2551996409893036,
      "step": 223
    },
    {
      "epoch": 0.05204158680374049,
      "grad_norm": 23.5734920501709,
      "learning_rate": 9.481997677119629e-06,
      "logits/chosen": 1.4373257160186768,
      "logits/rejected": 1.3470354080200195,
      "logps/chosen": -76.24015808105469,
      "logps/rejected": -96.11064910888672,
      "loss": 0.6201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12289832532405853,
      "rewards/margins": 0.323650598526001,
      "rewards/rejected": -0.4465489387512207,
      "step": 224
    },
    {
      "epoch": 0.052273915316257184,
      "grad_norm": 22.677249908447266,
      "learning_rate": 9.479674796747968e-06,
      "logits/chosen": 0.797163724899292,
      "logits/rejected": 0.7971245646476746,
      "logps/chosen": -74.92422485351562,
      "logps/rejected": -67.11775970458984,
      "loss": 0.6873,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1665925681591034,
      "rewards/margins": 0.028266338631510735,
      "rewards/rejected": -0.19485889375209808,
      "step": 225
    },
    {
      "epoch": 0.052506243828773884,
      "grad_norm": 22.052709579467773,
      "learning_rate": 9.477351916376307e-06,
      "logits/chosen": 1.1805821657180786,
      "logits/rejected": 1.3003482818603516,
      "logps/chosen": -78.12336730957031,
      "logps/rejected": -95.50572204589844,
      "loss": 0.5166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06373634934425354,
      "rewards/margins": 0.49462080001831055,
      "rewards/rejected": -0.5583571195602417,
      "step": 226
    },
    {
      "epoch": 0.052738572341290584,
      "grad_norm": 22.7008056640625,
      "learning_rate": 9.475029036004647e-06,
      "logits/chosen": 1.1976897716522217,
      "logits/rejected": 1.2526884078979492,
      "logps/chosen": -75.7769546508789,
      "logps/rejected": -84.09053039550781,
      "loss": 0.6128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05913214385509491,
      "rewards/margins": 0.18848958611488342,
      "rewards/rejected": -0.24762171506881714,
      "step": 227
    },
    {
      "epoch": 0.05297090085380728,
      "grad_norm": 24.290599822998047,
      "learning_rate": 9.472706155632986e-06,
      "logits/chosen": 1.450082778930664,
      "logits/rejected": 1.320256233215332,
      "logps/chosen": -82.68614959716797,
      "logps/rejected": -72.25157928466797,
      "loss": 0.7329,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20477160811424255,
      "rewards/margins": -0.05172766000032425,
      "rewards/rejected": -0.1530439704656601,
      "step": 228
    },
    {
      "epoch": 0.05320322936632398,
      "grad_norm": 24.177310943603516,
      "learning_rate": 9.470383275261325e-06,
      "logits/chosen": 2.0758728981018066,
      "logits/rejected": 1.881019115447998,
      "logps/chosen": -75.29872131347656,
      "logps/rejected": -80.09797668457031,
      "loss": 0.7114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21254903078079224,
      "rewards/margins": -0.026790481060743332,
      "rewards/rejected": -0.1857585608959198,
      "step": 229
    },
    {
      "epoch": 0.05343555787884068,
      "grad_norm": 25.93454360961914,
      "learning_rate": 9.468060394889664e-06,
      "logits/chosen": 1.5009284019470215,
      "logits/rejected": 1.5788259506225586,
      "logps/chosen": -86.96746826171875,
      "logps/rejected": -97.01960754394531,
      "loss": 0.6605,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19496294856071472,
      "rewards/margins": 0.08021178841590881,
      "rewards/rejected": -0.27517473697662354,
      "step": 230
    },
    {
      "epoch": 0.05366788639135738,
      "grad_norm": 25.060327529907227,
      "learning_rate": 9.465737514518003e-06,
      "logits/chosen": 1.9351660013198853,
      "logits/rejected": 1.8582054376602173,
      "logps/chosen": -102.83616638183594,
      "logps/rejected": -90.19717407226562,
      "loss": 0.6348,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06324052065610886,
      "rewards/margins": 0.24739500880241394,
      "rewards/rejected": -0.3106355369091034,
      "step": 231
    },
    {
      "epoch": 0.053900214903874075,
      "grad_norm": 26.746795654296875,
      "learning_rate": 9.463414634146342e-06,
      "logits/chosen": 0.9996305108070374,
      "logits/rejected": 0.870181679725647,
      "logps/chosen": -88.0987548828125,
      "logps/rejected": -86.24876403808594,
      "loss": 0.7379,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2112535536289215,
      "rewards/margins": -0.0733872652053833,
      "rewards/rejected": -0.1378662884235382,
      "step": 232
    },
    {
      "epoch": 0.054132543416390774,
      "grad_norm": 24.97496795654297,
      "learning_rate": 9.461091753774681e-06,
      "logits/chosen": 1.3558292388916016,
      "logits/rejected": 1.501130223274231,
      "logps/chosen": -79.86017608642578,
      "logps/rejected": -93.59545135498047,
      "loss": 0.6422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21076872944831848,
      "rewards/margins": 0.13602302968502045,
      "rewards/rejected": -0.3467917740345001,
      "step": 233
    },
    {
      "epoch": 0.054364871928907474,
      "grad_norm": 24.229764938354492,
      "learning_rate": 9.45876887340302e-06,
      "logits/chosen": 0.9713579416275024,
      "logits/rejected": 1.1407898664474487,
      "logps/chosen": -76.40094757080078,
      "logps/rejected": -91.38936614990234,
      "loss": 0.6161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15778684616088867,
      "rewards/margins": 0.20736216008663177,
      "rewards/rejected": -0.36514902114868164,
      "step": 234
    },
    {
      "epoch": 0.054597200441424174,
      "grad_norm": 24.471696853637695,
      "learning_rate": 9.45644599303136e-06,
      "logits/chosen": 1.6721343994140625,
      "logits/rejected": 1.8092221021652222,
      "logps/chosen": -72.79776000976562,
      "logps/rejected": -79.12478637695312,
      "loss": 0.6569,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15181155502796173,
      "rewards/margins": 0.10130995512008667,
      "rewards/rejected": -0.2531214952468872,
      "step": 235
    },
    {
      "epoch": 0.05482952895394087,
      "grad_norm": 26.8801326751709,
      "learning_rate": 9.4541231126597e-06,
      "logits/chosen": 1.4235434532165527,
      "logits/rejected": 1.563306212425232,
      "logps/chosen": -82.4010238647461,
      "logps/rejected": -92.59394073486328,
      "loss": 0.6661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2473740577697754,
      "rewards/margins": 0.08830185234546661,
      "rewards/rejected": -0.3356759250164032,
      "step": 236
    },
    {
      "epoch": 0.05506185746645757,
      "grad_norm": 27.463058471679688,
      "learning_rate": 9.451800232288037e-06,
      "logits/chosen": 1.3400388956069946,
      "logits/rejected": 1.4122618436813354,
      "logps/chosen": -86.28770446777344,
      "logps/rejected": -88.94383239746094,
      "loss": 0.7678,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1671684980392456,
      "rewards/margins": -0.13376259803771973,
      "rewards/rejected": -0.03340592607855797,
      "step": 237
    },
    {
      "epoch": 0.05529418597897427,
      "grad_norm": 25.72991180419922,
      "learning_rate": 9.449477351916377e-06,
      "logits/chosen": 2.3005521297454834,
      "logits/rejected": 2.190690279006958,
      "logps/chosen": -81.382080078125,
      "logps/rejected": -70.76031494140625,
      "loss": 0.7246,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1903112232685089,
      "rewards/margins": -0.0444798469543457,
      "rewards/rejected": -0.1458314061164856,
      "step": 238
    },
    {
      "epoch": 0.055526514491490965,
      "grad_norm": 24.991485595703125,
      "learning_rate": 9.447154471544716e-06,
      "logits/chosen": 1.052053451538086,
      "logits/rejected": 1.2196186780929565,
      "logps/chosen": -76.51312255859375,
      "logps/rejected": -88.5237045288086,
      "loss": 0.6692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10995927453041077,
      "rewards/margins": 0.05795592814683914,
      "rewards/rejected": -0.1679151952266693,
      "step": 239
    },
    {
      "epoch": 0.055758843004007665,
      "grad_norm": 23.161767959594727,
      "learning_rate": 9.444831591173055e-06,
      "logits/chosen": 1.2920268774032593,
      "logits/rejected": 1.2973453998565674,
      "logps/chosen": -82.72649383544922,
      "logps/rejected": -91.94039154052734,
      "loss": 0.6536,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11999014765024185,
      "rewards/margins": 0.0952644795179367,
      "rewards/rejected": -0.21525461971759796,
      "step": 240
    },
    {
      "epoch": 0.055991171516524364,
      "grad_norm": 25.25090980529785,
      "learning_rate": 9.442508710801394e-06,
      "logits/chosen": 0.9507448673248291,
      "logits/rejected": 0.9478709697723389,
      "logps/chosen": -97.75849914550781,
      "logps/rejected": -85.50361633300781,
      "loss": 0.7061,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3375244140625,
      "rewards/margins": -0.00152549147605896,
      "rewards/rejected": -0.33599892258644104,
      "step": 241
    },
    {
      "epoch": 0.056223500029041064,
      "grad_norm": 28.325868606567383,
      "learning_rate": 9.440185830429733e-06,
      "logits/chosen": 1.1931523084640503,
      "logits/rejected": 1.2921504974365234,
      "logps/chosen": -96.29569244384766,
      "logps/rejected": -97.69538116455078,
      "loss": 0.7418,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24708135426044464,
      "rewards/margins": -0.06270082294940948,
      "rewards/rejected": -0.18438054621219635,
      "step": 242
    },
    {
      "epoch": 0.056455828541557763,
      "grad_norm": 24.538795471191406,
      "learning_rate": 9.437862950058074e-06,
      "logits/chosen": 1.2546581029891968,
      "logits/rejected": 1.2406681776046753,
      "logps/chosen": -73.15858459472656,
      "logps/rejected": -69.15621948242188,
      "loss": 0.686,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.15799444913864136,
      "rewards/margins": 0.0371854305267334,
      "rewards/rejected": -0.19517987966537476,
      "step": 243
    },
    {
      "epoch": 0.05668815705407446,
      "grad_norm": 30.018823623657227,
      "learning_rate": 9.435540069686413e-06,
      "logits/chosen": 0.9548220634460449,
      "logits/rejected": 1.0148262977600098,
      "logps/chosen": -82.58935546875,
      "logps/rejected": -98.85728454589844,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33450403809547424,
      "rewards/margins": 0.02496056631207466,
      "rewards/rejected": -0.3594646155834198,
      "step": 244
    },
    {
      "epoch": 0.05692048556659116,
      "grad_norm": 55.98136901855469,
      "learning_rate": 9.43321718931475e-06,
      "logits/chosen": 1.443793773651123,
      "logits/rejected": 1.5033397674560547,
      "logps/chosen": -90.87355041503906,
      "logps/rejected": -95.08655548095703,
      "loss": 0.6701,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37244829535484314,
      "rewards/margins": 0.0826026201248169,
      "rewards/rejected": -0.45505085587501526,
      "step": 245
    },
    {
      "epoch": 0.057152814079107855,
      "grad_norm": 24.977794647216797,
      "learning_rate": 9.43089430894309e-06,
      "logits/chosen": 1.2242004871368408,
      "logits/rejected": 1.1889973878860474,
      "logps/chosen": -91.2010498046875,
      "logps/rejected": -97.5291519165039,
      "loss": 0.5377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06520523875951767,
      "rewards/margins": 0.5023460984230042,
      "rewards/rejected": -0.5675513744354248,
      "step": 246
    },
    {
      "epoch": 0.057385142591624555,
      "grad_norm": 24.605838775634766,
      "learning_rate": 9.42857142857143e-06,
      "logits/chosen": 1.2732224464416504,
      "logits/rejected": 1.2959538698196411,
      "logps/chosen": -81.98359680175781,
      "logps/rejected": -90.07846069335938,
      "loss": 0.6375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25593724846839905,
      "rewards/margins": 0.1283031404018402,
      "rewards/rejected": -0.38424035906791687,
      "step": 247
    },
    {
      "epoch": 0.057617471104141255,
      "grad_norm": 21.43461799621582,
      "learning_rate": 9.426248548199768e-06,
      "logits/chosen": 1.1149568557739258,
      "logits/rejected": 1.1094977855682373,
      "logps/chosen": -95.43555450439453,
      "logps/rejected": -112.39219665527344,
      "loss": 0.5343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04455437883734703,
      "rewards/margins": 0.5396963953971863,
      "rewards/rejected": -0.584250807762146,
      "step": 248
    },
    {
      "epoch": 0.057849799616657954,
      "grad_norm": 24.96822166442871,
      "learning_rate": 9.423925667828107e-06,
      "logits/chosen": 2.3049099445343018,
      "logits/rejected": 2.4599876403808594,
      "logps/chosen": -93.94032287597656,
      "logps/rejected": -98.322021484375,
      "loss": 0.6522,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19074571132659912,
      "rewards/margins": 0.09882239997386932,
      "rewards/rejected": -0.28956806659698486,
      "step": 249
    },
    {
      "epoch": 0.058082128129174654,
      "grad_norm": 24.42485809326172,
      "learning_rate": 9.421602787456446e-06,
      "logits/chosen": 1.6255440711975098,
      "logits/rejected": 1.771981954574585,
      "logps/chosen": -97.36038970947266,
      "logps/rejected": -80.53659057617188,
      "loss": 0.6419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1731569766998291,
      "rewards/margins": 0.11666981875896454,
      "rewards/rejected": -0.28982678055763245,
      "step": 250
    },
    {
      "epoch": 0.05831445664169135,
      "grad_norm": 23.54873275756836,
      "learning_rate": 9.419279907084787e-06,
      "logits/chosen": 1.883447289466858,
      "logits/rejected": 1.8346538543701172,
      "logps/chosen": -74.89922332763672,
      "logps/rejected": -89.65348815917969,
      "loss": 0.6652,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06643275916576385,
      "rewards/margins": 0.18886840343475342,
      "rewards/rejected": -0.2553011476993561,
      "step": 251
    },
    {
      "epoch": 0.05854678515420805,
      "grad_norm": 22.903364181518555,
      "learning_rate": 9.416957026713124e-06,
      "logits/chosen": 1.0142931938171387,
      "logits/rejected": 1.0608956813812256,
      "logps/chosen": -96.69383239746094,
      "logps/rejected": -90.87509155273438,
      "loss": 0.5818,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17117023468017578,
      "rewards/margins": 0.2615537941455841,
      "rewards/rejected": -0.4327240288257599,
      "step": 252
    },
    {
      "epoch": 0.058779113666724746,
      "grad_norm": 22.935508728027344,
      "learning_rate": 9.414634146341463e-06,
      "logits/chosen": 1.059821367263794,
      "logits/rejected": 1.2842278480529785,
      "logps/chosen": -76.71817779541016,
      "logps/rejected": -81.10167694091797,
      "loss": 0.6102,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19137519598007202,
      "rewards/margins": 0.1937275230884552,
      "rewards/rejected": -0.3851027488708496,
      "step": 253
    },
    {
      "epoch": 0.059011442179241445,
      "grad_norm": 24.8185977935791,
      "learning_rate": 9.412311265969804e-06,
      "logits/chosen": 1.7754141092300415,
      "logits/rejected": 1.5144609212875366,
      "logps/chosen": -85.98143768310547,
      "logps/rejected": -86.04600524902344,
      "loss": 0.7011,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18071527779102325,
      "rewards/margins": 0.010041959583759308,
      "rewards/rejected": -0.19075722992420197,
      "step": 254
    },
    {
      "epoch": 0.059243770691758145,
      "grad_norm": 30.455673217773438,
      "learning_rate": 9.409988385598143e-06,
      "logits/chosen": 1.0251027345657349,
      "logits/rejected": 1.1174815893173218,
      "logps/chosen": -84.17340850830078,
      "logps/rejected": -85.86695861816406,
      "loss": 0.6521,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2269105613231659,
      "rewards/margins": 0.10593805462121964,
      "rewards/rejected": -0.33284860849380493,
      "step": 255
    },
    {
      "epoch": 0.059476099204274845,
      "grad_norm": 24.374977111816406,
      "learning_rate": 9.407665505226482e-06,
      "logits/chosen": 1.756800651550293,
      "logits/rejected": 1.6404329538345337,
      "logps/chosen": -92.81446075439453,
      "logps/rejected": -92.22477722167969,
      "loss": 0.5878,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11798473447561264,
      "rewards/margins": 0.24944669008255005,
      "rewards/rejected": -0.3674314022064209,
      "step": 256
    },
    {
      "epoch": 0.059708427716791544,
      "grad_norm": 26.767602920532227,
      "learning_rate": 9.40534262485482e-06,
      "logits/chosen": 2.163294553756714,
      "logits/rejected": 2.3547861576080322,
      "logps/chosen": -91.3445816040039,
      "logps/rejected": -83.91513061523438,
      "loss": 0.6314,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11458630859851837,
      "rewards/margins": 0.17487283051013947,
      "rewards/rejected": -0.28945913910865784,
      "step": 257
    },
    {
      "epoch": 0.059940756229308244,
      "grad_norm": 22.457902908325195,
      "learning_rate": 9.40301974448316e-06,
      "logits/chosen": 1.2759802341461182,
      "logits/rejected": 1.1958986520767212,
      "logps/chosen": -76.57640075683594,
      "logps/rejected": -71.923583984375,
      "loss": 0.6805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15444834530353546,
      "rewards/margins": 0.04245833680033684,
      "rewards/rejected": -0.1969066858291626,
      "step": 258
    },
    {
      "epoch": 0.06017308474182494,
      "grad_norm": 21.313962936401367,
      "learning_rate": 9.400696864111498e-06,
      "logits/chosen": 1.0439332723617554,
      "logits/rejected": 0.8576859831809998,
      "logps/chosen": -79.42156219482422,
      "logps/rejected": -90.93929290771484,
      "loss": 0.6073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25621557235717773,
      "rewards/margins": 0.18666034936904907,
      "rewards/rejected": -0.4428759515285492,
      "step": 259
    },
    {
      "epoch": 0.060405413254341636,
      "grad_norm": 25.04743766784668,
      "learning_rate": 9.398373983739837e-06,
      "logits/chosen": 2.1633224487304688,
      "logits/rejected": 2.151156425476074,
      "logps/chosen": -93.56715393066406,
      "logps/rejected": -82.34738159179688,
      "loss": 0.6963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2459367960691452,
      "rewards/margins": 0.0215233713388443,
      "rewards/rejected": -0.2674601674079895,
      "step": 260
    },
    {
      "epoch": 0.060637741766858336,
      "grad_norm": 25.276798248291016,
      "learning_rate": 9.396051103368178e-06,
      "logits/chosen": 1.0729738473892212,
      "logits/rejected": 1.1912238597869873,
      "logps/chosen": -86.87132263183594,
      "logps/rejected": -81.49781799316406,
      "loss": 0.7314,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4792572855949402,
      "rewards/margins": -0.042765166610479355,
      "rewards/rejected": -0.4364921748638153,
      "step": 261
    },
    {
      "epoch": 0.060870070279375035,
      "grad_norm": 25.671598434448242,
      "learning_rate": 9.393728222996517e-06,
      "logits/chosen": 1.3035567998886108,
      "logits/rejected": 1.1957197189331055,
      "logps/chosen": -93.11483001708984,
      "logps/rejected": -73.17436218261719,
      "loss": 0.7179,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25290077924728394,
      "rewards/margins": -0.03669467568397522,
      "rewards/rejected": -0.2162061184644699,
      "step": 262
    },
    {
      "epoch": 0.061102398791891735,
      "grad_norm": 25.032119750976562,
      "learning_rate": 9.391405342624856e-06,
      "logits/chosen": 1.2738533020019531,
      "logits/rejected": 1.3065677881240845,
      "logps/chosen": -92.95804595947266,
      "logps/rejected": -120.79695892333984,
      "loss": 0.6154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17950578033924103,
      "rewards/margins": 0.22192618250846863,
      "rewards/rejected": -0.4014319181442261,
      "step": 263
    },
    {
      "epoch": 0.061334727304408435,
      "grad_norm": 28.281484603881836,
      "learning_rate": 9.389082462253195e-06,
      "logits/chosen": 0.6101279258728027,
      "logits/rejected": 0.6555444598197937,
      "logps/chosen": -86.58157348632812,
      "logps/rejected": -85.40066528320312,
      "loss": 0.7888,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.49376699328422546,
      "rewards/margins": -0.16959133744239807,
      "rewards/rejected": -0.3241756558418274,
      "step": 264
    },
    {
      "epoch": 0.061567055816925134,
      "grad_norm": 26.97138214111328,
      "learning_rate": 9.386759581881534e-06,
      "logits/chosen": 1.1646766662597656,
      "logits/rejected": 1.0490520000457764,
      "logps/chosen": -90.59959411621094,
      "logps/rejected": -96.8010025024414,
      "loss": 0.6901,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1879117488861084,
      "rewards/margins": 0.03151687979698181,
      "rewards/rejected": -0.2194286435842514,
      "step": 265
    },
    {
      "epoch": 0.061799384329441834,
      "grad_norm": 26.775358200073242,
      "learning_rate": 9.384436701509873e-06,
      "logits/chosen": 1.3477911949157715,
      "logits/rejected": 1.1487882137298584,
      "logps/chosen": -98.07902526855469,
      "logps/rejected": -100.70108795166016,
      "loss": 0.6253,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2857668697834015,
      "rewards/margins": 0.15301457047462463,
      "rewards/rejected": -0.43878135085105896,
      "step": 266
    },
    {
      "epoch": 0.062031712841958526,
      "grad_norm": 28.58702278137207,
      "learning_rate": 9.382113821138212e-06,
      "logits/chosen": 0.7032951712608337,
      "logits/rejected": 0.8045693039894104,
      "logps/chosen": -96.86177062988281,
      "logps/rejected": -81.73516082763672,
      "loss": 0.7768,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3347271680831909,
      "rewards/margins": -0.13792435824871063,
      "rewards/rejected": -0.19680280983448029,
      "step": 267
    },
    {
      "epoch": 0.062264041354475226,
      "grad_norm": 22.22774887084961,
      "learning_rate": 9.37979094076655e-06,
      "logits/chosen": 0.922122061252594,
      "logits/rejected": 0.8631289601325989,
      "logps/chosen": -81.10355377197266,
      "logps/rejected": -86.76280212402344,
      "loss": 0.6282,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27641597390174866,
      "rewards/margins": 0.19655628502368927,
      "rewards/rejected": -0.4729722738265991,
      "step": 268
    },
    {
      "epoch": 0.062496369866991926,
      "grad_norm": 24.546276092529297,
      "learning_rate": 9.377468060394891e-06,
      "logits/chosen": 1.5580905675888062,
      "logits/rejected": 1.5852751731872559,
      "logps/chosen": -82.15522766113281,
      "logps/rejected": -92.73299407958984,
      "loss": 0.6621,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30099305510520935,
      "rewards/margins": 0.08590465039014816,
      "rewards/rejected": -0.3868977129459381,
      "step": 269
    },
    {
      "epoch": 0.06272869837950862,
      "grad_norm": 27.16668701171875,
      "learning_rate": 9.37514518002323e-06,
      "logits/chosen": 0.8089917898178101,
      "logits/rejected": 0.7210565805435181,
      "logps/chosen": -100.27958679199219,
      "logps/rejected": -84.4389877319336,
      "loss": 0.711,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3277284502983093,
      "rewards/margins": -0.019289076328277588,
      "rewards/rejected": -0.30843937397003174,
      "step": 270
    },
    {
      "epoch": 0.06296102689202532,
      "grad_norm": 24.5843448638916,
      "learning_rate": 9.372822299651569e-06,
      "logits/chosen": 1.2380757331848145,
      "logits/rejected": 1.3388110399246216,
      "logps/chosen": -83.34083557128906,
      "logps/rejected": -103.16400146484375,
      "loss": 0.5437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07085716724395752,
      "rewards/margins": 0.3648220896720886,
      "rewards/rejected": -0.43567922711372375,
      "step": 271
    },
    {
      "epoch": 0.06319335540454202,
      "grad_norm": 24.85325050354004,
      "learning_rate": 9.370499419279908e-06,
      "logits/chosen": 0.6836954355239868,
      "logits/rejected": 0.8247714042663574,
      "logps/chosen": -78.73092651367188,
      "logps/rejected": -74.1185302734375,
      "loss": 0.7211,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2504175305366516,
      "rewards/margins": -0.028159860521554947,
      "rewards/rejected": -0.22225765883922577,
      "step": 272
    },
    {
      "epoch": 0.06342568391705872,
      "grad_norm": 22.095428466796875,
      "learning_rate": 9.368176538908247e-06,
      "logits/chosen": 0.9265656471252441,
      "logits/rejected": 1.041410207748413,
      "logps/chosen": -80.81743621826172,
      "logps/rejected": -90.32194519042969,
      "loss": 0.5563,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22730815410614014,
      "rewards/margins": 0.30590391159057617,
      "rewards/rejected": -0.5332120656967163,
      "step": 273
    },
    {
      "epoch": 0.06365801242957542,
      "grad_norm": 21.270788192749023,
      "learning_rate": 9.365853658536586e-06,
      "logits/chosen": 0.8501351475715637,
      "logits/rejected": 1.1687731742858887,
      "logps/chosen": -87.92684936523438,
      "logps/rejected": -72.49055480957031,
      "loss": 0.5999,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10004277527332306,
      "rewards/margins": 0.2068241536617279,
      "rewards/rejected": -0.30686691403388977,
      "step": 274
    },
    {
      "epoch": 0.06389034094209212,
      "grad_norm": 24.064071655273438,
      "learning_rate": 9.363530778164925e-06,
      "logits/chosen": 0.8899807929992676,
      "logits/rejected": 1.0092464685440063,
      "logps/chosen": -81.42794036865234,
      "logps/rejected": -80.25640869140625,
      "loss": 0.6903,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27012762427330017,
      "rewards/margins": 0.019590327516198158,
      "rewards/rejected": -0.28971797227859497,
      "step": 275
    },
    {
      "epoch": 0.06412266945460882,
      "grad_norm": 26.197978973388672,
      "learning_rate": 9.361207897793264e-06,
      "logits/chosen": 0.7622615694999695,
      "logits/rejected": 0.8955585360527039,
      "logps/chosen": -104.56193542480469,
      "logps/rejected": -93.79415893554688,
      "loss": 0.63,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1984696388244629,
      "rewards/margins": 0.17956143617630005,
      "rewards/rejected": -0.37803107500076294,
      "step": 276
    },
    {
      "epoch": 0.06435499796712552,
      "grad_norm": 27.20547866821289,
      "learning_rate": 9.358885017421604e-06,
      "logits/chosen": 0.8428409695625305,
      "logits/rejected": 0.9010639786720276,
      "logps/chosen": -86.81473541259766,
      "logps/rejected": -85.4675064086914,
      "loss": 0.696,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.18969464302062988,
      "rewards/margins": 0.024746662005782127,
      "rewards/rejected": -0.21444129943847656,
      "step": 277
    },
    {
      "epoch": 0.06458732647964222,
      "grad_norm": 26.8000545501709,
      "learning_rate": 9.356562137049943e-06,
      "logits/chosen": 1.088332176208496,
      "logits/rejected": 1.2040818929672241,
      "logps/chosen": -89.16706848144531,
      "logps/rejected": -78.08934020996094,
      "loss": 0.769,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.32687780261039734,
      "rewards/margins": -0.12984812259674072,
      "rewards/rejected": -0.1970296949148178,
      "step": 278
    },
    {
      "epoch": 0.06481965499215891,
      "grad_norm": 26.113859176635742,
      "learning_rate": 9.35423925667828e-06,
      "logits/chosen": 0.6561060547828674,
      "logits/rejected": 0.7228536009788513,
      "logps/chosen": -94.74264526367188,
      "logps/rejected": -92.26786804199219,
      "loss": 0.6155,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2625207304954529,
      "rewards/margins": 0.3763311803340912,
      "rewards/rejected": -0.6388519406318665,
      "step": 279
    },
    {
      "epoch": 0.06505198350467561,
      "grad_norm": 27.499595642089844,
      "learning_rate": 9.351916376306621e-06,
      "logits/chosen": 1.82683527469635,
      "logits/rejected": 1.8474757671356201,
      "logps/chosen": -87.77630615234375,
      "logps/rejected": -93.8446273803711,
      "loss": 0.7007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.246669203042984,
      "rewards/margins": 0.04392232745885849,
      "rewards/rejected": -0.2905915379524231,
      "step": 280
    },
    {
      "epoch": 0.06528431201719231,
      "grad_norm": 24.156408309936523,
      "learning_rate": 9.34959349593496e-06,
      "logits/chosen": 0.6997119188308716,
      "logits/rejected": 0.8866211175918579,
      "logps/chosen": -85.61001586914062,
      "logps/rejected": -77.63965606689453,
      "loss": 0.6752,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.051549725234508514,
      "rewards/margins": 0.09508730471134186,
      "rewards/rejected": -0.14663703739643097,
      "step": 281
    },
    {
      "epoch": 0.06551664052970901,
      "grad_norm": 21.721891403198242,
      "learning_rate": 9.347270615563299e-06,
      "logits/chosen": 1.5455853939056396,
      "logits/rejected": 1.3679957389831543,
      "logps/chosen": -72.55931091308594,
      "logps/rejected": -77.96286010742188,
      "loss": 0.602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.027166463434696198,
      "rewards/margins": 0.24431227147579193,
      "rewards/rejected": -0.27147868275642395,
      "step": 282
    },
    {
      "epoch": 0.06574896904222571,
      "grad_norm": 21.781641006469727,
      "learning_rate": 9.344947735191638e-06,
      "logits/chosen": 1.030091404914856,
      "logits/rejected": 0.9581285119056702,
      "logps/chosen": -78.5499496459961,
      "logps/rejected": -78.79859161376953,
      "loss": 0.6227,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05005364120006561,
      "rewards/margins": 0.21053841710090637,
      "rewards/rejected": -0.2605920433998108,
      "step": 283
    },
    {
      "epoch": 0.0659812975547424,
      "grad_norm": 30.800132751464844,
      "learning_rate": 9.342624854819977e-06,
      "logits/chosen": 0.9067666530609131,
      "logits/rejected": 0.83250492811203,
      "logps/chosen": -94.26411437988281,
      "logps/rejected": -79.99729919433594,
      "loss": 0.7695,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.32166415452957153,
      "rewards/margins": -0.10790358483791351,
      "rewards/rejected": -0.21376056969165802,
      "step": 284
    },
    {
      "epoch": 0.0662136260672591,
      "grad_norm": 27.854787826538086,
      "learning_rate": 9.340301974448318e-06,
      "logits/chosen": 0.8450517058372498,
      "logits/rejected": 0.9700777530670166,
      "logps/chosen": -83.91154479980469,
      "logps/rejected": -79.9000473022461,
      "loss": 0.7754,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3072691559791565,
      "rewards/margins": -0.1238579973578453,
      "rewards/rejected": -0.18341116607189178,
      "step": 285
    },
    {
      "epoch": 0.0664459545797758,
      "grad_norm": 21.55385398864746,
      "learning_rate": 9.337979094076656e-06,
      "logits/chosen": 0.8665450215339661,
      "logits/rejected": 0.8717779517173767,
      "logps/chosen": -79.03427124023438,
      "logps/rejected": -91.65467834472656,
      "loss": 0.623,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10645118355751038,
      "rewards/margins": 0.3651922047138214,
      "rewards/rejected": -0.4716433584690094,
      "step": 286
    },
    {
      "epoch": 0.0666782830922925,
      "grad_norm": 25.32794952392578,
      "learning_rate": 9.335656213704994e-06,
      "logits/chosen": 1.1170490980148315,
      "logits/rejected": 1.032767415046692,
      "logps/chosen": -96.92507934570312,
      "logps/rejected": -73.73995208740234,
      "loss": 0.6796,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.07273342460393906,
      "rewards/margins": 0.06511463969945908,
      "rewards/rejected": -0.13784807920455933,
      "step": 287
    },
    {
      "epoch": 0.0669106116048092,
      "grad_norm": 24.610822677612305,
      "learning_rate": 9.333333333333334e-06,
      "logits/chosen": 1.3323910236358643,
      "logits/rejected": 1.3352984189987183,
      "logps/chosen": -85.1610107421875,
      "logps/rejected": -90.07331085205078,
      "loss": 0.6441,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11907539516687393,
      "rewards/margins": 0.15455183386802673,
      "rewards/rejected": -0.27362722158432007,
      "step": 288
    },
    {
      "epoch": 0.0671429401173259,
      "grad_norm": 25.933979034423828,
      "learning_rate": 9.331010452961673e-06,
      "logits/chosen": 1.2413015365600586,
      "logits/rejected": 1.0946635007858276,
      "logps/chosen": -77.9164047241211,
      "logps/rejected": -91.19677734375,
      "loss": 0.6955,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.11987631767988205,
      "rewards/margins": 0.014223700389266014,
      "rewards/rejected": -0.13410000503063202,
      "step": 289
    },
    {
      "epoch": 0.0673752686298426,
      "grad_norm": 29.87858772277832,
      "learning_rate": 9.328687572590012e-06,
      "logits/chosen": 1.2814569473266602,
      "logits/rejected": 1.0578937530517578,
      "logps/chosen": -84.32884216308594,
      "logps/rejected": -92.46390533447266,
      "loss": 0.8087,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2800370156764984,
      "rewards/margins": -0.20572957396507263,
      "rewards/rejected": -0.07430744171142578,
      "step": 290
    },
    {
      "epoch": 0.0676075971423593,
      "grad_norm": 24.215871810913086,
      "learning_rate": 9.326364692218351e-06,
      "logits/chosen": 1.8690383434295654,
      "logits/rejected": 1.7580991983413696,
      "logps/chosen": -84.8453369140625,
      "logps/rejected": -89.26426696777344,
      "loss": 0.6077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06511914730072021,
      "rewards/margins": 0.3417985439300537,
      "rewards/rejected": -0.2766793966293335,
      "step": 291
    },
    {
      "epoch": 0.067839925654876,
      "grad_norm": 27.30272102355957,
      "learning_rate": 9.32404181184669e-06,
      "logits/chosen": 1.2847485542297363,
      "logits/rejected": 1.2965432405471802,
      "logps/chosen": -92.32637023925781,
      "logps/rejected": -84.51302337646484,
      "loss": 0.6586,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13196030259132385,
      "rewards/margins": 0.08134287595748901,
      "rewards/rejected": -0.21330319344997406,
      "step": 292
    },
    {
      "epoch": 0.0680722541673927,
      "grad_norm": 22.21808433532715,
      "learning_rate": 9.32171893147503e-06,
      "logits/chosen": 1.714162826538086,
      "logits/rejected": 1.7604329586029053,
      "logps/chosen": -77.7578125,
      "logps/rejected": -77.6015625,
      "loss": 0.6383,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06716108322143555,
      "rewards/margins": 0.1523125171661377,
      "rewards/rejected": -0.21947360038757324,
      "step": 293
    },
    {
      "epoch": 0.0683045826799094,
      "grad_norm": 29.297048568725586,
      "learning_rate": 9.319396051103368e-06,
      "logits/chosen": 0.9234082698822021,
      "logits/rejected": 0.989009439945221,
      "logps/chosen": -80.0303726196289,
      "logps/rejected": -87.93872833251953,
      "loss": 0.6317,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09330661594867706,
      "rewards/margins": 0.1765664517879486,
      "rewards/rejected": -0.2698730528354645,
      "step": 294
    },
    {
      "epoch": 0.0685369111924261,
      "grad_norm": 20.066614151000977,
      "learning_rate": 9.317073170731709e-06,
      "logits/chosen": 0.9493959546089172,
      "logits/rejected": 1.0341829061508179,
      "logps/chosen": -76.84292602539062,
      "logps/rejected": -73.11532592773438,
      "loss": 0.6164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08182427287101746,
      "rewards/margins": 0.20015712082386017,
      "rewards/rejected": -0.2819814085960388,
      "step": 295
    },
    {
      "epoch": 0.0687692397049428,
      "grad_norm": 24.61650848388672,
      "learning_rate": 9.314750290360047e-06,
      "logits/chosen": 1.5672144889831543,
      "logits/rejected": 1.730922818183899,
      "logps/chosen": -80.26622009277344,
      "logps/rejected": -89.16610717773438,
      "loss": 0.6243,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05904242396354675,
      "rewards/margins": 0.17019690573215485,
      "rewards/rejected": -0.2292393147945404,
      "step": 296
    },
    {
      "epoch": 0.0690015682174595,
      "grad_norm": 20.881929397583008,
      "learning_rate": 9.312427409988386e-06,
      "logits/chosen": 0.8778644800186157,
      "logits/rejected": 0.9908902645111084,
      "logps/chosen": -85.46332550048828,
      "logps/rejected": -81.63123321533203,
      "loss": 0.5553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.014569424092769623,
      "rewards/margins": 0.3457196056842804,
      "rewards/rejected": -0.3602890372276306,
      "step": 297
    },
    {
      "epoch": 0.06923389672997618,
      "grad_norm": 21.188888549804688,
      "learning_rate": 9.310104529616725e-06,
      "logits/chosen": 1.485657811164856,
      "logits/rejected": 1.5435373783111572,
      "logps/chosen": -73.0702133178711,
      "logps/rejected": -91.20503997802734,
      "loss": 0.5474,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10437928885221481,
      "rewards/margins": 0.4574993848800659,
      "rewards/rejected": -0.3531201481819153,
      "step": 298
    },
    {
      "epoch": 0.06946622524249288,
      "grad_norm": 25.324541091918945,
      "learning_rate": 9.307781649245064e-06,
      "logits/chosen": 1.135101318359375,
      "logits/rejected": 1.3170270919799805,
      "logps/chosen": -86.5981216430664,
      "logps/rejected": -83.8591079711914,
      "loss": 0.6289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11117443442344666,
      "rewards/margins": 0.1541796624660492,
      "rewards/rejected": -0.26535409688949585,
      "step": 299
    },
    {
      "epoch": 0.06969855375500958,
      "grad_norm": 22.401412963867188,
      "learning_rate": 9.305458768873403e-06,
      "logits/chosen": 0.7394976019859314,
      "logits/rejected": 0.7153785824775696,
      "logps/chosen": -79.60882568359375,
      "logps/rejected": -83.81566619873047,
      "loss": 0.6346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10546621680259705,
      "rewards/margins": 0.14704301953315735,
      "rewards/rejected": -0.252509206533432,
      "step": 300
    },
    {
      "epoch": 0.06993088226752628,
      "grad_norm": 26.760305404663086,
      "learning_rate": 9.303135888501744e-06,
      "logits/chosen": 0.8761122822761536,
      "logits/rejected": 1.0836844444274902,
      "logps/chosen": -75.83680725097656,
      "logps/rejected": -74.87562561035156,
      "loss": 0.7538,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.20225021243095398,
      "rewards/margins": -0.10796833038330078,
      "rewards/rejected": -0.0942818820476532,
      "step": 301
    },
    {
      "epoch": 0.07016321078004298,
      "grad_norm": 25.472694396972656,
      "learning_rate": 9.300813008130081e-06,
      "logits/chosen": 0.9822592735290527,
      "logits/rejected": 1.080685019493103,
      "logps/chosen": -81.53864288330078,
      "logps/rejected": -85.19805908203125,
      "loss": 0.6899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20826730132102966,
      "rewards/margins": 0.03063366189599037,
      "rewards/rejected": -0.23890097439289093,
      "step": 302
    },
    {
      "epoch": 0.07039553929255968,
      "grad_norm": 24.273427963256836,
      "learning_rate": 9.298490127758422e-06,
      "logits/chosen": 0.9860820770263672,
      "logits/rejected": 1.009285569190979,
      "logps/chosen": -93.56761169433594,
      "logps/rejected": -77.78841400146484,
      "loss": 0.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13255853950977325,
      "rewards/margins": -0.0035759396851062775,
      "rewards/rejected": -0.12898260354995728,
      "step": 303
    },
    {
      "epoch": 0.07062786780507638,
      "grad_norm": 27.262653350830078,
      "learning_rate": 9.29616724738676e-06,
      "logits/chosen": 1.9190473556518555,
      "logits/rejected": 1.8195306062698364,
      "logps/chosen": -81.5855712890625,
      "logps/rejected": -82.53794860839844,
      "loss": 0.709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08926527947187424,
      "rewards/margins": -0.006597496569156647,
      "rewards/rejected": -0.08266779035329819,
      "step": 304
    },
    {
      "epoch": 0.07086019631759308,
      "grad_norm": 24.130292892456055,
      "learning_rate": 9.2938443670151e-06,
      "logits/chosen": 0.9073941111564636,
      "logits/rejected": 0.9443073272705078,
      "logps/chosen": -80.19335174560547,
      "logps/rejected": -83.50019073486328,
      "loss": 0.6687,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1882946491241455,
      "rewards/margins": 0.06542392075061798,
      "rewards/rejected": -0.2537185847759247,
      "step": 305
    },
    {
      "epoch": 0.07109252483010978,
      "grad_norm": 24.300127029418945,
      "learning_rate": 9.291521486643439e-06,
      "logits/chosen": 0.8769561648368835,
      "logits/rejected": 0.8331964015960693,
      "logps/chosen": -80.32697296142578,
      "logps/rejected": -82.11312866210938,
      "loss": 0.6839,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.056980282068252563,
      "rewards/margins": 0.0755198523402214,
      "rewards/rejected": -0.13250012695789337,
      "step": 306
    },
    {
      "epoch": 0.07132485334262648,
      "grad_norm": 25.55893898010254,
      "learning_rate": 9.289198606271777e-06,
      "logits/chosen": 1.3956899642944336,
      "logits/rejected": 1.4136497974395752,
      "logps/chosen": -73.0401611328125,
      "logps/rejected": -84.73180389404297,
      "loss": 0.7229,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.22121047973632812,
      "rewards/margins": -0.049876585602760315,
      "rewards/rejected": -0.1713338941335678,
      "step": 307
    },
    {
      "epoch": 0.07155718185514318,
      "grad_norm": 25.034069061279297,
      "learning_rate": 9.286875725900118e-06,
      "logits/chosen": 1.3677916526794434,
      "logits/rejected": 1.2926225662231445,
      "logps/chosen": -83.85012817382812,
      "logps/rejected": -81.42173767089844,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12973462045192719,
      "rewards/margins": 0.018881753087043762,
      "rewards/rejected": -0.14861637353897095,
      "step": 308
    },
    {
      "epoch": 0.07178951036765988,
      "grad_norm": 22.06674575805664,
      "learning_rate": 9.284552845528455e-06,
      "logits/chosen": 0.7874094843864441,
      "logits/rejected": 0.9593048095703125,
      "logps/chosen": -81.1852798461914,
      "logps/rejected": -82.10470581054688,
      "loss": 0.6129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03730387985706329,
      "rewards/margins": 0.18311011791229248,
      "rewards/rejected": -0.22041399776935577,
      "step": 309
    },
    {
      "epoch": 0.07202183888017658,
      "grad_norm": 24.230487823486328,
      "learning_rate": 9.282229965156794e-06,
      "logits/chosen": 1.950911283493042,
      "logits/rejected": 2.025618553161621,
      "logps/chosen": -77.61517333984375,
      "logps/rejected": -75.9598159790039,
      "loss": 0.7376,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1763363927602768,
      "rewards/margins": -0.07725628465414047,
      "rewards/rejected": -0.09908010810613632,
      "step": 310
    },
    {
      "epoch": 0.07225416739269327,
      "grad_norm": 26.79959487915039,
      "learning_rate": 9.279907084785135e-06,
      "logits/chosen": 0.985507607460022,
      "logits/rejected": 0.8827049136161804,
      "logps/chosen": -80.87409973144531,
      "logps/rejected": -96.53418731689453,
      "loss": 0.7072,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.26288482546806335,
      "rewards/margins": -0.019575975835323334,
      "rewards/rejected": -0.2433088719844818,
      "step": 311
    },
    {
      "epoch": 0.07248649590520996,
      "grad_norm": 27.49338722229004,
      "learning_rate": 9.277584204413474e-06,
      "logits/chosen": 0.9357419013977051,
      "logits/rejected": 1.0248900651931763,
      "logps/chosen": -84.78581237792969,
      "logps/rejected": -90.01705169677734,
      "loss": 0.7564,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.22860723733901978,
      "rewards/margins": -0.11251893639564514,
      "rewards/rejected": -0.11608832329511642,
      "step": 312
    },
    {
      "epoch": 0.07271882441772666,
      "grad_norm": 27.12683868408203,
      "learning_rate": 9.275261324041813e-06,
      "logits/chosen": 1.6585729122161865,
      "logits/rejected": 1.4671108722686768,
      "logps/chosen": -94.31666564941406,
      "logps/rejected": -81.09420013427734,
      "loss": 0.6826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08689799159765244,
      "rewards/margins": 0.02713487297296524,
      "rewards/rejected": -0.11403286457061768,
      "step": 313
    },
    {
      "epoch": 0.07295115293024336,
      "grad_norm": 24.428133010864258,
      "learning_rate": 9.272938443670152e-06,
      "logits/chosen": 2.0082461833953857,
      "logits/rejected": 1.906229853630066,
      "logps/chosen": -84.42848205566406,
      "logps/rejected": -85.35287475585938,
      "loss": 0.6489,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18795938789844513,
      "rewards/margins": 0.12789683043956757,
      "rewards/rejected": -0.3158562183380127,
      "step": 314
    },
    {
      "epoch": 0.07318348144276006,
      "grad_norm": 20.852455139160156,
      "learning_rate": 9.27061556329849e-06,
      "logits/chosen": 1.2161389589309692,
      "logits/rejected": 1.1384611129760742,
      "logps/chosen": -74.84042358398438,
      "logps/rejected": -75.80134582519531,
      "loss": 0.5832,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1242719441652298,
      "rewards/margins": 0.25101107358932495,
      "rewards/rejected": -0.37528300285339355,
      "step": 315
    },
    {
      "epoch": 0.07341580995527676,
      "grad_norm": 27.14118003845215,
      "learning_rate": 9.268292682926831e-06,
      "logits/chosen": 1.3429756164550781,
      "logits/rejected": 1.269467830657959,
      "logps/chosen": -82.94624328613281,
      "logps/rejected": -87.8663558959961,
      "loss": 0.7836,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.22732041776180267,
      "rewards/margins": -0.16415786743164062,
      "rewards/rejected": -0.06316253542900085,
      "step": 316
    },
    {
      "epoch": 0.07364813846779346,
      "grad_norm": 23.582918167114258,
      "learning_rate": 9.265969802555169e-06,
      "logits/chosen": 1.2091805934906006,
      "logits/rejected": 1.3138576745986938,
      "logps/chosen": -77.18496704101562,
      "logps/rejected": -82.65235900878906,
      "loss": 0.6869,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14397770166397095,
      "rewards/margins": 0.028332147747278214,
      "rewards/rejected": -0.17230981588363647,
      "step": 317
    },
    {
      "epoch": 0.07388046698031016,
      "grad_norm": 21.610857009887695,
      "learning_rate": 9.263646922183507e-06,
      "logits/chosen": 0.9561871886253357,
      "logits/rejected": 1.0670157670974731,
      "logps/chosen": -77.54655456542969,
      "logps/rejected": -86.73898315429688,
      "loss": 0.5661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06176132708787918,
      "rewards/margins": 0.2848876416683197,
      "rewards/rejected": -0.3466489911079407,
      "step": 318
    },
    {
      "epoch": 0.07411279549282686,
      "grad_norm": 24.35181999206543,
      "learning_rate": 9.261324041811848e-06,
      "logits/chosen": 0.8452289700508118,
      "logits/rejected": 0.9029335379600525,
      "logps/chosen": -87.19078826904297,
      "logps/rejected": -83.58588409423828,
      "loss": 0.6769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18307772278785706,
      "rewards/margins": 0.04454917833209038,
      "rewards/rejected": -0.22762690484523773,
      "step": 319
    },
    {
      "epoch": 0.07434512400534356,
      "grad_norm": 23.61269187927246,
      "learning_rate": 9.259001161440187e-06,
      "logits/chosen": 0.9640538692474365,
      "logits/rejected": 0.997380793094635,
      "logps/chosen": -84.76210021972656,
      "logps/rejected": -83.02777099609375,
      "loss": 0.6641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14829173684120178,
      "rewards/margins": 0.07146275043487549,
      "rewards/rejected": -0.21975448727607727,
      "step": 320
    },
    {
      "epoch": 0.07457745251786026,
      "grad_norm": 20.11528968811035,
      "learning_rate": 9.256678281068526e-06,
      "logits/chosen": 1.562687635421753,
      "logits/rejected": 1.7433078289031982,
      "logps/chosen": -70.56607818603516,
      "logps/rejected": -66.55561828613281,
      "loss": 0.6497,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012163903564214706,
      "rewards/margins": 0.11544326692819595,
      "rewards/rejected": -0.12760716676712036,
      "step": 321
    },
    {
      "epoch": 0.07480978103037696,
      "grad_norm": 28.935041427612305,
      "learning_rate": 9.254355400696865e-06,
      "logits/chosen": 1.2428383827209473,
      "logits/rejected": 1.2174153327941895,
      "logps/chosen": -76.76209259033203,
      "logps/rejected": -86.84707641601562,
      "loss": 0.7619,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.21616476774215698,
      "rewards/margins": -0.11601950973272324,
      "rewards/rejected": -0.10014523565769196,
      "step": 322
    },
    {
      "epoch": 0.07504210954289366,
      "grad_norm": 24.378896713256836,
      "learning_rate": 9.252032520325204e-06,
      "logits/chosen": 1.1870204210281372,
      "logits/rejected": 1.2424275875091553,
      "logps/chosen": -85.41572570800781,
      "logps/rejected": -81.06929779052734,
      "loss": 0.6812,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1451551467180252,
      "rewards/margins": 0.051290158182382584,
      "rewards/rejected": -0.1964452862739563,
      "step": 323
    },
    {
      "epoch": 0.07527443805541036,
      "grad_norm": 24.013031005859375,
      "learning_rate": 9.249709639953543e-06,
      "logits/chosen": 1.6188745498657227,
      "logits/rejected": 1.618478775024414,
      "logps/chosen": -85.26432800292969,
      "logps/rejected": -85.11983489990234,
      "loss": 0.6158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03652805835008621,
      "rewards/margins": 0.21265415847301483,
      "rewards/rejected": -0.24918222427368164,
      "step": 324
    },
    {
      "epoch": 0.07550676656792706,
      "grad_norm": 29.657133102416992,
      "learning_rate": 9.247386759581882e-06,
      "logits/chosen": 2.5143990516662598,
      "logits/rejected": 2.5422275066375732,
      "logps/chosen": -98.864990234375,
      "logps/rejected": -102.26606750488281,
      "loss": 0.6826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09628072381019592,
      "rewards/margins": 0.0508522093296051,
      "rewards/rejected": -0.14713293313980103,
      "step": 325
    },
    {
      "epoch": 0.07573909508044374,
      "grad_norm": 23.35091781616211,
      "learning_rate": 9.24506387921022e-06,
      "logits/chosen": 0.8610386848449707,
      "logits/rejected": 1.007202386856079,
      "logps/chosen": -79.85843658447266,
      "logps/rejected": -76.33924865722656,
      "loss": 0.6702,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1514539271593094,
      "rewards/margins": 0.04876942187547684,
      "rewards/rejected": -0.20022335648536682,
      "step": 326
    },
    {
      "epoch": 0.07597142359296044,
      "grad_norm": 25.72856330871582,
      "learning_rate": 9.242740998838561e-06,
      "logits/chosen": 1.5697702169418335,
      "logits/rejected": 1.4387000799179077,
      "logps/chosen": -97.96452331542969,
      "logps/rejected": -86.41302490234375,
      "loss": 0.7213,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05583601072430611,
      "rewards/margins": -0.026649709790945053,
      "rewards/rejected": -0.029186297208070755,
      "step": 327
    },
    {
      "epoch": 0.07620375210547714,
      "grad_norm": 25.88642692565918,
      "learning_rate": 9.2404181184669e-06,
      "logits/chosen": 1.153348684310913,
      "logits/rejected": 1.1431220769882202,
      "logps/chosen": -81.79458618164062,
      "logps/rejected": -80.0313949584961,
      "loss": 0.7215,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.20546486973762512,
      "rewards/margins": 0.005708610638976097,
      "rewards/rejected": -0.21117347478866577,
      "step": 328
    },
    {
      "epoch": 0.07643608061799384,
      "grad_norm": 26.11725616455078,
      "learning_rate": 9.238095238095239e-06,
      "logits/chosen": 1.10659658908844,
      "logits/rejected": 1.203141212463379,
      "logps/chosen": -71.18525695800781,
      "logps/rejected": -88.12239074707031,
      "loss": 0.7709,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16117160022258759,
      "rewards/margins": -0.13740339875221252,
      "rewards/rejected": -0.023768212646245956,
      "step": 329
    },
    {
      "epoch": 0.07666840913051054,
      "grad_norm": 26.398527145385742,
      "learning_rate": 9.235772357723578e-06,
      "logits/chosen": 0.9929046034812927,
      "logits/rejected": 0.934783935546875,
      "logps/chosen": -93.0686264038086,
      "logps/rejected": -90.19357299804688,
      "loss": 0.7416,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.24072198569774628,
      "rewards/margins": -0.07570755481719971,
      "rewards/rejected": -0.16501443088054657,
      "step": 330
    },
    {
      "epoch": 0.07690073764302724,
      "grad_norm": 24.08504295349121,
      "learning_rate": 9.233449477351917e-06,
      "logits/chosen": 1.98326575756073,
      "logits/rejected": 1.9137343168258667,
      "logps/chosen": -89.89812469482422,
      "logps/rejected": -76.41596984863281,
      "loss": 0.6795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11752519756555557,
      "rewards/margins": 0.03747602924704552,
      "rewards/rejected": -0.15500125288963318,
      "step": 331
    },
    {
      "epoch": 0.07713306615554394,
      "grad_norm": 25.33626365661621,
      "learning_rate": 9.231126596980256e-06,
      "logits/chosen": 1.3925957679748535,
      "logits/rejected": 1.3064097166061401,
      "logps/chosen": -73.97454071044922,
      "logps/rejected": -76.52607727050781,
      "loss": 0.7519,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1375076323747635,
      "rewards/margins": -0.10937575995922089,
      "rewards/rejected": -0.028131861239671707,
      "step": 332
    },
    {
      "epoch": 0.07736539466806064,
      "grad_norm": 25.404359817504883,
      "learning_rate": 9.228803716608595e-06,
      "logits/chosen": 0.9882164001464844,
      "logits/rejected": 1.093759536743164,
      "logps/chosen": -87.29969024658203,
      "logps/rejected": -73.74002838134766,
      "loss": 0.6741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14535574615001678,
      "rewards/margins": 0.04290391132235527,
      "rewards/rejected": -0.18825964629650116,
      "step": 333
    },
    {
      "epoch": 0.07759772318057734,
      "grad_norm": 25.594070434570312,
      "learning_rate": 9.226480836236934e-06,
      "logits/chosen": 0.714486837387085,
      "logits/rejected": 0.7459316253662109,
      "logps/chosen": -90.81769561767578,
      "logps/rejected": -83.85246276855469,
      "loss": 0.696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16237039864063263,
      "rewards/margins": 0.00957460142672062,
      "rewards/rejected": -0.1719450056552887,
      "step": 334
    },
    {
      "epoch": 0.07783005169309404,
      "grad_norm": 23.641611099243164,
      "learning_rate": 9.224157955865274e-06,
      "logits/chosen": 1.343949794769287,
      "logits/rejected": 1.346126675605774,
      "logps/chosen": -94.23970794677734,
      "logps/rejected": -81.00080108642578,
      "loss": 0.6443,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1869789958000183,
      "rewards/margins": 0.11275763809680939,
      "rewards/rejected": -0.2997366189956665,
      "step": 335
    },
    {
      "epoch": 0.07806238020561074,
      "grad_norm": 25.009963989257812,
      "learning_rate": 9.221835075493613e-06,
      "logits/chosen": 1.5082629919052124,
      "logits/rejected": 1.6222928762435913,
      "logps/chosen": -81.4775390625,
      "logps/rejected": -81.56953430175781,
      "loss": 0.7009,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.19178377091884613,
      "rewards/margins": 0.00721721351146698,
      "rewards/rejected": -0.1990009844303131,
      "step": 336
    },
    {
      "epoch": 0.07829470871812744,
      "grad_norm": 24.998741149902344,
      "learning_rate": 9.219512195121952e-06,
      "logits/chosen": 0.8087542057037354,
      "logits/rejected": 0.7296180725097656,
      "logps/chosen": -84.93109130859375,
      "logps/rejected": -85.48914337158203,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15086017549037933,
      "rewards/margins": 0.005323648452758789,
      "rewards/rejected": -0.15618382394313812,
      "step": 337
    },
    {
      "epoch": 0.07852703723064414,
      "grad_norm": 21.045337677001953,
      "learning_rate": 9.217189314750291e-06,
      "logits/chosen": 1.0319147109985352,
      "logits/rejected": 0.8930114507675171,
      "logps/chosen": -75.32908630371094,
      "logps/rejected": -78.92794036865234,
      "loss": 0.5994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06158333644270897,
      "rewards/margins": 0.21243071556091309,
      "rewards/rejected": -0.27401402592658997,
      "step": 338
    },
    {
      "epoch": 0.07875936574316084,
      "grad_norm": 30.822547912597656,
      "learning_rate": 9.21486643437863e-06,
      "logits/chosen": 0.8441309928894043,
      "logits/rejected": 0.7347480654716492,
      "logps/chosen": -93.70718383789062,
      "logps/rejected": -92.27755737304688,
      "loss": 0.7259,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21869806945323944,
      "rewards/margins": -0.05553467199206352,
      "rewards/rejected": -0.16316337883472443,
      "step": 339
    },
    {
      "epoch": 0.07899169425567752,
      "grad_norm": 22.07221031188965,
      "learning_rate": 9.212543554006969e-06,
      "logits/chosen": 1.2798500061035156,
      "logits/rejected": 1.2503060102462769,
      "logps/chosen": -86.81575775146484,
      "logps/rejected": -79.360595703125,
      "loss": 0.592,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.023009397089481354,
      "rewards/margins": 0.2802523374557495,
      "rewards/rejected": -0.30326175689697266,
      "step": 340
    },
    {
      "epoch": 0.07922402276819422,
      "grad_norm": 21.00403594970703,
      "learning_rate": 9.210220673635308e-06,
      "logits/chosen": 0.7531572580337524,
      "logits/rejected": 0.7538022994995117,
      "logps/chosen": -71.47428894042969,
      "logps/rejected": -71.26575469970703,
      "loss": 0.6381,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05272434651851654,
      "rewards/margins": 0.14101256430149078,
      "rewards/rejected": -0.19373689591884613,
      "step": 341
    },
    {
      "epoch": 0.07945635128071092,
      "grad_norm": 23.17769432067871,
      "learning_rate": 9.207897793263649e-06,
      "logits/chosen": 0.868344247341156,
      "logits/rejected": 0.7333764433860779,
      "logps/chosen": -80.5214614868164,
      "logps/rejected": -79.45973205566406,
      "loss": 0.6245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07023134082555771,
      "rewards/margins": 0.17066656053066254,
      "rewards/rejected": -0.24089790880680084,
      "step": 342
    },
    {
      "epoch": 0.07968867979322762,
      "grad_norm": 33.533241271972656,
      "learning_rate": 9.205574912891988e-06,
      "logits/chosen": 1.0124915838241577,
      "logits/rejected": 0.9083446860313416,
      "logps/chosen": -92.12533569335938,
      "logps/rejected": -77.50403594970703,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08549842983484268,
      "rewards/margins": 0.08139531314373016,
      "rewards/rejected": -0.16689375042915344,
      "step": 343
    },
    {
      "epoch": 0.07992100830574432,
      "grad_norm": 22.73744773864746,
      "learning_rate": 9.203252032520325e-06,
      "logits/chosen": 1.7019327878952026,
      "logits/rejected": 1.6935453414916992,
      "logps/chosen": -63.47686004638672,
      "logps/rejected": -79.80366516113281,
      "loss": 0.6579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05206676945090294,
      "rewards/margins": 0.09087909013032913,
      "rewards/rejected": -0.14294587075710297,
      "step": 344
    },
    {
      "epoch": 0.08015333681826102,
      "grad_norm": 24.285964965820312,
      "learning_rate": 9.200929152148665e-06,
      "logits/chosen": 0.9220642447471619,
      "logits/rejected": 0.9577821493148804,
      "logps/chosen": -77.48619079589844,
      "logps/rejected": -86.66097259521484,
      "loss": 0.672,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13767622411251068,
      "rewards/margins": 0.11429288238286972,
      "rewards/rejected": -0.2519690990447998,
      "step": 345
    },
    {
      "epoch": 0.08038566533077772,
      "grad_norm": 23.049110412597656,
      "learning_rate": 9.198606271777004e-06,
      "logits/chosen": 1.3664324283599854,
      "logits/rejected": 1.3146847486495972,
      "logps/chosen": -82.36265563964844,
      "logps/rejected": -108.24588012695312,
      "loss": 0.5635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04379095882177353,
      "rewards/margins": 0.8302556276321411,
      "rewards/rejected": -0.8740465641021729,
      "step": 346
    },
    {
      "epoch": 0.08061799384329442,
      "grad_norm": 25.7449893951416,
      "learning_rate": 9.196283391405343e-06,
      "logits/chosen": 0.7316696047782898,
      "logits/rejected": 0.7972347736358643,
      "logps/chosen": -83.14390563964844,
      "logps/rejected": -93.69467163085938,
      "loss": 0.6951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13896310329437256,
      "rewards/margins": 0.015620730817317963,
      "rewards/rejected": -0.15458384156227112,
      "step": 347
    },
    {
      "epoch": 0.08085032235581112,
      "grad_norm": 27.491497039794922,
      "learning_rate": 9.193960511033682e-06,
      "logits/chosen": 1.4764257669448853,
      "logits/rejected": 1.2830020189285278,
      "logps/chosen": -88.11988830566406,
      "logps/rejected": -82.07515716552734,
      "loss": 0.7311,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1314089596271515,
      "rewards/margins": 0.007508949376642704,
      "rewards/rejected": -0.13891789317131042,
      "step": 348
    },
    {
      "epoch": 0.08108265086832782,
      "grad_norm": 20.718080520629883,
      "learning_rate": 9.191637630662021e-06,
      "logits/chosen": 1.1370083093643188,
      "logits/rejected": 1.040063738822937,
      "logps/chosen": -80.5224609375,
      "logps/rejected": -85.03816986083984,
      "loss": 0.5145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0248035229742527,
      "rewards/margins": 0.470783531665802,
      "rewards/rejected": -0.445980042219162,
      "step": 349
    },
    {
      "epoch": 0.08131497938084452,
      "grad_norm": 24.557119369506836,
      "learning_rate": 9.189314750290362e-06,
      "logits/chosen": 1.2348971366882324,
      "logits/rejected": 1.3918719291687012,
      "logps/chosen": -89.24726867675781,
      "logps/rejected": -83.89376831054688,
      "loss": 0.6861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1299123764038086,
      "rewards/margins": 0.04146895557641983,
      "rewards/rejected": -0.17138132452964783,
      "step": 350
    },
    {
      "epoch": 0.08154730789336122,
      "grad_norm": 25.194650650024414,
      "learning_rate": 9.1869918699187e-06,
      "logits/chosen": 1.3063033819198608,
      "logits/rejected": 1.256124496459961,
      "logps/chosen": -87.17273712158203,
      "logps/rejected": -111.54798889160156,
      "loss": 0.6527,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1976584792137146,
      "rewards/margins": 0.09784761071205139,
      "rewards/rejected": -0.295506089925766,
      "step": 351
    },
    {
      "epoch": 0.08177963640587792,
      "grad_norm": 24.483449935913086,
      "learning_rate": 9.184668989547038e-06,
      "logits/chosen": 0.6940838694572449,
      "logits/rejected": 0.848034679889679,
      "logps/chosen": -90.76899719238281,
      "logps/rejected": -80.60305786132812,
      "loss": 0.7,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.11418790370225906,
      "rewards/margins": 8.477456867694855e-05,
      "rewards/rejected": -0.11427266895771027,
      "step": 352
    },
    {
      "epoch": 0.0820119649183946,
      "grad_norm": 23.13422393798828,
      "learning_rate": 9.182346109175379e-06,
      "logits/chosen": 1.5637511014938354,
      "logits/rejected": 1.5815455913543701,
      "logps/chosen": -73.15337371826172,
      "logps/rejected": -91.04209899902344,
      "loss": 0.6021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.00857076421380043,
      "rewards/margins": 0.2272137999534607,
      "rewards/rejected": -0.21864300966262817,
      "step": 353
    },
    {
      "epoch": 0.0822442934309113,
      "grad_norm": 22.12407684326172,
      "learning_rate": 9.180023228803718e-06,
      "logits/chosen": 1.5402857065200806,
      "logits/rejected": 1.4737921953201294,
      "logps/chosen": -89.73516082763672,
      "logps/rejected": -95.72225189208984,
      "loss": 0.555,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0751676857471466,
      "rewards/margins": 0.38805755972862244,
      "rewards/rejected": -0.31288987398147583,
      "step": 354
    },
    {
      "epoch": 0.082476621943428,
      "grad_norm": 29.11765480041504,
      "learning_rate": 9.177700348432056e-06,
      "logits/chosen": 0.8288080096244812,
      "logits/rejected": 0.8778133988380432,
      "logps/chosen": -81.7333984375,
      "logps/rejected": -83.34222412109375,
      "loss": 0.5929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012845715507864952,
      "rewards/margins": 0.3253064751625061,
      "rewards/rejected": -0.3124607801437378,
      "step": 355
    },
    {
      "epoch": 0.0827089504559447,
      "grad_norm": 23.637075424194336,
      "learning_rate": 9.175377468060395e-06,
      "logits/chosen": 1.4349733591079712,
      "logits/rejected": 1.3955727815628052,
      "logps/chosen": -83.85388946533203,
      "logps/rejected": -74.30512237548828,
      "loss": 0.6659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06949999928474426,
      "rewards/margins": 0.05932219326496124,
      "rewards/rejected": -0.1288221925497055,
      "step": 356
    },
    {
      "epoch": 0.0829412789684614,
      "grad_norm": 23.515932083129883,
      "learning_rate": 9.173054587688734e-06,
      "logits/chosen": 0.796911895275116,
      "logits/rejected": 0.7410625219345093,
      "logps/chosen": -78.0911865234375,
      "logps/rejected": -78.56407165527344,
      "loss": 0.6751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1425042748451233,
      "rewards/margins": 0.05945930629968643,
      "rewards/rejected": -0.20196357369422913,
      "step": 357
    },
    {
      "epoch": 0.0831736074809781,
      "grad_norm": 22.49261474609375,
      "learning_rate": 9.170731707317075e-06,
      "logits/chosen": 0.7105243802070618,
      "logits/rejected": 0.5448044538497925,
      "logps/chosen": -80.62254333496094,
      "logps/rejected": -82.25557708740234,
      "loss": 0.6499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02967982366681099,
      "rewards/margins": 0.09724496304988861,
      "rewards/rejected": -0.1269247829914093,
      "step": 358
    },
    {
      "epoch": 0.0834059359934948,
      "grad_norm": 22.939538955688477,
      "learning_rate": 9.168408826945412e-06,
      "logits/chosen": 0.8418602347373962,
      "logits/rejected": 0.956823468208313,
      "logps/chosen": -85.26470947265625,
      "logps/rejected": -90.98609161376953,
      "loss": 0.5666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07983341068029404,
      "rewards/margins": 0.40673258900642395,
      "rewards/rejected": -0.3268992006778717,
      "step": 359
    },
    {
      "epoch": 0.0836382645060115,
      "grad_norm": 22.896167755126953,
      "learning_rate": 9.166085946573751e-06,
      "logits/chosen": 0.8614571690559387,
      "logits/rejected": 0.8850183486938477,
      "logps/chosen": -73.60377502441406,
      "logps/rejected": -94.03204345703125,
      "loss": 0.5974,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04927743226289749,
      "rewards/margins": 0.31046104431152344,
      "rewards/rejected": -0.3597384989261627,
      "step": 360
    },
    {
      "epoch": 0.0838705930185282,
      "grad_norm": 23.43048858642578,
      "learning_rate": 9.163763066202092e-06,
      "logits/chosen": 1.4657255411148071,
      "logits/rejected": 1.466817021369934,
      "logps/chosen": -72.63701629638672,
      "logps/rejected": -72.72714233398438,
      "loss": 0.6817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12407168745994568,
      "rewards/margins": 0.037010692059993744,
      "rewards/rejected": -0.1610824018716812,
      "step": 361
    },
    {
      "epoch": 0.0841029215310449,
      "grad_norm": 25.083850860595703,
      "learning_rate": 9.16144018583043e-06,
      "logits/chosen": 1.0579028129577637,
      "logits/rejected": 1.0358006954193115,
      "logps/chosen": -86.56989288330078,
      "logps/rejected": -94.35999298095703,
      "loss": 0.6503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03827672079205513,
      "rewards/margins": 0.09611516445875168,
      "rewards/rejected": -0.1343918740749359,
      "step": 362
    },
    {
      "epoch": 0.0843352500435616,
      "grad_norm": 25.846391677856445,
      "learning_rate": 9.15911730545877e-06,
      "logits/chosen": 1.5392184257507324,
      "logits/rejected": 1.3099111318588257,
      "logps/chosen": -81.7636947631836,
      "logps/rejected": -103.9048843383789,
      "loss": 0.7124,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11200332641601562,
      "rewards/margins": -0.016687586903572083,
      "rewards/rejected": -0.09531573951244354,
      "step": 363
    },
    {
      "epoch": 0.0845675785560783,
      "grad_norm": 25.38069725036621,
      "learning_rate": 9.156794425087109e-06,
      "logits/chosen": 1.0315518379211426,
      "logits/rejected": 1.1361671686172485,
      "logps/chosen": -93.16449737548828,
      "logps/rejected": -81.81612396240234,
      "loss": 0.7131,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18432459235191345,
      "rewards/margins": -0.025710150599479675,
      "rewards/rejected": -0.15861445665359497,
      "step": 364
    },
    {
      "epoch": 0.084799907068595,
      "grad_norm": 24.681480407714844,
      "learning_rate": 9.154471544715448e-06,
      "logits/chosen": 0.588445782661438,
      "logits/rejected": 0.5477443337440491,
      "logps/chosen": -86.55918884277344,
      "logps/rejected": -88.55259704589844,
      "loss": 0.7098,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1174391359090805,
      "rewards/margins": -0.01673765480518341,
      "rewards/rejected": -0.1007014811038971,
      "step": 365
    },
    {
      "epoch": 0.0850322355811117,
      "grad_norm": 23.54907989501953,
      "learning_rate": 9.152148664343788e-06,
      "logits/chosen": 1.1376203298568726,
      "logits/rejected": 1.1474194526672363,
      "logps/chosen": -76.89922332763672,
      "logps/rejected": -90.36830139160156,
      "loss": 0.6914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10527437180280685,
      "rewards/margins": 0.009580949321389198,
      "rewards/rejected": -0.1148553118109703,
      "step": 366
    },
    {
      "epoch": 0.08526456409362838,
      "grad_norm": 23.155872344970703,
      "learning_rate": 9.149825783972125e-06,
      "logits/chosen": 1.0289857387542725,
      "logits/rejected": 1.157084345817566,
      "logps/chosen": -81.31612396240234,
      "logps/rejected": -83.60746765136719,
      "loss": 0.6692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07545123994350433,
      "rewards/margins": 0.0808638408780098,
      "rewards/rejected": -0.15631508827209473,
      "step": 367
    },
    {
      "epoch": 0.08549689260614508,
      "grad_norm": 24.601655960083008,
      "learning_rate": 9.147502903600464e-06,
      "logits/chosen": 0.7808128595352173,
      "logits/rejected": 1.1070458889007568,
      "logps/chosen": -88.61378479003906,
      "logps/rejected": -80.59239196777344,
      "loss": 0.6069,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1177954226732254,
      "rewards/margins": 0.20690372586250305,
      "rewards/rejected": -0.32469913363456726,
      "step": 368
    },
    {
      "epoch": 0.08572922111866178,
      "grad_norm": 22.305206298828125,
      "learning_rate": 9.145180023228805e-06,
      "logits/chosen": 0.9709725379943848,
      "logits/rejected": 1.1223764419555664,
      "logps/chosen": -86.783447265625,
      "logps/rejected": -79.00598907470703,
      "loss": 0.5891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008849810808897018,
      "rewards/margins": 0.2546623945236206,
      "rewards/rejected": -0.26351219415664673,
      "step": 369
    },
    {
      "epoch": 0.08596154963117848,
      "grad_norm": 25.17638397216797,
      "learning_rate": 9.142857142857144e-06,
      "logits/chosen": 1.1854796409606934,
      "logits/rejected": 1.2414103746414185,
      "logps/chosen": -95.34709930419922,
      "logps/rejected": -78.523193359375,
      "loss": 0.6328,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13722991943359375,
      "rewards/margins": 0.17326879501342773,
      "rewards/rejected": -0.3104987144470215,
      "step": 370
    },
    {
      "epoch": 0.08619387814369518,
      "grad_norm": 25.67017936706543,
      "learning_rate": 9.140534262485483e-06,
      "logits/chosen": 0.9683698415756226,
      "logits/rejected": 0.9913468956947327,
      "logps/chosen": -72.44815826416016,
      "logps/rejected": -70.89802551269531,
      "loss": 0.8019,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.34941917657852173,
      "rewards/margins": -0.19670382142066956,
      "rewards/rejected": -0.15271535515785217,
      "step": 371
    },
    {
      "epoch": 0.08642620665621188,
      "grad_norm": 25.20633316040039,
      "learning_rate": 9.138211382113822e-06,
      "logits/chosen": 0.8778648376464844,
      "logits/rejected": 0.7722903490066528,
      "logps/chosen": -89.97575378417969,
      "logps/rejected": -93.88008117675781,
      "loss": 0.6636,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13634160161018372,
      "rewards/margins": 0.08358819782733917,
      "rewards/rejected": -0.2199297994375229,
      "step": 372
    },
    {
      "epoch": 0.08665853516872858,
      "grad_norm": 22.392154693603516,
      "learning_rate": 9.13588850174216e-06,
      "logits/chosen": 0.9561006426811218,
      "logits/rejected": 0.8600620627403259,
      "logps/chosen": -79.4534912109375,
      "logps/rejected": -80.82028198242188,
      "loss": 0.6664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09400897473096848,
      "rewards/margins": 0.06700766831636429,
      "rewards/rejected": -0.16101662814617157,
      "step": 373
    },
    {
      "epoch": 0.08689086368124528,
      "grad_norm": 23.152143478393555,
      "learning_rate": 9.1335656213705e-06,
      "logits/chosen": 0.7509196996688843,
      "logits/rejected": 0.7212533950805664,
      "logps/chosen": -84.93317413330078,
      "logps/rejected": -103.0205307006836,
      "loss": 0.5911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.015868734568357468,
      "rewards/margins": 0.2248777598142624,
      "rewards/rejected": -0.20900903642177582,
      "step": 374
    },
    {
      "epoch": 0.08712319219376198,
      "grad_norm": 23.576499938964844,
      "learning_rate": 9.131242740998839e-06,
      "logits/chosen": 0.9847970604896545,
      "logits/rejected": 1.1493738889694214,
      "logps/chosen": -84.1676025390625,
      "logps/rejected": -75.72502136230469,
      "loss": 0.6328,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01361251249909401,
      "rewards/margins": 0.19805651903152466,
      "rewards/rejected": -0.18444401025772095,
      "step": 375
    },
    {
      "epoch": 0.08735552070627868,
      "grad_norm": 23.26738739013672,
      "learning_rate": 9.12891986062718e-06,
      "logits/chosen": 0.5683489441871643,
      "logits/rejected": 0.6023372411727905,
      "logps/chosen": -73.34465026855469,
      "logps/rejected": -78.27710723876953,
      "loss": 0.626,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11983440071344376,
      "rewards/margins": 0.18736852705478668,
      "rewards/rejected": -0.30720293521881104,
      "step": 376
    },
    {
      "epoch": 0.08758784921879538,
      "grad_norm": 22.82176971435547,
      "learning_rate": 9.126596980255518e-06,
      "logits/chosen": 1.7373617887496948,
      "logits/rejected": 1.6191447973251343,
      "logps/chosen": -83.5445327758789,
      "logps/rejected": -110.58882141113281,
      "loss": 0.5889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03360123559832573,
      "rewards/margins": 0.7286075353622437,
      "rewards/rejected": -0.7622087597846985,
      "step": 377
    },
    {
      "epoch": 0.08782017773131208,
      "grad_norm": 24.523517608642578,
      "learning_rate": 9.124274099883857e-06,
      "logits/chosen": 1.3957968950271606,
      "logits/rejected": 1.3838039636611938,
      "logps/chosen": -81.06316375732422,
      "logps/rejected": -83.24882507324219,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.034908726811409,
      "rewards/margins": 0.0641777440905571,
      "rewards/rejected": -0.09908647835254669,
      "step": 378
    },
    {
      "epoch": 0.08805250624382878,
      "grad_norm": 21.76144790649414,
      "learning_rate": 9.121951219512196e-06,
      "logits/chosen": 0.5413928031921387,
      "logits/rejected": 0.6104039549827576,
      "logps/chosen": -79.99520874023438,
      "logps/rejected": -76.67191314697266,
      "loss": 0.5911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.005558589473366737,
      "rewards/margins": 0.22739166021347046,
      "rewards/rejected": -0.22183306515216827,
      "step": 379
    },
    {
      "epoch": 0.08828483475634548,
      "grad_norm": 28.96879005432129,
      "learning_rate": 9.119628339140535e-06,
      "logits/chosen": 1.6200275421142578,
      "logits/rejected": 1.7641355991363525,
      "logps/chosen": -106.60198974609375,
      "logps/rejected": -86.66020965576172,
      "loss": 0.7641,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2233758568763733,
      "rewards/margins": -0.12778379023075104,
      "rewards/rejected": -0.09559206664562225,
      "step": 380
    },
    {
      "epoch": 0.08851716326886216,
      "grad_norm": 24.966747283935547,
      "learning_rate": 9.117305458768874e-06,
      "logits/chosen": 0.7904449701309204,
      "logits/rejected": 0.7978812456130981,
      "logps/chosen": -76.84949493408203,
      "logps/rejected": -77.2691421508789,
      "loss": 0.6796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008621646091341972,
      "rewards/margins": 0.0499718002974987,
      "rewards/rejected": -0.058593444526195526,
      "step": 381
    },
    {
      "epoch": 0.08874949178137886,
      "grad_norm": 23.604177474975586,
      "learning_rate": 9.114982578397213e-06,
      "logits/chosen": 1.008932113647461,
      "logits/rejected": 0.9315681457519531,
      "logps/chosen": -83.93445587158203,
      "logps/rejected": -81.33647918701172,
      "loss": 0.6622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0943465456366539,
      "rewards/margins": 0.09153173118829727,
      "rewards/rejected": -0.18587827682495117,
      "step": 382
    },
    {
      "epoch": 0.08898182029389556,
      "grad_norm": 20.523775100708008,
      "learning_rate": 9.112659698025552e-06,
      "logits/chosen": 1.1617926359176636,
      "logits/rejected": 0.968107283115387,
      "logps/chosen": -80.64462280273438,
      "logps/rejected": -96.47709655761719,
      "loss": 0.4873,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.049767255783081055,
      "rewards/margins": 0.7834725975990295,
      "rewards/rejected": -0.7337052822113037,
      "step": 383
    },
    {
      "epoch": 0.08921414880641226,
      "grad_norm": 23.30335235595703,
      "learning_rate": 9.110336817653892e-06,
      "logits/chosen": 0.5518136024475098,
      "logits/rejected": 0.7242226004600525,
      "logps/chosen": -80.15338134765625,
      "logps/rejected": -87.90069580078125,
      "loss": 0.6347,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06538422405719757,
      "rewards/margins": 0.16108056902885437,
      "rewards/rejected": -0.22646482288837433,
      "step": 384
    },
    {
      "epoch": 0.08944647731892896,
      "grad_norm": 26.238683700561523,
      "learning_rate": 9.108013937282231e-06,
      "logits/chosen": 0.8428895473480225,
      "logits/rejected": 0.7111777067184448,
      "logps/chosen": -87.63347625732422,
      "logps/rejected": -89.19244384765625,
      "loss": 0.7059,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.023263927549123764,
      "rewards/margins": -0.017779521644115448,
      "rewards/rejected": -0.005484410561621189,
      "step": 385
    },
    {
      "epoch": 0.08967880583144566,
      "grad_norm": 22.788938522338867,
      "learning_rate": 9.10569105691057e-06,
      "logits/chosen": 1.3420099020004272,
      "logits/rejected": 1.3251045942306519,
      "logps/chosen": -76.03103637695312,
      "logps/rejected": -82.45321655273438,
      "loss": 0.6516,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.045308470726013184,
      "rewards/margins": 0.11879197508096695,
      "rewards/rejected": -0.16410046815872192,
      "step": 386
    },
    {
      "epoch": 0.08991113434396236,
      "grad_norm": 24.457366943359375,
      "learning_rate": 9.10336817653891e-06,
      "logits/chosen": 0.7011051177978516,
      "logits/rejected": 0.6999257802963257,
      "logps/chosen": -89.3139877319336,
      "logps/rejected": -88.21258544921875,
      "loss": 0.6786,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16762472689151764,
      "rewards/margins": 0.047376010566949844,
      "rewards/rejected": -0.215000718832016,
      "step": 387
    },
    {
      "epoch": 0.09014346285647906,
      "grad_norm": 22.95453453063965,
      "learning_rate": 9.101045296167248e-06,
      "logits/chosen": 0.7131365537643433,
      "logits/rejected": 0.6773615479469299,
      "logps/chosen": -91.0912857055664,
      "logps/rejected": -81.5689926147461,
      "loss": 0.6817,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.029850997030735016,
      "rewards/margins": 0.09390395879745483,
      "rewards/rejected": -0.12375496327877045,
      "step": 388
    },
    {
      "epoch": 0.09037579136899576,
      "grad_norm": 28.75022315979004,
      "learning_rate": 9.098722415795587e-06,
      "logits/chosen": 0.644774317741394,
      "logits/rejected": 0.7383732199668884,
      "logps/chosen": -97.2005844116211,
      "logps/rejected": -100.51121520996094,
      "loss": 0.738,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.19144582748413086,
      "rewards/margins": -0.07760784775018692,
      "rewards/rejected": -0.11383795738220215,
      "step": 389
    },
    {
      "epoch": 0.09060811988151246,
      "grad_norm": 27.029415130615234,
      "learning_rate": 9.096399535423926e-06,
      "logits/chosen": 1.5450034141540527,
      "logits/rejected": 1.5832834243774414,
      "logps/chosen": -105.3864974975586,
      "logps/rejected": -73.60846710205078,
      "loss": 0.7253,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1393902599811554,
      "rewards/margins": -0.04120950400829315,
      "rewards/rejected": -0.09818076342344284,
      "step": 390
    },
    {
      "epoch": 0.09084044839402916,
      "grad_norm": 23.207162857055664,
      "learning_rate": 9.094076655052265e-06,
      "logits/chosen": 0.8844873309135437,
      "logits/rejected": 0.8608018159866333,
      "logps/chosen": -71.59603881835938,
      "logps/rejected": -72.41341400146484,
      "loss": 0.643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13393189013004303,
      "rewards/margins": 0.11271005868911743,
      "rewards/rejected": -0.24664196372032166,
      "step": 391
    },
    {
      "epoch": 0.09107277690654586,
      "grad_norm": 22.685291290283203,
      "learning_rate": 9.091753774680606e-06,
      "logits/chosen": 0.906502366065979,
      "logits/rejected": 0.8742744326591492,
      "logps/chosen": -86.26383972167969,
      "logps/rejected": -77.66748046875,
      "loss": 0.5747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07189121842384338,
      "rewards/margins": 0.2914356589317322,
      "rewards/rejected": -0.21954447031021118,
      "step": 392
    },
    {
      "epoch": 0.09130510541906256,
      "grad_norm": 19.368818283081055,
      "learning_rate": 9.089430894308944e-06,
      "logits/chosen": 0.9409604072570801,
      "logits/rejected": 0.7815626263618469,
      "logps/chosen": -65.41036987304688,
      "logps/rejected": -83.8283920288086,
      "loss": 0.5703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06856724619865417,
      "rewards/margins": 0.3052413761615753,
      "rewards/rejected": -0.23667412996292114,
      "step": 393
    },
    {
      "epoch": 0.09153743393157926,
      "grad_norm": 22.23870277404785,
      "learning_rate": 9.087108013937282e-06,
      "logits/chosen": 0.6475763320922852,
      "logits/rejected": 0.6582375764846802,
      "logps/chosen": -72.9049072265625,
      "logps/rejected": -80.7071304321289,
      "loss": 0.6414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.015759238973259926,
      "rewards/margins": 0.1301044076681137,
      "rewards/rejected": -0.1458636373281479,
      "step": 394
    },
    {
      "epoch": 0.09176976244409595,
      "grad_norm": 24.827489852905273,
      "learning_rate": 9.084785133565622e-06,
      "logits/chosen": 0.4550788998603821,
      "logits/rejected": 0.5528865456581116,
      "logps/chosen": -78.17337799072266,
      "logps/rejected": -76.87850952148438,
      "loss": 0.7099,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.046370625495910645,
      "rewards/margins": -0.02105093188583851,
      "rewards/rejected": -0.025319695472717285,
      "step": 395
    },
    {
      "epoch": 0.09200209095661264,
      "grad_norm": 24.06397819519043,
      "learning_rate": 9.082462253193961e-06,
      "logits/chosen": 1.0390465259552002,
      "logits/rejected": 0.8550112843513489,
      "logps/chosen": -75.00308227539062,
      "logps/rejected": -95.52991485595703,
      "loss": 0.6502,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00985858216881752,
      "rewards/margins": 0.09085527062416077,
      "rewards/rejected": -0.08099668473005295,
      "step": 396
    },
    {
      "epoch": 0.09223441946912934,
      "grad_norm": 22.663930892944336,
      "learning_rate": 9.0801393728223e-06,
      "logits/chosen": 0.5355371236801147,
      "logits/rejected": 0.379521906375885,
      "logps/chosen": -78.75654602050781,
      "logps/rejected": -77.67450714111328,
      "loss": 0.6873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06215638294816017,
      "rewards/margins": 0.032582689076662064,
      "rewards/rejected": -0.09473907947540283,
      "step": 397
    },
    {
      "epoch": 0.09246674798164604,
      "grad_norm": 23.762178421020508,
      "learning_rate": 9.077816492450639e-06,
      "logits/chosen": 0.956570029258728,
      "logits/rejected": 0.9876409769058228,
      "logps/chosen": -86.01666259765625,
      "logps/rejected": -86.14423370361328,
      "loss": 0.564,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05628780275583267,
      "rewards/margins": 0.3463045060634613,
      "rewards/rejected": -0.29001665115356445,
      "step": 398
    },
    {
      "epoch": 0.09269907649416274,
      "grad_norm": 26.35443878173828,
      "learning_rate": 9.075493612078978e-06,
      "logits/chosen": 0.8780614733695984,
      "logits/rejected": 1.0021241903305054,
      "logps/chosen": -81.79537963867188,
      "logps/rejected": -86.7802734375,
      "loss": 0.737,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10340923070907593,
      "rewards/margins": -0.07686442136764526,
      "rewards/rejected": -0.026544811204075813,
      "step": 399
    },
    {
      "epoch": 0.09293140500667944,
      "grad_norm": 21.022974014282227,
      "learning_rate": 9.073170731707319e-06,
      "logits/chosen": 0.4029897451400757,
      "logits/rejected": 0.515538215637207,
      "logps/chosen": -77.41757202148438,
      "logps/rejected": -84.92047882080078,
      "loss": 0.5703,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08867838978767395,
      "rewards/margins": 0.30389752984046936,
      "rewards/rejected": -0.21521911025047302,
      "step": 400
    },
    {
      "epoch": 0.09316373351919614,
      "grad_norm": 21.98253059387207,
      "learning_rate": 9.070847851335658e-06,
      "logits/chosen": 0.700872540473938,
      "logits/rejected": 0.5802664756774902,
      "logps/chosen": -73.3678970336914,
      "logps/rejected": -113.0687255859375,
      "loss": 0.5885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07632983475923538,
      "rewards/margins": 0.6287835240364075,
      "rewards/rejected": -0.705113410949707,
      "step": 401
    },
    {
      "epoch": 0.09339606203171284,
      "grad_norm": 21.68434715270996,
      "learning_rate": 9.068524970963997e-06,
      "logits/chosen": 0.46697452664375305,
      "logits/rejected": 0.3794805705547333,
      "logps/chosen": -84.19971466064453,
      "logps/rejected": -91.1365966796875,
      "loss": 0.5588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.035828329622745514,
      "rewards/margins": 0.35554638504981995,
      "rewards/rejected": -0.3913746774196625,
      "step": 402
    },
    {
      "epoch": 0.09362839054422954,
      "grad_norm": 25.508441925048828,
      "learning_rate": 9.066202090592336e-06,
      "logits/chosen": 0.5020421147346497,
      "logits/rejected": 0.23597289621829987,
      "logps/chosen": -97.66352081298828,
      "logps/rejected": -81.15078735351562,
      "loss": 0.7046,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13976553082466125,
      "rewards/margins": 0.004269139841198921,
      "rewards/rejected": -0.14403466880321503,
      "step": 403
    },
    {
      "epoch": 0.09386071905674624,
      "grad_norm": 21.079816818237305,
      "learning_rate": 9.063879210220674e-06,
      "logits/chosen": 0.29623278975486755,
      "logits/rejected": 0.2996211051940918,
      "logps/chosen": -79.28385162353516,
      "logps/rejected": -78.96895599365234,
      "loss": 0.617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0661291852593422,
      "rewards/margins": 0.20307877659797668,
      "rewards/rejected": -0.2692079544067383,
      "step": 404
    },
    {
      "epoch": 0.09409304756926294,
      "grad_norm": 22.114013671875,
      "learning_rate": 9.061556329849013e-06,
      "logits/chosen": 0.4109417796134949,
      "logits/rejected": 0.3279249668121338,
      "logps/chosen": -74.74566650390625,
      "logps/rejected": -102.30718231201172,
      "loss": 0.552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02543507143855095,
      "rewards/margins": 0.825104832649231,
      "rewards/rejected": -0.7996697425842285,
      "step": 405
    },
    {
      "epoch": 0.09432537608177964,
      "grad_norm": 23.527727127075195,
      "learning_rate": 9.059233449477352e-06,
      "logits/chosen": 0.6546632647514343,
      "logits/rejected": 0.5120397806167603,
      "logps/chosen": -84.14280700683594,
      "logps/rejected": -81.28471374511719,
      "loss": 0.6888,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.08237385004758835,
      "rewards/margins": 0.017003679648041725,
      "rewards/rejected": -0.09937753528356552,
      "step": 406
    },
    {
      "epoch": 0.09455770459429634,
      "grad_norm": 19.46431541442871,
      "learning_rate": 9.056910569105691e-06,
      "logits/chosen": 0.5396019220352173,
      "logits/rejected": 0.45330509543418884,
      "logps/chosen": -77.28878784179688,
      "logps/rejected": -67.20320129394531,
      "loss": 0.5964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.025986455380916595,
      "rewards/margins": 0.2376820296049118,
      "rewards/rejected": -0.2116955667734146,
      "step": 407
    },
    {
      "epoch": 0.09479003310681304,
      "grad_norm": 20.442258834838867,
      "learning_rate": 9.054587688734032e-06,
      "logits/chosen": 0.3654536008834839,
      "logits/rejected": 0.3862881660461426,
      "logps/chosen": -67.35665130615234,
      "logps/rejected": -84.23423767089844,
      "loss": 0.6122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08975698053836823,
      "rewards/margins": 0.17811496555805206,
      "rewards/rejected": -0.2678719758987427,
      "step": 408
    },
    {
      "epoch": 0.09502236161932973,
      "grad_norm": 24.028772354125977,
      "learning_rate": 9.052264808362369e-06,
      "logits/chosen": 0.6663145422935486,
      "logits/rejected": 0.6297626495361328,
      "logps/chosen": -74.07549285888672,
      "logps/rejected": -90.25786590576172,
      "loss": 0.7375,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05208501219749451,
      "rewards/margins": -0.07433679699897766,
      "rewards/rejected": 0.0222517941147089,
      "step": 409
    },
    {
      "epoch": 0.09525469013184643,
      "grad_norm": 21.55181312561035,
      "learning_rate": 9.04994192799071e-06,
      "logits/chosen": 0.3614256680011749,
      "logits/rejected": 0.5570819973945618,
      "logps/chosen": -73.06782531738281,
      "logps/rejected": -86.47944641113281,
      "loss": 0.5964,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0035950196906924248,
      "rewards/margins": 0.24386370182037354,
      "rewards/rejected": -0.24026867747306824,
      "step": 410
    },
    {
      "epoch": 0.09548701864436313,
      "grad_norm": 21.7080078125,
      "learning_rate": 9.047619047619049e-06,
      "logits/chosen": 1.4921290874481201,
      "logits/rejected": 1.3973722457885742,
      "logps/chosen": -78.63534545898438,
      "logps/rejected": -74.3246841430664,
      "loss": 0.6314,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007103629410266876,
      "rewards/margins": 0.1537182629108429,
      "rewards/rejected": -0.16082186996936798,
      "step": 411
    },
    {
      "epoch": 0.09571934715687982,
      "grad_norm": 25.135366439819336,
      "learning_rate": 9.045296167247388e-06,
      "logits/chosen": 0.6148177981376648,
      "logits/rejected": 0.5719340443611145,
      "logps/chosen": -80.38299560546875,
      "logps/rejected": -81.18505096435547,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16937881708145142,
      "rewards/margins": -8.416082710027695e-05,
      "rewards/rejected": -0.16929465532302856,
      "step": 412
    },
    {
      "epoch": 0.09595167566939652,
      "grad_norm": 24.73611068725586,
      "learning_rate": 9.042973286875727e-06,
      "logits/chosen": 0.4129054546356201,
      "logits/rejected": 0.4165084958076477,
      "logps/chosen": -90.83561706542969,
      "logps/rejected": -73.88314056396484,
      "loss": 0.7106,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.12113608419895172,
      "rewards/margins": 0.0018896311521530151,
      "rewards/rejected": -0.12302572280168533,
      "step": 413
    },
    {
      "epoch": 0.09618400418191322,
      "grad_norm": 19.360340118408203,
      "learning_rate": 9.040650406504065e-06,
      "logits/chosen": 0.14024464786052704,
      "logits/rejected": 0.10872385650873184,
      "logps/chosen": -79.58175659179688,
      "logps/rejected": -72.0007095336914,
      "loss": 0.4885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16557136178016663,
      "rewards/margins": 0.521754801273346,
      "rewards/rejected": -0.3561834692955017,
      "step": 414
    },
    {
      "epoch": 0.09641633269442992,
      "grad_norm": 24.1959285736084,
      "learning_rate": 9.038327526132404e-06,
      "logits/chosen": 0.32480308413505554,
      "logits/rejected": 0.44510677456855774,
      "logps/chosen": -83.05706787109375,
      "logps/rejected": -73.74181365966797,
      "loss": 0.729,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009434277191758156,
      "rewards/margins": -0.0306788869202137,
      "rewards/rejected": 0.021244609728455544,
      "step": 415
    },
    {
      "epoch": 0.09664866120694662,
      "grad_norm": 25.20804214477539,
      "learning_rate": 9.036004645760745e-06,
      "logits/chosen": 0.003758717328310013,
      "logits/rejected": 0.20012950897216797,
      "logps/chosen": -85.85738372802734,
      "logps/rejected": -99.28764343261719,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09643711894750595,
      "rewards/margins": 0.04324159398674965,
      "rewards/rejected": -0.1396787166595459,
      "step": 416
    },
    {
      "epoch": 0.09688098971946332,
      "grad_norm": 21.932811737060547,
      "learning_rate": 9.033681765389082e-06,
      "logits/chosen": 0.14227330684661865,
      "logits/rejected": 0.21780811250209808,
      "logps/chosen": -84.56053161621094,
      "logps/rejected": -73.9457778930664,
      "loss": 0.6367,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.042675621807575226,
      "rewards/margins": 0.14528906345367432,
      "rewards/rejected": -0.18796469271183014,
      "step": 417
    },
    {
      "epoch": 0.09711331823198002,
      "grad_norm": 21.1175537109375,
      "learning_rate": 9.031358885017423e-06,
      "logits/chosen": 0.21144762635231018,
      "logits/rejected": 0.21697843074798584,
      "logps/chosen": -68.06011199951172,
      "logps/rejected": -68.51927947998047,
      "loss": 0.696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.007537852972745895,
      "rewards/margins": -0.00038982927799224854,
      "rewards/rejected": -0.00714802835136652,
      "step": 418
    },
    {
      "epoch": 0.09734564674449672,
      "grad_norm": 28.9465274810791,
      "learning_rate": 9.029036004645762e-06,
      "logits/chosen": 0.8147897124290466,
      "logits/rejected": 0.6995477676391602,
      "logps/chosen": -98.36351013183594,
      "logps/rejected": -102.37246704101562,
      "loss": 0.6344,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.13765554130077362,
      "rewards/margins": 0.22375930845737457,
      "rewards/rejected": -0.08610372245311737,
      "step": 419
    },
    {
      "epoch": 0.09757797525701342,
      "grad_norm": 20.87148666381836,
      "learning_rate": 9.0267131242741e-06,
      "logits/chosen": 0.7458413243293762,
      "logits/rejected": 0.7669036984443665,
      "logps/chosen": -80.78546142578125,
      "logps/rejected": -81.72825622558594,
      "loss": 0.5852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.038478970527648926,
      "rewards/margins": 0.3157610595226288,
      "rewards/rejected": -0.27728211879730225,
      "step": 420
    },
    {
      "epoch": 0.09781030376953012,
      "grad_norm": 22.68894386291504,
      "learning_rate": 9.02439024390244e-06,
      "logits/chosen": 0.3661412000656128,
      "logits/rejected": 0.2835877537727356,
      "logps/chosen": -75.59766387939453,
      "logps/rejected": -74.19371795654297,
      "loss": 0.675,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12281010299921036,
      "rewards/margins": 0.06590139865875244,
      "rewards/rejected": -0.1887115091085434,
      "step": 421
    },
    {
      "epoch": 0.09804263228204682,
      "grad_norm": 20.832704544067383,
      "learning_rate": 9.022067363530779e-06,
      "logits/chosen": 0.20455920696258545,
      "logits/rejected": 0.2471800148487091,
      "logps/chosen": -84.67782592773438,
      "logps/rejected": -89.74494934082031,
      "loss": 0.582,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.020766684785485268,
      "rewards/margins": 0.24901293218135834,
      "rewards/rejected": -0.26977965235710144,
      "step": 422
    },
    {
      "epoch": 0.0982749607945635,
      "grad_norm": 24.16189956665039,
      "learning_rate": 9.01974448315912e-06,
      "logits/chosen": 0.2238539755344391,
      "logits/rejected": 0.32216283679008484,
      "logps/chosen": -84.93081665039062,
      "logps/rejected": -87.4692611694336,
      "loss": 0.6787,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1355258822441101,
      "rewards/margins": 0.07720909267663956,
      "rewards/rejected": -0.21273498237133026,
      "step": 423
    },
    {
      "epoch": 0.0985072893070802,
      "grad_norm": 26.893415451049805,
      "learning_rate": 9.017421602787457e-06,
      "logits/chosen": 0.5631946325302124,
      "logits/rejected": 0.7779384255409241,
      "logps/chosen": -101.73206329345703,
      "logps/rejected": -96.21749877929688,
      "loss": 0.7027,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16751542687416077,
      "rewards/margins": 0.011951606720685959,
      "rewards/rejected": -0.17946703732013702,
      "step": 424
    },
    {
      "epoch": 0.0987396178195969,
      "grad_norm": 21.430803298950195,
      "learning_rate": 9.015098722415795e-06,
      "logits/chosen": 0.19268307089805603,
      "logits/rejected": 0.11991360038518906,
      "logps/chosen": -74.10458374023438,
      "logps/rejected": -86.59735870361328,
      "loss": 0.6459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04782268777489662,
      "rewards/margins": 0.11052921414375305,
      "rewards/rejected": -0.15835189819335938,
      "step": 425
    },
    {
      "epoch": 0.0989719463321136,
      "grad_norm": 25.878862380981445,
      "learning_rate": 9.012775842044136e-06,
      "logits/chosen": 0.370392769575119,
      "logits/rejected": 0.4388635754585266,
      "logps/chosen": -89.91728210449219,
      "logps/rejected": -83.10173797607422,
      "loss": 0.7289,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15239973366260529,
      "rewards/margins": -0.03171996399760246,
      "rewards/rejected": -0.12067975848913193,
      "step": 426
    },
    {
      "epoch": 0.0992042748446303,
      "grad_norm": 27.434040069580078,
      "learning_rate": 9.010452961672475e-06,
      "logits/chosen": 0.5210927724838257,
      "logits/rejected": 0.6092600226402283,
      "logps/chosen": -86.28050231933594,
      "logps/rejected": -86.20215606689453,
      "loss": 0.7394,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.031078197062015533,
      "rewards/margins": -0.05945464223623276,
      "rewards/rejected": 0.028376435860991478,
      "step": 427
    },
    {
      "epoch": 0.099436603357147,
      "grad_norm": 24.4539852142334,
      "learning_rate": 9.008130081300814e-06,
      "logits/chosen": 0.3506913483142853,
      "logits/rejected": 0.4093914031982422,
      "logps/chosen": -88.28239440917969,
      "logps/rejected": -85.28546142578125,
      "loss": 0.6796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11119413375854492,
      "rewards/margins": 0.04476914927363396,
      "rewards/rejected": 0.06642498821020126,
      "step": 428
    },
    {
      "epoch": 0.0996689318696637,
      "grad_norm": 23.50481414794922,
      "learning_rate": 9.005807200929153e-06,
      "logits/chosen": 0.756089448928833,
      "logits/rejected": 0.5684553980827332,
      "logps/chosen": -80.02229309082031,
      "logps/rejected": -68.2041015625,
      "loss": 0.6748,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04137308895587921,
      "rewards/margins": 0.04435471072793007,
      "rewards/rejected": -0.08572779595851898,
      "step": 429
    },
    {
      "epoch": 0.0999012603821804,
      "grad_norm": 23.13878059387207,
      "learning_rate": 9.003484320557492e-06,
      "logits/chosen": 0.4907287359237671,
      "logits/rejected": 0.5218780636787415,
      "logps/chosen": -77.19733428955078,
      "logps/rejected": -71.67033386230469,
      "loss": 0.6603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01562795601785183,
      "rewards/margins": 0.09658251702785492,
      "rewards/rejected": -0.08095455914735794,
      "step": 430
    },
    {
      "epoch": 0.1001335888946971,
      "grad_norm": 22.094417572021484,
      "learning_rate": 9.00116144018583e-06,
      "logits/chosen": 0.602331280708313,
      "logits/rejected": 0.5329594016075134,
      "logps/chosen": -81.57435607910156,
      "logps/rejected": -81.0116958618164,
      "loss": 0.6304,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0059920065104961395,
      "rewards/margins": 0.15766987204551697,
      "rewards/rejected": -0.1636618822813034,
      "step": 431
    },
    {
      "epoch": 0.1003659174072138,
      "grad_norm": 25.03439712524414,
      "learning_rate": 8.99883855981417e-06,
      "logits/chosen": 0.4657560884952545,
      "logits/rejected": 0.5743473768234253,
      "logps/chosen": -82.10270690917969,
      "logps/rejected": -82.3171615600586,
      "loss": 0.7041,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.07591184973716736,
      "rewards/margins": 0.04632449150085449,
      "rewards/rejected": -0.12223634123802185,
      "step": 432
    },
    {
      "epoch": 0.1005982459197305,
      "grad_norm": 26.291784286499023,
      "learning_rate": 8.996515679442509e-06,
      "logits/chosen": 0.2543153464794159,
      "logits/rejected": 0.32852596044540405,
      "logps/chosen": -98.2467041015625,
      "logps/rejected": -86.23063659667969,
      "loss": 0.7353,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07687436044216156,
      "rewards/margins": -0.06831381469964981,
      "rewards/rejected": -0.008560544811189175,
      "step": 433
    },
    {
      "epoch": 0.1008305744322472,
      "grad_norm": 23.292362213134766,
      "learning_rate": 8.99419279907085e-06,
      "logits/chosen": 1.0630826950073242,
      "logits/rejected": 1.0603371858596802,
      "logps/chosen": -72.28781127929688,
      "logps/rejected": -77.46063232421875,
      "loss": 0.7371,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07525002956390381,
      "rewards/margins": -0.0762644112110138,
      "rewards/rejected": 0.0010143760591745377,
      "step": 434
    },
    {
      "epoch": 0.1010629029447639,
      "grad_norm": 23.60221290588379,
      "learning_rate": 8.991869918699188e-06,
      "logits/chosen": 1.1403751373291016,
      "logits/rejected": 1.032752513885498,
      "logps/chosen": -89.35420989990234,
      "logps/rejected": -73.89395904541016,
      "loss": 0.6377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0071369195356965065,
      "rewards/margins": 0.13021470606327057,
      "rewards/rejected": -0.12307778000831604,
      "step": 435
    },
    {
      "epoch": 0.10129523145728059,
      "grad_norm": 24.59775161743164,
      "learning_rate": 8.989547038327527e-06,
      "logits/chosen": 0.7042232155799866,
      "logits/rejected": 0.8591244220733643,
      "logps/chosen": -87.67108154296875,
      "logps/rejected": -79.4881591796875,
      "loss": 0.6475,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.020775556564331055,
      "rewards/margins": 0.1110389456152916,
      "rewards/rejected": -0.13181449472904205,
      "step": 436
    },
    {
      "epoch": 0.10152755996979729,
      "grad_norm": 32.24711227416992,
      "learning_rate": 8.987224157955866e-06,
      "logits/chosen": 0.060243185609579086,
      "logits/rejected": -0.007116702385246754,
      "logps/chosen": -102.01651000976562,
      "logps/rejected": -74.20005798339844,
      "loss": 0.8539,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.20569965243339539,
      "rewards/margins": -0.25994598865509033,
      "rewards/rejected": 0.05424628406763077,
      "step": 437
    },
    {
      "epoch": 0.10175988848231399,
      "grad_norm": 18.85454559326172,
      "learning_rate": 8.984901277584205e-06,
      "logits/chosen": 0.7945140600204468,
      "logits/rejected": 0.9103537797927856,
      "logps/chosen": -72.70145416259766,
      "logps/rejected": -81.88967895507812,
      "loss": 0.5336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.196775421500206,
      "rewards/margins": 0.44442737102508545,
      "rewards/rejected": -0.24765194952487946,
      "step": 438
    },
    {
      "epoch": 0.10199221699483069,
      "grad_norm": 20.118579864501953,
      "learning_rate": 8.982578397212544e-06,
      "logits/chosen": 0.45108214020729065,
      "logits/rejected": 0.5283178091049194,
      "logps/chosen": -70.62033081054688,
      "logps/rejected": -79.62662506103516,
      "loss": 0.634,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02656765840947628,
      "rewards/margins": 0.16208653151988983,
      "rewards/rejected": -0.18865418434143066,
      "step": 439
    },
    {
      "epoch": 0.10222454550734739,
      "grad_norm": 27.388635635375977,
      "learning_rate": 8.980255516840883e-06,
      "logits/chosen": 0.4416181743144989,
      "logits/rejected": 0.4638019800186157,
      "logps/chosen": -89.61970520019531,
      "logps/rejected": -89.10430908203125,
      "loss": 0.7665,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1301228553056717,
      "rewards/margins": -0.12692192196846008,
      "rewards/rejected": -0.0032009370625019073,
      "step": 440
    },
    {
      "epoch": 0.10245687401986409,
      "grad_norm": 24.024412155151367,
      "learning_rate": 8.977932636469222e-06,
      "logits/chosen": 0.5597981214523315,
      "logits/rejected": 0.45134663581848145,
      "logps/chosen": -79.26133728027344,
      "logps/rejected": -73.66809844970703,
      "loss": 0.7231,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.06743651628494263,
      "rewards/margins": -0.05408361554145813,
      "rewards/rejected": -0.013352897949516773,
      "step": 441
    },
    {
      "epoch": 0.10268920253238079,
      "grad_norm": 19.56511688232422,
      "learning_rate": 8.975609756097562e-06,
      "logits/chosen": 0.5880265235900879,
      "logits/rejected": 0.7027972936630249,
      "logps/chosen": -86.59356689453125,
      "logps/rejected": -87.96171569824219,
      "loss": 0.4675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18309198319911957,
      "rewards/margins": 0.6306878328323364,
      "rewards/rejected": -0.4475959241390228,
      "step": 442
    },
    {
      "epoch": 0.10292153104489749,
      "grad_norm": 27.187795639038086,
      "learning_rate": 8.973286875725901e-06,
      "logits/chosen": 0.5778418183326721,
      "logits/rejected": 0.5450075268745422,
      "logps/chosen": -99.81893920898438,
      "logps/rejected": -101.3408203125,
      "loss": 0.6979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03599896281957626,
      "rewards/margins": -0.003816366195678711,
      "rewards/rejected": -0.03218259662389755,
      "step": 443
    },
    {
      "epoch": 0.10315385955741418,
      "grad_norm": 21.54132652282715,
      "learning_rate": 8.97096399535424e-06,
      "logits/chosen": 0.180043563246727,
      "logits/rejected": 0.36187946796417236,
      "logps/chosen": -72.89905548095703,
      "logps/rejected": -73.52293395996094,
      "loss": 0.64,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04898788779973984,
      "rewards/margins": 0.12539732456207275,
      "rewards/rejected": -0.07640943676233292,
      "step": 444
    },
    {
      "epoch": 0.10338618806993088,
      "grad_norm": 23.578628540039062,
      "learning_rate": 8.96864111498258e-06,
      "logits/chosen": 0.7149763107299805,
      "logits/rejected": 0.7233058214187622,
      "logps/chosen": -70.5165786743164,
      "logps/rejected": -92.27685546875,
      "loss": 0.6098,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05780830606818199,
      "rewards/margins": 0.233826145529747,
      "rewards/rejected": -0.2916344404220581,
      "step": 445
    },
    {
      "epoch": 0.10361851658244758,
      "grad_norm": 24.473114013671875,
      "learning_rate": 8.966318234610918e-06,
      "logits/chosen": 0.8095352053642273,
      "logits/rejected": 1.0350209474563599,
      "logps/chosen": -87.25642395019531,
      "logps/rejected": -79.87873077392578,
      "loss": 0.6442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07841727882623672,
      "rewards/margins": 0.1370541751384735,
      "rewards/rejected": -0.05863690376281738,
      "step": 446
    },
    {
      "epoch": 0.10385084509496428,
      "grad_norm": 22.00398063659668,
      "learning_rate": 8.963995354239257e-06,
      "logits/chosen": 0.5218835473060608,
      "logits/rejected": 0.609647810459137,
      "logps/chosen": -87.69849395751953,
      "logps/rejected": -71.45243835449219,
      "loss": 0.6631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.030060481280088425,
      "rewards/margins": 0.0689854696393013,
      "rewards/rejected": -0.03892498090863228,
      "step": 447
    },
    {
      "epoch": 0.10408317360748098,
      "grad_norm": 23.706661224365234,
      "learning_rate": 8.961672473867596e-06,
      "logits/chosen": 0.7919798493385315,
      "logits/rejected": 0.6503695249557495,
      "logps/chosen": -85.2048110961914,
      "logps/rejected": -76.1549072265625,
      "loss": 0.6419,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05032820627093315,
      "rewards/margins": 0.14404280483722687,
      "rewards/rejected": -0.09371459484100342,
      "step": 448
    },
    {
      "epoch": 0.10431550211999768,
      "grad_norm": 22.53598403930664,
      "learning_rate": 8.959349593495935e-06,
      "logits/chosen": 0.3894888460636139,
      "logits/rejected": 0.39290404319763184,
      "logps/chosen": -75.56258392333984,
      "logps/rejected": -71.83605194091797,
      "loss": 0.6229,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08396835625171661,
      "rewards/margins": 0.2128678262233734,
      "rewards/rejected": -0.1288994699716568,
      "step": 449
    },
    {
      "epoch": 0.10454783063251437,
      "grad_norm": 24.372684478759766,
      "learning_rate": 8.957026713124276e-06,
      "logits/chosen": 0.7951987981796265,
      "logits/rejected": 0.6503618359565735,
      "logps/chosen": -79.0759048461914,
      "logps/rejected": -80.62834167480469,
      "loss": 0.6026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02154717780649662,
      "rewards/margins": 0.24810196459293365,
      "rewards/rejected": -0.2265547811985016,
      "step": 450
    },
    {
      "epoch": 0.10478015914503107,
      "grad_norm": 22.776165008544922,
      "learning_rate": 8.954703832752613e-06,
      "logits/chosen": 0.4885343611240387,
      "logits/rejected": 0.4054243266582489,
      "logps/chosen": -79.61209869384766,
      "logps/rejected": -70.63929748535156,
      "loss": 0.6534,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04821305349469185,
      "rewards/margins": 0.09967882931232452,
      "rewards/rejected": -0.14789187908172607,
      "step": 451
    },
    {
      "epoch": 0.10501248765754777,
      "grad_norm": 25.76335334777832,
      "learning_rate": 8.952380952380953e-06,
      "logits/chosen": 0.30700597167015076,
      "logits/rejected": 0.2678235173225403,
      "logps/chosen": -83.8179702758789,
      "logps/rejected": -82.4179916381836,
      "loss": 0.6802,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10686245560646057,
      "rewards/margins": 0.04338392987847328,
      "rewards/rejected": -0.15024638175964355,
      "step": 452
    },
    {
      "epoch": 0.10524481617006447,
      "grad_norm": 21.214012145996094,
      "learning_rate": 8.950058072009292e-06,
      "logits/chosen": 0.6923484802246094,
      "logits/rejected": 0.4013485312461853,
      "logps/chosen": -78.15069580078125,
      "logps/rejected": -77.65005493164062,
      "loss": 0.599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.030487827956676483,
      "rewards/margins": 0.2664094865322113,
      "rewards/rejected": -0.296897292137146,
      "step": 453
    },
    {
      "epoch": 0.10547714468258117,
      "grad_norm": 22.642290115356445,
      "learning_rate": 8.947735191637631e-06,
      "logits/chosen": 1.0379387140274048,
      "logits/rejected": 1.0803974866867065,
      "logps/chosen": -74.32615661621094,
      "logps/rejected": -85.83509063720703,
      "loss": 0.67,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08948996663093567,
      "rewards/margins": 0.08641217648983002,
      "rewards/rejected": 0.003077790141105652,
      "step": 454
    },
    {
      "epoch": 0.10570947319509787,
      "grad_norm": 24.889657974243164,
      "learning_rate": 8.94541231126597e-06,
      "logits/chosen": 0.39104482531547546,
      "logits/rejected": 0.455843985080719,
      "logps/chosen": -80.44881439208984,
      "logps/rejected": -81.70523071289062,
      "loss": 0.7245,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06878636032342911,
      "rewards/margins": -0.04375976324081421,
      "rewards/rejected": -0.025026585906744003,
      "step": 455
    },
    {
      "epoch": 0.10594180170761457,
      "grad_norm": 23.08271026611328,
      "learning_rate": 8.94308943089431e-06,
      "logits/chosen": 0.5025799870491028,
      "logits/rejected": 0.5343688726425171,
      "logps/chosen": -73.91494750976562,
      "logps/rejected": -87.9511489868164,
      "loss": 0.6406,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07537160068750381,
      "rewards/margins": 0.1723795384168625,
      "rewards/rejected": -0.09700794517993927,
      "step": 456
    },
    {
      "epoch": 0.10617413022013127,
      "grad_norm": 20.74022102355957,
      "learning_rate": 8.94076655052265e-06,
      "logits/chosen": 0.5793872475624084,
      "logits/rejected": 0.5753535628318787,
      "logps/chosen": -75.76802062988281,
      "logps/rejected": -80.43960571289062,
      "loss": 0.653,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05139363557100296,
      "rewards/margins": 0.10815741121768951,
      "rewards/rejected": -0.15955103933811188,
      "step": 457
    },
    {
      "epoch": 0.10640645873264797,
      "grad_norm": 25.047815322875977,
      "learning_rate": 8.938443670150989e-06,
      "logits/chosen": 0.3217410743236542,
      "logits/rejected": 0.16510409116744995,
      "logps/chosen": -72.27009582519531,
      "logps/rejected": -74.17317962646484,
      "loss": 0.7256,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.061369575560092926,
      "rewards/margins": 0.02717088907957077,
      "rewards/rejected": -0.0885404497385025,
      "step": 458
    },
    {
      "epoch": 0.10663878724516467,
      "grad_norm": 22.219749450683594,
      "learning_rate": 8.936120789779326e-06,
      "logits/chosen": 1.143118143081665,
      "logits/rejected": 0.9932928681373596,
      "logps/chosen": -68.25509643554688,
      "logps/rejected": -88.870849609375,
      "loss": 0.5461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.054256685078144073,
      "rewards/margins": 0.40179920196533203,
      "rewards/rejected": -0.34754252433776855,
      "step": 459
    },
    {
      "epoch": 0.10687111575768136,
      "grad_norm": 22.224992752075195,
      "learning_rate": 8.933797909407667e-06,
      "logits/chosen": 0.9051030874252319,
      "logits/rejected": 0.8272618055343628,
      "logps/chosen": -70.59434509277344,
      "logps/rejected": -87.2183837890625,
      "loss": 0.6008,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07074296474456787,
      "rewards/margins": 0.27003204822540283,
      "rewards/rejected": -0.19928905367851257,
      "step": 460
    },
    {
      "epoch": 0.10710344427019806,
      "grad_norm": 24.818256378173828,
      "learning_rate": 8.931475029036006e-06,
      "logits/chosen": 0.45153799653053284,
      "logits/rejected": 0.3160092830657959,
      "logps/chosen": -82.952392578125,
      "logps/rejected": -85.44847869873047,
      "loss": 0.6438,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04815707355737686,
      "rewards/margins": 0.20909686386585236,
      "rewards/rejected": -0.1609397828578949,
      "step": 461
    },
    {
      "epoch": 0.10733577278271476,
      "grad_norm": 22.45347023010254,
      "learning_rate": 8.929152148664345e-06,
      "logits/chosen": 0.5095810294151306,
      "logits/rejected": 0.6376430988311768,
      "logps/chosen": -86.09229278564453,
      "logps/rejected": -86.14775848388672,
      "loss": 0.6236,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.014837399125099182,
      "rewards/margins": 0.18069928884506226,
      "rewards/rejected": -0.19553667306900024,
      "step": 462
    },
    {
      "epoch": 0.10756810129523146,
      "grad_norm": 22.92831039428711,
      "learning_rate": 8.926829268292683e-06,
      "logits/chosen": 0.1899597942829132,
      "logits/rejected": 0.1977877914905548,
      "logps/chosen": -75.92855072021484,
      "logps/rejected": -84.4046630859375,
      "loss": 0.6436,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014385269954800606,
      "rewards/margins": 0.11055684089660645,
      "rewards/rejected": -0.1249421164393425,
      "step": 463
    },
    {
      "epoch": 0.10780042980774815,
      "grad_norm": 27.608640670776367,
      "learning_rate": 8.924506387921022e-06,
      "logits/chosen": 0.45995014905929565,
      "logits/rejected": 0.4444942772388458,
      "logps/chosen": -87.98919677734375,
      "logps/rejected": -81.46575927734375,
      "loss": 0.7535,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16997769474983215,
      "rewards/margins": -0.05878525227308273,
      "rewards/rejected": -0.11119247227907181,
      "step": 464
    },
    {
      "epoch": 0.10803275832026485,
      "grad_norm": 24.110994338989258,
      "learning_rate": 8.922183507549363e-06,
      "logits/chosen": 0.8741798400878906,
      "logits/rejected": 0.8921769261360168,
      "logps/chosen": -72.26093292236328,
      "logps/rejected": -85.76150512695312,
      "loss": 0.7211,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08211855590343475,
      "rewards/margins": -0.026025380939245224,
      "rewards/rejected": -0.056093163788318634,
      "step": 465
    },
    {
      "epoch": 0.10826508683278155,
      "grad_norm": 20.73505973815918,
      "learning_rate": 8.9198606271777e-06,
      "logits/chosen": 0.5548875331878662,
      "logits/rejected": 0.7220085263252258,
      "logps/chosen": -73.59353637695312,
      "logps/rejected": -70.711181640625,
      "loss": 0.6019,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15461879968643188,
      "rewards/margins": 0.2639106214046478,
      "rewards/rejected": -0.10929181426763535,
      "step": 466
    },
    {
      "epoch": 0.10849741534529825,
      "grad_norm": 21.074949264526367,
      "learning_rate": 8.91753774680604e-06,
      "logits/chosen": 0.5990058183670044,
      "logits/rejected": 0.8651700615882874,
      "logps/chosen": -71.8155288696289,
      "logps/rejected": -76.73161315917969,
      "loss": 0.573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1156579777598381,
      "rewards/margins": 0.271240234375,
      "rewards/rejected": -0.1555822640657425,
      "step": 467
    },
    {
      "epoch": 0.10872974385781495,
      "grad_norm": 28.332080841064453,
      "learning_rate": 8.91521486643438e-06,
      "logits/chosen": 1.3006962537765503,
      "logits/rejected": 1.2632811069488525,
      "logps/chosen": -89.13056945800781,
      "logps/rejected": -75.92925262451172,
      "loss": 0.7829,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.16198983788490295,
      "rewards/margins": -0.15518495440483093,
      "rewards/rejected": -0.006804873235523701,
      "step": 468
    },
    {
      "epoch": 0.10896207237033165,
      "grad_norm": 25.524686813354492,
      "learning_rate": 8.912891986062719e-06,
      "logits/chosen": 0.4459306299686432,
      "logits/rejected": 0.6123607754707336,
      "logps/chosen": -80.75562286376953,
      "logps/rejected": -95.75565338134766,
      "loss": 0.6703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05636361986398697,
      "rewards/margins": 0.05004559084773064,
      "rewards/rejected": -0.1064092218875885,
      "step": 469
    },
    {
      "epoch": 0.10919440088284835,
      "grad_norm": 27.7990665435791,
      "learning_rate": 8.910569105691058e-06,
      "logits/chosen": 0.6301128268241882,
      "logits/rejected": 0.6670284271240234,
      "logps/chosen": -91.63126373291016,
      "logps/rejected": -84.5386734008789,
      "loss": 0.7286,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.019716713577508926,
      "rewards/margins": -0.051406532526016235,
      "rewards/rejected": 0.07112325727939606,
      "step": 470
    },
    {
      "epoch": 0.10942672939536505,
      "grad_norm": 23.34882354736328,
      "learning_rate": 8.908246225319397e-06,
      "logits/chosen": 0.33419978618621826,
      "logits/rejected": 0.3251442611217499,
      "logps/chosen": -75.98375701904297,
      "logps/rejected": -98.94439697265625,
      "loss": 0.6803,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08910971879959106,
      "rewards/margins": 0.14516228437423706,
      "rewards/rejected": -0.23427198827266693,
      "step": 471
    },
    {
      "epoch": 0.10965905790788175,
      "grad_norm": 22.269750595092773,
      "learning_rate": 8.905923344947736e-06,
      "logits/chosen": 0.22199788689613342,
      "logits/rejected": 0.2030666172504425,
      "logps/chosen": -77.91324615478516,
      "logps/rejected": -83.78511047363281,
      "loss": 0.637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02017228491604328,
      "rewards/margins": 0.15274092555046082,
      "rewards/rejected": -0.17291319370269775,
      "step": 472
    },
    {
      "epoch": 0.10989138642039845,
      "grad_norm": 24.19915008544922,
      "learning_rate": 8.903600464576076e-06,
      "logits/chosen": 0.3432970345020294,
      "logits/rejected": 0.40427345037460327,
      "logps/chosen": -85.09931945800781,
      "logps/rejected": -84.00233459472656,
      "loss": 0.6319,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019599201157689095,
      "rewards/margins": 0.20238646864891052,
      "rewards/rejected": -0.18278725445270538,
      "step": 473
    },
    {
      "epoch": 0.11012371493291515,
      "grad_norm": 19.460752487182617,
      "learning_rate": 8.901277584204413e-06,
      "logits/chosen": 0.6382564902305603,
      "logits/rejected": 0.7882046103477478,
      "logps/chosen": -81.05936431884766,
      "logps/rejected": -78.48954010009766,
      "loss": 0.5144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25197136402130127,
      "rewards/margins": 0.4338686764240265,
      "rewards/rejected": -0.18189728260040283,
      "step": 474
    },
    {
      "epoch": 0.11035604344543185,
      "grad_norm": 22.84834861755371,
      "learning_rate": 8.898954703832752e-06,
      "logits/chosen": 0.6228610277175903,
      "logits/rejected": 0.6862332820892334,
      "logps/chosen": -77.97683715820312,
      "logps/rejected": -76.79689025878906,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025508426129817963,
      "rewards/margins": 0.016618799418210983,
      "rewards/rejected": 0.00888962484896183,
      "step": 475
    },
    {
      "epoch": 0.11058837195794854,
      "grad_norm": 22.930803298950195,
      "learning_rate": 8.896631823461093e-06,
      "logits/chosen": 0.38467633724212646,
      "logits/rejected": 0.6637954711914062,
      "logps/chosen": -78.75468444824219,
      "logps/rejected": -99.79228210449219,
      "loss": 0.5755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.016314052045345306,
      "rewards/margins": 0.27536728978157043,
      "rewards/rejected": -0.29168134927749634,
      "step": 476
    },
    {
      "epoch": 0.11082070047046524,
      "grad_norm": 29.89509391784668,
      "learning_rate": 8.894308943089432e-06,
      "logits/chosen": 0.6779375076293945,
      "logits/rejected": 0.6771290898323059,
      "logps/chosen": -86.72396087646484,
      "logps/rejected": -78.77659606933594,
      "loss": 0.748,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14584103226661682,
      "rewards/margins": -0.05984945595264435,
      "rewards/rejected": -0.08599156886339188,
      "step": 477
    },
    {
      "epoch": 0.11105302898298193,
      "grad_norm": 22.730880737304688,
      "learning_rate": 8.891986062717771e-06,
      "logits/chosen": 0.5767961740493774,
      "logits/rejected": 0.6767807602882385,
      "logps/chosen": -92.49686431884766,
      "logps/rejected": -72.57461547851562,
      "loss": 0.6376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.027013063430786133,
      "rewards/margins": 0.13301165401935577,
      "rewards/rejected": -0.1600247025489807,
      "step": 478
    },
    {
      "epoch": 0.11128535749549863,
      "grad_norm": 23.189241409301758,
      "learning_rate": 8.88966318234611e-06,
      "logits/chosen": 0.5019114017486572,
      "logits/rejected": 0.6048083305358887,
      "logps/chosen": -78.23165130615234,
      "logps/rejected": -67.78890991210938,
      "loss": 0.7477,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.17233271896839142,
      "rewards/margins": -0.08968986570835114,
      "rewards/rejected": -0.08264284580945969,
      "step": 479
    },
    {
      "epoch": 0.11151768600801533,
      "grad_norm": 20.31208610534668,
      "learning_rate": 8.887340301974449e-06,
      "logits/chosen": 0.4884869158267975,
      "logits/rejected": 0.6769218444824219,
      "logps/chosen": -70.65135955810547,
      "logps/rejected": -84.077392578125,
      "loss": 0.5894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10712041705846786,
      "rewards/margins": 0.2813119888305664,
      "rewards/rejected": -0.17419156432151794,
      "step": 480
    },
    {
      "epoch": 0.11175001452053203,
      "grad_norm": 22.304407119750977,
      "learning_rate": 8.885017421602788e-06,
      "logits/chosen": 0.2451362907886505,
      "logits/rejected": 0.2422465980052948,
      "logps/chosen": -85.68864440917969,
      "logps/rejected": -71.93067169189453,
      "loss": 0.6081,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04158001020550728,
      "rewards/margins": 0.21537554264068604,
      "rewards/rejected": -0.17379553616046906,
      "step": 481
    },
    {
      "epoch": 0.11198234303304873,
      "grad_norm": 23.81793212890625,
      "learning_rate": 8.882694541231127e-06,
      "logits/chosen": 0.9167830348014832,
      "logits/rejected": 0.7467952966690063,
      "logps/chosen": -72.00251770019531,
      "logps/rejected": -78.72877502441406,
      "loss": 0.7083,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.10456131398677826,
      "rewards/margins": -3.654882311820984e-05,
      "rewards/rejected": -0.10452475398778915,
      "step": 482
    },
    {
      "epoch": 0.11221467154556543,
      "grad_norm": 23.759502410888672,
      "learning_rate": 8.880371660859467e-06,
      "logits/chosen": 0.034272514283657074,
      "logits/rejected": 0.18362554907798767,
      "logps/chosen": -86.14647674560547,
      "logps/rejected": -73.32188415527344,
      "loss": 0.6987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08941483497619629,
      "rewards/margins": 0.009232426062226295,
      "rewards/rejected": -0.09864726662635803,
      "step": 483
    },
    {
      "epoch": 0.11244700005808213,
      "grad_norm": 22.942659378051758,
      "learning_rate": 8.878048780487806e-06,
      "logits/chosen": 0.3140602111816406,
      "logits/rejected": 0.3311956226825714,
      "logps/chosen": -76.9742431640625,
      "logps/rejected": -70.41972351074219,
      "loss": 0.7073,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08237694203853607,
      "rewards/margins": -0.012934306636452675,
      "rewards/rejected": -0.06944262981414795,
      "step": 484
    },
    {
      "epoch": 0.11267932857059883,
      "grad_norm": 21.234821319580078,
      "learning_rate": 8.875725900116145e-06,
      "logits/chosen": 0.08511479943990707,
      "logits/rejected": 0.1725986748933792,
      "logps/chosen": -74.08683013916016,
      "logps/rejected": -82.13677215576172,
      "loss": 0.5485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08034567534923553,
      "rewards/margins": 0.40049830079078674,
      "rewards/rejected": -0.3201526403427124,
      "step": 485
    },
    {
      "epoch": 0.11291165708311553,
      "grad_norm": 27.768218994140625,
      "learning_rate": 8.873403019744484e-06,
      "logits/chosen": 0.1414823830127716,
      "logits/rejected": 0.1745224893093109,
      "logps/chosen": -89.57657623291016,
      "logps/rejected": -90.56962585449219,
      "loss": 0.6561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07723617553710938,
      "rewards/margins": 0.11081691086292267,
      "rewards/rejected": -0.18805310130119324,
      "step": 486
    },
    {
      "epoch": 0.11314398559563223,
      "grad_norm": 24.769332885742188,
      "learning_rate": 8.871080139372823e-06,
      "logits/chosen": 0.2134062796831131,
      "logits/rejected": 0.1465064436197281,
      "logps/chosen": -77.53733825683594,
      "logps/rejected": -77.59017181396484,
      "loss": 0.7133,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03420763462781906,
      "rewards/margins": -0.0015962868928909302,
      "rewards/rejected": -0.03261134400963783,
      "step": 487
    },
    {
      "epoch": 0.11337631410814893,
      "grad_norm": 24.75571060180664,
      "learning_rate": 8.868757259001162e-06,
      "logits/chosen": -0.10606688261032104,
      "logits/rejected": 0.04264908283948898,
      "logps/chosen": -76.0092544555664,
      "logps/rejected": -87.68616485595703,
      "loss": 0.7219,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09010619670152664,
      "rewards/margins": -0.010969070717692375,
      "rewards/rejected": -0.07913713902235031,
      "step": 488
    },
    {
      "epoch": 0.11360864262066563,
      "grad_norm": 23.261455535888672,
      "learning_rate": 8.8664343786295e-06,
      "logits/chosen": 0.5790551900863647,
      "logits/rejected": 0.47026482224464417,
      "logps/chosen": -84.73387908935547,
      "logps/rejected": -92.26966857910156,
      "loss": 0.6184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.037991903722286224,
      "rewards/margins": 0.19907939434051514,
      "rewards/rejected": -0.1610874980688095,
      "step": 489
    },
    {
      "epoch": 0.11384097113318233,
      "grad_norm": 20.98050880432129,
      "learning_rate": 8.86411149825784e-06,
      "logits/chosen": -0.10888335853815079,
      "logits/rejected": -0.01572130247950554,
      "logps/chosen": -83.21401977539062,
      "logps/rejected": -82.90782165527344,
      "loss": 0.5954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04567398875951767,
      "rewards/margins": 0.24783265590667725,
      "rewards/rejected": -0.20215864479541779,
      "step": 490
    },
    {
      "epoch": 0.11407329964569903,
      "grad_norm": 23.19867706298828,
      "learning_rate": 8.86178861788618e-06,
      "logits/chosen": 0.41496384143829346,
      "logits/rejected": 0.5305248498916626,
      "logps/chosen": -84.09992980957031,
      "logps/rejected": -83.03948211669922,
      "loss": 0.66,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.026010489091277122,
      "rewards/margins": 0.08851991593837738,
      "rewards/rejected": -0.11453038454055786,
      "step": 491
    },
    {
      "epoch": 0.11430562815821571,
      "grad_norm": 22.58319091796875,
      "learning_rate": 8.85946573751452e-06,
      "logits/chosen": 0.8716623783111572,
      "logits/rejected": 1.0704598426818848,
      "logps/chosen": -81.55248260498047,
      "logps/rejected": -90.32362365722656,
      "loss": 0.625,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.26777273416519165,
      "rewards/margins": 0.4085102379322052,
      "rewards/rejected": -0.14073753356933594,
      "step": 492
    },
    {
      "epoch": 0.11453795667073241,
      "grad_norm": 21.245361328125,
      "learning_rate": 8.857142857142858e-06,
      "logits/chosen": 0.14522767066955566,
      "logits/rejected": 0.1485324203968048,
      "logps/chosen": -75.21356201171875,
      "logps/rejected": -77.52655029296875,
      "loss": 0.618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.22048719227313995,
      "rewards/margins": 0.2466171234846115,
      "rewards/rejected": -0.026129933074116707,
      "step": 493
    },
    {
      "epoch": 0.11477028518324911,
      "grad_norm": 23.927900314331055,
      "learning_rate": 8.854819976771197e-06,
      "logits/chosen": 0.8812259435653687,
      "logits/rejected": 1.1006301641464233,
      "logps/chosen": -87.86512756347656,
      "logps/rejected": -73.07049560546875,
      "loss": 0.7427,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.19199469685554504,
      "rewards/margins": -0.08270163834095001,
      "rewards/rejected": -0.10929305851459503,
      "step": 494
    },
    {
      "epoch": 0.11500261369576581,
      "grad_norm": 16.787837982177734,
      "learning_rate": 8.852497096399536e-06,
      "logits/chosen": 0.4336768388748169,
      "logits/rejected": 0.2841884195804596,
      "logps/chosen": -68.4066162109375,
      "logps/rejected": -92.10099792480469,
      "loss": 0.4631,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17892423272132874,
      "rewards/margins": 0.5708433985710144,
      "rewards/rejected": -0.39191916584968567,
      "step": 495
    },
    {
      "epoch": 0.11523494220828251,
      "grad_norm": 25.728824615478516,
      "learning_rate": 8.850174216027875e-06,
      "logits/chosen": 0.7218519449234009,
      "logits/rejected": 0.7188904285430908,
      "logps/chosen": -87.84168243408203,
      "logps/rejected": -82.05064392089844,
      "loss": 0.6953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015089228749275208,
      "rewards/margins": 0.029854565858840942,
      "rewards/rejected": -0.014765359461307526,
      "step": 496
    },
    {
      "epoch": 0.11546727072079921,
      "grad_norm": 23.74083137512207,
      "learning_rate": 8.847851335656214e-06,
      "logits/chosen": 0.3989861011505127,
      "logits/rejected": 0.38363951444625854,
      "logps/chosen": -82.3178939819336,
      "logps/rejected": -70.1900405883789,
      "loss": 0.6663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07025904953479767,
      "rewards/margins": 0.09355810284614563,
      "rewards/rejected": -0.1638171374797821,
      "step": 497
    },
    {
      "epoch": 0.11569959923331591,
      "grad_norm": 23.004758834838867,
      "learning_rate": 8.845528455284553e-06,
      "logits/chosen": 0.03333902359008789,
      "logits/rejected": 0.1821252554655075,
      "logps/chosen": -72.11629486083984,
      "logps/rejected": -84.41514587402344,
      "loss": 0.6395,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008546683005988598,
      "rewards/margins": 0.12988585233688354,
      "rewards/rejected": -0.12133915722370148,
      "step": 498
    },
    {
      "epoch": 0.11593192774583261,
      "grad_norm": 21.153465270996094,
      "learning_rate": 8.843205574912894e-06,
      "logits/chosen": 1.255466103553772,
      "logits/rejected": 1.346278190612793,
      "logps/chosen": -75.76646423339844,
      "logps/rejected": -83.30604553222656,
      "loss": 0.618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.028608065098524094,
      "rewards/margins": 0.2011643350124359,
      "rewards/rejected": -0.17255626618862152,
      "step": 499
    },
    {
      "epoch": 0.11616425625834931,
      "grad_norm": 26.420791625976562,
      "learning_rate": 8.840882694541232e-06,
      "logits/chosen": -0.0010559502989053726,
      "logits/rejected": 0.06421857327222824,
      "logps/chosen": -84.25804138183594,
      "logps/rejected": -65.4998779296875,
      "loss": 0.7204,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15115267038345337,
      "rewards/margins": -0.04125736653804779,
      "rewards/rejected": -0.10989528894424438,
      "step": 500
    },
    {
      "epoch": 0.11639658477086601,
      "grad_norm": 21.30701446533203,
      "learning_rate": 8.83855981416957e-06,
      "logits/chosen": 0.38765400648117065,
      "logits/rejected": 0.4112858772277832,
      "logps/chosen": -74.01205444335938,
      "logps/rejected": -69.82799530029297,
      "loss": 0.6738,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0008471719920635223,
      "rewards/margins": 0.07913658022880554,
      "rewards/rejected": -0.07998375594615936,
      "step": 501
    },
    {
      "epoch": 0.1166289132833827,
      "grad_norm": 24.800128936767578,
      "learning_rate": 8.83623693379791e-06,
      "logits/chosen": -0.09908172488212585,
      "logits/rejected": -0.020038852468132973,
      "logps/chosen": -79.73785400390625,
      "logps/rejected": -104.02249145507812,
      "loss": 0.622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07752318680286407,
      "rewards/margins": 0.16634322702884674,
      "rewards/rejected": -0.24386641383171082,
      "step": 502
    },
    {
      "epoch": 0.1168612417958994,
      "grad_norm": 26.39325714111328,
      "learning_rate": 8.83391405342625e-06,
      "logits/chosen": 0.5208216905593872,
      "logits/rejected": 0.42916589975357056,
      "logps/chosen": -78.06038665771484,
      "logps/rejected": -75.31382751464844,
      "loss": 0.6264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01581600122153759,
      "rewards/margins": 0.21134254336357117,
      "rewards/rejected": -0.2271585762500763,
      "step": 503
    },
    {
      "epoch": 0.1170935703084161,
      "grad_norm": 22.140548706054688,
      "learning_rate": 8.831591173054588e-06,
      "logits/chosen": 0.4262514114379883,
      "logits/rejected": 0.38635605573654175,
      "logps/chosen": -85.88623046875,
      "logps/rejected": -93.17095184326172,
      "loss": 0.5824,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00851951539516449,
      "rewards/margins": 0.2671278119087219,
      "rewards/rejected": -0.25860828161239624,
      "step": 504
    },
    {
      "epoch": 0.1173258988209328,
      "grad_norm": 25.78154945373535,
      "learning_rate": 8.829268292682927e-06,
      "logits/chosen": 0.2874882221221924,
      "logits/rejected": 0.39860406517982483,
      "logps/chosen": -81.12842559814453,
      "logps/rejected": -76.88544464111328,
      "loss": 0.7414,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07903032004833221,
      "rewards/margins": -0.07298137992620468,
      "rewards/rejected": -0.006048941984772682,
      "step": 505
    },
    {
      "epoch": 0.11755822733344949,
      "grad_norm": 23.37728500366211,
      "learning_rate": 8.826945412311266e-06,
      "logits/chosen": 0.32733508944511414,
      "logits/rejected": 0.3057281970977783,
      "logps/chosen": -80.95214080810547,
      "logps/rejected": -77.72320556640625,
      "loss": 0.6349,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.010912507772445679,
      "rewards/margins": 0.17345543205738068,
      "rewards/rejected": -0.18436795473098755,
      "step": 506
    },
    {
      "epoch": 0.11779055584596619,
      "grad_norm": 24.014360427856445,
      "learning_rate": 8.824622531939607e-06,
      "logits/chosen": 0.05523233860731125,
      "logits/rejected": -0.08429844677448273,
      "logps/chosen": -86.08943176269531,
      "logps/rejected": -74.76446533203125,
      "loss": 0.6566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10063472390174866,
      "rewards/margins": 0.11664783209562302,
      "rewards/rejected": -0.21728254854679108,
      "step": 507
    },
    {
      "epoch": 0.11802288435848289,
      "grad_norm": 26.388526916503906,
      "learning_rate": 8.822299651567946e-06,
      "logits/chosen": 0.15842565894126892,
      "logits/rejected": 0.14573854207992554,
      "logps/chosen": -88.28862762451172,
      "logps/rejected": -74.64271545410156,
      "loss": 0.7841,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08428120613098145,
      "rewards/margins": -0.1156751811504364,
      "rewards/rejected": 0.031393952667713165,
      "step": 508
    },
    {
      "epoch": 0.11825521287099959,
      "grad_norm": 23.72325325012207,
      "learning_rate": 8.819976771196283e-06,
      "logits/chosen": 0.2723575830459595,
      "logits/rejected": 0.23846201598644257,
      "logps/chosen": -78.51179504394531,
      "logps/rejected": -82.66687774658203,
      "loss": 0.6362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16677238047122955,
      "rewards/margins": 0.14056289196014404,
      "rewards/rejected": -0.3073352575302124,
      "step": 509
    },
    {
      "epoch": 0.11848754138351629,
      "grad_norm": 17.359189987182617,
      "learning_rate": 8.817653890824624e-06,
      "logits/chosen": 0.1600056290626526,
      "logits/rejected": 0.21995395421981812,
      "logps/chosen": -65.96358489990234,
      "logps/rejected": -83.382568359375,
      "loss": 0.4484,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33173897862434387,
      "rewards/margins": 0.7617268562316895,
      "rewards/rejected": -0.4299878776073456,
      "step": 510
    },
    {
      "epoch": 0.11871986989603299,
      "grad_norm": 24.941810607910156,
      "learning_rate": 8.815331010452962e-06,
      "logits/chosen": -0.008098334074020386,
      "logits/rejected": -0.07166478037834167,
      "logps/chosen": -82.21424102783203,
      "logps/rejected": -72.99012756347656,
      "loss": 0.7572,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.18257752060890198,
      "rewards/margins": -0.0820470005273819,
      "rewards/rejected": -0.10053049772977829,
      "step": 511
    },
    {
      "epoch": 0.11895219840854969,
      "grad_norm": 25.281620025634766,
      "learning_rate": 8.813008130081301e-06,
      "logits/chosen": 0.6509397625923157,
      "logits/rejected": 0.6982929110527039,
      "logps/chosen": -84.65084838867188,
      "logps/rejected": -65.17253112792969,
      "loss": 0.7349,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.09025845676660538,
      "rewards/margins": -0.07648828625679016,
      "rewards/rejected": -0.013770174235105515,
      "step": 512
    },
    {
      "epoch": 0.11918452692106639,
      "grad_norm": 22.599491119384766,
      "learning_rate": 8.81068524970964e-06,
      "logits/chosen": 0.9991588592529297,
      "logits/rejected": 0.8433800339698792,
      "logps/chosen": -66.7741928100586,
      "logps/rejected": -75.62455749511719,
      "loss": 0.6709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07416488975286484,
      "rewards/margins": 0.08364804089069366,
      "rewards/rejected": -0.009483144618570805,
      "step": 513
    },
    {
      "epoch": 0.11941685543358309,
      "grad_norm": 27.206619262695312,
      "learning_rate": 8.80836236933798e-06,
      "logits/chosen": -0.20572426915168762,
      "logits/rejected": -0.06504887342453003,
      "logps/chosen": -86.87677764892578,
      "logps/rejected": -78.49156951904297,
      "loss": 0.7981,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.2003730833530426,
      "rewards/margins": -0.1849183738231659,
      "rewards/rejected": -0.015454694628715515,
      "step": 514
    },
    {
      "epoch": 0.11964918394609979,
      "grad_norm": 27.461755752563477,
      "learning_rate": 8.80603948896632e-06,
      "logits/chosen": 0.7149969339370728,
      "logits/rejected": 0.6168245077133179,
      "logps/chosen": -88.73117065429688,
      "logps/rejected": -89.26789855957031,
      "loss": 0.7624,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.14653289318084717,
      "rewards/margins": -0.12621647119522095,
      "rewards/rejected": -0.020316412672400475,
      "step": 515
    },
    {
      "epoch": 0.11988151245861649,
      "grad_norm": 23.821191787719727,
      "learning_rate": 8.803716608594657e-06,
      "logits/chosen": -0.2293388843536377,
      "logits/rejected": -0.25602853298187256,
      "logps/chosen": -81.30329132080078,
      "logps/rejected": -77.6747055053711,
      "loss": 0.6323,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03400454670190811,
      "rewards/margins": 0.14472374320030212,
      "rewards/rejected": -0.11071920394897461,
      "step": 516
    },
    {
      "epoch": 0.12011384097113319,
      "grad_norm": 25.256319046020508,
      "learning_rate": 8.801393728222998e-06,
      "logits/chosen": 0.16352258622646332,
      "logits/rejected": 0.1662711352109909,
      "logps/chosen": -78.52633666992188,
      "logps/rejected": -83.82100677490234,
      "loss": 0.7298,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.043384816497564316,
      "rewards/margins": -0.05279390513896942,
      "rewards/rejected": 0.009409092366695404,
      "step": 517
    },
    {
      "epoch": 0.12034616948364989,
      "grad_norm": 25.29903793334961,
      "learning_rate": 8.799070847851337e-06,
      "logits/chosen": -0.1750541627407074,
      "logits/rejected": -0.19631463289260864,
      "logps/chosen": -78.57528686523438,
      "logps/rejected": -79.68632507324219,
      "loss": 0.7515,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.14325442910194397,
      "rewards/margins": -0.07135650515556335,
      "rewards/rejected": -0.07189791649580002,
      "step": 518
    },
    {
      "epoch": 0.12057849799616659,
      "grad_norm": 22.198850631713867,
      "learning_rate": 8.796747967479676e-06,
      "logits/chosen": 0.8335707783699036,
      "logits/rejected": 0.9457336068153381,
      "logps/chosen": -77.18115997314453,
      "logps/rejected": -99.771728515625,
      "loss": 0.6326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09311022609472275,
      "rewards/margins": 0.15685264766216278,
      "rewards/rejected": -0.24996285140514374,
      "step": 519
    },
    {
      "epoch": 0.12081082650868327,
      "grad_norm": 21.900426864624023,
      "learning_rate": 8.794425087108015e-06,
      "logits/chosen": 0.604762852191925,
      "logits/rejected": 0.8242324590682983,
      "logps/chosen": -80.67391967773438,
      "logps/rejected": -84.16902923583984,
      "loss": 0.6036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07630649209022522,
      "rewards/margins": 0.2160661667585373,
      "rewards/rejected": -0.13975967466831207,
      "step": 520
    },
    {
      "epoch": 0.12104315502119997,
      "grad_norm": 21.975032806396484,
      "learning_rate": 8.792102206736354e-06,
      "logits/chosen": 0.3629342317581177,
      "logits/rejected": 0.3951454758644104,
      "logps/chosen": -80.76271057128906,
      "logps/rejected": -81.46244812011719,
      "loss": 0.6584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05729994177818298,
      "rewards/margins": 0.09038850665092468,
      "rewards/rejected": -0.14768843352794647,
      "step": 521
    },
    {
      "epoch": 0.12127548353371667,
      "grad_norm": 22.36244010925293,
      "learning_rate": 8.789779326364692e-06,
      "logits/chosen": -0.03353564441204071,
      "logits/rejected": 0.005616353824734688,
      "logps/chosen": -76.58441925048828,
      "logps/rejected": -79.23927307128906,
      "loss": 0.646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.019018080085515976,
      "rewards/margins": 0.1657731533050537,
      "rewards/rejected": -0.18479123711585999,
      "step": 522
    },
    {
      "epoch": 0.12150781204623337,
      "grad_norm": 25.140308380126953,
      "learning_rate": 8.787456445993033e-06,
      "logits/chosen": 0.8108692765235901,
      "logits/rejected": 0.825223982334137,
      "logps/chosen": -82.4893798828125,
      "logps/rejected": -83.90406799316406,
      "loss": 0.6467,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00836134236305952,
      "rewards/margins": 0.14547857642173767,
      "rewards/rejected": -0.15383993089199066,
      "step": 523
    },
    {
      "epoch": 0.12174014055875007,
      "grad_norm": 23.369733810424805,
      "learning_rate": 8.78513356562137e-06,
      "logits/chosen": 0.5029531717300415,
      "logits/rejected": 0.5932786464691162,
      "logps/chosen": -83.93492126464844,
      "logps/rejected": -94.32832336425781,
      "loss": 0.5716,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.25577133893966675,
      "rewards/margins": 0.6736885905265808,
      "rewards/rejected": -0.41791725158691406,
      "step": 524
    },
    {
      "epoch": 0.12197246907126677,
      "grad_norm": 24.560543060302734,
      "learning_rate": 8.782810685249711e-06,
      "logits/chosen": 0.27367737889289856,
      "logits/rejected": 0.45838624238967896,
      "logps/chosen": -89.32664489746094,
      "logps/rejected": -82.24210357666016,
      "loss": 0.608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01906397193670273,
      "rewards/margins": 0.2787178158760071,
      "rewards/rejected": -0.2977817952632904,
      "step": 525
    },
    {
      "epoch": 0.12220479758378347,
      "grad_norm": 22.310712814331055,
      "learning_rate": 8.78048780487805e-06,
      "logits/chosen": 0.3140117824077606,
      "logits/rejected": 0.37083351612091064,
      "logps/chosen": -82.24732971191406,
      "logps/rejected": -69.67571258544922,
      "loss": 0.6404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03767653927206993,
      "rewards/margins": 0.1772838532924652,
      "rewards/rejected": -0.1396072804927826,
      "step": 526
    },
    {
      "epoch": 0.12243712609630017,
      "grad_norm": 25.3028621673584,
      "learning_rate": 8.778164924506389e-06,
      "logits/chosen": 0.44101694226264954,
      "logits/rejected": 0.6244397163391113,
      "logps/chosen": -91.8450927734375,
      "logps/rejected": -95.22999572753906,
      "loss": 0.641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05150681361556053,
      "rewards/margins": 0.11507010459899902,
      "rewards/rejected": -0.16657690703868866,
      "step": 527
    },
    {
      "epoch": 0.12266945460881687,
      "grad_norm": 23.785045623779297,
      "learning_rate": 8.775842044134728e-06,
      "logits/chosen": 0.6218142509460449,
      "logits/rejected": 0.9742294549942017,
      "logps/chosen": -84.52031707763672,
      "logps/rejected": -75.11819458007812,
      "loss": 0.6987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06524400413036346,
      "rewards/margins": 0.012973830103874207,
      "rewards/rejected": -0.07821783423423767,
      "step": 528
    },
    {
      "epoch": 0.12290178312133357,
      "grad_norm": 22.80458641052246,
      "learning_rate": 8.773519163763067e-06,
      "logits/chosen": 0.29546087980270386,
      "logits/rejected": 0.15573786199092865,
      "logps/chosen": -79.30583190917969,
      "logps/rejected": -79.69530487060547,
      "loss": 0.6286,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12457969784736633,
      "rewards/margins": 0.16873782873153687,
      "rewards/rejected": -0.044158127158880234,
      "step": 529
    },
    {
      "epoch": 0.12313411163385027,
      "grad_norm": 23.989408493041992,
      "learning_rate": 8.771196283391407e-06,
      "logits/chosen": 0.4030238389968872,
      "logits/rejected": 0.38435325026512146,
      "logps/chosen": -91.88068389892578,
      "logps/rejected": -73.94031524658203,
      "loss": 0.6418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.033588845282793045,
      "rewards/margins": 0.11687584966421127,
      "rewards/rejected": -0.15046468377113342,
      "step": 530
    },
    {
      "epoch": 0.12336644014636697,
      "grad_norm": 23.98807144165039,
      "learning_rate": 8.768873403019745e-06,
      "logits/chosen": 0.7385097742080688,
      "logits/rejected": 0.7634755969047546,
      "logps/chosen": -72.02794647216797,
      "logps/rejected": -68.99581146240234,
      "loss": 0.78,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.17682351171970367,
      "rewards/margins": -0.15684820711612701,
      "rewards/rejected": -0.01997530460357666,
      "step": 531
    },
    {
      "epoch": 0.12359876865888367,
      "grad_norm": 21.920326232910156,
      "learning_rate": 8.766550522648083e-06,
      "logits/chosen": 0.2036622166633606,
      "logits/rejected": 0.486649751663208,
      "logps/chosen": -86.54335021972656,
      "logps/rejected": -86.35151672363281,
      "loss": 0.5872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07684414833784103,
      "rewards/margins": 0.26059937477111816,
      "rewards/rejected": -0.18375520408153534,
      "step": 532
    },
    {
      "epoch": 0.12383109717140035,
      "grad_norm": 25.624439239501953,
      "learning_rate": 8.764227642276424e-06,
      "logits/chosen": -0.000851738266646862,
      "logits/rejected": 0.06634148955345154,
      "logps/chosen": -97.7831802368164,
      "logps/rejected": -78.37278747558594,
      "loss": 0.6856,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03526044264435768,
      "rewards/margins": 0.06769105046987534,
      "rewards/rejected": -0.10295149683952332,
      "step": 533
    },
    {
      "epoch": 0.12406342568391705,
      "grad_norm": 23.797285079956055,
      "learning_rate": 8.761904761904763e-06,
      "logits/chosen": 0.45037779211997986,
      "logits/rejected": 0.5191367864608765,
      "logps/chosen": -83.06246948242188,
      "logps/rejected": -80.0791244506836,
      "loss": 0.6693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0461614616215229,
      "rewards/margins": 0.15419092774391174,
      "rewards/rejected": -0.20035238564014435,
      "step": 534
    },
    {
      "epoch": 0.12429575419643375,
      "grad_norm": 28.30113410949707,
      "learning_rate": 8.759581881533102e-06,
      "logits/chosen": 0.503392219543457,
      "logits/rejected": 0.6724566221237183,
      "logps/chosen": -82.828125,
      "logps/rejected": -83.0694351196289,
      "loss": 0.8135,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1535700410604477,
      "rewards/margins": -0.16880950331687927,
      "rewards/rejected": 0.015239447355270386,
      "step": 535
    },
    {
      "epoch": 0.12452808270895045,
      "grad_norm": 26.13671875,
      "learning_rate": 8.757259001161441e-06,
      "logits/chosen": 0.5398272275924683,
      "logits/rejected": 0.6120855808258057,
      "logps/chosen": -87.09710693359375,
      "logps/rejected": -80.97848510742188,
      "loss": 0.7194,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.049874525517225266,
      "rewards/margins": 0.0067908018827438354,
      "rewards/rejected": -0.0566653311252594,
      "step": 536
    },
    {
      "epoch": 0.12476041122146715,
      "grad_norm": 21.222576141357422,
      "learning_rate": 8.75493612078978e-06,
      "logits/chosen": 0.17630642652511597,
      "logits/rejected": 0.16212418675422668,
      "logps/chosen": -64.09971618652344,
      "logps/rejected": -85.58241271972656,
      "loss": 0.6139,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03271918371319771,
      "rewards/margins": 0.21271437406539917,
      "rewards/rejected": -0.17999517917633057,
      "step": 537
    },
    {
      "epoch": 0.12499273973398385,
      "grad_norm": 18.118227005004883,
      "learning_rate": 8.75261324041812e-06,
      "logits/chosen": 0.19216391444206238,
      "logits/rejected": 0.20931366086006165,
      "logps/chosen": -64.81742858886719,
      "logps/rejected": -78.21102142333984,
      "loss": 0.5176,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29676172137260437,
      "rewards/margins": 0.4791452884674072,
      "rewards/rejected": -0.18238356709480286,
      "step": 538
    },
    {
      "epoch": 0.12522506824650056,
      "grad_norm": 20.480356216430664,
      "learning_rate": 8.750290360046458e-06,
      "logits/chosen": 0.6718565225601196,
      "logits/rejected": 0.6503617763519287,
      "logps/chosen": -76.43397521972656,
      "logps/rejected": -92.8166732788086,
      "loss": 0.5128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16030831634998322,
      "rewards/margins": 0.5479004979133606,
      "rewards/rejected": -0.3875921964645386,
      "step": 539
    },
    {
      "epoch": 0.12545739675901724,
      "grad_norm": 24.727388381958008,
      "learning_rate": 8.747967479674797e-06,
      "logits/chosen": 1.00051748752594,
      "logits/rejected": 1.089518427848816,
      "logps/chosen": -90.670654296875,
      "logps/rejected": -90.19279479980469,
      "loss": 0.7365,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14858970046043396,
      "rewards/margins": 0.13247013092041016,
      "rewards/rejected": -0.2810598611831665,
      "step": 540
    },
    {
      "epoch": 0.12568972527153394,
      "grad_norm": 21.014392852783203,
      "learning_rate": 8.745644599303137e-06,
      "logits/chosen": 0.5700863599777222,
      "logits/rejected": 0.5808189511299133,
      "logps/chosen": -72.24142456054688,
      "logps/rejected": -102.52884674072266,
      "loss": 0.5573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22493940591812134,
      "rewards/margins": 0.8050758838653564,
      "rewards/rejected": -0.5801364779472351,
      "step": 541
    },
    {
      "epoch": 0.12592205378405064,
      "grad_norm": 23.939905166625977,
      "learning_rate": 8.743321718931476e-06,
      "logits/chosen": 1.079419493675232,
      "logits/rejected": 1.217639446258545,
      "logps/chosen": -89.70893096923828,
      "logps/rejected": -86.05738830566406,
      "loss": 0.6538,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01579158566892147,
      "rewards/margins": 0.13335300981998444,
      "rewards/rejected": -0.11756143718957901,
      "step": 542
    },
    {
      "epoch": 0.12615438229656734,
      "grad_norm": 24.87330436706543,
      "learning_rate": 8.740998838559815e-06,
      "logits/chosen": 0.6427921056747437,
      "logits/rejected": 0.7494451999664307,
      "logps/chosen": -84.28861999511719,
      "logps/rejected": -90.85090637207031,
      "loss": 0.6754,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03645595908164978,
      "rewards/margins": 0.047291629016399384,
      "rewards/rejected": -0.010835668072104454,
      "step": 543
    },
    {
      "epoch": 0.12638671080908404,
      "grad_norm": 21.219676971435547,
      "learning_rate": 8.738675958188154e-06,
      "logits/chosen": 0.37853503227233887,
      "logits/rejected": 0.4084469676017761,
      "logps/chosen": -72.76285552978516,
      "logps/rejected": -72.22976684570312,
      "loss": 0.5819,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.039008114486932755,
      "rewards/margins": 0.2812792956829071,
      "rewards/rejected": -0.24227117002010345,
      "step": 544
    },
    {
      "epoch": 0.12661903932160073,
      "grad_norm": 22.64232063293457,
      "learning_rate": 8.736353077816493e-06,
      "logits/chosen": 0.3383888900279999,
      "logits/rejected": 0.3554133474826813,
      "logps/chosen": -85.99208068847656,
      "logps/rejected": -97.73290252685547,
      "loss": 0.6254,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12585297226905823,
      "rewards/margins": 0.5755083560943604,
      "rewards/rejected": -0.7013612985610962,
      "step": 545
    },
    {
      "epoch": 0.12685136783411743,
      "grad_norm": 25.058366775512695,
      "learning_rate": 8.734030197444832e-06,
      "logits/chosen": 0.6501442193984985,
      "logits/rejected": 0.676155149936676,
      "logps/chosen": -98.84406280517578,
      "logps/rejected": -98.70603942871094,
      "loss": 0.6425,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16253572702407837,
      "rewards/margins": 0.14683276414871216,
      "rewards/rejected": -0.30936846137046814,
      "step": 546
    },
    {
      "epoch": 0.12708369634663413,
      "grad_norm": 22.253082275390625,
      "learning_rate": 8.731707317073171e-06,
      "logits/chosen": 0.42336902022361755,
      "logits/rejected": 0.3249603509902954,
      "logps/chosen": -84.03495788574219,
      "logps/rejected": -80.11445617675781,
      "loss": 0.6591,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08847036957740784,
      "rewards/margins": 0.14276817440986633,
      "rewards/rejected": -0.054297808557748795,
      "step": 547
    },
    {
      "epoch": 0.12731602485915083,
      "grad_norm": 23.354141235351562,
      "learning_rate": 8.72938443670151e-06,
      "logits/chosen": 0.4838576912879944,
      "logits/rejected": 0.5332684516906738,
      "logps/chosen": -77.96820831298828,
      "logps/rejected": -78.98007202148438,
      "loss": 0.6027,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1107269823551178,
      "rewards/margins": 0.2939005196094513,
      "rewards/rejected": -0.1831735372543335,
      "step": 548
    },
    {
      "epoch": 0.12754835337166753,
      "grad_norm": 25.668174743652344,
      "learning_rate": 8.72706155632985e-06,
      "logits/chosen": 1.9536212682724,
      "logits/rejected": 2.0489134788513184,
      "logps/chosen": -85.19493865966797,
      "logps/rejected": -83.2860107421875,
      "loss": 0.7123,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17526310682296753,
      "rewards/margins": 0.004743322730064392,
      "rewards/rejected": -0.18000641465187073,
      "step": 549
    },
    {
      "epoch": 0.12778068188418423,
      "grad_norm": 24.715614318847656,
      "learning_rate": 8.72473867595819e-06,
      "logits/chosen": 0.37745600938796997,
      "logits/rejected": 0.08677195012569427,
      "logps/chosen": -76.25567626953125,
      "logps/rejected": -105.64627838134766,
      "loss": 0.6566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.031120017170906067,
      "rewards/margins": 0.08609211444854736,
      "rewards/rejected": -0.11721213161945343,
      "step": 550
    },
    {
      "epoch": 0.12801301039670093,
      "grad_norm": 22.665287017822266,
      "learning_rate": 8.722415795586528e-06,
      "logits/chosen": 0.532903790473938,
      "logits/rejected": 0.5766276121139526,
      "logps/chosen": -79.2843246459961,
      "logps/rejected": -86.39204406738281,
      "loss": 0.6093,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03627428784966469,
      "rewards/margins": 0.21149732172489166,
      "rewards/rejected": -0.17522300779819489,
      "step": 551
    },
    {
      "epoch": 0.12824533890921763,
      "grad_norm": 24.26068687438965,
      "learning_rate": 8.720092915214867e-06,
      "logits/chosen": 0.6242964267730713,
      "logits/rejected": 0.626185417175293,
      "logps/chosen": -74.04817199707031,
      "logps/rejected": -86.6812744140625,
      "loss": 0.6662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07316651195287704,
      "rewards/margins": 0.0650002583861351,
      "rewards/rejected": -0.13816677033901215,
      "step": 552
    },
    {
      "epoch": 0.12847766742173433,
      "grad_norm": 23.965829849243164,
      "learning_rate": 8.717770034843206e-06,
      "logits/chosen": 0.5285652279853821,
      "logits/rejected": 0.49335435032844543,
      "logps/chosen": -81.23477172851562,
      "logps/rejected": -88.36480712890625,
      "loss": 0.6056,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12847286462783813,
      "rewards/margins": 0.33581191301345825,
      "rewards/rejected": -0.20733904838562012,
      "step": 553
    },
    {
      "epoch": 0.12870999593425103,
      "grad_norm": 22.54737663269043,
      "learning_rate": 8.715447154471545e-06,
      "logits/chosen": 0.8201939463615417,
      "logits/rejected": 0.7242713570594788,
      "logps/chosen": -82.11538696289062,
      "logps/rejected": -81.70091247558594,
      "loss": 0.6494,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06083095073699951,
      "rewards/margins": 0.1072455495595932,
      "rewards/rejected": -0.04641461744904518,
      "step": 554
    },
    {
      "epoch": 0.12894232444676773,
      "grad_norm": 23.515981674194336,
      "learning_rate": 8.713124274099884e-06,
      "logits/chosen": 1.0310269594192505,
      "logits/rejected": 1.0805600881576538,
      "logps/chosen": -73.30989074707031,
      "logps/rejected": -80.57134246826172,
      "loss": 0.6737,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.11726661026477814,
      "rewards/margins": 0.069911427795887,
      "rewards/rejected": 0.047355178743600845,
      "step": 555
    },
    {
      "epoch": 0.12917465295928443,
      "grad_norm": 22.19605255126953,
      "learning_rate": 8.710801393728223e-06,
      "logits/chosen": 1.257031798362732,
      "logits/rejected": 1.0438451766967773,
      "logps/chosen": -64.03507995605469,
      "logps/rejected": -86.89727783203125,
      "loss": 0.5649,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24236005544662476,
      "rewards/margins": 0.5399588346481323,
      "rewards/rejected": -0.2975987493991852,
      "step": 556
    },
    {
      "epoch": 0.12940698147180113,
      "grad_norm": 24.121170043945312,
      "learning_rate": 8.708478513356564e-06,
      "logits/chosen": 0.6886366009712219,
      "logits/rejected": 0.7320111393928528,
      "logps/chosen": -82.05078125,
      "logps/rejected": -92.26451110839844,
      "loss": 0.5403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09053217619657516,
      "rewards/margins": 0.41122105717658997,
      "rewards/rejected": -0.3206888735294342,
      "step": 557
    },
    {
      "epoch": 0.12963930998431783,
      "grad_norm": 23.687442779541016,
      "learning_rate": 8.706155632984903e-06,
      "logits/chosen": 0.1203341856598854,
      "logits/rejected": 0.08697082847356796,
      "logps/chosen": -72.87451934814453,
      "logps/rejected": -86.0804672241211,
      "loss": 0.6827,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01776833087205887,
      "rewards/margins": 0.05198178440332413,
      "rewards/rejected": -0.06975012272596359,
      "step": 558
    },
    {
      "epoch": 0.12987163849683453,
      "grad_norm": 23.69328498840332,
      "learning_rate": 8.703832752613241e-06,
      "logits/chosen": 1.0979868173599243,
      "logits/rejected": 0.9597607851028442,
      "logps/chosen": -78.90464782714844,
      "logps/rejected": -82.19535064697266,
      "loss": 0.6764,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04742243140935898,
      "rewards/margins": 0.05344469100236893,
      "rewards/rejected": -0.1008671298623085,
      "step": 559
    },
    {
      "epoch": 0.13010396700935123,
      "grad_norm": 23.674518585205078,
      "learning_rate": 8.70150987224158e-06,
      "logits/chosen": 0.23871417343616486,
      "logits/rejected": 0.23671571910381317,
      "logps/chosen": -80.48612213134766,
      "logps/rejected": -84.36575317382812,
      "loss": 0.7105,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04207267612218857,
      "rewards/margins": -0.025444697588682175,
      "rewards/rejected": -0.016627974808216095,
      "step": 560
    },
    {
      "epoch": 0.13033629552186793,
      "grad_norm": 25.048908233642578,
      "learning_rate": 8.69918699186992e-06,
      "logits/chosen": 0.5115377902984619,
      "logits/rejected": 0.507036566734314,
      "logps/chosen": -99.52218627929688,
      "logps/rejected": -105.9583511352539,
      "loss": 0.5959,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.021596193313598633,
      "rewards/margins": 0.47017335891723633,
      "rewards/rejected": -0.49176955223083496,
      "step": 561
    },
    {
      "epoch": 0.13056862403438463,
      "grad_norm": 24.84480094909668,
      "learning_rate": 8.696864111498258e-06,
      "logits/chosen": 0.3044716417789459,
      "logits/rejected": 0.08370860666036606,
      "logps/chosen": -72.6453857421875,
      "logps/rejected": -103.91632080078125,
      "loss": 0.6379,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023732569068670273,
      "rewards/margins": 0.14705580472946167,
      "rewards/rejected": -0.12332325428724289,
      "step": 562
    },
    {
      "epoch": 0.13080095254690133,
      "grad_norm": 20.509920120239258,
      "learning_rate": 8.694541231126597e-06,
      "logits/chosen": 0.5437986850738525,
      "logits/rejected": 0.7216073274612427,
      "logps/chosen": -80.95158386230469,
      "logps/rejected": -85.23023986816406,
      "loss": 0.5747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2855551540851593,
      "rewards/margins": 0.38682833313941956,
      "rewards/rejected": -0.10127312690019608,
      "step": 563
    },
    {
      "epoch": 0.13103328105941803,
      "grad_norm": 26.024703979492188,
      "learning_rate": 8.692218350754938e-06,
      "logits/chosen": 0.5552992820739746,
      "logits/rejected": 0.4469509422779083,
      "logps/chosen": -97.3768081665039,
      "logps/rejected": -85.38184356689453,
      "loss": 0.6502,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0052433498203754425,
      "rewards/margins": 0.1991616189479828,
      "rewards/rejected": -0.19391828775405884,
      "step": 564
    },
    {
      "epoch": 0.13126560957193473,
      "grad_norm": 23.222383499145508,
      "learning_rate": 8.689895470383277e-06,
      "logits/chosen": 0.031402960419654846,
      "logits/rejected": 0.13382834196090698,
      "logps/chosen": -81.17802429199219,
      "logps/rejected": -80.03092193603516,
      "loss": 0.6553,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08132036030292511,
      "rewards/margins": 0.08523771166801453,
      "rewards/rejected": -0.16655808687210083,
      "step": 565
    },
    {
      "epoch": 0.13149793808445143,
      "grad_norm": 22.6265811920166,
      "learning_rate": 8.687572590011614e-06,
      "logits/chosen": -0.006730616092681885,
      "logits/rejected": 0.02525077573955059,
      "logps/chosen": -69.15617370605469,
      "logps/rejected": -77.56734466552734,
      "loss": 0.5969,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05235423892736435,
      "rewards/margins": 0.24950745701789856,
      "rewards/rejected": -0.197153240442276,
      "step": 566
    },
    {
      "epoch": 0.13173026659696813,
      "grad_norm": 22.822717666625977,
      "learning_rate": 8.685249709639955e-06,
      "logits/chosen": 0.18416516482830048,
      "logits/rejected": 0.23104903101921082,
      "logps/chosen": -79.74395751953125,
      "logps/rejected": -84.16267395019531,
      "loss": 0.547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13446158170700073,
      "rewards/margins": 0.38442254066467285,
      "rewards/rejected": -0.2499609887599945,
      "step": 567
    },
    {
      "epoch": 0.1319625951094848,
      "grad_norm": 24.46898651123047,
      "learning_rate": 8.682926829268294e-06,
      "logits/chosen": 0.1486358791589737,
      "logits/rejected": -0.009972227737307549,
      "logps/chosen": -79.84512329101562,
      "logps/rejected": -80.2905044555664,
      "loss": 0.7107,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.032372210174798965,
      "rewards/margins": -0.02262284606695175,
      "rewards/rejected": -0.009749367833137512,
      "step": 568
    },
    {
      "epoch": 0.1321949236220015,
      "grad_norm": 20.911357879638672,
      "learning_rate": 8.680603948896633e-06,
      "logits/chosen": 0.02602256089448929,
      "logits/rejected": 0.1264408975839615,
      "logps/chosen": -74.1099853515625,
      "logps/rejected": -84.59329986572266,
      "loss": 0.6085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07757416367530823,
      "rewards/margins": 0.23982879519462585,
      "rewards/rejected": -0.16225463151931763,
      "step": 569
    },
    {
      "epoch": 0.1324272521345182,
      "grad_norm": 24.317331314086914,
      "learning_rate": 8.678281068524971e-06,
      "logits/chosen": 0.8721495270729065,
      "logits/rejected": 0.992946207523346,
      "logps/chosen": -89.37006378173828,
      "logps/rejected": -86.99954223632812,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13358443975448608,
      "rewards/margins": 0.02060697041451931,
      "rewards/rejected": -0.15419141948223114,
      "step": 570
    },
    {
      "epoch": 0.1326595806470349,
      "grad_norm": 22.80604362487793,
      "learning_rate": 8.67595818815331e-06,
      "logits/chosen": 0.5260655283927917,
      "logits/rejected": 0.5996692180633545,
      "logps/chosen": -75.68953704833984,
      "logps/rejected": -82.74667358398438,
      "loss": 0.6505,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016774792224168777,
      "rewards/margins": 0.14412912726402283,
      "rewards/rejected": -0.12735432386398315,
      "step": 571
    },
    {
      "epoch": 0.1328919091595516,
      "grad_norm": 24.484893798828125,
      "learning_rate": 8.673635307781651e-06,
      "logits/chosen": 0.26054689288139343,
      "logits/rejected": 0.6427192687988281,
      "logps/chosen": -84.93595123291016,
      "logps/rejected": -76.83537292480469,
      "loss": 0.6695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01507829874753952,
      "rewards/margins": 0.08763006329536438,
      "rewards/rejected": -0.07255174964666367,
      "step": 572
    },
    {
      "epoch": 0.1331242376720683,
      "grad_norm": 26.38052749633789,
      "learning_rate": 8.67131242740999e-06,
      "logits/chosen": 0.24545219540596008,
      "logits/rejected": 0.3526630103588104,
      "logps/chosen": -79.96672821044922,
      "logps/rejected": -80.79080200195312,
      "loss": 0.6557,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021848510950803757,
      "rewards/margins": 0.1360602229833603,
      "rewards/rejected": -0.15790872275829315,
      "step": 573
    },
    {
      "epoch": 0.133356566184585,
      "grad_norm": 27.471899032592773,
      "learning_rate": 8.668989547038327e-06,
      "logits/chosen": 0.3465893268585205,
      "logits/rejected": 0.3386125862598419,
      "logps/chosen": -84.08483123779297,
      "logps/rejected": -94.60747528076172,
      "loss": 0.7299,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.07734479755163193,
      "rewards/margins": -0.05080196261405945,
      "rewards/rejected": -0.02654283121228218,
      "step": 574
    },
    {
      "epoch": 0.1335888946971017,
      "grad_norm": 22.628583908081055,
      "learning_rate": 8.666666666666668e-06,
      "logits/chosen": -0.006933167576789856,
      "logits/rejected": 0.024470489472150803,
      "logps/chosen": -77.53765106201172,
      "logps/rejected": -105.8631362915039,
      "loss": 0.6216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02941306121647358,
      "rewards/margins": 0.25842612981796265,
      "rewards/rejected": -0.2878391742706299,
      "step": 575
    },
    {
      "epoch": 0.1338212232096184,
      "grad_norm": 20.274080276489258,
      "learning_rate": 8.664343786295007e-06,
      "logits/chosen": 0.49881425499916077,
      "logits/rejected": 0.6223765015602112,
      "logps/chosen": -71.94429779052734,
      "logps/rejected": -71.90206909179688,
      "loss": 0.5825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09643368422985077,
      "rewards/margins": 0.2932032346725464,
      "rewards/rejected": -0.19676955044269562,
      "step": 576
    },
    {
      "epoch": 0.1340535517221351,
      "grad_norm": 22.266284942626953,
      "learning_rate": 8.662020905923346e-06,
      "logits/chosen": 0.2462896704673767,
      "logits/rejected": 0.37320059537887573,
      "logps/chosen": -78.28266143798828,
      "logps/rejected": -79.86727905273438,
      "loss": 0.6195,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04446844011545181,
      "rewards/margins": 0.17338338494300842,
      "rewards/rejected": -0.21785181760787964,
      "step": 577
    },
    {
      "epoch": 0.1342858802346518,
      "grad_norm": 25.21910285949707,
      "learning_rate": 8.659698025551685e-06,
      "logits/chosen": 0.9272564053535461,
      "logits/rejected": 1.040421962738037,
      "logps/chosen": -75.04714965820312,
      "logps/rejected": -71.6305160522461,
      "loss": 0.7058,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 1.9431114196777344e-05,
      "rewards/margins": 0.002296239137649536,
      "rewards/rejected": -0.002276793122291565,
      "step": 578
    },
    {
      "epoch": 0.1345182087471685,
      "grad_norm": 26.494054794311523,
      "learning_rate": 8.657375145180024e-06,
      "logits/chosen": 0.33419492840766907,
      "logits/rejected": 0.4939333498477936,
      "logps/chosen": -81.0933609008789,
      "logps/rejected": -90.06700897216797,
      "loss": 0.7061,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07143183052539825,
      "rewards/margins": -0.0010411739349365234,
      "rewards/rejected": 0.07247301191091537,
      "step": 579
    },
    {
      "epoch": 0.1347505372596852,
      "grad_norm": 23.55759620666504,
      "learning_rate": 8.655052264808364e-06,
      "logits/chosen": 0.3710951805114746,
      "logits/rejected": 0.5065730214118958,
      "logps/chosen": -77.71990966796875,
      "logps/rejected": -80.43793487548828,
      "loss": 0.6465,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.033986713737249374,
      "rewards/margins": 0.16484305262565613,
      "rewards/rejected": -0.13085633516311646,
      "step": 580
    },
    {
      "epoch": 0.1349828657722019,
      "grad_norm": 23.162578582763672,
      "learning_rate": 8.652729384436701e-06,
      "logits/chosen": 0.04986168444156647,
      "logits/rejected": 0.004877574276179075,
      "logps/chosen": -85.5745849609375,
      "logps/rejected": -82.02391052246094,
      "loss": 0.6368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.005022665951400995,
      "rewards/margins": 0.21131859719753265,
      "rewards/rejected": -0.21634124219417572,
      "step": 581
    },
    {
      "epoch": 0.1352151942847186,
      "grad_norm": 22.047964096069336,
      "learning_rate": 8.65040650406504e-06,
      "logits/chosen": 1.2216522693634033,
      "logits/rejected": 1.3980563879013062,
      "logps/chosen": -75.49059295654297,
      "logps/rejected": -86.3515853881836,
      "loss": 0.5577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19587761163711548,
      "rewards/margins": 0.5883809328079224,
      "rewards/rejected": -0.3925033211708069,
      "step": 582
    },
    {
      "epoch": 0.1354475227972353,
      "grad_norm": 17.783794403076172,
      "learning_rate": 8.648083623693381e-06,
      "logits/chosen": -0.043755654245615005,
      "logits/rejected": -0.030786005780100822,
      "logps/chosen": -73.81185913085938,
      "logps/rejected": -86.4852066040039,
      "loss": 0.4351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15890108048915863,
      "rewards/margins": 0.6842694282531738,
      "rewards/rejected": -0.5253683924674988,
      "step": 583
    },
    {
      "epoch": 0.135679851309752,
      "grad_norm": 19.121116638183594,
      "learning_rate": 8.64576074332172e-06,
      "logits/chosen": -0.11130407452583313,
      "logits/rejected": -0.07987996935844421,
      "logps/chosen": -71.8948974609375,
      "logps/rejected": -89.10261535644531,
      "loss": 0.4679,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2974192500114441,
      "rewards/margins": 0.573948085308075,
      "rewards/rejected": -0.27652883529663086,
      "step": 584
    },
    {
      "epoch": 0.1359121798222687,
      "grad_norm": 21.050886154174805,
      "learning_rate": 8.643437862950059e-06,
      "logits/chosen": 0.3606015741825104,
      "logits/rejected": 0.24677115678787231,
      "logps/chosen": -69.42543029785156,
      "logps/rejected": -80.55502319335938,
      "loss": 0.6048,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12106907367706299,
      "rewards/margins": 0.25442925095558167,
      "rewards/rejected": -0.1333601474761963,
      "step": 585
    },
    {
      "epoch": 0.1361445083347854,
      "grad_norm": 19.158647537231445,
      "learning_rate": 8.641114982578398e-06,
      "logits/chosen": -0.03530559688806534,
      "logits/rejected": -0.011369075626134872,
      "logps/chosen": -74.73128509521484,
      "logps/rejected": -82.15069580078125,
      "loss": 0.509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1202985942363739,
      "rewards/margins": 0.4772355556488037,
      "rewards/rejected": -0.3569369614124298,
      "step": 586
    },
    {
      "epoch": 0.1363768368473021,
      "grad_norm": 23.908334732055664,
      "learning_rate": 8.638792102206737e-06,
      "logits/chosen": 0.49299371242523193,
      "logits/rejected": 0.6372601389884949,
      "logps/chosen": -81.5633316040039,
      "logps/rejected": -80.48040771484375,
      "loss": 0.6687,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06477221101522446,
      "rewards/margins": 0.1490744650363922,
      "rewards/rejected": -0.08430229127407074,
      "step": 587
    },
    {
      "epoch": 0.1366091653598188,
      "grad_norm": 26.528608322143555,
      "learning_rate": 8.636469221835077e-06,
      "logits/chosen": -0.15357443690299988,
      "logits/rejected": -0.11309854686260223,
      "logps/chosen": -79.13450622558594,
      "logps/rejected": -80.89141845703125,
      "loss": 0.6544,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06394829601049423,
      "rewards/margins": 0.09373831748962402,
      "rewards/rejected": -0.029790017753839493,
      "step": 588
    },
    {
      "epoch": 0.1368414938723355,
      "grad_norm": 24.678556442260742,
      "learning_rate": 8.634146341463415e-06,
      "logits/chosen": -0.3125719428062439,
      "logits/rejected": -0.07780162990093231,
      "logps/chosen": -74.81671905517578,
      "logps/rejected": -85.58979797363281,
      "loss": 0.6792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03345717862248421,
      "rewards/margins": 0.03698401898145676,
      "rewards/rejected": -0.0035268308129161596,
      "step": 589
    },
    {
      "epoch": 0.1370738223848522,
      "grad_norm": 25.536800384521484,
      "learning_rate": 8.631823461091754e-06,
      "logits/chosen": -0.27999839186668396,
      "logits/rejected": -0.269050657749176,
      "logps/chosen": -80.29639434814453,
      "logps/rejected": -77.236083984375,
      "loss": 0.7082,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.025420164689421654,
      "rewards/margins": 0.04519219696521759,
      "rewards/rejected": -0.0706123635172844,
      "step": 590
    },
    {
      "epoch": 0.1373061508973689,
      "grad_norm": 23.12672233581543,
      "learning_rate": 8.629500580720094e-06,
      "logits/chosen": 0.44496339559555054,
      "logits/rejected": 0.5962374210357666,
      "logps/chosen": -71.26696014404297,
      "logps/rejected": -77.49341583251953,
      "loss": 0.7068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.043479084968566895,
      "rewards/margins": 0.07048483192920685,
      "rewards/rejected": -0.027005746960639954,
      "step": 591
    },
    {
      "epoch": 0.1375384794098856,
      "grad_norm": 21.85063934326172,
      "learning_rate": 8.627177700348433e-06,
      "logits/chosen": -0.265463262796402,
      "logits/rejected": -0.1978680044412613,
      "logps/chosen": -77.01426696777344,
      "logps/rejected": -79.62828063964844,
      "loss": 0.6329,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05772972106933594,
      "rewards/margins": 0.1892668455839157,
      "rewards/rejected": -0.24699656665325165,
      "step": 592
    },
    {
      "epoch": 0.1377708079224023,
      "grad_norm": 22.02198600769043,
      "learning_rate": 8.624854819976772e-06,
      "logits/chosen": -0.16484025120735168,
      "logits/rejected": -0.145332470536232,
      "logps/chosen": -82.9950180053711,
      "logps/rejected": -80.85955810546875,
      "loss": 0.5817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05215387046337128,
      "rewards/margins": 0.301286906003952,
      "rewards/rejected": -0.24913306534290314,
      "step": 593
    },
    {
      "epoch": 0.138003136434919,
      "grad_norm": 23.354734420776367,
      "learning_rate": 8.622531939605111e-06,
      "logits/chosen": 0.5551037788391113,
      "logits/rejected": 0.5900536179542542,
      "logps/chosen": -73.3014144897461,
      "logps/rejected": -85.37810516357422,
      "loss": 0.6426,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07789413630962372,
      "rewards/margins": 0.1307157725095749,
      "rewards/rejected": -0.05282163619995117,
      "step": 594
    },
    {
      "epoch": 0.1382354649474357,
      "grad_norm": 25.96752166748047,
      "learning_rate": 8.62020905923345e-06,
      "logits/chosen": 0.13793224096298218,
      "logits/rejected": 0.17446477711200714,
      "logps/chosen": -77.80081176757812,
      "logps/rejected": -77.35057067871094,
      "loss": 0.7317,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0820891484618187,
      "rewards/margins": -0.025476932525634766,
      "rewards/rejected": -0.056612201035022736,
      "step": 595
    },
    {
      "epoch": 0.13846779345995236,
      "grad_norm": 33.496402740478516,
      "learning_rate": 8.617886178861789e-06,
      "logits/chosen": 0.2947934567928314,
      "logits/rejected": 0.36450648307800293,
      "logps/chosen": -98.26433563232422,
      "logps/rejected": -81.25420379638672,
      "loss": 0.8172,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.2180672287940979,
      "rewards/margins": -0.16354592144489288,
      "rewards/rejected": -0.05452127754688263,
      "step": 596
    },
    {
      "epoch": 0.13870012197246906,
      "grad_norm": 20.01543426513672,
      "learning_rate": 8.615563298490128e-06,
      "logits/chosen": -0.35370805859565735,
      "logits/rejected": -0.3649233877658844,
      "logps/chosen": -81.58207702636719,
      "logps/rejected": -80.13805389404297,
      "loss": 0.5726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11759376525878906,
      "rewards/margins": 0.3453190326690674,
      "rewards/rejected": -0.22772526741027832,
      "step": 597
    },
    {
      "epoch": 0.13893245048498576,
      "grad_norm": 23.170923233032227,
      "learning_rate": 8.613240418118468e-06,
      "logits/chosen": 0.928661048412323,
      "logits/rejected": 0.6789621710777283,
      "logps/chosen": -82.96235656738281,
      "logps/rejected": -69.38684844970703,
      "loss": 0.6988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11939531564712524,
      "rewards/margins": 0.03703214228153229,
      "rewards/rejected": -0.15642745792865753,
      "step": 598
    },
    {
      "epoch": 0.13916477899750246,
      "grad_norm": 28.197948455810547,
      "learning_rate": 8.610917537746807e-06,
      "logits/chosen": -0.39316990971565247,
      "logits/rejected": -0.31033051013946533,
      "logps/chosen": -85.68014526367188,
      "logps/rejected": -89.97817993164062,
      "loss": 0.7831,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.23342344164848328,
      "rewards/margins": -0.15595375001430511,
      "rewards/rejected": -0.07746967673301697,
      "step": 599
    },
    {
      "epoch": 0.13939710751001916,
      "grad_norm": 23.17230224609375,
      "learning_rate": 8.608594657375146e-06,
      "logits/chosen": -0.49037569761276245,
      "logits/rejected": -0.32981523871421814,
      "logps/chosen": -77.62451934814453,
      "logps/rejected": -82.32162475585938,
      "loss": 0.6481,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06847431510686874,
      "rewards/margins": 0.15721943974494934,
      "rewards/rejected": -0.2256937325000763,
      "step": 600
    },
    {
      "epoch": 0.13962943602253586,
      "grad_norm": 25.057819366455078,
      "learning_rate": 8.606271777003485e-06,
      "logits/chosen": 0.13330450654029846,
      "logits/rejected": 0.06885156035423279,
      "logps/chosen": -91.26261901855469,
      "logps/rejected": -79.83019256591797,
      "loss": 0.6654,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03377387672662735,
      "rewards/margins": 0.0856657475233078,
      "rewards/rejected": -0.11943962424993515,
      "step": 601
    },
    {
      "epoch": 0.13986176453505256,
      "grad_norm": 25.39658546447754,
      "learning_rate": 8.603948896631824e-06,
      "logits/chosen": 0.4679776132106781,
      "logits/rejected": 0.5753879547119141,
      "logps/chosen": -70.84107971191406,
      "logps/rejected": -84.85750579833984,
      "loss": 0.7335,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.04287745803594589,
      "rewards/margins": -0.07351770251989365,
      "rewards/rejected": 0.030640240758657455,
      "step": 602
    },
    {
      "epoch": 0.14009409304756926,
      "grad_norm": 21.006519317626953,
      "learning_rate": 8.601626016260163e-06,
      "logits/chosen": -0.10910596698522568,
      "logits/rejected": -0.18441450595855713,
      "logps/chosen": -73.87394714355469,
      "logps/rejected": -71.05423736572266,
      "loss": 0.6214,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.21237245202064514,
      "rewards/margins": 0.2592678964138031,
      "rewards/rejected": -0.04689546301960945,
      "step": 603
    },
    {
      "epoch": 0.14032642156008596,
      "grad_norm": 23.060060501098633,
      "learning_rate": 8.599303135888502e-06,
      "logits/chosen": 0.5369699597358704,
      "logits/rejected": 0.44573307037353516,
      "logps/chosen": -72.92021179199219,
      "logps/rejected": -99.73809051513672,
      "loss": 0.6113,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13351160287857056,
      "rewards/margins": 0.2514183521270752,
      "rewards/rejected": -0.11790674924850464,
      "step": 604
    },
    {
      "epoch": 0.14055875007260266,
      "grad_norm": 21.624181747436523,
      "learning_rate": 8.596980255516841e-06,
      "logits/chosen": 0.23285573720932007,
      "logits/rejected": 0.21008817851543427,
      "logps/chosen": -79.52802276611328,
      "logps/rejected": -77.38002014160156,
      "loss": 0.5973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025502562522888184,
      "rewards/margins": 0.26423147320747375,
      "rewards/rejected": -0.23872891068458557,
      "step": 605
    },
    {
      "epoch": 0.14079107858511936,
      "grad_norm": 19.043903350830078,
      "learning_rate": 8.594657375145182e-06,
      "logits/chosen": 0.06741093099117279,
      "logits/rejected": 0.06256907433271408,
      "logps/chosen": -75.7883071899414,
      "logps/rejected": -81.62615203857422,
      "loss": 0.558,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1558675467967987,
      "rewards/margins": 0.36665982007980347,
      "rewards/rejected": -0.21079230308532715,
      "step": 606
    },
    {
      "epoch": 0.14102340709763606,
      "grad_norm": 24.369232177734375,
      "learning_rate": 8.59233449477352e-06,
      "logits/chosen": 0.41103595495224,
      "logits/rejected": 0.22451937198638916,
      "logps/chosen": -76.192138671875,
      "logps/rejected": -77.28544616699219,
      "loss": 0.7189,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06859025359153748,
      "rewards/margins": -0.02983374148607254,
      "rewards/rejected": -0.038756512105464935,
      "step": 607
    },
    {
      "epoch": 0.14125573561015276,
      "grad_norm": 23.18673324584961,
      "learning_rate": 8.59001161440186e-06,
      "logits/chosen": 0.17596983909606934,
      "logits/rejected": 0.20248636603355408,
      "logps/chosen": -77.16468811035156,
      "logps/rejected": -71.236572265625,
      "loss": 0.6531,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029780961573123932,
      "rewards/margins": 0.11214125156402588,
      "rewards/rejected": -0.08236029744148254,
      "step": 608
    },
    {
      "epoch": 0.14148806412266945,
      "grad_norm": 24.933197021484375,
      "learning_rate": 8.587688734030198e-06,
      "logits/chosen": -0.4485866129398346,
      "logits/rejected": -0.25600942969322205,
      "logps/chosen": -68.56037902832031,
      "logps/rejected": -77.36195373535156,
      "loss": 0.6949,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.07707007229328156,
      "rewards/margins": 0.03696971759200096,
      "rewards/rejected": 0.04010036215186119,
      "step": 609
    },
    {
      "epoch": 0.14172039263518615,
      "grad_norm": 19.87164306640625,
      "learning_rate": 8.585365853658537e-06,
      "logits/chosen": -0.3054876923561096,
      "logits/rejected": -0.17142489552497864,
      "logps/chosen": -77.3244857788086,
      "logps/rejected": -85.48760223388672,
      "loss": 0.5337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.059755466878414154,
      "rewards/margins": 0.3868666887283325,
      "rewards/rejected": -0.32711124420166016,
      "step": 610
    },
    {
      "epoch": 0.14195272114770285,
      "grad_norm": 23.442176818847656,
      "learning_rate": 8.583042973286876e-06,
      "logits/chosen": -0.3037230968475342,
      "logits/rejected": -0.5278717279434204,
      "logps/chosen": -83.1868667602539,
      "logps/rejected": -78.30570220947266,
      "loss": 0.6792,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04448135197162628,
      "rewards/margins": 0.04858224093914032,
      "rewards/rejected": -0.0930635929107666,
      "step": 611
    },
    {
      "epoch": 0.14218504966021955,
      "grad_norm": 24.168121337890625,
      "learning_rate": 8.580720092915215e-06,
      "logits/chosen": 0.2972860336303711,
      "logits/rejected": 0.2219795137643814,
      "logps/chosen": -81.01253509521484,
      "logps/rejected": -79.48394012451172,
      "loss": 0.5722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05367477238178253,
      "rewards/margins": 0.29460084438323975,
      "rewards/rejected": -0.24092605710029602,
      "step": 612
    },
    {
      "epoch": 0.14241737817273625,
      "grad_norm": 21.80843162536621,
      "learning_rate": 8.578397212543554e-06,
      "logits/chosen": 0.0016882121562957764,
      "logits/rejected": 0.14960609376430511,
      "logps/chosen": -77.99195861816406,
      "logps/rejected": -89.90005493164062,
      "loss": 0.6017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.041881490498781204,
      "rewards/margins": 0.28262531757354736,
      "rewards/rejected": -0.3245067894458771,
      "step": 613
    },
    {
      "epoch": 0.14264970668525295,
      "grad_norm": 22.019601821899414,
      "learning_rate": 8.576074332171895e-06,
      "logits/chosen": -0.2860247790813446,
      "logits/rejected": -0.07073080539703369,
      "logps/chosen": -82.59012603759766,
      "logps/rejected": -77.56452178955078,
      "loss": 0.6236,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07563353329896927,
      "rewards/margins": 0.18583601713180542,
      "rewards/rejected": -0.11020250618457794,
      "step": 614
    },
    {
      "epoch": 0.14288203519776965,
      "grad_norm": 22.736032485961914,
      "learning_rate": 8.573751451800234e-06,
      "logits/chosen": 0.04109008610248566,
      "logits/rejected": 0.2500365674495697,
      "logps/chosen": -76.92902374267578,
      "logps/rejected": -75.06143951416016,
      "loss": 0.6071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06442301720380783,
      "rewards/margins": 0.19340859353542328,
      "rewards/rejected": -0.12898556888103485,
      "step": 615
    },
    {
      "epoch": 0.14311436371028635,
      "grad_norm": 23.40386199951172,
      "learning_rate": 8.571428571428571e-06,
      "logits/chosen": 0.635275661945343,
      "logits/rejected": 0.6591968536376953,
      "logps/chosen": -78.28147888183594,
      "logps/rejected": -81.10152435302734,
      "loss": 0.6455,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04141230508685112,
      "rewards/margins": 0.1323058307170868,
      "rewards/rejected": -0.1737181544303894,
      "step": 616
    },
    {
      "epoch": 0.14334669222280305,
      "grad_norm": 24.828344345092773,
      "learning_rate": 8.569105691056912e-06,
      "logits/chosen": 0.08573839068412781,
      "logits/rejected": 0.08324771374464035,
      "logps/chosen": -93.69918060302734,
      "logps/rejected": -82.71591186523438,
      "loss": 0.6539,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0022867899388074875,
      "rewards/margins": 0.09347248822450638,
      "rewards/rejected": -0.09575928002595901,
      "step": 617
    },
    {
      "epoch": 0.14357902073531975,
      "grad_norm": 30.13585090637207,
      "learning_rate": 8.56678281068525e-06,
      "logits/chosen": -0.4016076326370239,
      "logits/rejected": -0.4029979705810547,
      "logps/chosen": -93.38761901855469,
      "logps/rejected": -86.40432739257812,
      "loss": 0.7424,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.19547826051712036,
      "rewards/margins": -0.08306261897087097,
      "rewards/rejected": -0.11241564899682999,
      "step": 618
    },
    {
      "epoch": 0.14381134924783645,
      "grad_norm": 25.879398345947266,
      "learning_rate": 8.56445993031359e-06,
      "logits/chosen": -0.2765452265739441,
      "logits/rejected": -0.11111204326152802,
      "logps/chosen": -90.7781753540039,
      "logps/rejected": -81.49108123779297,
      "loss": 0.731,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05483713373541832,
      "rewards/margins": -0.0504070483148098,
      "rewards/rejected": -0.004430077970027924,
      "step": 619
    },
    {
      "epoch": 0.14404367776035315,
      "grad_norm": 26.846019744873047,
      "learning_rate": 8.562137049941928e-06,
      "logits/chosen": 0.368734210729599,
      "logits/rejected": 0.39568179845809937,
      "logps/chosen": -81.5524673461914,
      "logps/rejected": -85.31108856201172,
      "loss": 0.6425,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.08862758427858353,
      "rewards/margins": 0.2496047466993332,
      "rewards/rejected": -0.16097716987133026,
      "step": 620
    },
    {
      "epoch": 0.14427600627286985,
      "grad_norm": 20.893329620361328,
      "learning_rate": 8.559814169570267e-06,
      "logits/chosen": 0.638338029384613,
      "logits/rejected": 0.47362926602363586,
      "logps/chosen": -70.49893188476562,
      "logps/rejected": -86.25874328613281,
      "loss": 0.6044,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.043883245438337326,
      "rewards/margins": 0.2609848380088806,
      "rewards/rejected": -0.217101588845253,
      "step": 621
    },
    {
      "epoch": 0.14450833478538655,
      "grad_norm": 24.00556182861328,
      "learning_rate": 8.557491289198608e-06,
      "logits/chosen": -0.29004818201065063,
      "logits/rejected": -0.2702922821044922,
      "logps/chosen": -78.67828369140625,
      "logps/rejected": -94.1122817993164,
      "loss": 0.6329,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1267620474100113,
      "rewards/margins": 0.14035114645957947,
      "rewards/rejected": -0.26711320877075195,
      "step": 622
    },
    {
      "epoch": 0.14474066329790322,
      "grad_norm": 20.018085479736328,
      "learning_rate": 8.555168408826945e-06,
      "logits/chosen": 1.0093696117401123,
      "logits/rejected": 0.8968405723571777,
      "logps/chosen": -79.43142700195312,
      "logps/rejected": -79.36038208007812,
      "loss": 0.491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25013911724090576,
      "rewards/margins": 0.5779056549072266,
      "rewards/rejected": -0.32776662707328796,
      "step": 623
    },
    {
      "epoch": 0.14497299181041992,
      "grad_norm": 20.12184715270996,
      "learning_rate": 8.552845528455284e-06,
      "logits/chosen": 0.37777411937713623,
      "logits/rejected": 0.49259695410728455,
      "logps/chosen": -83.47976684570312,
      "logps/rejected": -95.62360382080078,
      "loss": 0.5245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11964022368192673,
      "rewards/margins": 0.42038267850875854,
      "rewards/rejected": -0.3007424473762512,
      "step": 624
    },
    {
      "epoch": 0.14520532032293662,
      "grad_norm": 19.76944351196289,
      "learning_rate": 8.550522648083625e-06,
      "logits/chosen": 0.1578509509563446,
      "logits/rejected": 0.2643626928329468,
      "logps/chosen": -78.01579284667969,
      "logps/rejected": -82.5289306640625,
      "loss": 0.5499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16885587573051453,
      "rewards/margins": 0.4869260787963867,
      "rewards/rejected": -0.3180702328681946,
      "step": 625
    },
    {
      "epoch": 0.14543764883545332,
      "grad_norm": 25.00440788269043,
      "learning_rate": 8.548199767711964e-06,
      "logits/chosen": 0.7270134687423706,
      "logits/rejected": 0.8433186411857605,
      "logps/chosen": -71.23796844482422,
      "logps/rejected": -99.71991729736328,
      "loss": 0.7061,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0719643384218216,
      "rewards/margins": 0.13946865499019623,
      "rewards/rejected": -0.21143294870853424,
      "step": 626
    },
    {
      "epoch": 0.14566997734797002,
      "grad_norm": 24.275190353393555,
      "learning_rate": 8.545876887340303e-06,
      "logits/chosen": 0.184414803981781,
      "logits/rejected": 0.1025269627571106,
      "logps/chosen": -82.21334838867188,
      "logps/rejected": -82.92107391357422,
      "loss": 0.6241,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03395172208547592,
      "rewards/margins": 0.26852917671203613,
      "rewards/rejected": -0.3024808466434479,
      "step": 627
    },
    {
      "epoch": 0.14590230586048672,
      "grad_norm": 25.704504013061523,
      "learning_rate": 8.543554006968642e-06,
      "logits/chosen": -0.2075965404510498,
      "logits/rejected": -0.23477870225906372,
      "logps/chosen": -85.4442138671875,
      "logps/rejected": -84.4361801147461,
      "loss": 0.6584,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.049238525331020355,
      "rewards/margins": 0.16258011758327484,
      "rewards/rejected": -0.11334157735109329,
      "step": 628
    },
    {
      "epoch": 0.14613463437300342,
      "grad_norm": 20.35263442993164,
      "learning_rate": 8.54123112659698e-06,
      "logits/chosen": 0.08447454124689102,
      "logits/rejected": -0.029117561876773834,
      "logps/chosen": -70.39878845214844,
      "logps/rejected": -71.51302337646484,
      "loss": 0.5756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04659204185009003,
      "rewards/margins": 0.2725226879119873,
      "rewards/rejected": -0.22593063116073608,
      "step": 629
    },
    {
      "epoch": 0.14636696288552012,
      "grad_norm": 24.28004264831543,
      "learning_rate": 8.538908246225321e-06,
      "logits/chosen": -0.08631223440170288,
      "logits/rejected": -0.02713608555495739,
      "logps/chosen": -70.53822326660156,
      "logps/rejected": -68.19776916503906,
      "loss": 0.774,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.016637444496154785,
      "rewards/margins": -0.14488260447978973,
      "rewards/rejected": 0.1615200638771057,
      "step": 630
    },
    {
      "epoch": 0.14659929139803682,
      "grad_norm": 26.216236114501953,
      "learning_rate": 8.536585365853658e-06,
      "logits/chosen": -0.06332609802484512,
      "logits/rejected": 0.026132158935070038,
      "logps/chosen": -95.01773071289062,
      "logps/rejected": -96.34671783447266,
      "loss": 0.6651,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15536855161190033,
      "rewards/margins": 0.08115637302398682,
      "rewards/rejected": -0.23652490973472595,
      "step": 631
    },
    {
      "epoch": 0.14683161991055352,
      "grad_norm": 20.50075340270996,
      "learning_rate": 8.534262485481999e-06,
      "logits/chosen": 1.1581652164459229,
      "logits/rejected": 1.1240649223327637,
      "logps/chosen": -76.7276382446289,
      "logps/rejected": -77.94880676269531,
      "loss": 0.5748,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11279003322124481,
      "rewards/margins": 0.3293229341506958,
      "rewards/rejected": -0.2165328562259674,
      "step": 632
    },
    {
      "epoch": 0.14706394842307022,
      "grad_norm": 48.67193603515625,
      "learning_rate": 8.531939605110338e-06,
      "logits/chosen": -0.0557730458676815,
      "logits/rejected": -0.2311905026435852,
      "logps/chosen": -106.13555908203125,
      "logps/rejected": -90.39732360839844,
      "loss": 0.6469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00856504775583744,
      "rewards/margins": 0.1092992052435875,
      "rewards/rejected": -0.11786425113677979,
      "step": 633
    },
    {
      "epoch": 0.14729627693558692,
      "grad_norm": 23.304683685302734,
      "learning_rate": 8.529616724738677e-06,
      "logits/chosen": -0.022517118602991104,
      "logits/rejected": -0.04848940670490265,
      "logps/chosen": -65.61835479736328,
      "logps/rejected": -82.09515380859375,
      "loss": 0.6612,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06610694527626038,
      "rewards/margins": 0.09764274954795837,
      "rewards/rejected": -0.16374969482421875,
      "step": 634
    },
    {
      "epoch": 0.14752860544810362,
      "grad_norm": 21.204118728637695,
      "learning_rate": 8.527293844367016e-06,
      "logits/chosen": 0.009289365261793137,
      "logits/rejected": 0.1320609748363495,
      "logps/chosen": -77.50738525390625,
      "logps/rejected": -76.28822326660156,
      "loss": 0.571,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05845758691430092,
      "rewards/margins": 0.2835923731327057,
      "rewards/rejected": -0.22513480484485626,
      "step": 635
    },
    {
      "epoch": 0.14776093396062032,
      "grad_norm": 20.30323028564453,
      "learning_rate": 8.524970963995355e-06,
      "logits/chosen": 0.5931076407432556,
      "logits/rejected": 0.5217759013175964,
      "logps/chosen": -68.9444580078125,
      "logps/rejected": -75.53872680664062,
      "loss": 0.6364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.014619157649576664,
      "rewards/margins": 0.13041304051876068,
      "rewards/rejected": -0.14503218233585358,
      "step": 636
    },
    {
      "epoch": 0.14799326247313702,
      "grad_norm": 21.5361270904541,
      "learning_rate": 8.522648083623694e-06,
      "logits/chosen": -0.3450290858745575,
      "logits/rejected": -0.24382995069026947,
      "logps/chosen": -76.71365356445312,
      "logps/rejected": -80.04039001464844,
      "loss": 0.5574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09835996478796005,
      "rewards/margins": 0.32288146018981934,
      "rewards/rejected": -0.2245214879512787,
      "step": 637
    },
    {
      "epoch": 0.14822559098565372,
      "grad_norm": 21.79676628112793,
      "learning_rate": 8.520325203252033e-06,
      "logits/chosen": 0.12286494672298431,
      "logits/rejected": 0.29513102769851685,
      "logps/chosen": -70.25090026855469,
      "logps/rejected": -67.18949127197266,
      "loss": 0.6536,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025179170072078705,
      "rewards/margins": 0.10818489640951157,
      "rewards/rejected": -0.08300571888685226,
      "step": 638
    },
    {
      "epoch": 0.14845791949817042,
      "grad_norm": 22.846881866455078,
      "learning_rate": 8.518002322880371e-06,
      "logits/chosen": 0.2464008331298828,
      "logits/rejected": 0.09060642868280411,
      "logps/chosen": -84.14823913574219,
      "logps/rejected": -84.17994689941406,
      "loss": 0.6714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04464889317750931,
      "rewards/margins": 0.08644923567771912,
      "rewards/rejected": -0.13109812140464783,
      "step": 639
    },
    {
      "epoch": 0.14869024801068711,
      "grad_norm": 28.220951080322266,
      "learning_rate": 8.515679442508712e-06,
      "logits/chosen": 0.27688443660736084,
      "logits/rejected": 0.3343322277069092,
      "logps/chosen": -82.04885864257812,
      "logps/rejected": -77.82852172851562,
      "loss": 0.7228,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04868675768375397,
      "rewards/margins": -0.016905002295970917,
      "rewards/rejected": -0.031781744211912155,
      "step": 640
    },
    {
      "epoch": 0.14892257652320381,
      "grad_norm": 27.00749969482422,
      "learning_rate": 8.513356562137051e-06,
      "logits/chosen": 0.509133517742157,
      "logits/rejected": 0.44311684370040894,
      "logps/chosen": -79.50404357910156,
      "logps/rejected": -78.525390625,
      "loss": 0.8151,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.19766707718372345,
      "rewards/margins": -0.21890103816986084,
      "rewards/rejected": 0.02123396098613739,
      "step": 641
    },
    {
      "epoch": 0.14915490503572051,
      "grad_norm": 23.244600296020508,
      "learning_rate": 8.51103368176539e-06,
      "logits/chosen": 0.46354562044143677,
      "logits/rejected": 0.399345725774765,
      "logps/chosen": -99.3228988647461,
      "logps/rejected": -84.3253173828125,
      "loss": 0.6313,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17288994789123535,
      "rewards/margins": 0.1379242241382599,
      "rewards/rejected": -0.31081417202949524,
      "step": 642
    },
    {
      "epoch": 0.1493872335482372,
      "grad_norm": 23.29707145690918,
      "learning_rate": 8.508710801393729e-06,
      "logits/chosen": -0.12474417686462402,
      "logits/rejected": -0.24133187532424927,
      "logps/chosen": -80.81331634521484,
      "logps/rejected": -86.77325439453125,
      "loss": 0.6666,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12138812988996506,
      "rewards/margins": 0.12618610262870789,
      "rewards/rejected": -0.24757423996925354,
      "step": 643
    },
    {
      "epoch": 0.1496195620607539,
      "grad_norm": 22.50655174255371,
      "learning_rate": 8.506387921022068e-06,
      "logits/chosen": 0.11520753055810928,
      "logits/rejected": 0.22911056876182556,
      "logps/chosen": -79.12584686279297,
      "logps/rejected": -75.92240142822266,
      "loss": 0.6357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08730578422546387,
      "rewards/margins": 0.212834894657135,
      "rewards/rejected": -0.3001406788825989,
      "step": 644
    },
    {
      "epoch": 0.1498518905732706,
      "grad_norm": 24.659008026123047,
      "learning_rate": 8.504065040650408e-06,
      "logits/chosen": -0.5158414244651794,
      "logits/rejected": -0.18867778778076172,
      "logps/chosen": -82.51493072509766,
      "logps/rejected": -81.96643829345703,
      "loss": 0.6853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05301683023571968,
      "rewards/margins": 0.07437458634376526,
      "rewards/rejected": -0.12739141285419464,
      "step": 645
    },
    {
      "epoch": 0.1500842190857873,
      "grad_norm": 21.715742111206055,
      "learning_rate": 8.501742160278746e-06,
      "logits/chosen": -0.14395789802074432,
      "logits/rejected": 0.009822501800954342,
      "logps/chosen": -83.68539428710938,
      "logps/rejected": -80.06852722167969,
      "loss": 0.6003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06002860516309738,
      "rewards/margins": 0.21696528792381287,
      "rewards/rejected": -0.15693670511245728,
      "step": 646
    },
    {
      "epoch": 0.150316547598304,
      "grad_norm": 21.170921325683594,
      "learning_rate": 8.499419279907085e-06,
      "logits/chosen": -0.19085010886192322,
      "logits/rejected": -0.14022153615951538,
      "logps/chosen": -73.987060546875,
      "logps/rejected": -93.37654876708984,
      "loss": 0.5593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.021455097943544388,
      "rewards/margins": 0.688306450843811,
      "rewards/rejected": -0.6668513417243958,
      "step": 647
    },
    {
      "epoch": 0.1505488761108207,
      "grad_norm": 20.654541015625,
      "learning_rate": 8.497096399535425e-06,
      "logits/chosen": 0.1982463300228119,
      "logits/rejected": 0.07512398064136505,
      "logps/chosen": -92.90771484375,
      "logps/rejected": -57.53947448730469,
      "loss": 0.6442,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07438333332538605,
      "rewards/margins": 0.12248680740594864,
      "rewards/rejected": -0.04810348153114319,
      "step": 648
    },
    {
      "epoch": 0.1507812046233374,
      "grad_norm": 25.818910598754883,
      "learning_rate": 8.494773519163764e-06,
      "logits/chosen": 0.511877179145813,
      "logits/rejected": 0.6157098412513733,
      "logps/chosen": -80.70824432373047,
      "logps/rejected": -78.76429748535156,
      "loss": 0.7239,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03864654153585434,
      "rewards/margins": 0.03549492359161377,
      "rewards/rejected": -0.0741414800286293,
      "step": 649
    },
    {
      "epoch": 0.1510135331358541,
      "grad_norm": 25.816808700561523,
      "learning_rate": 8.492450638792103e-06,
      "logits/chosen": 1.0037479400634766,
      "logits/rejected": 1.0248316526412964,
      "logps/chosen": -85.50949096679688,
      "logps/rejected": -99.75967407226562,
      "loss": 0.6661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025061320513486862,
      "rewards/margins": 0.08811421692371368,
      "rewards/rejected": -0.06305288523435593,
      "step": 650
    },
    {
      "epoch": 0.15124586164837078,
      "grad_norm": 26.706581115722656,
      "learning_rate": 8.490127758420442e-06,
      "logits/chosen": -0.36634883284568787,
      "logits/rejected": -0.4055863618850708,
      "logps/chosen": -92.02867889404297,
      "logps/rejected": -76.76692962646484,
      "loss": 0.6739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0881112664937973,
      "rewards/margins": 0.07181108742952347,
      "rewards/rejected": 0.016300180926918983,
      "step": 651
    },
    {
      "epoch": 0.15147819016088748,
      "grad_norm": 23.769641876220703,
      "learning_rate": 8.487804878048781e-06,
      "logits/chosen": -0.25827866792678833,
      "logits/rejected": -0.08556684106588364,
      "logps/chosen": -85.35665130615234,
      "logps/rejected": -83.61936950683594,
      "loss": 0.6334,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.034114591777324677,
      "rewards/margins": 0.2433108389377594,
      "rewards/rejected": -0.2774254381656647,
      "step": 652
    },
    {
      "epoch": 0.15171051867340418,
      "grad_norm": 20.918729782104492,
      "learning_rate": 8.48548199767712e-06,
      "logits/chosen": -0.37731629610061646,
      "logits/rejected": -0.2114875614643097,
      "logps/chosen": -68.47372436523438,
      "logps/rejected": -82.95279693603516,
      "loss": 0.5781,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11084714531898499,
      "rewards/margins": 0.39534229040145874,
      "rewards/rejected": -0.28449511528015137,
      "step": 653
    },
    {
      "epoch": 0.15194284718592088,
      "grad_norm": 19.62190055847168,
      "learning_rate": 8.483159117305459e-06,
      "logits/chosen": -0.12900671362876892,
      "logits/rejected": -0.06455705314874649,
      "logps/chosen": -69.3332748413086,
      "logps/rejected": -86.34624481201172,
      "loss": 0.5725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12448650598526001,
      "rewards/margins": 0.34243863821029663,
      "rewards/rejected": -0.21795213222503662,
      "step": 654
    },
    {
      "epoch": 0.15217517569843758,
      "grad_norm": 24.879470825195312,
      "learning_rate": 8.480836236933798e-06,
      "logits/chosen": 0.47526121139526367,
      "logits/rejected": 0.5377706289291382,
      "logps/chosen": -77.93294525146484,
      "logps/rejected": -71.81316375732422,
      "loss": 0.7339,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.037421517074108124,
      "rewards/margins": -0.04689643532037735,
      "rewards/rejected": 0.08431794494390488,
      "step": 655
    },
    {
      "epoch": 0.15240750421095428,
      "grad_norm": 21.605024337768555,
      "learning_rate": 8.478513356562138e-06,
      "logits/chosen": 0.8524705171585083,
      "logits/rejected": 0.7928670048713684,
      "logps/chosen": -79.31400299072266,
      "logps/rejected": -84.22560119628906,
      "loss": 0.5718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14517822861671448,
      "rewards/margins": 0.3752343952655792,
      "rewards/rejected": -0.23005616664886475,
      "step": 656
    },
    {
      "epoch": 0.15263983272347098,
      "grad_norm": 23.800369262695312,
      "learning_rate": 8.476190476190477e-06,
      "logits/chosen": -0.040778785943984985,
      "logits/rejected": -0.04323573410511017,
      "logps/chosen": -75.5810317993164,
      "logps/rejected": -80.36686706542969,
      "loss": 0.7326,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.20457516610622406,
      "rewards/margins": -0.006010755896568298,
      "rewards/rejected": -0.19856441020965576,
      "step": 657
    },
    {
      "epoch": 0.15287216123598768,
      "grad_norm": 25.79787826538086,
      "learning_rate": 8.473867595818816e-06,
      "logits/chosen": 0.384400337934494,
      "logits/rejected": 0.1838141679763794,
      "logps/chosen": -72.96517181396484,
      "logps/rejected": -104.35211181640625,
      "loss": 0.648,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.028184793889522552,
      "rewards/margins": 0.2026260793209076,
      "rewards/rejected": -0.17444127798080444,
      "step": 658
    },
    {
      "epoch": 0.15310448974850438,
      "grad_norm": 21.072168350219727,
      "learning_rate": 8.471544715447155e-06,
      "logits/chosen": 0.17715921998023987,
      "logits/rejected": 0.17082923650741577,
      "logps/chosen": -81.70732116699219,
      "logps/rejected": -77.14014434814453,
      "loss": 0.596,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004640199244022369,
      "rewards/margins": 0.27495521306991577,
      "rewards/rejected": -0.2703149914741516,
      "step": 659
    },
    {
      "epoch": 0.15333681826102108,
      "grad_norm": 25.263452529907227,
      "learning_rate": 8.469221835075494e-06,
      "logits/chosen": 0.23129528760910034,
      "logits/rejected": 0.38079002499580383,
      "logps/chosen": -77.33694458007812,
      "logps/rejected": -82.13500213623047,
      "loss": 0.7578,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1489052176475525,
      "rewards/margins": -0.10484282672405243,
      "rewards/rejected": -0.04406239464879036,
      "step": 660
    },
    {
      "epoch": 0.15356914677353778,
      "grad_norm": 19.329341888427734,
      "learning_rate": 8.466898954703833e-06,
      "logits/chosen": 0.0696054995059967,
      "logits/rejected": 0.10706362128257751,
      "logps/chosen": -69.48976135253906,
      "logps/rejected": -72.6402587890625,
      "loss": 0.5492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04047803953289986,
      "rewards/margins": 0.33144667744636536,
      "rewards/rejected": -0.2909686863422394,
      "step": 661
    },
    {
      "epoch": 0.15380147528605448,
      "grad_norm": 22.225751876831055,
      "learning_rate": 8.464576074332172e-06,
      "logits/chosen": -0.13051050901412964,
      "logits/rejected": -0.1675831377506256,
      "logps/chosen": -90.56402587890625,
      "logps/rejected": -83.40279388427734,
      "loss": 0.6613,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13194268941879272,
      "rewards/margins": 0.08466261625289917,
      "rewards/rejected": -0.2166052907705307,
      "step": 662
    },
    {
      "epoch": 0.15403380379857118,
      "grad_norm": 22.096681594848633,
      "learning_rate": 8.462253193960511e-06,
      "logits/chosen": -0.10923111438751221,
      "logits/rejected": -0.00882045179605484,
      "logps/chosen": -77.91278839111328,
      "logps/rejected": -76.03702545166016,
      "loss": 0.6067,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05550701171159744,
      "rewards/margins": 0.20541870594024658,
      "rewards/rejected": -0.14991168677806854,
      "step": 663
    },
    {
      "epoch": 0.15426613231108788,
      "grad_norm": 23.520355224609375,
      "learning_rate": 8.459930313588852e-06,
      "logits/chosen": 0.11986687034368515,
      "logits/rejected": 0.008954189717769623,
      "logps/chosen": -87.0836181640625,
      "logps/rejected": -80.9559326171875,
      "loss": 0.6562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02745341882109642,
      "rewards/margins": 0.13952499628067017,
      "rewards/rejected": -0.1669784039258957,
      "step": 664
    },
    {
      "epoch": 0.15449846082360458,
      "grad_norm": 21.101497650146484,
      "learning_rate": 8.45760743321719e-06,
      "logits/chosen": 1.0655423402786255,
      "logits/rejected": 1.2099578380584717,
      "logps/chosen": -95.4037857055664,
      "logps/rejected": -86.2781753540039,
      "loss": 0.5444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17778803408145905,
      "rewards/margins": 0.4657377302646637,
      "rewards/rejected": -0.28794974088668823,
      "step": 665
    },
    {
      "epoch": 0.15473078933612128,
      "grad_norm": 27.592620849609375,
      "learning_rate": 8.45528455284553e-06,
      "logits/chosen": 0.7680114507675171,
      "logits/rejected": 0.728050172328949,
      "logps/chosen": -94.74818420410156,
      "logps/rejected": -84.02093505859375,
      "loss": 0.6786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05015126243233681,
      "rewards/margins": 0.09025402367115021,
      "rewards/rejected": -0.14040526747703552,
      "step": 666
    },
    {
      "epoch": 0.15496311784863798,
      "grad_norm": 22.454492568969727,
      "learning_rate": 8.452961672473868e-06,
      "logits/chosen": 0.6623930931091309,
      "logits/rejected": 0.6126430630683899,
      "logps/chosen": -75.60944366455078,
      "logps/rejected": -81.97124481201172,
      "loss": 0.6078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08121271431446075,
      "rewards/margins": 0.20915673673152924,
      "rewards/rejected": -0.12794402241706848,
      "step": 667
    },
    {
      "epoch": 0.15519544636115468,
      "grad_norm": 22.736738204956055,
      "learning_rate": 8.450638792102207e-06,
      "logits/chosen": 0.33837130665779114,
      "logits/rejected": 0.2956307530403137,
      "logps/chosen": -85.41712188720703,
      "logps/rejected": -96.99951171875,
      "loss": 0.6124,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008271262049674988,
      "rewards/margins": 0.20828337967395782,
      "rewards/rejected": -0.2165546417236328,
      "step": 668
    },
    {
      "epoch": 0.15542777487367138,
      "grad_norm": 24.377046585083008,
      "learning_rate": 8.448315911730546e-06,
      "logits/chosen": -0.09621114283800125,
      "logits/rejected": 0.09412233531475067,
      "logps/chosen": -82.70411682128906,
      "logps/rejected": -69.71898651123047,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06077919155359268,
      "rewards/margins": 0.011595649644732475,
      "rewards/rejected": -0.0723748430609703,
      "step": 669
    },
    {
      "epoch": 0.15566010338618808,
      "grad_norm": 20.792987823486328,
      "learning_rate": 8.445993031358885e-06,
      "logits/chosen": -0.06501352041959763,
      "logits/rejected": -0.056536704301834106,
      "logps/chosen": -72.96672058105469,
      "logps/rejected": -93.23681640625,
      "loss": 0.5043,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23704029619693756,
      "rewards/margins": 0.5359333157539368,
      "rewards/rejected": -0.2988930344581604,
      "step": 670
    },
    {
      "epoch": 0.15589243189870478,
      "grad_norm": 20.0699520111084,
      "learning_rate": 8.443670150987224e-06,
      "logits/chosen": 0.01975998282432556,
      "logits/rejected": 0.027362225577235222,
      "logps/chosen": -82.3515625,
      "logps/rejected": -82.45512390136719,
      "loss": 0.5361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08321237564086914,
      "rewards/margins": 0.3924759328365326,
      "rewards/rejected": -0.30926358699798584,
      "step": 671
    },
    {
      "epoch": 0.15612476041122147,
      "grad_norm": 47.35782241821289,
      "learning_rate": 8.441347270615565e-06,
      "logits/chosen": 0.7233083844184875,
      "logits/rejected": 0.6547330021858215,
      "logps/chosen": -82.27053833007812,
      "logps/rejected": -85.17713928222656,
      "loss": 0.6268,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003002883866429329,
      "rewards/margins": 0.17662377655506134,
      "rewards/rejected": -0.1796266734600067,
      "step": 672
    },
    {
      "epoch": 0.15635708892373817,
      "grad_norm": 23.765127182006836,
      "learning_rate": 8.439024390243902e-06,
      "logits/chosen": -0.012452242895960808,
      "logits/rejected": 0.019455719739198685,
      "logps/chosen": -79.93020629882812,
      "logps/rejected": -79.26791381835938,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009912343695759773,
      "rewards/margins": 0.0331365168094635,
      "rewards/rejected": -0.043048858642578125,
      "step": 673
    },
    {
      "epoch": 0.15658941743625487,
      "grad_norm": 24.937353134155273,
      "learning_rate": 8.436701509872243e-06,
      "logits/chosen": 0.299154132604599,
      "logits/rejected": 0.484592467546463,
      "logps/chosen": -80.0935287475586,
      "logps/rejected": -80.00631713867188,
      "loss": 0.645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02174854651093483,
      "rewards/margins": 0.12610864639282227,
      "rewards/rejected": -0.1478571891784668,
      "step": 674
    },
    {
      "epoch": 0.15682174594877157,
      "grad_norm": 23.046113967895508,
      "learning_rate": 8.434378629500582e-06,
      "logits/chosen": 0.051739975810050964,
      "logits/rejected": 0.17090928554534912,
      "logps/chosen": -75.84355163574219,
      "logps/rejected": -88.29325103759766,
      "loss": 0.634,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.19837556779384613,
      "rewards/margins": 0.2898674011230469,
      "rewards/rejected": -0.09149183332920074,
      "step": 675
    },
    {
      "epoch": 0.15705407446128827,
      "grad_norm": 28.9276065826416,
      "learning_rate": 8.43205574912892e-06,
      "logits/chosen": 0.3301566243171692,
      "logits/rejected": 0.36095577478408813,
      "logps/chosen": -80.52767944335938,
      "logps/rejected": -87.06893157958984,
      "loss": 0.7875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16918975114822388,
      "rewards/margins": -0.09785240888595581,
      "rewards/rejected": -0.07133734226226807,
      "step": 676
    },
    {
      "epoch": 0.15728640297380497,
      "grad_norm": 25.516361236572266,
      "learning_rate": 8.42973286875726e-06,
      "logits/chosen": -0.15945664048194885,
      "logits/rejected": 0.19369018077850342,
      "logps/chosen": -88.71087646484375,
      "logps/rejected": -92.43907165527344,
      "loss": 0.6848,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.17700481414794922,
      "rewards/margins": 0.04052644222974777,
      "rewards/rejected": -0.2175312638282776,
      "step": 677
    },
    {
      "epoch": 0.15751873148632167,
      "grad_norm": 24.303409576416016,
      "learning_rate": 8.427409988385598e-06,
      "logits/chosen": 0.2018924355506897,
      "logits/rejected": 0.07893441617488861,
      "logps/chosen": -84.48060607910156,
      "logps/rejected": -87.81209564208984,
      "loss": 0.6561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09932035207748413,
      "rewards/margins": 0.11676628887653351,
      "rewards/rejected": -0.21608662605285645,
      "step": 678
    },
    {
      "epoch": 0.15775105999883834,
      "grad_norm": 26.310501098632812,
      "learning_rate": 8.425087108013939e-06,
      "logits/chosen": 0.0807633325457573,
      "logits/rejected": -0.037228867411613464,
      "logps/chosen": -81.57320404052734,
      "logps/rejected": -87.75563049316406,
      "loss": 0.7709,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1259903609752655,
      "rewards/margins": -0.11407943069934845,
      "rewards/rejected": -0.011910922825336456,
      "step": 679
    },
    {
      "epoch": 0.15798338851135504,
      "grad_norm": 21.37674903869629,
      "learning_rate": 8.422764227642278e-06,
      "logits/chosen": 0.5856508612632751,
      "logits/rejected": 0.7079684138298035,
      "logps/chosen": -74.29039001464844,
      "logps/rejected": -97.15601348876953,
      "loss": 0.5445,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007181428372859955,
      "rewards/margins": 0.4444969594478607,
      "rewards/rejected": -0.43731552362442017,
      "step": 680
    },
    {
      "epoch": 0.15821571702387174,
      "grad_norm": 23.088825225830078,
      "learning_rate": 8.420441347270615e-06,
      "logits/chosen": -0.009016208350658417,
      "logits/rejected": 0.11091044545173645,
      "logps/chosen": -88.22551727294922,
      "logps/rejected": -81.05107879638672,
      "loss": 0.5951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1717231124639511,
      "rewards/margins": 0.262572705745697,
      "rewards/rejected": -0.0908496156334877,
      "step": 681
    },
    {
      "epoch": 0.15844804553638844,
      "grad_norm": 18.812114715576172,
      "learning_rate": 8.418118466898956e-06,
      "logits/chosen": 0.06745859235525131,
      "logits/rejected": 0.19376297295093536,
      "logps/chosen": -76.224365234375,
      "logps/rejected": -76.76646423339844,
      "loss": 0.5096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09200455993413925,
      "rewards/margins": 0.47741201519966125,
      "rewards/rejected": -0.3854074776172638,
      "step": 682
    },
    {
      "epoch": 0.15868037404890514,
      "grad_norm": 24.339828491210938,
      "learning_rate": 8.415795586527295e-06,
      "logits/chosen": -0.14971692860126495,
      "logits/rejected": -0.18442302942276,
      "logps/chosen": -75.61358642578125,
      "logps/rejected": -83.67524719238281,
      "loss": 0.6753,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03206803649663925,
      "rewards/margins": 0.10035207122564316,
      "rewards/rejected": -0.1324201077222824,
      "step": 683
    },
    {
      "epoch": 0.15891270256142184,
      "grad_norm": 19.798643112182617,
      "learning_rate": 8.413472706155634e-06,
      "logits/chosen": 0.7724091410636902,
      "logits/rejected": 0.8844639658927917,
      "logps/chosen": -82.2089614868164,
      "logps/rejected": -77.68865966796875,
      "loss": 0.5537,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1738329380750656,
      "rewards/margins": 0.38563987612724304,
      "rewards/rejected": -0.21180692315101624,
      "step": 684
    },
    {
      "epoch": 0.15914503107393854,
      "grad_norm": 24.984291076660156,
      "learning_rate": 8.411149825783973e-06,
      "logits/chosen": 1.2059454917907715,
      "logits/rejected": 1.2238492965698242,
      "logps/chosen": -86.0113296508789,
      "logps/rejected": -86.49085235595703,
      "loss": 0.6484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12355995178222656,
      "rewards/margins": 0.197457954287529,
      "rewards/rejected": -0.07389800995588303,
      "step": 685
    },
    {
      "epoch": 0.15937735958645524,
      "grad_norm": 21.784292221069336,
      "learning_rate": 8.408826945412312e-06,
      "logits/chosen": 0.5601926445960999,
      "logits/rejected": 0.4636000394821167,
      "logps/chosen": -77.67981719970703,
      "logps/rejected": -81.38725280761719,
      "loss": 0.5115,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3042314350605011,
      "rewards/margins": 0.4798666834831238,
      "rewards/rejected": -0.17563524842262268,
      "step": 686
    },
    {
      "epoch": 0.15960968809897194,
      "grad_norm": 23.828439712524414,
      "learning_rate": 8.406504065040652e-06,
      "logits/chosen": 0.2522677481174469,
      "logits/rejected": 0.3396127223968506,
      "logps/chosen": -78.57131958007812,
      "logps/rejected": -73.12852478027344,
      "loss": 0.7399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10325689613819122,
      "rewards/margins": -0.041381850838661194,
      "rewards/rejected": -0.06187503784894943,
      "step": 687
    },
    {
      "epoch": 0.15984201661148864,
      "grad_norm": 19.68539047241211,
      "learning_rate": 8.40418118466899e-06,
      "logits/chosen": 0.1888187825679779,
      "logits/rejected": 0.25480055809020996,
      "logps/chosen": -69.38717651367188,
      "logps/rejected": -82.55201721191406,
      "loss": 0.544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06989538669586182,
      "rewards/margins": 0.3694547414779663,
      "rewards/rejected": -0.2995593845844269,
      "step": 688
    },
    {
      "epoch": 0.16007434512400534,
      "grad_norm": 26.547754287719727,
      "learning_rate": 8.401858304297328e-06,
      "logits/chosen": 0.6905965209007263,
      "logits/rejected": 0.6797113418579102,
      "logps/chosen": -85.41082763671875,
      "logps/rejected": -83.95585632324219,
      "loss": 0.7065,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15753093361854553,
      "rewards/margins": 0.004807613790035248,
      "rewards/rejected": -0.16233853995800018,
      "step": 689
    },
    {
      "epoch": 0.16030667363652204,
      "grad_norm": 25.503332138061523,
      "learning_rate": 8.399535423925669e-06,
      "logits/chosen": 0.5344926714897156,
      "logits/rejected": 0.5596413612365723,
      "logps/chosen": -83.54624938964844,
      "logps/rejected": -77.10096740722656,
      "loss": 0.7429,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017401602119207382,
      "rewards/margins": -0.011395767331123352,
      "rewards/rejected": -0.0060058352537453175,
      "step": 690
    },
    {
      "epoch": 0.16053900214903874,
      "grad_norm": 25.705373764038086,
      "learning_rate": 8.397212543554008e-06,
      "logits/chosen": -0.15338920056819916,
      "logits/rejected": -0.16108323633670807,
      "logps/chosen": -89.12097930908203,
      "logps/rejected": -80.61348724365234,
      "loss": 0.6643,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12265697866678238,
      "rewards/margins": 0.16775870323181152,
      "rewards/rejected": -0.2904156744480133,
      "step": 691
    },
    {
      "epoch": 0.16077133066155544,
      "grad_norm": 24.042131423950195,
      "learning_rate": 8.394889663182347e-06,
      "logits/chosen": -0.06949944794178009,
      "logits/rejected": -0.20074398815631866,
      "logps/chosen": -78.71907043457031,
      "logps/rejected": -80.4403076171875,
      "loss": 0.6454,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1015615314245224,
      "rewards/margins": 0.17733247578144073,
      "rewards/rejected": -0.27889397740364075,
      "step": 692
    },
    {
      "epoch": 0.16100365917407214,
      "grad_norm": 25.0599308013916,
      "learning_rate": 8.392566782810686e-06,
      "logits/chosen": -0.4330262243747711,
      "logits/rejected": -0.7189555764198303,
      "logps/chosen": -77.32080841064453,
      "logps/rejected": -88.19353485107422,
      "loss": 0.7562,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.060532767325639725,
      "rewards/margins": -0.09118493646383286,
      "rewards/rejected": 0.03065219148993492,
      "step": 693
    },
    {
      "epoch": 0.16123598768658884,
      "grad_norm": 21.696964263916016,
      "learning_rate": 8.390243902439025e-06,
      "logits/chosen": -0.23069322109222412,
      "logits/rejected": -0.2874224781990051,
      "logps/chosen": -80.35103607177734,
      "logps/rejected": -91.08917236328125,
      "loss": 0.5893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09832938015460968,
      "rewards/margins": 0.23240846395492554,
      "rewards/rejected": -0.13407909870147705,
      "step": 694
    },
    {
      "epoch": 0.16146831619910554,
      "grad_norm": 19.581172943115234,
      "learning_rate": 8.387921022067365e-06,
      "logits/chosen": -0.10093928128480911,
      "logits/rejected": -0.0737067237496376,
      "logps/chosen": -69.64189910888672,
      "logps/rejected": -73.49653625488281,
      "loss": 0.5985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.150145024061203,
      "rewards/margins": 0.33165112137794495,
      "rewards/rejected": -0.18150609731674194,
      "step": 695
    },
    {
      "epoch": 0.16170064471162224,
      "grad_norm": 22.701030731201172,
      "learning_rate": 8.385598141695703e-06,
      "logits/chosen": 0.053656190633773804,
      "logits/rejected": -0.10891350358724594,
      "logps/chosen": -78.44345092773438,
      "logps/rejected": -80.6293716430664,
      "loss": 0.6357,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.14693166315555573,
      "rewards/margins": 0.2049812376499176,
      "rewards/rejected": -0.05804961174726486,
      "step": 696
    },
    {
      "epoch": 0.16193297322413894,
      "grad_norm": 25.77927589416504,
      "learning_rate": 8.383275261324042e-06,
      "logits/chosen": 0.38498783111572266,
      "logits/rejected": 0.3362152874469757,
      "logps/chosen": -83.4942398071289,
      "logps/rejected": -74.08016204833984,
      "loss": 0.7766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04788140580058098,
      "rewards/margins": -0.13456542789936066,
      "rewards/rejected": 0.08668401837348938,
      "step": 697
    },
    {
      "epoch": 0.16216530173665564,
      "grad_norm": 27.727930068969727,
      "learning_rate": 8.380952380952382e-06,
      "logits/chosen": 0.1604718416929245,
      "logits/rejected": 0.1858188807964325,
      "logps/chosen": -91.9381332397461,
      "logps/rejected": -82.62590789794922,
      "loss": 0.7996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12262062728404999,
      "rewards/margins": -0.1668618619441986,
      "rewards/rejected": 0.04424123466014862,
      "step": 698
    },
    {
      "epoch": 0.16239763024917234,
      "grad_norm": 21.49900245666504,
      "learning_rate": 8.378629500580721e-06,
      "logits/chosen": 0.003995709121227264,
      "logits/rejected": 0.16183513402938843,
      "logps/chosen": -64.88667297363281,
      "logps/rejected": -75.44808197021484,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03212590515613556,
      "rewards/margins": 0.08834338188171387,
      "rewards/rejected": -0.12046928703784943,
      "step": 699
    },
    {
      "epoch": 0.16262995876168904,
      "grad_norm": 22.442062377929688,
      "learning_rate": 8.37630662020906e-06,
      "logits/chosen": -0.04211962968111038,
      "logits/rejected": 0.06877011060714722,
      "logps/chosen": -81.17303466796875,
      "logps/rejected": -78.88316345214844,
      "loss": 0.6122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10060921311378479,
      "rewards/margins": 0.3023100793361664,
      "rewards/rejected": -0.20170088112354279,
      "step": 700
    },
    {
      "epoch": 0.16286228727420574,
      "grad_norm": 23.216564178466797,
      "learning_rate": 8.373983739837399e-06,
      "logits/chosen": 0.2861010730266571,
      "logits/rejected": 0.2159116417169571,
      "logps/chosen": -78.80681610107422,
      "logps/rejected": -86.09156799316406,
      "loss": 0.6364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04293389245867729,
      "rewards/margins": 0.19676707684993744,
      "rewards/rejected": -0.15383321046829224,
      "step": 701
    },
    {
      "epoch": 0.16309461578672244,
      "grad_norm": 22.81436538696289,
      "learning_rate": 8.371660859465738e-06,
      "logits/chosen": 0.07547184824943542,
      "logits/rejected": -0.10230099409818649,
      "logps/chosen": -72.05152893066406,
      "logps/rejected": -99.78462982177734,
      "loss": 0.6011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0970328226685524,
      "rewards/margins": 0.7772139310836792,
      "rewards/rejected": -0.6801811456680298,
      "step": 702
    },
    {
      "epoch": 0.16332694429923914,
      "grad_norm": 24.16386604309082,
      "learning_rate": 8.369337979094077e-06,
      "logits/chosen": 0.272489070892334,
      "logits/rejected": 0.16114354133605957,
      "logps/chosen": -84.50703430175781,
      "logps/rejected": -83.35912322998047,
      "loss": 0.6126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1584041863679886,
      "rewards/margins": 0.19682343304157257,
      "rewards/rejected": -0.038419246673583984,
      "step": 703
    },
    {
      "epoch": 0.16355927281175583,
      "grad_norm": 25.226255416870117,
      "learning_rate": 8.367015098722416e-06,
      "logits/chosen": -0.0027802959084510803,
      "logits/rejected": -0.028067169710993767,
      "logps/chosen": -83.11859130859375,
      "logps/rejected": -72.2121353149414,
      "loss": 0.716,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.07486166805028915,
      "rewards/margins": 0.0159059576690197,
      "rewards/rejected": 0.05895572155714035,
      "step": 704
    },
    {
      "epoch": 0.16379160132427253,
      "grad_norm": 25.352767944335938,
      "learning_rate": 8.364692218350755e-06,
      "logits/chosen": 0.14650724828243256,
      "logits/rejected": 0.01321747899055481,
      "logps/chosen": -81.50723266601562,
      "logps/rejected": -81.25752258300781,
      "loss": 0.7159,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08507566154003143,
      "rewards/margins": -0.001149917021393776,
      "rewards/rejected": 0.08622556179761887,
      "step": 705
    },
    {
      "epoch": 0.1640239298367892,
      "grad_norm": 24.653099060058594,
      "learning_rate": 8.362369337979095e-06,
      "logits/chosen": 0.27825671434402466,
      "logits/rejected": 0.25305622816085815,
      "logps/chosen": -90.47004699707031,
      "logps/rejected": -74.67526245117188,
      "loss": 0.6355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16272693872451782,
      "rewards/margins": 0.2530606985092163,
      "rewards/rejected": -0.09033374488353729,
      "step": 706
    },
    {
      "epoch": 0.1642562583493059,
      "grad_norm": 22.156097412109375,
      "learning_rate": 8.360046457607434e-06,
      "logits/chosen": -0.09881971776485443,
      "logits/rejected": -0.10433905571699142,
      "logps/chosen": -82.31522369384766,
      "logps/rejected": -70.7885971069336,
      "loss": 0.6733,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.037571150809526443,
      "rewards/margins": 0.0852377787232399,
      "rewards/rejected": -0.04766664281487465,
      "step": 707
    },
    {
      "epoch": 0.1644885868618226,
      "grad_norm": 29.207311630249023,
      "learning_rate": 8.357723577235773e-06,
      "logits/chosen": -0.30172020196914673,
      "logits/rejected": -0.29835277795791626,
      "logps/chosen": -92.45186614990234,
      "logps/rejected": -82.17243957519531,
      "loss": 0.816,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.25896376371383667,
      "rewards/margins": -0.14543908834457397,
      "rewards/rejected": -0.11352464556694031,
      "step": 708
    },
    {
      "epoch": 0.1647209153743393,
      "grad_norm": 22.783226013183594,
      "learning_rate": 8.355400696864112e-06,
      "logits/chosen": -0.22858551144599915,
      "logits/rejected": -0.2023286074399948,
      "logps/chosen": -99.12579345703125,
      "logps/rejected": -102.41514587402344,
      "loss": 0.5602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03624714910984039,
      "rewards/margins": 0.34125450253486633,
      "rewards/rejected": -0.37750160694122314,
      "step": 709
    },
    {
      "epoch": 0.164953243886856,
      "grad_norm": 20.314708709716797,
      "learning_rate": 8.353077816492451e-06,
      "logits/chosen": 0.11933956295251846,
      "logits/rejected": 0.07481923699378967,
      "logps/chosen": -79.09252166748047,
      "logps/rejected": -82.1579818725586,
      "loss": 0.5562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27929890155792236,
      "rewards/margins": 0.32152652740478516,
      "rewards/rejected": -0.04222762584686279,
      "step": 710
    },
    {
      "epoch": 0.1651855723993727,
      "grad_norm": 20.506715774536133,
      "learning_rate": 8.35075493612079e-06,
      "logits/chosen": 0.3099430203437805,
      "logits/rejected": 0.170841783285141,
      "logps/chosen": -78.0134048461914,
      "logps/rejected": -99.97337341308594,
      "loss": 0.5198,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09756380319595337,
      "rewards/margins": 0.47947433590888977,
      "rewards/rejected": -0.381910502910614,
      "step": 711
    },
    {
      "epoch": 0.1654179009118894,
      "grad_norm": 23.39671516418457,
      "learning_rate": 8.348432055749129e-06,
      "logits/chosen": 1.3638148307800293,
      "logits/rejected": 1.4197452068328857,
      "logps/chosen": -77.26903533935547,
      "logps/rejected": -91.97624206542969,
      "loss": 0.6531,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.3793083727359772,
      "rewards/margins": 0.7690519094467163,
      "rewards/rejected": -0.3897435665130615,
      "step": 712
    },
    {
      "epoch": 0.1656502294244061,
      "grad_norm": 20.901214599609375,
      "learning_rate": 8.34610917537747e-06,
      "logits/chosen": -0.0028216876089572906,
      "logits/rejected": 0.06113665550947189,
      "logps/chosen": -76.80061340332031,
      "logps/rejected": -81.8978042602539,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19295591115951538,
      "rewards/margins": 0.4281481206417084,
      "rewards/rejected": -0.2351921796798706,
      "step": 713
    },
    {
      "epoch": 0.1658825579369228,
      "grad_norm": 22.947656631469727,
      "learning_rate": 8.343786295005809e-06,
      "logits/chosen": 0.02117352932691574,
      "logits/rejected": 0.1529691517353058,
      "logps/chosen": -79.31185150146484,
      "logps/rejected": -82.58627319335938,
      "loss": 0.6542,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14157558977603912,
      "rewards/margins": 0.17579148709774017,
      "rewards/rejected": -0.034215930849313736,
      "step": 714
    },
    {
      "epoch": 0.1661148864494395,
      "grad_norm": 19.13530921936035,
      "learning_rate": 8.341463414634147e-06,
      "logits/chosen": 0.44928592443466187,
      "logits/rejected": 0.6615853905677795,
      "logps/chosen": -77.54855346679688,
      "logps/rejected": -78.98802185058594,
      "loss": 0.5132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09475810825824738,
      "rewards/margins": 0.4474576711654663,
      "rewards/rejected": -0.3526995778083801,
      "step": 715
    },
    {
      "epoch": 0.1663472149619562,
      "grad_norm": 19.04230308532715,
      "learning_rate": 8.339140534262486e-06,
      "logits/chosen": -0.28430965542793274,
      "logits/rejected": -0.19407187402248383,
      "logps/chosen": -73.51065063476562,
      "logps/rejected": -80.63059997558594,
      "loss": 0.5368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14469583332538605,
      "rewards/margins": 0.3654215633869171,
      "rewards/rejected": -0.22072570025920868,
      "step": 716
    },
    {
      "epoch": 0.1665795434744729,
      "grad_norm": 26.357572555541992,
      "learning_rate": 8.336817653890825e-06,
      "logits/chosen": 0.41933274269104004,
      "logits/rejected": 0.4105478823184967,
      "logps/chosen": -73.34920501708984,
      "logps/rejected": -91.9828109741211,
      "loss": 0.6675,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10298056900501251,
      "rewards/margins": 0.07200789451599121,
      "rewards/rejected": 0.030972670763731003,
      "step": 717
    },
    {
      "epoch": 0.1668118719869896,
      "grad_norm": 27.92857551574707,
      "learning_rate": 8.334494773519164e-06,
      "logits/chosen": 0.20649364590644836,
      "logits/rejected": 0.15706714987754822,
      "logps/chosen": -78.67579650878906,
      "logps/rejected": -85.51228332519531,
      "loss": 0.7008,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0240938700735569,
      "rewards/margins": 0.09826923161745071,
      "rewards/rejected": -0.07417535781860352,
      "step": 718
    },
    {
      "epoch": 0.1670442004995063,
      "grad_norm": 21.19866943359375,
      "learning_rate": 8.332171893147503e-06,
      "logits/chosen": -0.13383159041404724,
      "logits/rejected": -0.06420104205608368,
      "logps/chosen": -80.56597137451172,
      "logps/rejected": -89.35447692871094,
      "loss": 0.5962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.24827197194099426,
      "rewards/margins": 0.3257269263267517,
      "rewards/rejected": -0.07745495438575745,
      "step": 719
    },
    {
      "epoch": 0.167276529012023,
      "grad_norm": 24.568603515625,
      "learning_rate": 8.329849012775842e-06,
      "logits/chosen": -0.07419166713953018,
      "logits/rejected": -0.0327882319688797,
      "logps/chosen": -87.00716400146484,
      "logps/rejected": -81.32746124267578,
      "loss": 0.6969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05787024274468422,
      "rewards/margins": 0.018133878707885742,
      "rewards/rejected": 0.039736371487379074,
      "step": 720
    },
    {
      "epoch": 0.1675088575245397,
      "grad_norm": 28.18291473388672,
      "learning_rate": 8.327526132404183e-06,
      "logits/chosen": -0.09146641194820404,
      "logits/rejected": -0.019465390592813492,
      "logps/chosen": -84.1988525390625,
      "logps/rejected": -84.07971954345703,
      "loss": 0.6507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.028564929962158203,
      "rewards/margins": 0.14255544543266296,
      "rewards/rejected": -0.11399053037166595,
      "step": 721
    },
    {
      "epoch": 0.1677411860370564,
      "grad_norm": 26.610197067260742,
      "learning_rate": 8.325203252032522e-06,
      "logits/chosen": 0.31183090806007385,
      "logits/rejected": 0.4351661205291748,
      "logps/chosen": -77.57181549072266,
      "logps/rejected": -93.2269515991211,
      "loss": 0.7297,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.05444176867604256,
      "rewards/margins": 0.03984639793634415,
      "rewards/rejected": 0.014595365151762962,
      "step": 722
    },
    {
      "epoch": 0.1679735145495731,
      "grad_norm": 19.297975540161133,
      "learning_rate": 8.322880371660859e-06,
      "logits/chosen": 0.6137267351150513,
      "logits/rejected": 0.555136501789093,
      "logps/chosen": -80.31099700927734,
      "logps/rejected": -85.81575012207031,
      "loss": 0.5344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10176806151866913,
      "rewards/margins": 0.3727414309978485,
      "rewards/rejected": -0.2709733545780182,
      "step": 723
    },
    {
      "epoch": 0.1682058430620898,
      "grad_norm": 18.002243041992188,
      "learning_rate": 8.3205574912892e-06,
      "logits/chosen": -0.06054910272359848,
      "logits/rejected": 0.09430979937314987,
      "logps/chosen": -74.61066436767578,
      "logps/rejected": -79.0970458984375,
      "loss": 0.4846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20654357969760895,
      "rewards/margins": 0.5155163407325745,
      "rewards/rejected": -0.3089727461338043,
      "step": 724
    },
    {
      "epoch": 0.1684381715746065,
      "grad_norm": 49.66355895996094,
      "learning_rate": 8.318234610917539e-06,
      "logits/chosen": -0.36454129219055176,
      "logits/rejected": -0.41361942887306213,
      "logps/chosen": -90.220947265625,
      "logps/rejected": -82.79724884033203,
      "loss": 0.628,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06287539005279541,
      "rewards/margins": 0.23291485011577606,
      "rewards/rejected": -0.17003947496414185,
      "step": 725
    },
    {
      "epoch": 0.1686705000871232,
      "grad_norm": 18.890869140625,
      "learning_rate": 8.315911730545877e-06,
      "logits/chosen": -0.2611384093761444,
      "logits/rejected": -0.23830735683441162,
      "logps/chosen": -68.70162200927734,
      "logps/rejected": -89.18763732910156,
      "loss": 0.5158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.266448438167572,
      "rewards/margins": 0.4131610095500946,
      "rewards/rejected": -0.14671257138252258,
      "step": 726
    },
    {
      "epoch": 0.1689028285996399,
      "grad_norm": 27.219799041748047,
      "learning_rate": 8.313588850174216e-06,
      "logits/chosen": -0.13793811202049255,
      "logits/rejected": -0.2321828007698059,
      "logps/chosen": -78.8183822631836,
      "logps/rejected": -88.12704467773438,
      "loss": 0.7499,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10940268635749817,
      "rewards/margins": -0.09624770283699036,
      "rewards/rejected": -0.013154986314475536,
      "step": 727
    },
    {
      "epoch": 0.1691351571121566,
      "grad_norm": 23.002090454101562,
      "learning_rate": 8.311265969802555e-06,
      "logits/chosen": 0.09750426560640335,
      "logits/rejected": 0.0581352636218071,
      "logps/chosen": -77.81565856933594,
      "logps/rejected": -76.6611328125,
      "loss": 0.7058,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03610661253333092,
      "rewards/margins": -0.007598251104354858,
      "rewards/rejected": -0.028508353978395462,
      "step": 728
    },
    {
      "epoch": 0.1693674856246733,
      "grad_norm": 24.310232162475586,
      "learning_rate": 8.308943089430896e-06,
      "logits/chosen": 0.9608585834503174,
      "logits/rejected": 0.9563791751861572,
      "logps/chosen": -74.1238784790039,
      "logps/rejected": -103.45049285888672,
      "loss": 0.5854,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.4721525311470032,
      "rewards/margins": 0.7293386459350586,
      "rewards/rejected": -0.2571861147880554,
      "step": 729
    },
    {
      "epoch": 0.16959981413719,
      "grad_norm": 25.2594051361084,
      "learning_rate": 8.306620209059235e-06,
      "logits/chosen": -0.09117969125509262,
      "logits/rejected": 0.0033488571643829346,
      "logps/chosen": -76.54127502441406,
      "logps/rejected": -76.73277282714844,
      "loss": 0.7176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03753533214330673,
      "rewards/margins": 0.008595170453190804,
      "rewards/rejected": -0.04613049328327179,
      "step": 730
    },
    {
      "epoch": 0.1698321426497067,
      "grad_norm": 20.560535430908203,
      "learning_rate": 8.304297328687572e-06,
      "logits/chosen": -0.16374555230140686,
      "logits/rejected": -0.07719279080629349,
      "logps/chosen": -69.61857604980469,
      "logps/rejected": -78.5965805053711,
      "loss": 0.5575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16455906629562378,
      "rewards/margins": 0.3945172429084778,
      "rewards/rejected": -0.229958176612854,
      "step": 731
    },
    {
      "epoch": 0.1700644711622234,
      "grad_norm": 24.030200958251953,
      "learning_rate": 8.301974448315913e-06,
      "logits/chosen": -0.3131306767463684,
      "logits/rejected": -0.342695415019989,
      "logps/chosen": -74.38147735595703,
      "logps/rejected": -90.74665832519531,
      "loss": 0.5809,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.23247170448303223,
      "rewards/margins": 0.3516954183578491,
      "rewards/rejected": -0.11922372877597809,
      "step": 732
    },
    {
      "epoch": 0.1702967996747401,
      "grad_norm": 23.186817169189453,
      "learning_rate": 8.299651567944252e-06,
      "logits/chosen": -0.4102976322174072,
      "logits/rejected": -0.09277505427598953,
      "logps/chosen": -80.30313873291016,
      "logps/rejected": -84.14013671875,
      "loss": 0.6103,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18501117825508118,
      "rewards/margins": 0.2550555169582367,
      "rewards/rejected": -0.07004431635141373,
      "step": 733
    },
    {
      "epoch": 0.17052912818725677,
      "grad_norm": 22.685697555541992,
      "learning_rate": 8.29732868757259e-06,
      "logits/chosen": -0.16575053334236145,
      "logits/rejected": 0.03918227553367615,
      "logps/chosen": -76.29857635498047,
      "logps/rejected": -89.83946228027344,
      "loss": 0.5857,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.278737872838974,
      "rewards/margins": 0.30629077553749084,
      "rewards/rejected": -0.027552910149097443,
      "step": 734
    },
    {
      "epoch": 0.17076145669977347,
      "grad_norm": 26.548622131347656,
      "learning_rate": 8.29500580720093e-06,
      "logits/chosen": -0.07206542044878006,
      "logits/rejected": -0.0076250736601650715,
      "logps/chosen": -76.94113159179688,
      "logps/rejected": -74.01183319091797,
      "loss": 0.6939,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08802661299705505,
      "rewards/margins": 0.07315398752689362,
      "rewards/rejected": 0.014872618019580841,
      "step": 735
    },
    {
      "epoch": 0.17099378521229017,
      "grad_norm": 23.942413330078125,
      "learning_rate": 8.292682926829268e-06,
      "logits/chosen": -0.3033811151981354,
      "logits/rejected": -0.13812394440174103,
      "logps/chosen": -74.97543334960938,
      "logps/rejected": -75.33729553222656,
      "loss": 0.638,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.23810097575187683,
      "rewards/margins": 0.17200927436351776,
      "rewards/rejected": 0.06609170138835907,
      "step": 736
    },
    {
      "epoch": 0.17122611372480687,
      "grad_norm": 17.489181518554688,
      "learning_rate": 8.290360046457609e-06,
      "logits/chosen": -0.3411638140678406,
      "logits/rejected": -0.37998515367507935,
      "logps/chosen": -72.57567596435547,
      "logps/rejected": -79.63890075683594,
      "loss": 0.4794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.40576112270355225,
      "rewards/margins": 0.5778809785842896,
      "rewards/rejected": -0.1721199005842209,
      "step": 737
    },
    {
      "epoch": 0.17145844223732357,
      "grad_norm": 23.352460861206055,
      "learning_rate": 8.288037166085946e-06,
      "logits/chosen": -0.376697301864624,
      "logits/rejected": -0.35518524050712585,
      "logps/chosen": -95.14320373535156,
      "logps/rejected": -85.85488891601562,
      "loss": 0.5064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1626092493534088,
      "rewards/margins": 0.47137755155563354,
      "rewards/rejected": -0.30876827239990234,
      "step": 738
    },
    {
      "epoch": 0.17169077074984027,
      "grad_norm": 23.66398811340332,
      "learning_rate": 8.285714285714287e-06,
      "logits/chosen": -0.2467108964920044,
      "logits/rejected": -0.23374126851558685,
      "logps/chosen": -73.64501190185547,
      "logps/rejected": -74.38227844238281,
      "loss": 0.7386,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.028203679248690605,
      "rewards/margins": -0.0028403252363204956,
      "rewards/rejected": 0.0310440044850111,
      "step": 739
    },
    {
      "epoch": 0.17192309926235697,
      "grad_norm": 19.900327682495117,
      "learning_rate": 8.283391405342626e-06,
      "logits/chosen": 0.7277423143386841,
      "logits/rejected": 0.7575611472129822,
      "logps/chosen": -72.3604736328125,
      "logps/rejected": -92.603759765625,
      "loss": 0.5051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2774285078048706,
      "rewards/margins": 0.5803908109664917,
      "rewards/rejected": -0.3029623031616211,
      "step": 740
    },
    {
      "epoch": 0.17215542777487366,
      "grad_norm": 26.944019317626953,
      "learning_rate": 8.281068524970965e-06,
      "logits/chosen": -0.36619624495506287,
      "logits/rejected": -0.19463130831718445,
      "logps/chosen": -84.62538146972656,
      "logps/rejected": -79.3871078491211,
      "loss": 0.6504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1272851824760437,
      "rewards/margins": 0.1345614194869995,
      "rewards/rejected": -0.007276227697730064,
      "step": 741
    },
    {
      "epoch": 0.17238775628739036,
      "grad_norm": 24.519420623779297,
      "learning_rate": 8.278745644599304e-06,
      "logits/chosen": -0.3164449632167816,
      "logits/rejected": -0.19192194938659668,
      "logps/chosen": -88.47440338134766,
      "logps/rejected": -81.82880401611328,
      "loss": 0.6547,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13644623756408691,
      "rewards/margins": 0.1246349886059761,
      "rewards/rejected": -0.2610812187194824,
      "step": 742
    },
    {
      "epoch": 0.17262008479990706,
      "grad_norm": 26.30563735961914,
      "learning_rate": 8.276422764227643e-06,
      "logits/chosen": 0.8221890926361084,
      "logits/rejected": 0.6825031042098999,
      "logps/chosen": -77.85603332519531,
      "logps/rejected": -85.72526550292969,
      "loss": 0.6808,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.25881367921829224,
      "rewards/margins": 0.15517404675483704,
      "rewards/rejected": 0.10363966226577759,
      "step": 743
    },
    {
      "epoch": 0.17285241331242376,
      "grad_norm": 23.43466567993164,
      "learning_rate": 8.274099883855982e-06,
      "logits/chosen": 0.21529990434646606,
      "logits/rejected": 0.23716507852077484,
      "logps/chosen": -74.95793914794922,
      "logps/rejected": -71.75148010253906,
      "loss": 0.7165,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023845739662647247,
      "rewards/margins": -0.01685134321451187,
      "rewards/rejected": 0.04069708287715912,
      "step": 744
    },
    {
      "epoch": 0.17308474182494046,
      "grad_norm": 25.4796085357666,
      "learning_rate": 8.271777003484322e-06,
      "logits/chosen": -0.29714950919151306,
      "logits/rejected": -0.372209370136261,
      "logps/chosen": -75.62786102294922,
      "logps/rejected": -74.50252532958984,
      "loss": 0.6761,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11373412609100342,
      "rewards/margins": 0.09017583727836609,
      "rewards/rejected": 0.023558281362056732,
      "step": 745
    },
    {
      "epoch": 0.17331707033745716,
      "grad_norm": 22.181421279907227,
      "learning_rate": 8.26945412311266e-06,
      "logits/chosen": -0.523225724697113,
      "logits/rejected": -0.6364769339561462,
      "logps/chosen": -67.23194122314453,
      "logps/rejected": -83.67874908447266,
      "loss": 0.6087,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.25349172949790955,
      "rewards/margins": 0.25062912702560425,
      "rewards/rejected": 0.00286262109875679,
      "step": 746
    },
    {
      "epoch": 0.17354939884997386,
      "grad_norm": 24.833810806274414,
      "learning_rate": 8.267131242741e-06,
      "logits/chosen": -0.5331619381904602,
      "logits/rejected": -0.6234796643257141,
      "logps/chosen": -76.13045501708984,
      "logps/rejected": -81.18171691894531,
      "loss": 0.7478,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06001630425453186,
      "rewards/margins": -0.06604719161987305,
      "rewards/rejected": 0.1260634809732437,
      "step": 747
    },
    {
      "epoch": 0.17378172736249056,
      "grad_norm": 34.38259506225586,
      "learning_rate": 8.264808362369339e-06,
      "logits/chosen": 0.11933938413858414,
      "logits/rejected": 0.29578936100006104,
      "logps/chosen": -87.26593017578125,
      "logps/rejected": -78.11593627929688,
      "loss": 0.6883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017174243927001953,
      "rewards/margins": 0.14761967957019806,
      "rewards/rejected": -0.1647939234972,
      "step": 748
    },
    {
      "epoch": 0.17401405587500726,
      "grad_norm": 26.702838897705078,
      "learning_rate": 8.262485481997678e-06,
      "logits/chosen": -0.15091082453727722,
      "logits/rejected": -0.1886855810880661,
      "logps/chosen": -83.9268798828125,
      "logps/rejected": -75.05609130859375,
      "loss": 0.8227,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02254573255777359,
      "rewards/margins": -0.17904077470302582,
      "rewards/rejected": 0.15649503469467163,
      "step": 749
    },
    {
      "epoch": 0.17424638438752396,
      "grad_norm": 24.381235122680664,
      "learning_rate": 8.260162601626017e-06,
      "logits/chosen": -0.5207265615463257,
      "logits/rejected": -0.41773802042007446,
      "logps/chosen": -77.35188293457031,
      "logps/rejected": -69.64706420898438,
      "loss": 0.6788,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14900511503219604,
      "rewards/margins": 0.0711771696805954,
      "rewards/rejected": 0.07782792299985886,
      "step": 750
    },
    {
      "epoch": 0.17447871290004066,
      "grad_norm": 26.506860733032227,
      "learning_rate": 8.257839721254356e-06,
      "logits/chosen": 0.04663879796862602,
      "logits/rejected": 0.042755406349897385,
      "logps/chosen": -80.6396484375,
      "logps/rejected": -85.54789733886719,
      "loss": 0.6641,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16399559378623962,
      "rewards/margins": 0.2737092971801758,
      "rewards/rejected": -0.10971365123987198,
      "step": 751
    },
    {
      "epoch": 0.17471104141255736,
      "grad_norm": 19.79906463623047,
      "learning_rate": 8.255516840882695e-06,
      "logits/chosen": 0.2946588397026062,
      "logits/rejected": 0.3672434687614441,
      "logps/chosen": -73.07723999023438,
      "logps/rejected": -69.39877319335938,
      "loss": 0.5707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3304017186164856,
      "rewards/margins": 0.31602242588996887,
      "rewards/rejected": 0.014379281550645828,
      "step": 752
    },
    {
      "epoch": 0.17494336992507406,
      "grad_norm": 59.621307373046875,
      "learning_rate": 8.253193960511034e-06,
      "logits/chosen": -0.3108311593532562,
      "logits/rejected": -0.24239633977413177,
      "logps/chosen": -68.91011810302734,
      "logps/rejected": -76.63442993164062,
      "loss": 0.6346,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3078824281692505,
      "rewards/margins": 0.2945549488067627,
      "rewards/rejected": 0.013327496126294136,
      "step": 753
    },
    {
      "epoch": 0.17517569843759076,
      "grad_norm": 23.656021118164062,
      "learning_rate": 8.250871080139373e-06,
      "logits/chosen": -0.11541511863470078,
      "logits/rejected": -0.17188699543476105,
      "logps/chosen": -73.68346405029297,
      "logps/rejected": -87.24044036865234,
      "loss": 0.6103,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.010550525039434433,
      "rewards/margins": 0.31884461641311646,
      "rewards/rejected": -0.30829405784606934,
      "step": 754
    },
    {
      "epoch": 0.17540802695010746,
      "grad_norm": 23.569141387939453,
      "learning_rate": 8.248548199767713e-06,
      "logits/chosen": -0.5877402424812317,
      "logits/rejected": -0.5078723430633545,
      "logps/chosen": -82.1037368774414,
      "logps/rejected": -81.9984359741211,
      "loss": 0.7273,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.11899634450674057,
      "rewards/margins": -0.024775460362434387,
      "rewards/rejected": -0.09422086924314499,
      "step": 755
    },
    {
      "epoch": 0.17564035546262416,
      "grad_norm": 20.49896240234375,
      "learning_rate": 8.246225319396052e-06,
      "logits/chosen": -0.11634428054094315,
      "logits/rejected": -0.15114250779151917,
      "logps/chosen": -82.32699584960938,
      "logps/rejected": -74.93852996826172,
      "loss": 0.5763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29731258749961853,
      "rewards/margins": 0.2988969385623932,
      "rewards/rejected": -0.0015843603760004044,
      "step": 756
    },
    {
      "epoch": 0.17587268397514086,
      "grad_norm": 20.75507354736328,
      "learning_rate": 8.243902439024391e-06,
      "logits/chosen": -0.23924686014652252,
      "logits/rejected": -0.25941863656044006,
      "logps/chosen": -77.37828063964844,
      "logps/rejected": -69.26859283447266,
      "loss": 0.6128,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.20778870582580566,
      "rewards/margins": 0.21770521998405457,
      "rewards/rejected": -0.00991649180650711,
      "step": 757
    },
    {
      "epoch": 0.17610501248765756,
      "grad_norm": 24.889503479003906,
      "learning_rate": 8.24157955865273e-06,
      "logits/chosen": 0.10881856083869934,
      "logits/rejected": 0.18601486086845398,
      "logps/chosen": -78.4817123413086,
      "logps/rejected": -84.1408462524414,
      "loss": 0.6324,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.1531088501214981,
      "rewards/margins": 0.15118756890296936,
      "rewards/rejected": 0.0019212998449802399,
      "step": 758
    },
    {
      "epoch": 0.17633734100017426,
      "grad_norm": 21.108959197998047,
      "learning_rate": 8.239256678281069e-06,
      "logits/chosen": -0.088274285197258,
      "logits/rejected": 0.08027803897857666,
      "logps/chosen": -82.28981018066406,
      "logps/rejected": -77.36331176757812,
      "loss": 0.5364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3196713626384735,
      "rewards/margins": 0.5378549695014954,
      "rewards/rejected": -0.21818360686302185,
      "step": 759
    },
    {
      "epoch": 0.17656966951269096,
      "grad_norm": 21.711721420288086,
      "learning_rate": 8.23693379790941e-06,
      "logits/chosen": 0.2594926059246063,
      "logits/rejected": 0.47076842188835144,
      "logps/chosen": -81.75556945800781,
      "logps/rejected": -76.50395965576172,
      "loss": 0.5984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13552431762218475,
      "rewards/margins": 0.3199866712093353,
      "rewards/rejected": -0.18446235358715057,
      "step": 760
    },
    {
      "epoch": 0.17680199802520766,
      "grad_norm": 22.48493194580078,
      "learning_rate": 8.234610917537747e-06,
      "logits/chosen": -0.2898542881011963,
      "logits/rejected": -0.3333670496940613,
      "logps/chosen": -75.62360382080078,
      "logps/rejected": -81.3843994140625,
      "loss": 0.6102,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2637799382209778,
      "rewards/margins": 0.21013441681861877,
      "rewards/rejected": 0.05364551395177841,
      "step": 761
    },
    {
      "epoch": 0.17703432653772433,
      "grad_norm": 24.793746948242188,
      "learning_rate": 8.232288037166086e-06,
      "logits/chosen": 0.019470904022455215,
      "logits/rejected": 0.07756539434194565,
      "logps/chosen": -79.64059448242188,
      "logps/rejected": -77.13795471191406,
      "loss": 0.6247,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24476152658462524,
      "rewards/margins": 0.15660089254379272,
      "rewards/rejected": 0.08816065639257431,
      "step": 762
    },
    {
      "epoch": 0.17726665505024103,
      "grad_norm": 23.21087646484375,
      "learning_rate": 8.229965156794426e-06,
      "logits/chosen": -0.024853594601154327,
      "logits/rejected": -0.18324674665927887,
      "logps/chosen": -80.07682037353516,
      "logps/rejected": -102.40608978271484,
      "loss": 0.5527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08473721146583557,
      "rewards/margins": 0.4088268578052521,
      "rewards/rejected": -0.3240896463394165,
      "step": 763
    },
    {
      "epoch": 0.17749898356275773,
      "grad_norm": 25.120864868164062,
      "learning_rate": 8.227642276422765e-06,
      "logits/chosen": -0.2660306692123413,
      "logits/rejected": -0.4781646430492401,
      "logps/chosen": -77.2087173461914,
      "logps/rejected": -73.77998352050781,
      "loss": 0.7626,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.07543833553791046,
      "rewards/margins": -0.08206425607204437,
      "rewards/rejected": 0.15750259160995483,
      "step": 764
    },
    {
      "epoch": 0.17773131207527443,
      "grad_norm": 22.579509735107422,
      "learning_rate": 8.225319396051104e-06,
      "logits/chosen": -0.535947859287262,
      "logits/rejected": -0.4505681097507477,
      "logps/chosen": -81.73282623291016,
      "logps/rejected": -84.27196502685547,
      "loss": 0.5722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1820523738861084,
      "rewards/margins": 0.3142489492893219,
      "rewards/rejected": -0.1321965903043747,
      "step": 765
    },
    {
      "epoch": 0.17796364058779113,
      "grad_norm": 24.732999801635742,
      "learning_rate": 8.222996515679443e-06,
      "logits/chosen": -0.04726065695285797,
      "logits/rejected": -0.04639767110347748,
      "logps/chosen": -66.90035247802734,
      "logps/rejected": -72.60381317138672,
      "loss": 0.4881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24301576614379883,
      "rewards/margins": 0.5324968099594116,
      "rewards/rejected": -0.2894810736179352,
      "step": 766
    },
    {
      "epoch": 0.17819596910030783,
      "grad_norm": 21.42509651184082,
      "learning_rate": 8.220673635307782e-06,
      "logits/chosen": 0.4396021068096161,
      "logits/rejected": 0.29670804738998413,
      "logps/chosen": -78.50973510742188,
      "logps/rejected": -89.07379150390625,
      "loss": 0.5221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20838774740695953,
      "rewards/margins": 0.4115291237831116,
      "rewards/rejected": -0.20314137637615204,
      "step": 767
    },
    {
      "epoch": 0.17842829761282453,
      "grad_norm": 20.8935604095459,
      "learning_rate": 8.218350754936121e-06,
      "logits/chosen": 0.07286012172698975,
      "logits/rejected": 0.29912713170051575,
      "logps/chosen": -75.26860046386719,
      "logps/rejected": -90.63082122802734,
      "loss": 0.5645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.22996683418750763,
      "rewards/margins": 0.823668360710144,
      "rewards/rejected": -0.5937016010284424,
      "step": 768
    },
    {
      "epoch": 0.17866062612534123,
      "grad_norm": 19.212358474731445,
      "learning_rate": 8.21602787456446e-06,
      "logits/chosen": -0.31551480293273926,
      "logits/rejected": -0.207400381565094,
      "logps/chosen": -78.82243347167969,
      "logps/rejected": -74.29905700683594,
      "loss": 0.4915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2768308222293854,
      "rewards/margins": 0.5470989346504211,
      "rewards/rejected": -0.2702680826187134,
      "step": 769
    },
    {
      "epoch": 0.17889295463785793,
      "grad_norm": 23.581525802612305,
      "learning_rate": 8.213704994192799e-06,
      "logits/chosen": -0.3171570897102356,
      "logits/rejected": -0.4152328372001648,
      "logps/chosen": -84.6849594116211,
      "logps/rejected": -80.88201904296875,
      "loss": 0.6468,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05825307220220566,
      "rewards/margins": 0.15332330763339996,
      "rewards/rejected": -0.09507020562887192,
      "step": 770
    },
    {
      "epoch": 0.17912528315037463,
      "grad_norm": 40.41072082519531,
      "learning_rate": 8.21138211382114e-06,
      "logits/chosen": -0.4712865352630615,
      "logits/rejected": -0.4650665819644928,
      "logps/chosen": -91.64659881591797,
      "logps/rejected": -83.62992095947266,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004374312236905098,
      "rewards/margins": 0.07032735645771027,
      "rewards/rejected": -0.07470168173313141,
      "step": 771
    },
    {
      "epoch": 0.17935761166289133,
      "grad_norm": 25.464582443237305,
      "learning_rate": 8.209059233449479e-06,
      "logits/chosen": 0.17024125158786774,
      "logits/rejected": -0.012060049921274185,
      "logps/chosen": -74.00485229492188,
      "logps/rejected": -84.10018920898438,
      "loss": 0.6633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.050519756972789764,
      "rewards/margins": 0.16537588834762573,
      "rewards/rejected": -0.11485612392425537,
      "step": 772
    },
    {
      "epoch": 0.17958994017540802,
      "grad_norm": 34.56949996948242,
      "learning_rate": 8.206736353077818e-06,
      "logits/chosen": -0.4713660478591919,
      "logits/rejected": -0.6333848237991333,
      "logps/chosen": -76.7824935913086,
      "logps/rejected": -80.37248229980469,
      "loss": 0.654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13389571011066437,
      "rewards/margins": 0.10431007295846939,
      "rewards/rejected": 0.02958565019071102,
      "step": 773
    },
    {
      "epoch": 0.17982226868792472,
      "grad_norm": 21.11847686767578,
      "learning_rate": 8.204413472706156e-06,
      "logits/chosen": -0.09456748515367508,
      "logits/rejected": -0.04698591306805611,
      "logps/chosen": -73.82102966308594,
      "logps/rejected": -68.85324096679688,
      "loss": 0.6351,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2547415494918823,
      "rewards/margins": 0.2138509750366211,
      "rewards/rejected": 0.04089060053229332,
      "step": 774
    },
    {
      "epoch": 0.18005459720044142,
      "grad_norm": 25.30293846130371,
      "learning_rate": 8.202090592334495e-06,
      "logits/chosen": 0.5362528562545776,
      "logits/rejected": 0.33682602643966675,
      "logps/chosen": -83.87904357910156,
      "logps/rejected": -69.29501342773438,
      "loss": 0.7097,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.09259846061468124,
      "rewards/margins": 0.019158490002155304,
      "rewards/rejected": 0.07343997806310654,
      "step": 775
    },
    {
      "epoch": 0.18028692571295812,
      "grad_norm": 19.568111419677734,
      "learning_rate": 8.199767711962834e-06,
      "logits/chosen": -0.35699307918548584,
      "logits/rejected": -0.31041866540908813,
      "logps/chosen": -76.78394317626953,
      "logps/rejected": -85.83417510986328,
      "loss": 0.519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3718656897544861,
      "rewards/margins": 0.5629252791404724,
      "rewards/rejected": -0.19105958938598633,
      "step": 776
    },
    {
      "epoch": 0.18051925422547482,
      "grad_norm": 22.64267349243164,
      "learning_rate": 8.197444831591173e-06,
      "logits/chosen": -0.2034284770488739,
      "logits/rejected": -0.1510372757911682,
      "logps/chosen": -72.67146301269531,
      "logps/rejected": -79.69503021240234,
      "loss": 0.6231,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16343393921852112,
      "rewards/margins": 0.2674618363380432,
      "rewards/rejected": -0.1040278822183609,
      "step": 777
    },
    {
      "epoch": 0.18075158273799152,
      "grad_norm": 26.906675338745117,
      "learning_rate": 8.195121951219512e-06,
      "logits/chosen": 0.4391039311885834,
      "logits/rejected": 0.5144011974334717,
      "logps/chosen": -90.14134216308594,
      "logps/rejected": -102.93312072753906,
      "loss": 0.6889,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05046357959508896,
      "rewards/margins": 0.10181489586830139,
      "rewards/rejected": -0.15227846801280975,
      "step": 778
    },
    {
      "epoch": 0.18098391125050822,
      "grad_norm": 38.24856185913086,
      "learning_rate": 8.192799070847853e-06,
      "logits/chosen": 1.05272376537323,
      "logits/rejected": 1.0769069194793701,
      "logps/chosen": -80.6378173828125,
      "logps/rejected": -81.9725112915039,
      "loss": 0.7702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.13418371975421906,
      "rewards/margins": -0.1415463238954544,
      "rewards/rejected": 0.2757300138473511,
      "step": 779
    },
    {
      "epoch": 0.18121623976302492,
      "grad_norm": 20.673830032348633,
      "learning_rate": 8.190476190476192e-06,
      "logits/chosen": -0.17939850687980652,
      "logits/rejected": -0.10933949053287506,
      "logps/chosen": -70.77928924560547,
      "logps/rejected": -82.31614685058594,
      "loss": 0.5332,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.37282970547676086,
      "rewards/margins": 0.5137088894844055,
      "rewards/rejected": -0.14087915420532227,
      "step": 780
    },
    {
      "epoch": 0.18144856827554162,
      "grad_norm": 22.79326057434082,
      "learning_rate": 8.18815331010453e-06,
      "logits/chosen": 0.13386276364326477,
      "logits/rejected": -0.14616456627845764,
      "logps/chosen": -65.68412017822266,
      "logps/rejected": -75.45928955078125,
      "loss": 0.7537,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07034219056367874,
      "rewards/margins": -0.06228011101484299,
      "rewards/rejected": -0.00806207675486803,
      "step": 781
    },
    {
      "epoch": 0.18168089678805832,
      "grad_norm": 26.633617401123047,
      "learning_rate": 8.18583042973287e-06,
      "logits/chosen": -0.031134143471717834,
      "logits/rejected": 0.05872190743684769,
      "logps/chosen": -73.18522644042969,
      "logps/rejected": -78.1227798461914,
      "loss": 0.676,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12839451432228088,
      "rewards/margins": 0.07415322959423065,
      "rewards/rejected": 0.05424128845334053,
      "step": 782
    },
    {
      "epoch": 0.18191322530057502,
      "grad_norm": 25.724260330200195,
      "learning_rate": 8.183507549361209e-06,
      "logits/chosen": 0.49560409784317017,
      "logits/rejected": 0.6276540756225586,
      "logps/chosen": -75.12273406982422,
      "logps/rejected": -69.4104232788086,
      "loss": 0.7556,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04616248980164528,
      "rewards/margins": -0.04268375039100647,
      "rewards/rejected": -0.0034787431359291077,
      "step": 783
    },
    {
      "epoch": 0.18214555381309172,
      "grad_norm": 24.247098922729492,
      "learning_rate": 8.181184668989547e-06,
      "logits/chosen": -0.3776763081550598,
      "logits/rejected": -0.1689251810312271,
      "logps/chosen": -72.78850555419922,
      "logps/rejected": -82.29085540771484,
      "loss": 0.5976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22530889511108398,
      "rewards/margins": 0.21422280371189117,
      "rewards/rejected": 0.011086102575063705,
      "step": 784
    },
    {
      "epoch": 0.18237788232560842,
      "grad_norm": 22.163921356201172,
      "learning_rate": 8.178861788617886e-06,
      "logits/chosen": -0.5139271020889282,
      "logits/rejected": -0.5758270025253296,
      "logps/chosen": -79.11148834228516,
      "logps/rejected": -93.47535705566406,
      "loss": 0.5847,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1044473648071289,
      "rewards/margins": 0.3266183137893677,
      "rewards/rejected": -0.22217093408107758,
      "step": 785
    },
    {
      "epoch": 0.18261021083812512,
      "grad_norm": 25.331724166870117,
      "learning_rate": 8.176538908246227e-06,
      "logits/chosen": 0.014999188482761383,
      "logits/rejected": -0.04283475503325462,
      "logps/chosen": -78.3228988647461,
      "logps/rejected": -88.49056243896484,
      "loss": 0.697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1487063467502594,
      "rewards/margins": 0.06371736526489258,
      "rewards/rejected": 0.08498898893594742,
      "step": 786
    },
    {
      "epoch": 0.18284253935064182,
      "grad_norm": 26.895179748535156,
      "learning_rate": 8.174216027874566e-06,
      "logits/chosen": 0.03435856103897095,
      "logits/rejected": -0.07646939158439636,
      "logps/chosen": -85.10966491699219,
      "logps/rejected": -77.14558410644531,
      "loss": 0.7557,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.07116790115833282,
      "rewards/margins": -0.1107703223824501,
      "rewards/rejected": 0.18193823099136353,
      "step": 787
    },
    {
      "epoch": 0.18307486786315852,
      "grad_norm": 25.5053768157959,
      "learning_rate": 8.171893147502903e-06,
      "logits/chosen": 0.08605419099330902,
      "logits/rejected": 0.2601016163825989,
      "logps/chosen": -82.625,
      "logps/rejected": -86.40605163574219,
      "loss": 0.6491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.20061813294887543,
      "rewards/margins": 0.18319398164749146,
      "rewards/rejected": 0.01742413267493248,
      "step": 788
    },
    {
      "epoch": 0.1833071963756752,
      "grad_norm": 24.89459800720215,
      "learning_rate": 8.169570267131244e-06,
      "logits/chosen": -0.3783077597618103,
      "logits/rejected": -0.25991517305374146,
      "logps/chosen": -78.36112213134766,
      "logps/rejected": -94.02457427978516,
      "loss": 0.743,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00929085910320282,
      "rewards/margins": 0.012966156005859375,
      "rewards/rejected": -0.0036752745509147644,
      "step": 789
    },
    {
      "epoch": 0.1835395248881919,
      "grad_norm": 21.623668670654297,
      "learning_rate": 8.167247386759583e-06,
      "logits/chosen": -0.2789974510669708,
      "logits/rejected": -0.16233700513839722,
      "logps/chosen": -70.20834350585938,
      "logps/rejected": -79.00541687011719,
      "loss": 0.6606,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1220630407333374,
      "rewards/margins": 0.09831195324659348,
      "rewards/rejected": 0.02375108189880848,
      "step": 790
    },
    {
      "epoch": 0.1837718534007086,
      "grad_norm": 20.235546112060547,
      "learning_rate": 8.164924506387922e-06,
      "logits/chosen": 0.5913870930671692,
      "logits/rejected": 0.445438027381897,
      "logps/chosen": -75.94792938232422,
      "logps/rejected": -83.54081726074219,
      "loss": 0.476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4217465817928314,
      "rewards/margins": 0.5758633017539978,
      "rewards/rejected": -0.15411677956581116,
      "step": 791
    },
    {
      "epoch": 0.1840041819132253,
      "grad_norm": 23.19337272644043,
      "learning_rate": 8.16260162601626e-06,
      "logits/chosen": -0.3808854818344116,
      "logits/rejected": -0.18564891815185547,
      "logps/chosen": -76.97637176513672,
      "logps/rejected": -78.4201431274414,
      "loss": 0.6451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07107566297054291,
      "rewards/margins": 0.15232107043266296,
      "rewards/rejected": -0.08124542236328125,
      "step": 792
    },
    {
      "epoch": 0.184236510425742,
      "grad_norm": 24.79413604736328,
      "learning_rate": 8.1602787456446e-06,
      "logits/chosen": -0.2608306407928467,
      "logits/rejected": -0.3267984092235565,
      "logps/chosen": -67.93357849121094,
      "logps/rejected": -77.99378967285156,
      "loss": 0.7001,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06051956117153168,
      "rewards/margins": 0.058641254901885986,
      "rewards/rejected": 0.0018782978877425194,
      "step": 793
    },
    {
      "epoch": 0.1844688389382587,
      "grad_norm": 23.612037658691406,
      "learning_rate": 8.15795586527294e-06,
      "logits/chosen": -0.054079797118902206,
      "logits/rejected": -0.10942570120096207,
      "logps/chosen": -83.73347473144531,
      "logps/rejected": -87.72097778320312,
      "loss": 0.6128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12300892174243927,
      "rewards/margins": 0.33158060908317566,
      "rewards/rejected": -0.2085716724395752,
      "step": 794
    },
    {
      "epoch": 0.1847011674507754,
      "grad_norm": 27.735595703125,
      "learning_rate": 8.155632984901277e-06,
      "logits/chosen": 0.3197880685329437,
      "logits/rejected": 0.18157461285591125,
      "logps/chosen": -71.68573760986328,
      "logps/rejected": -106.6421890258789,
      "loss": 0.4483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.302488774061203,
      "rewards/margins": 0.785118818283081,
      "rewards/rejected": -0.48263001441955566,
      "step": 795
    },
    {
      "epoch": 0.1849334959632921,
      "grad_norm": 22.53548240661621,
      "learning_rate": 8.153310104529616e-06,
      "logits/chosen": -0.38891828060150146,
      "logits/rejected": -0.32318416237831116,
      "logps/chosen": -82.53925323486328,
      "logps/rejected": -78.19677734375,
      "loss": 0.6295,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09329865127801895,
      "rewards/margins": 0.25259464979171753,
      "rewards/rejected": -0.15929602086544037,
      "step": 796
    },
    {
      "epoch": 0.1851658244758088,
      "grad_norm": 21.76309585571289,
      "learning_rate": 8.150987224157957e-06,
      "logits/chosen": -0.6317245960235596,
      "logits/rejected": -0.6517712473869324,
      "logps/chosen": -85.21772766113281,
      "logps/rejected": -93.43519592285156,
      "loss": 0.5999,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14471736550331116,
      "rewards/margins": 0.2904369831085205,
      "rewards/rejected": -0.14571961760520935,
      "step": 797
    },
    {
      "epoch": 0.1853981529883255,
      "grad_norm": 23.613943099975586,
      "learning_rate": 8.148664343786296e-06,
      "logits/chosen": 0.6292772889137268,
      "logits/rejected": 0.7036432027816772,
      "logps/chosen": -77.85882568359375,
      "logps/rejected": -75.86702728271484,
      "loss": 0.6688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.20360112190246582,
      "rewards/margins": 0.07571744918823242,
      "rewards/rejected": 0.1278836727142334,
      "step": 798
    },
    {
      "epoch": 0.1856304815008422,
      "grad_norm": 24.677162170410156,
      "learning_rate": 8.146341463414635e-06,
      "logits/chosen": -0.37247785925865173,
      "logits/rejected": -0.31933656334877014,
      "logps/chosen": -81.78827667236328,
      "logps/rejected": -64.7642822265625,
      "loss": 0.7024,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06412003189325333,
      "rewards/margins": 0.04496186971664429,
      "rewards/rejected": 0.019158147275447845,
      "step": 799
    },
    {
      "epoch": 0.1858628100133589,
      "grad_norm": 26.86172866821289,
      "learning_rate": 8.144018583042974e-06,
      "logits/chosen": -0.21546372771263123,
      "logits/rejected": -0.2441418319940567,
      "logps/chosen": -69.4280014038086,
      "logps/rejected": -86.33353424072266,
      "loss": 0.6698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.041128456592559814,
      "rewards/margins": 0.12309328466653824,
      "rewards/rejected": -0.08196482807397842,
      "step": 800
    },
    {
      "epoch": 0.1860951385258756,
      "grad_norm": 18.562355041503906,
      "learning_rate": 8.141695702671313e-06,
      "logits/chosen": -0.5462385416030884,
      "logits/rejected": -0.4833720326423645,
      "logps/chosen": -80.64117431640625,
      "logps/rejected": -96.07633972167969,
      "loss": 0.4694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1926976889371872,
      "rewards/margins": 0.7564640045166016,
      "rewards/rejected": -0.563766360282898,
      "step": 801
    },
    {
      "epoch": 0.18632746703839229,
      "grad_norm": 22.61098861694336,
      "learning_rate": 8.139372822299653e-06,
      "logits/chosen": -0.49210724234580994,
      "logits/rejected": -0.5933005213737488,
      "logps/chosen": -73.5125732421875,
      "logps/rejected": -93.67327117919922,
      "loss": 0.6055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1328997164964676,
      "rewards/margins": 0.4024280607700348,
      "rewards/rejected": -0.2695283591747284,
      "step": 802
    },
    {
      "epoch": 0.18655979555090899,
      "grad_norm": 24.015321731567383,
      "learning_rate": 8.13704994192799e-06,
      "logits/chosen": -0.3316969871520996,
      "logits/rejected": -0.3895215094089508,
      "logps/chosen": -80.47512817382812,
      "logps/rejected": -83.09080505371094,
      "loss": 0.7111,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04851634055376053,
      "rewards/margins": 0.03221695125102997,
      "rewards/rejected": 0.016299398615956306,
      "step": 803
    },
    {
      "epoch": 0.18679212406342569,
      "grad_norm": 24.626359939575195,
      "learning_rate": 8.13472706155633e-06,
      "logits/chosen": 0.056776903569698334,
      "logits/rejected": 0.09948520362377167,
      "logps/chosen": -73.9101333618164,
      "logps/rejected": -82.85562896728516,
      "loss": 0.7036,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07415519654750824,
      "rewards/margins": 0.05472612380981445,
      "rewards/rejected": 0.01942906528711319,
      "step": 804
    },
    {
      "epoch": 0.18702445257594238,
      "grad_norm": 26.77496910095215,
      "learning_rate": 8.13240418118467e-06,
      "logits/chosen": -0.34365198016166687,
      "logits/rejected": -0.16042394936084747,
      "logps/chosen": -98.55648040771484,
      "logps/rejected": -90.69039916992188,
      "loss": 0.7166,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06382922828197479,
      "rewards/margins": 0.026292402297258377,
      "rewards/rejected": 0.037536829710006714,
      "step": 805
    },
    {
      "epoch": 0.18725678108845908,
      "grad_norm": 24.74738121032715,
      "learning_rate": 8.130081300813009e-06,
      "logits/chosen": 0.6197289228439331,
      "logits/rejected": 0.5876274704933167,
      "logps/chosen": -68.14359283447266,
      "logps/rejected": -85.92601776123047,
      "loss": 0.7272,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.06903062015771866,
      "rewards/margins": 0.06362355500459671,
      "rewards/rejected": 0.005407050251960754,
      "step": 806
    },
    {
      "epoch": 0.18748910960097578,
      "grad_norm": 21.866653442382812,
      "learning_rate": 8.127758420441348e-06,
      "logits/chosen": -0.45514294505119324,
      "logits/rejected": -0.2729644179344177,
      "logps/chosen": -67.34883880615234,
      "logps/rejected": -72.90765380859375,
      "loss": 0.6649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14316968619823456,
      "rewards/margins": 0.10481658577919006,
      "rewards/rejected": 0.03835310414433479,
      "step": 807
    },
    {
      "epoch": 0.18772143811349248,
      "grad_norm": 24.000062942504883,
      "learning_rate": 8.125435540069687e-06,
      "logits/chosen": -0.4480117857456207,
      "logits/rejected": -0.5870133638381958,
      "logps/chosen": -87.11152648925781,
      "logps/rejected": -100.37736511230469,
      "loss": 0.622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06500203907489777,
      "rewards/margins": 0.21971189975738525,
      "rewards/rejected": -0.1547098606824875,
      "step": 808
    },
    {
      "epoch": 0.18795376662600918,
      "grad_norm": 25.958574295043945,
      "learning_rate": 8.123112659698026e-06,
      "logits/chosen": -0.5343766808509827,
      "logits/rejected": -0.5302034616470337,
      "logps/chosen": -78.94505310058594,
      "logps/rejected": -90.99588012695312,
      "loss": 0.6407,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.20356862246990204,
      "rewards/margins": 0.20060952007770538,
      "rewards/rejected": 0.0029591135680675507,
      "step": 809
    },
    {
      "epoch": 0.18818609513852588,
      "grad_norm": 22.27794075012207,
      "learning_rate": 8.120789779326365e-06,
      "logits/chosen": 0.06913289427757263,
      "logits/rejected": 0.14766618609428406,
      "logps/chosen": -81.09961700439453,
      "logps/rejected": -80.21050262451172,
      "loss": 0.5162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21599198877811432,
      "rewards/margins": 0.4361895024776459,
      "rewards/rejected": -0.22019755840301514,
      "step": 810
    },
    {
      "epoch": 0.18841842365104258,
      "grad_norm": 23.931501388549805,
      "learning_rate": 8.118466898954704e-06,
      "logits/chosen": -0.6747130751609802,
      "logits/rejected": -0.7899734973907471,
      "logps/chosen": -78.08145904541016,
      "logps/rejected": -78.73460388183594,
      "loss": 0.7025,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.07019834965467453,
      "rewards/margins": 0.08392644673585892,
      "rewards/rejected": -0.013728129677474499,
      "step": 811
    },
    {
      "epoch": 0.18865075216355928,
      "grad_norm": 22.03333854675293,
      "learning_rate": 8.116144018583043e-06,
      "logits/chosen": -0.8625714778900146,
      "logits/rejected": -0.9022699594497681,
      "logps/chosen": -68.12196350097656,
      "logps/rejected": -75.30596923828125,
      "loss": 0.6535,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.15338711440563202,
      "rewards/margins": 0.16907599568367004,
      "rewards/rejected": -0.015688864514231682,
      "step": 812
    },
    {
      "epoch": 0.18888308067607598,
      "grad_norm": 23.684139251708984,
      "learning_rate": 8.113821138211383e-06,
      "logits/chosen": -0.9287048578262329,
      "logits/rejected": -0.7375731468200684,
      "logps/chosen": -78.53730773925781,
      "logps/rejected": -91.3725357055664,
      "loss": 0.5995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06107330694794655,
      "rewards/margins": 0.34746479988098145,
      "rewards/rejected": -0.2863914966583252,
      "step": 813
    },
    {
      "epoch": 0.18911540918859268,
      "grad_norm": 24.41145133972168,
      "learning_rate": 8.111498257839722e-06,
      "logits/chosen": 1.2110810279846191,
      "logits/rejected": 1.4455819129943848,
      "logps/chosen": -87.52544403076172,
      "logps/rejected": -84.48982238769531,
      "loss": 0.6046,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05264608934521675,
      "rewards/margins": 0.2845069169998169,
      "rewards/rejected": -0.23186083137989044,
      "step": 814
    },
    {
      "epoch": 0.18934773770110938,
      "grad_norm": 20.80415916442871,
      "learning_rate": 8.109175377468061e-06,
      "logits/chosen": -0.383584201335907,
      "logits/rejected": -0.24873048067092896,
      "logps/chosen": -79.86162567138672,
      "logps/rejected": -72.86119079589844,
      "loss": 0.5866,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23695988953113556,
      "rewards/margins": 0.27073901891708374,
      "rewards/rejected": -0.033779121935367584,
      "step": 815
    },
    {
      "epoch": 0.18958006621362608,
      "grad_norm": 24.64765167236328,
      "learning_rate": 8.1068524970964e-06,
      "logits/chosen": -0.8312644958496094,
      "logits/rejected": -0.7982953786849976,
      "logps/chosen": -82.74801635742188,
      "logps/rejected": -85.6102066040039,
      "loss": 0.6331,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1170910894870758,
      "rewards/margins": 0.20958013832569122,
      "rewards/rejected": -0.09248904883861542,
      "step": 816
    },
    {
      "epoch": 0.18981239472614275,
      "grad_norm": 23.449243545532227,
      "learning_rate": 8.104529616724739e-06,
      "logits/chosen": -0.5795250535011292,
      "logits/rejected": -0.569233775138855,
      "logps/chosen": -91.56912231445312,
      "logps/rejected": -96.54329681396484,
      "loss": 0.5864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.19021344184875488,
      "rewards/margins": 0.583183765411377,
      "rewards/rejected": -0.3929702937602997,
      "step": 817
    },
    {
      "epoch": 0.19004472323865945,
      "grad_norm": 24.123023986816406,
      "learning_rate": 8.102206736353078e-06,
      "logits/chosen": -0.11700591444969177,
      "logits/rejected": -0.2580530047416687,
      "logps/chosen": -77.26534271240234,
      "logps/rejected": -96.89768981933594,
      "loss": 0.6649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15013113617897034,
      "rewards/margins": 0.11922936141490936,
      "rewards/rejected": 0.03090181201696396,
      "step": 818
    },
    {
      "epoch": 0.19027705175117615,
      "grad_norm": 23.01740264892578,
      "learning_rate": 8.099883855981417e-06,
      "logits/chosen": -0.28791239857673645,
      "logits/rejected": -0.18498417735099792,
      "logps/chosen": -82.8801498413086,
      "logps/rejected": -99.876708984375,
      "loss": 0.5636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.261966735124588,
      "rewards/margins": 0.5089007019996643,
      "rewards/rejected": -0.2469339817762375,
      "step": 819
    },
    {
      "epoch": 0.19050938026369285,
      "grad_norm": 24.487680435180664,
      "learning_rate": 8.097560975609758e-06,
      "logits/chosen": -0.2933594584465027,
      "logits/rejected": -0.3151476979255676,
      "logps/chosen": -75.97407531738281,
      "logps/rejected": -76.43608856201172,
      "loss": 0.6671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05211859196424484,
      "rewards/margins": 0.10383124649524689,
      "rewards/rejected": -0.05171266198158264,
      "step": 820
    },
    {
      "epoch": 0.19074170877620955,
      "grad_norm": 20.994258880615234,
      "learning_rate": 8.095238095238097e-06,
      "logits/chosen": -0.7753283381462097,
      "logits/rejected": -0.7663830518722534,
      "logps/chosen": -86.25344848632812,
      "logps/rejected": -86.01301574707031,
      "loss": 0.5802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1320098340511322,
      "rewards/margins": 0.3397328853607178,
      "rewards/rejected": -0.20772305130958557,
      "step": 821
    },
    {
      "epoch": 0.19097403728872625,
      "grad_norm": 24.529191970825195,
      "learning_rate": 8.092915214866435e-06,
      "logits/chosen": 0.25092142820358276,
      "logits/rejected": 0.33518800139427185,
      "logps/chosen": -81.25886535644531,
      "logps/rejected": -75.05304718017578,
      "loss": 0.7167,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.14442014694213867,
      "rewards/margins": 0.08740413933992386,
      "rewards/rejected": 0.057016029953956604,
      "step": 822
    },
    {
      "epoch": 0.19120636580124295,
      "grad_norm": 22.67327117919922,
      "learning_rate": 8.090592334494774e-06,
      "logits/chosen": -0.5008730292320251,
      "logits/rejected": -0.5096276998519897,
      "logps/chosen": -85.30908966064453,
      "logps/rejected": -80.17388153076172,
      "loss": 0.5751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11304757744073868,
      "rewards/margins": 0.31191879510879517,
      "rewards/rejected": -0.1988712102174759,
      "step": 823
    },
    {
      "epoch": 0.19143869431375965,
      "grad_norm": 27.42055320739746,
      "learning_rate": 8.088269454123113e-06,
      "logits/chosen": 0.7928783893585205,
      "logits/rejected": 0.6868176460266113,
      "logps/chosen": -84.81336975097656,
      "logps/rejected": -95.7723159790039,
      "loss": 0.7168,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.09074817597866058,
      "rewards/margins": 0.2772435247898102,
      "rewards/rejected": -0.1864953488111496,
      "step": 824
    },
    {
      "epoch": 0.19167102282627635,
      "grad_norm": 23.384395599365234,
      "learning_rate": 8.085946573751452e-06,
      "logits/chosen": 0.14196118712425232,
      "logits/rejected": -0.01456347107887268,
      "logps/chosen": -87.24124908447266,
      "logps/rejected": -87.79147338867188,
      "loss": 0.5562,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24249175190925598,
      "rewards/margins": 0.5371143221855164,
      "rewards/rejected": -0.294622540473938,
      "step": 825
    },
    {
      "epoch": 0.19190335133879305,
      "grad_norm": 23.448564529418945,
      "learning_rate": 8.083623693379791e-06,
      "logits/chosen": -0.13553759455680847,
      "logits/rejected": -0.07155433297157288,
      "logps/chosen": -79.54940795898438,
      "logps/rejected": -74.9133529663086,
      "loss": 0.6768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09848815947771072,
      "rewards/margins": 0.06925804913043976,
      "rewards/rejected": 0.029230115935206413,
      "step": 826
    },
    {
      "epoch": 0.19213567985130975,
      "grad_norm": 22.07485008239746,
      "learning_rate": 8.08130081300813e-06,
      "logits/chosen": 0.9494229555130005,
      "logits/rejected": 0.9843077659606934,
      "logps/chosen": -87.48223114013672,
      "logps/rejected": -78.59685516357422,
      "loss": 0.5884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19486205279827118,
      "rewards/margins": 0.29630011320114136,
      "rewards/rejected": -0.10143804550170898,
      "step": 827
    },
    {
      "epoch": 0.19236800836382645,
      "grad_norm": 21.695022583007812,
      "learning_rate": 8.07897793263647e-06,
      "logits/chosen": -0.8223528861999512,
      "logits/rejected": -0.7118065357208252,
      "logps/chosen": -74.03140258789062,
      "logps/rejected": -68.53224182128906,
      "loss": 0.5847,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.286588191986084,
      "rewards/margins": 0.3816586136817932,
      "rewards/rejected": -0.09507042169570923,
      "step": 828
    },
    {
      "epoch": 0.19260033687634315,
      "grad_norm": 21.387304306030273,
      "learning_rate": 8.07665505226481e-06,
      "logits/chosen": -0.8208708167076111,
      "logits/rejected": -0.8380972146987915,
      "logps/chosen": -81.89909362792969,
      "logps/rejected": -75.49115753173828,
      "loss": 0.6117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2633858919143677,
      "rewards/margins": 0.20168064534664154,
      "rewards/rejected": 0.061705250293016434,
      "step": 829
    },
    {
      "epoch": 0.19283266538885985,
      "grad_norm": 24.55653953552246,
      "learning_rate": 8.074332171893147e-06,
      "logits/chosen": -0.30059850215911865,
      "logits/rejected": -0.1995815932750702,
      "logps/chosen": -79.68965911865234,
      "logps/rejected": -91.796142578125,
      "loss": 0.6118,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.016033321619033813,
      "rewards/margins": 0.41998469829559326,
      "rewards/rejected": -0.4360180199146271,
      "step": 830
    },
    {
      "epoch": 0.19306499390137655,
      "grad_norm": 23.50952911376953,
      "learning_rate": 8.072009291521488e-06,
      "logits/chosen": -0.9808313846588135,
      "logits/rejected": -0.8332194685935974,
      "logps/chosen": -78.69232940673828,
      "logps/rejected": -85.7540283203125,
      "loss": 0.6002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.028714390471577644,
      "rewards/margins": 0.3107217848300934,
      "rewards/rejected": -0.2820073962211609,
      "step": 831
    },
    {
      "epoch": 0.19329732241389325,
      "grad_norm": 22.82854461669922,
      "learning_rate": 8.069686411149827e-06,
      "logits/chosen": -0.08761680126190186,
      "logits/rejected": -0.15524224936962128,
      "logps/chosen": -69.885498046875,
      "logps/rejected": -78.9539566040039,
      "loss": 0.6775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04682181030511856,
      "rewards/margins": 0.059139151126146317,
      "rewards/rejected": -0.10596096515655518,
      "step": 832
    },
    {
      "epoch": 0.19352965092640995,
      "grad_norm": 22.589319229125977,
      "learning_rate": 8.067363530778165e-06,
      "logits/chosen": -0.5795852541923523,
      "logits/rejected": -0.5692289471626282,
      "logps/chosen": -79.69085693359375,
      "logps/rejected": -95.96678161621094,
      "loss": 0.5637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14474138617515564,
      "rewards/margins": 0.40701916813850403,
      "rewards/rejected": -0.2622778117656708,
      "step": 833
    },
    {
      "epoch": 0.19376197943892665,
      "grad_norm": 26.980941772460938,
      "learning_rate": 8.065040650406504e-06,
      "logits/chosen": 0.4355832636356354,
      "logits/rejected": 0.3829522728919983,
      "logps/chosen": -76.57738494873047,
      "logps/rejected": -77.26934814453125,
      "loss": 0.7973,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.045145705342292786,
      "rewards/margins": -0.14447632431983948,
      "rewards/rejected": 0.18962202966213226,
      "step": 834
    },
    {
      "epoch": 0.19399430795144335,
      "grad_norm": 25.667587280273438,
      "learning_rate": 8.062717770034843e-06,
      "logits/chosen": -0.4838998317718506,
      "logits/rejected": -0.5337504148483276,
      "logps/chosen": -79.29192352294922,
      "logps/rejected": -76.15377044677734,
      "loss": 0.7313,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.10945043712854385,
      "rewards/margins": -0.06539402902126312,
      "rewards/rejected": 0.17484445869922638,
      "step": 835
    },
    {
      "epoch": 0.19422663646396005,
      "grad_norm": 26.253725051879883,
      "learning_rate": 8.060394889663184e-06,
      "logits/chosen": -0.17711108922958374,
      "logits/rejected": -0.1746141016483307,
      "logps/chosen": -72.39907836914062,
      "logps/rejected": -78.20918273925781,
      "loss": 0.638,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03724364936351776,
      "rewards/margins": 0.22931107878684998,
      "rewards/rejected": -0.19206736981868744,
      "step": 836
    },
    {
      "epoch": 0.19445896497647674,
      "grad_norm": 28.661663055419922,
      "learning_rate": 8.058072009291523e-06,
      "logits/chosen": -0.7294145822525024,
      "logits/rejected": -0.6591504812240601,
      "logps/chosen": -77.46949005126953,
      "logps/rejected": -81.04215240478516,
      "loss": 0.6008,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.20920391380786896,
      "rewards/margins": 0.25944820046424866,
      "rewards/rejected": -0.0502442829310894,
      "step": 837
    },
    {
      "epoch": 0.19469129348899344,
      "grad_norm": 24.657428741455078,
      "learning_rate": 8.05574912891986e-06,
      "logits/chosen": -0.5348845720291138,
      "logits/rejected": -0.3263010084629059,
      "logps/chosen": -89.14270782470703,
      "logps/rejected": -74.62812805175781,
      "loss": 0.7332,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02167649194598198,
      "rewards/margins": 0.035457804799079895,
      "rewards/rejected": -0.057134296745061874,
      "step": 838
    },
    {
      "epoch": 0.19492362200151014,
      "grad_norm": 21.60019874572754,
      "learning_rate": 8.0534262485482e-06,
      "logits/chosen": 0.47620701789855957,
      "logits/rejected": 0.5798327326774597,
      "logps/chosen": -67.96821594238281,
      "logps/rejected": -91.7848129272461,
      "loss": 0.5619,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2658502459526062,
      "rewards/margins": 0.719111442565918,
      "rewards/rejected": -0.45326119661331177,
      "step": 839
    },
    {
      "epoch": 0.19515595051402684,
      "grad_norm": 19.413043975830078,
      "learning_rate": 8.05110336817654e-06,
      "logits/chosen": -0.5797943472862244,
      "logits/rejected": -0.4857272505760193,
      "logps/chosen": -81.00841522216797,
      "logps/rejected": -97.13473510742188,
      "loss": 0.4684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2403772473335266,
      "rewards/margins": 0.9370006918907166,
      "rewards/rejected": -0.6966234445571899,
      "step": 840
    },
    {
      "epoch": 0.19538827902654354,
      "grad_norm": 25.548084259033203,
      "learning_rate": 8.048780487804879e-06,
      "logits/chosen": -0.26180073618888855,
      "logits/rejected": -0.5177832245826721,
      "logps/chosen": -92.04769897460938,
      "logps/rejected": -90.8794174194336,
      "loss": 0.6528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05112376809120178,
      "rewards/margins": 0.14807820320129395,
      "rewards/rejected": -0.09695444256067276,
      "step": 841
    },
    {
      "epoch": 0.19562060753906024,
      "grad_norm": 23.607526779174805,
      "learning_rate": 8.046457607433218e-06,
      "logits/chosen": -0.17901092767715454,
      "logits/rejected": -0.1298472136259079,
      "logps/chosen": -72.5686264038086,
      "logps/rejected": -66.43617248535156,
      "loss": 0.6663,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07457366585731506,
      "rewards/margins": 0.15266990661621094,
      "rewards/rejected": -0.07809624075889587,
      "step": 842
    },
    {
      "epoch": 0.19585293605157694,
      "grad_norm": 26.465044021606445,
      "learning_rate": 8.044134727061556e-06,
      "logits/chosen": -0.47259387373924255,
      "logits/rejected": -0.5490302443504333,
      "logps/chosen": -90.50312805175781,
      "logps/rejected": -91.26374053955078,
      "loss": 0.7124,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.023060746490955353,
      "rewards/margins": 0.01851639524102211,
      "rewards/rejected": 0.00454435870051384,
      "step": 843
    },
    {
      "epoch": 0.19608526456409364,
      "grad_norm": 26.008228302001953,
      "learning_rate": 8.041811846689897e-06,
      "logits/chosen": -0.824093759059906,
      "logits/rejected": -0.9359332323074341,
      "logps/chosen": -83.28427124023438,
      "logps/rejected": -89.8829116821289,
      "loss": 0.6986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05692296102643013,
      "rewards/margins": 0.03712143748998642,
      "rewards/rejected": 0.01980152167379856,
      "step": 844
    },
    {
      "epoch": 0.19631759307661031,
      "grad_norm": 28.947938919067383,
      "learning_rate": 8.039488966318234e-06,
      "logits/chosen": -0.5481814742088318,
      "logits/rejected": -0.34396445751190186,
      "logps/chosen": -93.35441589355469,
      "logps/rejected": -89.31706237792969,
      "loss": 0.6764,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15168815851211548,
      "rewards/margins": 0.05267864093184471,
      "rewards/rejected": 0.09900951385498047,
      "step": 845
    },
    {
      "epoch": 0.196549921589127,
      "grad_norm": 27.003490447998047,
      "learning_rate": 8.037166085946573e-06,
      "logits/chosen": -0.10691317915916443,
      "logits/rejected": 0.05256972461938858,
      "logps/chosen": -90.67156219482422,
      "logps/rejected": -93.96174621582031,
      "loss": 0.7114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.46701332926750183,
      "rewards/margins": 0.8131923079490662,
      "rewards/rejected": -0.3461790084838867,
      "step": 846
    },
    {
      "epoch": 0.1967822501016437,
      "grad_norm": 23.82795524597168,
      "learning_rate": 8.034843205574914e-06,
      "logits/chosen": -0.3537796139717102,
      "logits/rejected": -0.3622978925704956,
      "logps/chosen": -74.83047485351562,
      "logps/rejected": -85.41641998291016,
      "loss": 0.692,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.14325568079948425,
      "rewards/margins": 0.06846200674772263,
      "rewards/rejected": 0.07479369640350342,
      "step": 847
    },
    {
      "epoch": 0.1970145786141604,
      "grad_norm": 28.062997817993164,
      "learning_rate": 8.032520325203253e-06,
      "logits/chosen": -1.0331251621246338,
      "logits/rejected": -0.9577901363372803,
      "logps/chosen": -81.82799530029297,
      "logps/rejected": -91.92301940917969,
      "loss": 0.7685,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14101029932498932,
      "rewards/margins": -0.07324764877557755,
      "rewards/rejected": -0.06776265799999237,
      "step": 848
    },
    {
      "epoch": 0.1972469071266771,
      "grad_norm": 22.87285804748535,
      "learning_rate": 8.030197444831592e-06,
      "logits/chosen": -0.0012741237878799438,
      "logits/rejected": -0.12751078605651855,
      "logps/chosen": -86.38250732421875,
      "logps/rejected": -72.079833984375,
      "loss": 0.623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03909165412187576,
      "rewards/margins": 0.17008109390735626,
      "rewards/rejected": -0.1309894323348999,
      "step": 849
    },
    {
      "epoch": 0.1974792356391938,
      "grad_norm": 35.13692092895508,
      "learning_rate": 8.02787456445993e-06,
      "logits/chosen": -0.9773246645927429,
      "logits/rejected": -0.9621289372444153,
      "logps/chosen": -84.56968688964844,
      "logps/rejected": -70.14317321777344,
      "loss": 0.7461,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.037453871220350266,
      "rewards/margins": -0.06470096111297607,
      "rewards/rejected": 0.10215482860803604,
      "step": 850
    },
    {
      "epoch": 0.1977115641517105,
      "grad_norm": 22.13910484313965,
      "learning_rate": 8.02555168408827e-06,
      "logits/chosen": -1.0170236825942993,
      "logits/rejected": -0.9293851852416992,
      "logps/chosen": -81.91470336914062,
      "logps/rejected": -102.1513671875,
      "loss": 0.5469,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03662855923175812,
      "rewards/margins": 0.9135376811027527,
      "rewards/rejected": -0.9501662254333496,
      "step": 851
    },
    {
      "epoch": 0.1979438926642272,
      "grad_norm": 22.170164108276367,
      "learning_rate": 8.02322880371661e-06,
      "logits/chosen": 0.18850018084049225,
      "logits/rejected": 0.08712136745452881,
      "logps/chosen": -81.96273040771484,
      "logps/rejected": -74.90133666992188,
      "loss": 0.5929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2361712008714676,
      "rewards/margins": 0.3502746820449829,
      "rewards/rejected": -0.11410346627235413,
      "step": 852
    },
    {
      "epoch": 0.1981762211767439,
      "grad_norm": 23.969112396240234,
      "learning_rate": 8.020905923344948e-06,
      "logits/chosen": -0.5432956218719482,
      "logits/rejected": -0.4985630512237549,
      "logps/chosen": -87.84437561035156,
      "logps/rejected": -75.29183197021484,
      "loss": 0.6771,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0601005032658577,
      "rewards/margins": 0.0807509571313858,
      "rewards/rejected": -0.02065047062933445,
      "step": 853
    },
    {
      "epoch": 0.1984085496892606,
      "grad_norm": 21.732938766479492,
      "learning_rate": 8.018583042973288e-06,
      "logits/chosen": 0.34770071506500244,
      "logits/rejected": 0.6038612723350525,
      "logps/chosen": -65.55848693847656,
      "logps/rejected": -83.08191680908203,
      "loss": 0.5778,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12717045843601227,
      "rewards/margins": 0.5803642868995667,
      "rewards/rejected": -0.45319387316703796,
      "step": 854
    },
    {
      "epoch": 0.1986408782017773,
      "grad_norm": 22.729263305664062,
      "learning_rate": 8.016260162601627e-06,
      "logits/chosen": -0.2481522560119629,
      "logits/rejected": -0.051406487822532654,
      "logps/chosen": -87.1717300415039,
      "logps/rejected": -81.01853942871094,
      "loss": 0.6009,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.179100900888443,
      "rewards/margins": 0.21851973235607147,
      "rewards/rejected": -0.039418838918209076,
      "step": 855
    },
    {
      "epoch": 0.198873206714294,
      "grad_norm": 22.003530502319336,
      "learning_rate": 8.013937282229966e-06,
      "logits/chosen": -0.5295941829681396,
      "logits/rejected": -0.5436968803405762,
      "logps/chosen": -75.55631256103516,
      "logps/rejected": -85.22992706298828,
      "loss": 0.5411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24973875284194946,
      "rewards/margins": 0.41474318504333496,
      "rewards/rejected": -0.16500446200370789,
      "step": 856
    },
    {
      "epoch": 0.1991055352268107,
      "grad_norm": 23.52433204650879,
      "learning_rate": 8.011614401858305e-06,
      "logits/chosen": -0.03286188840866089,
      "logits/rejected": -0.13135427236557007,
      "logps/chosen": -82.41082763671875,
      "logps/rejected": -89.25103759765625,
      "loss": 0.6159,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05298778787255287,
      "rewards/margins": 0.1859554648399353,
      "rewards/rejected": -0.13296768069267273,
      "step": 857
    },
    {
      "epoch": 0.1993378637393274,
      "grad_norm": 19.81428337097168,
      "learning_rate": 8.009291521486644e-06,
      "logits/chosen": -0.823571503162384,
      "logits/rejected": -0.8581105470657349,
      "logps/chosen": -72.09713745117188,
      "logps/rejected": -74.33517456054688,
      "loss": 0.5618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3015791177749634,
      "rewards/margins": 0.32069090008735657,
      "rewards/rejected": -0.01911180093884468,
      "step": 858
    },
    {
      "epoch": 0.1995701922518441,
      "grad_norm": 20.874353408813477,
      "learning_rate": 8.006968641114983e-06,
      "logits/chosen": -0.8374466300010681,
      "logits/rejected": -0.9205967783927917,
      "logps/chosen": -65.10233306884766,
      "logps/rejected": -77.75361633300781,
      "loss": 0.6263,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06376036256551743,
      "rewards/margins": 0.19612851738929749,
      "rewards/rejected": -0.13236816227436066,
      "step": 859
    },
    {
      "epoch": 0.1998025207643608,
      "grad_norm": 19.92304229736328,
      "learning_rate": 8.004645760743322e-06,
      "logits/chosen": -0.7399406433105469,
      "logits/rejected": -0.7832372188568115,
      "logps/chosen": -79.59661102294922,
      "logps/rejected": -79.79933166503906,
      "loss": 0.5503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24658361077308655,
      "rewards/margins": 0.33150726556777954,
      "rewards/rejected": -0.0849236473441124,
      "step": 860
    },
    {
      "epoch": 0.2000348492768775,
      "grad_norm": 23.02298355102539,
      "learning_rate": 8.00232288037166e-06,
      "logits/chosen": -0.4035312533378601,
      "logits/rejected": -0.19792887568473816,
      "logps/chosen": -84.17840576171875,
      "logps/rejected": -79.42838287353516,
      "loss": 0.6133,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.031208299100399017,
      "rewards/margins": 0.2436218410730362,
      "rewards/rejected": -0.21241351962089539,
      "step": 861
    },
    {
      "epoch": 0.2002671777893942,
      "grad_norm": 23.09166145324707,
      "learning_rate": 8.000000000000001e-06,
      "logits/chosen": -0.839642345905304,
      "logits/rejected": -0.8712789416313171,
      "logps/chosen": -76.39836120605469,
      "logps/rejected": -78.23857116699219,
      "loss": 0.618,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07738529145717621,
      "rewards/margins": 0.2962949275970459,
      "rewards/rejected": -0.2189096212387085,
      "step": 862
    },
    {
      "epoch": 0.2004995063019109,
      "grad_norm": 19.414718627929688,
      "learning_rate": 7.99767711962834e-06,
      "logits/chosen": -0.006913691759109497,
      "logits/rejected": -0.41335445642471313,
      "logps/chosen": -84.75169372558594,
      "logps/rejected": -88.7717514038086,
      "loss": 0.5099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18471166491508484,
      "rewards/margins": 0.6517432928085327,
      "rewards/rejected": -0.4670315980911255,
      "step": 863
    },
    {
      "epoch": 0.2007318348144276,
      "grad_norm": 20.48731231689453,
      "learning_rate": 7.99535423925668e-06,
      "logits/chosen": -0.6588104367256165,
      "logits/rejected": -0.8023021221160889,
      "logps/chosen": -75.18508911132812,
      "logps/rejected": -77.34671783447266,
      "loss": 0.6258,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0540054589509964,
      "rewards/margins": 0.156671941280365,
      "rewards/rejected": -0.1026664674282074,
      "step": 864
    },
    {
      "epoch": 0.2009641633269443,
      "grad_norm": 19.063919067382812,
      "learning_rate": 7.993031358885018e-06,
      "logits/chosen": -0.7840511798858643,
      "logits/rejected": -0.8148788809776306,
      "logps/chosen": -71.86768341064453,
      "logps/rejected": -82.25794982910156,
      "loss": 0.5169,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26335251331329346,
      "rewards/margins": 0.5060529112815857,
      "rewards/rejected": -0.24270036816596985,
      "step": 865
    },
    {
      "epoch": 0.201196491839461,
      "grad_norm": 18.4134578704834,
      "learning_rate": 7.990708478513357e-06,
      "logits/chosen": 1.1088852882385254,
      "logits/rejected": 0.8309265375137329,
      "logps/chosen": -75.78044891357422,
      "logps/rejected": -79.91043090820312,
      "loss": 0.5393,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3642112910747528,
      "rewards/margins": 0.9178220629692078,
      "rewards/rejected": -0.5536108016967773,
      "step": 866
    },
    {
      "epoch": 0.2014288203519777,
      "grad_norm": 27.29509162902832,
      "learning_rate": 7.988385598141698e-06,
      "logits/chosen": -0.5162484049797058,
      "logits/rejected": -0.6464741230010986,
      "logps/chosen": -82.56819152832031,
      "logps/rejected": -84.57034301757812,
      "loss": 0.8571,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2718014419078827,
      "rewards/margins": -0.25163111090660095,
      "rewards/rejected": -0.020170411095023155,
      "step": 867
    },
    {
      "epoch": 0.2016611488644944,
      "grad_norm": 23.545654296875,
      "learning_rate": 7.986062717770035e-06,
      "logits/chosen": -0.5307892560958862,
      "logits/rejected": -0.4928606450557709,
      "logps/chosen": -71.07484436035156,
      "logps/rejected": -96.90351867675781,
      "loss": 0.5924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05306616052985191,
      "rewards/margins": 0.2627298831939697,
      "rewards/rejected": -0.2096637338399887,
      "step": 868
    },
    {
      "epoch": 0.2018934773770111,
      "grad_norm": 19.739757537841797,
      "learning_rate": 7.983739837398374e-06,
      "logits/chosen": -1.056250810623169,
      "logits/rejected": -1.0540322065353394,
      "logps/chosen": -75.90089416503906,
      "logps/rejected": -85.13875579833984,
      "loss": 0.5703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12012577056884766,
      "rewards/margins": 0.35367417335510254,
      "rewards/rejected": -0.23354841768741608,
      "step": 869
    },
    {
      "epoch": 0.2021258058895278,
      "grad_norm": 22.92657470703125,
      "learning_rate": 7.981416957026715e-06,
      "logits/chosen": -0.8310425281524658,
      "logits/rejected": -0.595466136932373,
      "logps/chosen": -73.60350036621094,
      "logps/rejected": -73.90788269042969,
      "loss": 0.7053,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.054293226450681686,
      "rewards/margins": 0.0014492496848106384,
      "rewards/rejected": -0.05574247986078262,
      "step": 870
    },
    {
      "epoch": 0.2023581344020445,
      "grad_norm": 20.709686279296875,
      "learning_rate": 7.979094076655053e-06,
      "logits/chosen": -0.7594259977340698,
      "logits/rejected": -0.7945791482925415,
      "logps/chosen": -72.2728042602539,
      "logps/rejected": -95.16126251220703,
      "loss": 0.5387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1126580685377121,
      "rewards/margins": 0.45373621582984924,
      "rewards/rejected": -0.34107816219329834,
      "step": 871
    },
    {
      "epoch": 0.20259046291456118,
      "grad_norm": 16.974945068359375,
      "learning_rate": 7.976771196283392e-06,
      "logits/chosen": -0.6802023649215698,
      "logits/rejected": -0.5479783415794373,
      "logps/chosen": -73.11430358886719,
      "logps/rejected": -82.69995880126953,
      "loss": 0.4208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23828238248825073,
      "rewards/margins": 0.697239100933075,
      "rewards/rejected": -0.4589567184448242,
      "step": 872
    },
    {
      "epoch": 0.20282279142707788,
      "grad_norm": 22.16388511657715,
      "learning_rate": 7.974448315911731e-06,
      "logits/chosen": -0.6661506295204163,
      "logits/rejected": -0.8157879114151001,
      "logps/chosen": -80.99192810058594,
      "logps/rejected": -96.40803527832031,
      "loss": 0.5656,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16631335020065308,
      "rewards/margins": 0.33709585666656494,
      "rewards/rejected": -0.17078247666358948,
      "step": 873
    },
    {
      "epoch": 0.20305511993959457,
      "grad_norm": 26.047746658325195,
      "learning_rate": 7.97212543554007e-06,
      "logits/chosen": -0.8377823829650879,
      "logits/rejected": -0.9308815598487854,
      "logps/chosen": -90.08232116699219,
      "logps/rejected": -73.7096939086914,
      "loss": 0.7353,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.11157336831092834,
      "rewards/margins": -0.019534720107913017,
      "rewards/rejected": -0.09203862398862839,
      "step": 874
    },
    {
      "epoch": 0.20328744845211127,
      "grad_norm": 28.315929412841797,
      "learning_rate": 7.96980255516841e-06,
      "logits/chosen": -0.8152628540992737,
      "logits/rejected": -0.6731648445129395,
      "logps/chosen": -79.78770446777344,
      "logps/rejected": -79.02450561523438,
      "loss": 0.6946,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06585321575403214,
      "rewards/margins": 0.04807448387145996,
      "rewards/rejected": -0.1139276996254921,
      "step": 875
    },
    {
      "epoch": 0.20351977696462797,
      "grad_norm": 27.415010452270508,
      "learning_rate": 7.967479674796748e-06,
      "logits/chosen": -0.9629467129707336,
      "logits/rejected": -0.989372193813324,
      "logps/chosen": -96.45414733886719,
      "logps/rejected": -95.42484283447266,
      "loss": 0.6694,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04111604019999504,
      "rewards/margins": 0.11205607652664185,
      "rewards/rejected": -0.15317213535308838,
      "step": 876
    },
    {
      "epoch": 0.20375210547714467,
      "grad_norm": 19.72804832458496,
      "learning_rate": 7.965156794425087e-06,
      "logits/chosen": -0.29733091592788696,
      "logits/rejected": -0.5312460660934448,
      "logps/chosen": -66.75254821777344,
      "logps/rejected": -87.81651306152344,
      "loss": 0.5061,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23718300461769104,
      "rewards/margins": 0.466636598110199,
      "rewards/rejected": -0.22945360839366913,
      "step": 877
    },
    {
      "epoch": 0.20398443398966137,
      "grad_norm": 22.478832244873047,
      "learning_rate": 7.962833914053428e-06,
      "logits/chosen": 0.19688019156455994,
      "logits/rejected": 0.34686028957366943,
      "logps/chosen": -92.22531127929688,
      "logps/rejected": -86.13903045654297,
      "loss": 0.5943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.016967952251434326,
      "rewards/margins": 0.28514364361763,
      "rewards/rejected": -0.30211156606674194,
      "step": 878
    },
    {
      "epoch": 0.20421676250217807,
      "grad_norm": 24.035728454589844,
      "learning_rate": 7.960511033681767e-06,
      "logits/chosen": 0.057804688811302185,
      "logits/rejected": 0.2061220407485962,
      "logps/chosen": -82.8088607788086,
      "logps/rejected": -90.58158874511719,
      "loss": 0.6512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009542755782604218,
      "rewards/margins": 0.12863889336585999,
      "rewards/rejected": -0.1381816416978836,
      "step": 879
    },
    {
      "epoch": 0.20444909101469477,
      "grad_norm": 28.48656463623047,
      "learning_rate": 7.958188153310104e-06,
      "logits/chosen": -0.8702530264854431,
      "logits/rejected": -0.9295247793197632,
      "logps/chosen": -87.2864990234375,
      "logps/rejected": -92.33045196533203,
      "loss": 0.7512,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10653962939977646,
      "rewards/margins": -0.041303716599941254,
      "rewards/rejected": -0.06523590534925461,
      "step": 880
    },
    {
      "epoch": 0.20468141952721147,
      "grad_norm": 21.14015007019043,
      "learning_rate": 7.955865272938444e-06,
      "logits/chosen": -0.2160872370004654,
      "logits/rejected": -0.3969658315181732,
      "logps/chosen": -84.0805435180664,
      "logps/rejected": -84.43810272216797,
      "loss": 0.5902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015330193564295769,
      "rewards/margins": 0.27399635314941406,
      "rewards/rejected": -0.25866612792015076,
      "step": 881
    },
    {
      "epoch": 0.20491374803972817,
      "grad_norm": 23.07240867614746,
      "learning_rate": 7.953542392566783e-06,
      "logits/chosen": -0.5334939360618591,
      "logits/rejected": -0.4039747714996338,
      "logps/chosen": -79.15428161621094,
      "logps/rejected": -87.76299285888672,
      "loss": 0.6447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03659161925315857,
      "rewards/margins": 0.18292385339736938,
      "rewards/rejected": -0.21951547265052795,
      "step": 882
    },
    {
      "epoch": 0.20514607655224487,
      "grad_norm": 25.3311824798584,
      "learning_rate": 7.951219512195122e-06,
      "logits/chosen": -0.5225567817687988,
      "logits/rejected": -0.5236049294471741,
      "logps/chosen": -86.02236938476562,
      "logps/rejected": -87.99942016601562,
      "loss": 0.6986,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008267618715763092,
      "rewards/margins": 0.11872327327728271,
      "rewards/rejected": -0.1269908994436264,
      "step": 883
    },
    {
      "epoch": 0.20537840506476157,
      "grad_norm": 22.161785125732422,
      "learning_rate": 7.948896631823461e-06,
      "logits/chosen": 0.10622347891330719,
      "logits/rejected": -0.09129099547863007,
      "logps/chosen": -79.21681213378906,
      "logps/rejected": -88.88282775878906,
      "loss": 0.5212,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1634858101606369,
      "rewards/margins": 0.48432910442352295,
      "rewards/rejected": -0.32084324955940247,
      "step": 884
    },
    {
      "epoch": 0.20561073357727827,
      "grad_norm": 18.739295959472656,
      "learning_rate": 7.9465737514518e-06,
      "logits/chosen": -0.4338792562484741,
      "logits/rejected": -0.57745361328125,
      "logps/chosen": -73.85655212402344,
      "logps/rejected": -87.55302429199219,
      "loss": 0.4701,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3687000274658203,
      "rewards/margins": 0.5749610662460327,
      "rewards/rejected": -0.2062610387802124,
      "step": 885
    },
    {
      "epoch": 0.20584306208979497,
      "grad_norm": 22.71162986755371,
      "learning_rate": 7.94425087108014e-06,
      "logits/chosen": -0.2054063230752945,
      "logits/rejected": -0.33142590522766113,
      "logps/chosen": -94.21980285644531,
      "logps/rejected": -85.48384094238281,
      "loss": 0.6067,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.055302999913692474,
      "rewards/margins": 0.20534636080265045,
      "rewards/rejected": -0.15004335343837738,
      "step": 886
    },
    {
      "epoch": 0.20607539060231167,
      "grad_norm": 24.233722686767578,
      "learning_rate": 7.94192799070848e-06,
      "logits/chosen": -0.41522738337516785,
      "logits/rejected": -0.4237940013408661,
      "logps/chosen": -84.19623565673828,
      "logps/rejected": -88.06806945800781,
      "loss": 0.6053,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10127827525138855,
      "rewards/margins": 0.31285959482192993,
      "rewards/rejected": -0.21158131957054138,
      "step": 887
    },
    {
      "epoch": 0.20630771911482837,
      "grad_norm": 23.70760154724121,
      "learning_rate": 7.939605110336819e-06,
      "logits/chosen": -1.0198003053665161,
      "logits/rejected": -0.957191526889801,
      "logps/chosen": -84.07917785644531,
      "logps/rejected": -78.98822784423828,
      "loss": 0.6476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02319159545004368,
      "rewards/margins": 0.12228932976722717,
      "rewards/rejected": -0.1454809308052063,
      "step": 888
    },
    {
      "epoch": 0.20654004762734507,
      "grad_norm": 20.839433670043945,
      "learning_rate": 7.937282229965158e-06,
      "logits/chosen": -0.9716277122497559,
      "logits/rejected": -0.8381446003913879,
      "logps/chosen": -76.12564849853516,
      "logps/rejected": -76.60198211669922,
      "loss": 0.5794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14437322318553925,
      "rewards/margins": 0.27984994649887085,
      "rewards/rejected": -0.1354767084121704,
      "step": 889
    },
    {
      "epoch": 0.20677237613986177,
      "grad_norm": 28.083086013793945,
      "learning_rate": 7.934959349593497e-06,
      "logits/chosen": -0.17894551157951355,
      "logits/rejected": -0.08662886917591095,
      "logps/chosen": -81.8117446899414,
      "logps/rejected": -71.54007720947266,
      "loss": 0.7147,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06842422485351562,
      "rewards/margins": -0.009854085743427277,
      "rewards/rejected": -0.058570146560668945,
      "step": 890
    },
    {
      "epoch": 0.20700470465237847,
      "grad_norm": 21.207063674926758,
      "learning_rate": 7.932636469221836e-06,
      "logits/chosen": -0.1210971474647522,
      "logits/rejected": -0.4070953130722046,
      "logps/chosen": -80.04373168945312,
      "logps/rejected": -84.4737319946289,
      "loss": 0.5765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02861928939819336,
      "rewards/margins": 0.292206734418869,
      "rewards/rejected": -0.26358744502067566,
      "step": 891
    },
    {
      "epoch": 0.20723703316489517,
      "grad_norm": 23.274154663085938,
      "learning_rate": 7.930313588850174e-06,
      "logits/chosen": 0.2791869640350342,
      "logits/rejected": 0.49281996488571167,
      "logps/chosen": -84.09004974365234,
      "logps/rejected": -80.83656311035156,
      "loss": 0.595,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.19319754838943481,
      "rewards/margins": 0.24768483638763428,
      "rewards/rejected": -0.05448732525110245,
      "step": 892
    },
    {
      "epoch": 0.20746936167741187,
      "grad_norm": 24.633146286010742,
      "learning_rate": 7.927990708478513e-06,
      "logits/chosen": -0.8190498948097229,
      "logits/rejected": -0.6183092594146729,
      "logps/chosen": -77.8225326538086,
      "logps/rejected": -81.40491485595703,
      "loss": 0.6033,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.17191436886787415,
      "rewards/margins": 0.3617711663246155,
      "rewards/rejected": -0.18985682725906372,
      "step": 893
    },
    {
      "epoch": 0.20770169018992857,
      "grad_norm": 23.476882934570312,
      "learning_rate": 7.925667828106854e-06,
      "logits/chosen": -0.6261897087097168,
      "logits/rejected": -0.6624265909194946,
      "logps/chosen": -69.05707550048828,
      "logps/rejected": -80.03633880615234,
      "loss": 0.6491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14786498248577118,
      "rewards/margins": 0.13940605521202087,
      "rewards/rejected": 0.008458925411105156,
      "step": 894
    },
    {
      "epoch": 0.20793401870244527,
      "grad_norm": 21.72330093383789,
      "learning_rate": 7.923344947735191e-06,
      "logits/chosen": -0.9498690962791443,
      "logits/rejected": -1.0045416355133057,
      "logps/chosen": -72.72396850585938,
      "logps/rejected": -69.39665222167969,
      "loss": 0.6663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.138575479388237,
      "rewards/margins": 0.15901033580303192,
      "rewards/rejected": -0.02043483778834343,
      "step": 895
    },
    {
      "epoch": 0.20816634721496197,
      "grad_norm": 25.10753631591797,
      "learning_rate": 7.921022067363532e-06,
      "logits/chosen": -0.7205557227134705,
      "logits/rejected": -0.6712722778320312,
      "logps/chosen": -94.7532958984375,
      "logps/rejected": -85.8125,
      "loss": 0.6871,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04600212723016739,
      "rewards/margins": 0.09721681475639343,
      "rewards/rejected": -0.1432189643383026,
      "step": 896
    },
    {
      "epoch": 0.20839867572747867,
      "grad_norm": 21.923805236816406,
      "learning_rate": 7.91869918699187e-06,
      "logits/chosen": 0.6684099435806274,
      "logits/rejected": 0.6391215920448303,
      "logps/chosen": -84.40144348144531,
      "logps/rejected": -87.56600189208984,
      "loss": 0.6181,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12472167611122131,
      "rewards/margins": 0.44145843386650085,
      "rewards/rejected": -0.31673672795295715,
      "step": 897
    },
    {
      "epoch": 0.20863100423999537,
      "grad_norm": 21.27451515197754,
      "learning_rate": 7.91637630662021e-06,
      "logits/chosen": -0.5987492203712463,
      "logits/rejected": -0.6173799633979797,
      "logps/chosen": -73.7431869506836,
      "logps/rejected": -76.87438201904297,
      "loss": 0.5778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27901384234428406,
      "rewards/margins": 0.3957355320453644,
      "rewards/rejected": -0.11672169715166092,
      "step": 898
    },
    {
      "epoch": 0.20886333275251207,
      "grad_norm": 22.38945198059082,
      "learning_rate": 7.914053426248549e-06,
      "logits/chosen": -0.7248865365982056,
      "logits/rejected": -0.7065495252609253,
      "logps/chosen": -70.1832046508789,
      "logps/rejected": -83.4065170288086,
      "loss": 0.6102,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08990535140037537,
      "rewards/margins": 0.3189082145690918,
      "rewards/rejected": -0.22900286316871643,
      "step": 899
    },
    {
      "epoch": 0.20909566126502874,
      "grad_norm": 15.448760032653809,
      "learning_rate": 7.911730545876888e-06,
      "logits/chosen": -0.4994238018989563,
      "logits/rejected": -0.5176914930343628,
      "logps/chosen": -75.1869888305664,
      "logps/rejected": -88.08674621582031,
      "loss": 0.4001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3628585934638977,
      "rewards/margins": 0.7907916903495789,
      "rewards/rejected": -0.42793309688568115,
      "step": 900
    },
    {
      "epoch": 0.20932798977754544,
      "grad_norm": 24.86018943786621,
      "learning_rate": 7.909407665505228e-06,
      "logits/chosen": -0.8771405816078186,
      "logits/rejected": -0.9046273231506348,
      "logps/chosen": -78.78056335449219,
      "logps/rejected": -76.72078704833984,
      "loss": 0.7032,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19392555952072144,
      "rewards/margins": 0.11964480578899384,
      "rewards/rejected": 0.0742807611823082,
      "step": 901
    },
    {
      "epoch": 0.20956031829006214,
      "grad_norm": 21.644498825073242,
      "learning_rate": 7.907084785133567e-06,
      "logits/chosen": -0.7060079574584961,
      "logits/rejected": -0.5146605372428894,
      "logps/chosen": -71.0947494506836,
      "logps/rejected": -74.77094268798828,
      "loss": 0.6568,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.019915690645575523,
      "rewards/margins": 0.15849417448043823,
      "rewards/rejected": -0.13857848942279816,
      "step": 902
    },
    {
      "epoch": 0.20979264680257884,
      "grad_norm": 21.61927032470703,
      "learning_rate": 7.904761904761904e-06,
      "logits/chosen": -0.25225409865379333,
      "logits/rejected": -0.3343490660190582,
      "logps/chosen": -79.09943389892578,
      "logps/rejected": -88.86243438720703,
      "loss": 0.5593,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029096461832523346,
      "rewards/margins": 0.43166986107826233,
      "rewards/rejected": -0.4025733768939972,
      "step": 903
    },
    {
      "epoch": 0.21002497531509554,
      "grad_norm": 22.642047882080078,
      "learning_rate": 7.902439024390245e-06,
      "logits/chosen": -0.1169249564409256,
      "logits/rejected": -0.1788511723279953,
      "logps/chosen": -71.47620391845703,
      "logps/rejected": -79.70345306396484,
      "loss": 0.5664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4078930616378784,
      "rewards/margins": 0.5705036520957947,
      "rewards/rejected": -0.16261059045791626,
      "step": 904
    },
    {
      "epoch": 0.21025730382761224,
      "grad_norm": 18.65274429321289,
      "learning_rate": 7.900116144018584e-06,
      "logits/chosen": -0.3578840494155884,
      "logits/rejected": -0.14977216720581055,
      "logps/chosen": -83.3825454711914,
      "logps/rejected": -96.84711456298828,
      "loss": 0.4868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1845327913761139,
      "rewards/margins": 0.7631503939628601,
      "rewards/rejected": -0.5786176323890686,
      "step": 905
    },
    {
      "epoch": 0.21048963234012893,
      "grad_norm": 24.48946762084961,
      "learning_rate": 7.897793263646923e-06,
      "logits/chosen": -0.9460026025772095,
      "logits/rejected": -0.745090126991272,
      "logps/chosen": -94.66888427734375,
      "logps/rejected": -99.39985656738281,
      "loss": 0.5575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13212376832962036,
      "rewards/margins": 0.6472446918487549,
      "rewards/rejected": -0.5151209235191345,
      "step": 906
    },
    {
      "epoch": 0.21072196085264563,
      "grad_norm": 26.39508628845215,
      "learning_rate": 7.895470383275262e-06,
      "logits/chosen": -0.4939410090446472,
      "logits/rejected": -0.4376338720321655,
      "logps/chosen": -90.06697082519531,
      "logps/rejected": -76.32867431640625,
      "loss": 0.7892,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06071286275982857,
      "rewards/margins": -0.155049130320549,
      "rewards/rejected": 0.09433627128601074,
      "step": 907
    },
    {
      "epoch": 0.21095428936516233,
      "grad_norm": 25.90627098083496,
      "learning_rate": 7.8931475029036e-06,
      "logits/chosen": -0.15590086579322815,
      "logits/rejected": -0.010087251663208008,
      "logps/chosen": -85.42738342285156,
      "logps/rejected": -81.8475341796875,
      "loss": 0.7128,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02834014967083931,
      "rewards/margins": -0.015546884387731552,
      "rewards/rejected": -0.012793254107236862,
      "step": 908
    },
    {
      "epoch": 0.21118661787767903,
      "grad_norm": 20.965362548828125,
      "learning_rate": 7.890824622531941e-06,
      "logits/chosen": -1.1494903564453125,
      "logits/rejected": -0.9497573971748352,
      "logps/chosen": -80.31179809570312,
      "logps/rejected": -72.16482543945312,
      "loss": 0.6231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10466791689395905,
      "rewards/margins": 0.22657352685928345,
      "rewards/rejected": -0.12190559506416321,
      "step": 909
    },
    {
      "epoch": 0.21141894639019573,
      "grad_norm": 23.933101654052734,
      "learning_rate": 7.888501742160279e-06,
      "logits/chosen": -0.6200497150421143,
      "logits/rejected": -0.6154159903526306,
      "logps/chosen": -82.799560546875,
      "logps/rejected": -75.46829223632812,
      "loss": 0.6926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13628558814525604,
      "rewards/margins": 0.028684396296739578,
      "rewards/rejected": -0.16496999561786652,
      "step": 910
    },
    {
      "epoch": 0.21165127490271243,
      "grad_norm": 26.86187171936035,
      "learning_rate": 7.886178861788618e-06,
      "logits/chosen": -0.6501588821411133,
      "logits/rejected": -0.9813646078109741,
      "logps/chosen": -83.34765625,
      "logps/rejected": -105.41395568847656,
      "loss": 0.6594,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1359923630952835,
      "rewards/margins": 0.6903501152992249,
      "rewards/rejected": -0.8263425827026367,
      "step": 911
    },
    {
      "epoch": 0.21188360341522913,
      "grad_norm": 18.61419105529785,
      "learning_rate": 7.883855981416958e-06,
      "logits/chosen": -1.0148001909255981,
      "logits/rejected": -0.862337589263916,
      "logps/chosen": -72.59893798828125,
      "logps/rejected": -80.96234130859375,
      "loss": 0.4962,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2409244179725647,
      "rewards/margins": 0.48497387766838074,
      "rewards/rejected": -0.24404951930046082,
      "step": 912
    },
    {
      "epoch": 0.21211593192774583,
      "grad_norm": 24.494686126708984,
      "learning_rate": 7.881533101045297e-06,
      "logits/chosen": -0.5938686728477478,
      "logits/rejected": -0.5852104425430298,
      "logps/chosen": -70.3730697631836,
      "logps/rejected": -71.05720520019531,
      "loss": 0.7845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09885306656360626,
      "rewards/margins": -0.07152199745178223,
      "rewards/rejected": -0.027331069111824036,
      "step": 913
    },
    {
      "epoch": 0.21234826044026253,
      "grad_norm": 22.113109588623047,
      "learning_rate": 7.879210220673636e-06,
      "logits/chosen": -1.0678380727767944,
      "logits/rejected": -0.9743248224258423,
      "logps/chosen": -77.20037078857422,
      "logps/rejected": -85.54364013671875,
      "loss": 0.6398,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02013351581990719,
      "rewards/margins": 0.15732143819332123,
      "rewards/rejected": -0.1371879130601883,
      "step": 914
    },
    {
      "epoch": 0.21258058895277923,
      "grad_norm": 22.941143035888672,
      "learning_rate": 7.876887340301975e-06,
      "logits/chosen": -0.7372113466262817,
      "logits/rejected": -0.6619598269462585,
      "logps/chosen": -81.58432006835938,
      "logps/rejected": -88.77085876464844,
      "loss": 0.5426,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09931023418903351,
      "rewards/margins": 0.4998599886894226,
      "rewards/rejected": -0.4005497395992279,
      "step": 915
    },
    {
      "epoch": 0.21281291746529593,
      "grad_norm": 27.83687400817871,
      "learning_rate": 7.874564459930314e-06,
      "logits/chosen": -0.547705888748169,
      "logits/rejected": -0.5325852632522583,
      "logps/chosen": -98.03385925292969,
      "logps/rejected": -95.77386474609375,
      "loss": 0.7615,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.28205323219299316,
      "rewards/margins": -0.08092032372951508,
      "rewards/rejected": -0.20113292336463928,
      "step": 916
    },
    {
      "epoch": 0.21304524597781263,
      "grad_norm": 21.342525482177734,
      "learning_rate": 7.872241579558655e-06,
      "logits/chosen": -0.8219766616821289,
      "logits/rejected": -0.8764755725860596,
      "logps/chosen": -77.11480712890625,
      "logps/rejected": -82.96772766113281,
      "loss": 0.5971,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13825711607933044,
      "rewards/margins": 0.29106566309928894,
      "rewards/rejected": -0.1528085470199585,
      "step": 917
    },
    {
      "epoch": 0.21327757449032933,
      "grad_norm": 28.8781795501709,
      "learning_rate": 7.869918699186992e-06,
      "logits/chosen": -0.47861573100090027,
      "logits/rejected": -0.48845934867858887,
      "logps/chosen": -80.47924041748047,
      "logps/rejected": -87.21031188964844,
      "loss": 0.8435,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.14829513430595398,
      "rewards/margins": -0.24411490559577942,
      "rewards/rejected": 0.09581978619098663,
      "step": 918
    },
    {
      "epoch": 0.21350990300284603,
      "grad_norm": 33.03474044799805,
      "learning_rate": 7.86759581881533e-06,
      "logits/chosen": -0.029748521745204926,
      "logits/rejected": -0.06239555776119232,
      "logps/chosen": -107.43412017822266,
      "logps/rejected": -98.70914459228516,
      "loss": 0.9196,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.34311050176620483,
      "rewards/margins": -0.34073397517204285,
      "rewards/rejected": -0.002376551739871502,
      "step": 919
    },
    {
      "epoch": 0.21374223151536273,
      "grad_norm": 24.87542724609375,
      "learning_rate": 7.865272938443671e-06,
      "logits/chosen": -0.713062047958374,
      "logits/rejected": -0.715892493724823,
      "logps/chosen": -86.2467041015625,
      "logps/rejected": -90.71318054199219,
      "loss": 0.6628,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.125158429145813,
      "rewards/margins": 0.1376226246356964,
      "rewards/rejected": -0.2627810537815094,
      "step": 920
    },
    {
      "epoch": 0.21397456002787943,
      "grad_norm": 23.765207290649414,
      "learning_rate": 7.86295005807201e-06,
      "logits/chosen": -0.17131248116493225,
      "logits/rejected": -0.07573379576206207,
      "logps/chosen": -76.35671997070312,
      "logps/rejected": -87.35369873046875,
      "loss": 0.6166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.006659765727818012,
      "rewards/margins": 0.21814745664596558,
      "rewards/rejected": -0.21148768067359924,
      "step": 921
    },
    {
      "epoch": 0.21420688854039613,
      "grad_norm": 19.48139190673828,
      "learning_rate": 7.86062717770035e-06,
      "logits/chosen": -0.30128800868988037,
      "logits/rejected": -0.30229565501213074,
      "logps/chosen": -77.59674835205078,
      "logps/rejected": -85.29449462890625,
      "loss": 0.4875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06792409718036652,
      "rewards/margins": 0.4969373047351837,
      "rewards/rejected": -0.429013192653656,
      "step": 922
    },
    {
      "epoch": 0.21443921705291283,
      "grad_norm": 24.421579360961914,
      "learning_rate": 7.858304297328688e-06,
      "logits/chosen": -0.5916696190834045,
      "logits/rejected": -0.4831475615501404,
      "logps/chosen": -74.58367919921875,
      "logps/rejected": -95.76651000976562,
      "loss": 0.6923,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012562772259116173,
      "rewards/margins": 0.14306095242500305,
      "rewards/rejected": -0.15562371909618378,
      "step": 923
    },
    {
      "epoch": 0.21467154556542953,
      "grad_norm": 21.08375358581543,
      "learning_rate": 7.855981416957027e-06,
      "logits/chosen": -0.34723034501075745,
      "logits/rejected": -0.4760580062866211,
      "logps/chosen": -78.81573486328125,
      "logps/rejected": -82.94107818603516,
      "loss": 0.5398,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18281635642051697,
      "rewards/margins": 0.44603851437568665,
      "rewards/rejected": -0.2632221281528473,
      "step": 924
    },
    {
      "epoch": 0.21490387407794623,
      "grad_norm": 29.586313247680664,
      "learning_rate": 7.853658536585366e-06,
      "logits/chosen": -0.566565215587616,
      "logits/rejected": -0.8085084557533264,
      "logps/chosen": -86.81016540527344,
      "logps/rejected": -85.89056396484375,
      "loss": 0.8941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.32576677203178406,
      "rewards/margins": -0.30129748582839966,
      "rewards/rejected": -0.0244692862033844,
      "step": 925
    },
    {
      "epoch": 0.21513620259046293,
      "grad_norm": 27.26796531677246,
      "learning_rate": 7.851335656213705e-06,
      "logits/chosen": -0.313856303691864,
      "logits/rejected": -0.30264803767204285,
      "logps/chosen": -89.49078369140625,
      "logps/rejected": -81.91187286376953,
      "loss": 0.7718,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.019283518195152283,
      "rewards/margins": -0.09641207754611969,
      "rewards/rejected": 0.11569560319185257,
      "step": 926
    },
    {
      "epoch": 0.21536853110297963,
      "grad_norm": 21.686859130859375,
      "learning_rate": 7.849012775842044e-06,
      "logits/chosen": -0.3755851686000824,
      "logits/rejected": -0.479732871055603,
      "logps/chosen": -79.27639770507812,
      "logps/rejected": -92.44061279296875,
      "loss": 0.5537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.035687338560819626,
      "rewards/margins": 0.367856502532959,
      "rewards/rejected": -0.4035438895225525,
      "step": 927
    },
    {
      "epoch": 0.2156008596154963,
      "grad_norm": 22.09327507019043,
      "learning_rate": 7.846689895470385e-06,
      "logits/chosen": -0.9891065359115601,
      "logits/rejected": -0.7831617593765259,
      "logps/chosen": -79.59266662597656,
      "logps/rejected": -92.61043548583984,
      "loss": 0.6323,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13569407165050507,
      "rewards/margins": 0.24636760354042053,
      "rewards/rejected": -0.3820616602897644,
      "step": 928
    },
    {
      "epoch": 0.215833188128013,
      "grad_norm": 20.507320404052734,
      "learning_rate": 7.844367015098723e-06,
      "logits/chosen": -0.15532203018665314,
      "logits/rejected": -0.17899790406227112,
      "logps/chosen": -89.94269561767578,
      "logps/rejected": -84.43154907226562,
      "loss": 0.4927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.038221027702093124,
      "rewards/margins": 0.5432811975479126,
      "rewards/rejected": -0.5815021991729736,
      "step": 929
    },
    {
      "epoch": 0.2160655166405297,
      "grad_norm": 25.31558609008789,
      "learning_rate": 7.842044134727062e-06,
      "logits/chosen": -0.8040270805358887,
      "logits/rejected": -0.9116278886795044,
      "logps/chosen": -88.84362030029297,
      "logps/rejected": -87.0703353881836,
      "loss": 0.7583,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1720985472202301,
      "rewards/margins": -0.023252207785844803,
      "rewards/rejected": -0.1488463282585144,
      "step": 930
    },
    {
      "epoch": 0.2162978451530464,
      "grad_norm": 22.1485652923584,
      "learning_rate": 7.839721254355401e-06,
      "logits/chosen": -0.622153103351593,
      "logits/rejected": -0.7032917141914368,
      "logps/chosen": -77.25602722167969,
      "logps/rejected": -89.1799087524414,
      "loss": 0.6167,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06486985832452774,
      "rewards/margins": 0.28351789712905884,
      "rewards/rejected": -0.3483877182006836,
      "step": 931
    },
    {
      "epoch": 0.2165301736655631,
      "grad_norm": 26.246170043945312,
      "learning_rate": 7.83739837398374e-06,
      "logits/chosen": -0.2793387770652771,
      "logits/rejected": -0.2677103579044342,
      "logps/chosen": -75.74824523925781,
      "logps/rejected": -76.99212646484375,
      "loss": 0.7094,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15420182049274445,
      "rewards/margins": 0.005254283547401428,
      "rewards/rejected": -0.15945611894130707,
      "step": 932
    },
    {
      "epoch": 0.2167625021780798,
      "grad_norm": 18.40263557434082,
      "learning_rate": 7.83507549361208e-06,
      "logits/chosen": -0.8127117156982422,
      "logits/rejected": -0.7077860832214355,
      "logps/chosen": -90.77767944335938,
      "logps/rejected": -87.74602508544922,
      "loss": 0.4788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14282269775867462,
      "rewards/margins": 0.5265930891036987,
      "rewards/rejected": -0.3837704062461853,
      "step": 933
    },
    {
      "epoch": 0.2169948306905965,
      "grad_norm": 21.93915557861328,
      "learning_rate": 7.832752613240418e-06,
      "logits/chosen": -0.025363072752952576,
      "logits/rejected": 0.052511438727378845,
      "logps/chosen": -83.2998046875,
      "logps/rejected": -109.95060729980469,
      "loss": 0.5019,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3155182898044586,
      "rewards/margins": 0.5076965689659119,
      "rewards/rejected": -0.19217820465564728,
      "step": 934
    },
    {
      "epoch": 0.2172271592031132,
      "grad_norm": 22.253339767456055,
      "learning_rate": 7.830429732868759e-06,
      "logits/chosen": -0.09324899315834045,
      "logits/rejected": -0.13179515302181244,
      "logps/chosen": -72.18490600585938,
      "logps/rejected": -74.26616668701172,
      "loss": 0.6695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06171334534883499,
      "rewards/margins": 0.2570783793926239,
      "rewards/rejected": -0.1953650265932083,
      "step": 935
    },
    {
      "epoch": 0.2174594877156299,
      "grad_norm": 27.429590225219727,
      "learning_rate": 7.828106852497098e-06,
      "logits/chosen": -0.7037116289138794,
      "logits/rejected": -0.7429527640342712,
      "logps/chosen": -84.30326843261719,
      "logps/rejected": -83.57148742675781,
      "loss": 0.8365,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.4487157166004181,
      "rewards/margins": -0.24444304406642914,
      "rewards/rejected": -0.20427271723747253,
      "step": 936
    },
    {
      "epoch": 0.2176918162281466,
      "grad_norm": 20.87291717529297,
      "learning_rate": 7.825783972125437e-06,
      "logits/chosen": -0.54308021068573,
      "logits/rejected": -0.5514176487922668,
      "logps/chosen": -81.28280639648438,
      "logps/rejected": -88.79070281982422,
      "loss": 0.5301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10972262918949127,
      "rewards/margins": 0.4160670340061188,
      "rewards/rejected": -0.5257896184921265,
      "step": 937
    },
    {
      "epoch": 0.2179241447406633,
      "grad_norm": 20.03630828857422,
      "learning_rate": 7.823461091753776e-06,
      "logits/chosen": -0.7059678435325623,
      "logits/rejected": -0.6878108382225037,
      "logps/chosen": -77.2635269165039,
      "logps/rejected": -83.1082992553711,
      "loss": 0.5914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.027828877791762352,
      "rewards/margins": 0.34595274925231934,
      "rewards/rejected": -0.3181239068508148,
      "step": 938
    },
    {
      "epoch": 0.21815647325318,
      "grad_norm": 21.613109588623047,
      "learning_rate": 7.821138211382115e-06,
      "logits/chosen": -0.6482673287391663,
      "logits/rejected": -0.8100574016571045,
      "logps/chosen": -75.02273559570312,
      "logps/rejected": -81.72688293457031,
      "loss": 0.6251,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04524388909339905,
      "rewards/margins": 0.20436140894889832,
      "rewards/rejected": -0.24960529804229736,
      "step": 939
    },
    {
      "epoch": 0.2183888017656967,
      "grad_norm": 23.81171226501465,
      "learning_rate": 7.818815331010453e-06,
      "logits/chosen": -0.7028401494026184,
      "logits/rejected": -0.8380053043365479,
      "logps/chosen": -84.85182189941406,
      "logps/rejected": -78.70325469970703,
      "loss": 0.6612,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.017518416047096252,
      "rewards/margins": 0.14572587609291077,
      "rewards/rejected": -0.1632443219423294,
      "step": 940
    },
    {
      "epoch": 0.2186211302782134,
      "grad_norm": 24.63398551940918,
      "learning_rate": 7.816492450638792e-06,
      "logits/chosen": 0.7353252172470093,
      "logits/rejected": 0.7831894159317017,
      "logps/chosen": -74.11343383789062,
      "logps/rejected": -89.07218933105469,
      "loss": 0.6956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1821790337562561,
      "rewards/margins": 0.03077104315161705,
      "rewards/rejected": -0.21295006573200226,
      "step": 941
    },
    {
      "epoch": 0.2188534587907301,
      "grad_norm": 25.148635864257812,
      "learning_rate": 7.814169570267131e-06,
      "logits/chosen": -0.06439919769763947,
      "logits/rejected": -0.14743341505527496,
      "logps/chosen": -64.59439086914062,
      "logps/rejected": -90.89530944824219,
      "loss": 0.6705,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06855502724647522,
      "rewards/margins": 0.08831825107336044,
      "rewards/rejected": -0.15687328577041626,
      "step": 942
    },
    {
      "epoch": 0.2190857873032468,
      "grad_norm": 26.24050521850586,
      "learning_rate": 7.811846689895472e-06,
      "logits/chosen": -0.419461190700531,
      "logits/rejected": -0.44009482860565186,
      "logps/chosen": -78.17839050292969,
      "logps/rejected": -80.39757537841797,
      "loss": 0.707,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12670139968395233,
      "rewards/margins": 0.021197982132434845,
      "rewards/rejected": -0.14789938926696777,
      "step": 943
    },
    {
      "epoch": 0.2193181158157635,
      "grad_norm": 19.640806198120117,
      "learning_rate": 7.809523809523811e-06,
      "logits/chosen": -0.592170238494873,
      "logits/rejected": -0.6509079933166504,
      "logps/chosen": -81.62947082519531,
      "logps/rejected": -82.9218978881836,
      "loss": 0.5148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15589074790477753,
      "rewards/margins": 0.4605313837528229,
      "rewards/rejected": -0.30464065074920654,
      "step": 944
    },
    {
      "epoch": 0.2195504443282802,
      "grad_norm": 24.897029876708984,
      "learning_rate": 7.807200929152148e-06,
      "logits/chosen": -0.7821695804595947,
      "logits/rejected": -0.7391118407249451,
      "logps/chosen": -73.93461608886719,
      "logps/rejected": -80.28517150878906,
      "loss": 0.7126,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07849199324846268,
      "rewards/margins": -0.007332485169172287,
      "rewards/rejected": -0.07115951180458069,
      "step": 945
    },
    {
      "epoch": 0.2197827728407969,
      "grad_norm": 25.58285903930664,
      "learning_rate": 7.804878048780489e-06,
      "logits/chosen": -0.6326808333396912,
      "logits/rejected": -0.5852510929107666,
      "logps/chosen": -80.03606414794922,
      "logps/rejected": -91.53986358642578,
      "loss": 0.6712,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2407814860343933,
      "rewards/margins": 0.06407658755779266,
      "rewards/rejected": -0.30485811829566956,
      "step": 946
    },
    {
      "epoch": 0.2200151013533136,
      "grad_norm": 19.575468063354492,
      "learning_rate": 7.802555168408828e-06,
      "logits/chosen": 0.5650081634521484,
      "logits/rejected": 0.5210551023483276,
      "logps/chosen": -72.04645538330078,
      "logps/rejected": -69.86727142333984,
      "loss": 0.5805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16247431933879852,
      "rewards/margins": 0.30848702788352966,
      "rewards/rejected": -0.14601270854473114,
      "step": 947
    },
    {
      "epoch": 0.2202474298658303,
      "grad_norm": 23.639482498168945,
      "learning_rate": 7.800232288037167e-06,
      "logits/chosen": -0.31677019596099854,
      "logits/rejected": -0.4648243188858032,
      "logps/chosen": -94.54623413085938,
      "logps/rejected": -81.01885986328125,
      "loss": 0.6489,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2177647203207016,
      "rewards/margins": 0.14968574047088623,
      "rewards/rejected": -0.36745044589042664,
      "step": 948
    },
    {
      "epoch": 0.220479758378347,
      "grad_norm": 22.65177345275879,
      "learning_rate": 7.797909407665506e-06,
      "logits/chosen": -0.19932666420936584,
      "logits/rejected": -0.1366136223077774,
      "logps/chosen": -79.9024658203125,
      "logps/rejected": -71.84180450439453,
      "loss": 0.6061,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.090978242456913,
      "rewards/margins": 0.23899319767951965,
      "rewards/rejected": -0.14801497757434845,
      "step": 949
    },
    {
      "epoch": 0.2207120868908637,
      "grad_norm": 23.950624465942383,
      "learning_rate": 7.795586527293845e-06,
      "logits/chosen": -0.479947030544281,
      "logits/rejected": -0.3928360044956207,
      "logps/chosen": -86.75383758544922,
      "logps/rejected": -78.16825103759766,
      "loss": 0.6628,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15972095727920532,
      "rewards/margins": 0.1363351047039032,
      "rewards/rejected": -0.2960560619831085,
      "step": 950
    },
    {
      "epoch": 0.2209444154033804,
      "grad_norm": 24.24115753173828,
      "learning_rate": 7.793263646922185e-06,
      "logits/chosen": -0.7311063408851624,
      "logits/rejected": -0.43126922845840454,
      "logps/chosen": -92.20816040039062,
      "logps/rejected": -94.67634582519531,
      "loss": 0.6082,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05548899993300438,
      "rewards/margins": 0.3235000669956207,
      "rewards/rejected": -0.378989040851593,
      "step": 951
    },
    {
      "epoch": 0.2211767439158971,
      "grad_norm": 24.918529510498047,
      "learning_rate": 7.790940766550522e-06,
      "logits/chosen": -0.5569818019866943,
      "logits/rejected": -0.6027792096138,
      "logps/chosen": -86.60403442382812,
      "logps/rejected": -95.2483139038086,
      "loss": 0.6761,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05445832386612892,
      "rewards/margins": 0.14766302704811096,
      "rewards/rejected": -0.20212137699127197,
      "step": 952
    },
    {
      "epoch": 0.2214090724284138,
      "grad_norm": 21.356651306152344,
      "learning_rate": 7.788617886178861e-06,
      "logits/chosen": -0.06865143775939941,
      "logits/rejected": -0.13584190607070923,
      "logps/chosen": -75.70645904541016,
      "logps/rejected": -88.63623046875,
      "loss": 0.594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.034250471740961075,
      "rewards/margins": 0.25101450085639954,
      "rewards/rejected": -0.21676403284072876,
      "step": 953
    },
    {
      "epoch": 0.2216414009409305,
      "grad_norm": 19.4932918548584,
      "learning_rate": 7.786295005807202e-06,
      "logits/chosen": -0.6099170446395874,
      "logits/rejected": -0.7025131583213806,
      "logps/chosen": -67.23204803466797,
      "logps/rejected": -82.75639343261719,
      "loss": 0.6232,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016679774969816208,
      "rewards/margins": 0.2337370365858078,
      "rewards/rejected": -0.2170572280883789,
      "step": 954
    },
    {
      "epoch": 0.22187372945344716,
      "grad_norm": 17.959156036376953,
      "learning_rate": 7.783972125435541e-06,
      "logits/chosen": -0.397189199924469,
      "logits/rejected": -0.355393648147583,
      "logps/chosen": -82.30484008789062,
      "logps/rejected": -98.04838562011719,
      "loss": 0.4447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1450493335723877,
      "rewards/margins": 0.6828833222389221,
      "rewards/rejected": -0.5378339886665344,
      "step": 955
    },
    {
      "epoch": 0.22210605796596386,
      "grad_norm": 24.115856170654297,
      "learning_rate": 7.78164924506388e-06,
      "logits/chosen": -0.2134746015071869,
      "logits/rejected": -0.20785020291805267,
      "logps/chosen": -83.14790344238281,
      "logps/rejected": -83.93058776855469,
      "loss": 0.6884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09655118733644485,
      "rewards/margins": 0.1575535237789154,
      "rewards/rejected": -0.061002351343631744,
      "step": 956
    },
    {
      "epoch": 0.22233838647848056,
      "grad_norm": 21.80962371826172,
      "learning_rate": 7.779326364692219e-06,
      "logits/chosen": 0.27246251702308655,
      "logits/rejected": 0.21967504918575287,
      "logps/chosen": -70.98834228515625,
      "logps/rejected": -81.66680145263672,
      "loss": 0.6237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13950154185295105,
      "rewards/margins": 0.2461753785610199,
      "rewards/rejected": -0.38567692041397095,
      "step": 957
    },
    {
      "epoch": 0.22257071499099726,
      "grad_norm": 23.294836044311523,
      "learning_rate": 7.777003484320558e-06,
      "logits/chosen": 0.2317788004875183,
      "logits/rejected": 0.23024645447731018,
      "logps/chosen": -81.01332092285156,
      "logps/rejected": -111.32685089111328,
      "loss": 0.5613,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1405211240053177,
      "rewards/margins": 1.0630818605422974,
      "rewards/rejected": -0.9225608110427856,
      "step": 958
    },
    {
      "epoch": 0.22280304350351396,
      "grad_norm": 20.6252498626709,
      "learning_rate": 7.774680603948898e-06,
      "logits/chosen": -0.25378715991973877,
      "logits/rejected": -0.21961715817451477,
      "logps/chosen": -86.56575012207031,
      "logps/rejected": -73.80168914794922,
      "loss": 0.5866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10583916306495667,
      "rewards/margins": 0.32235100865364075,
      "rewards/rejected": -0.2165118157863617,
      "step": 959
    },
    {
      "epoch": 0.22303537201603066,
      "grad_norm": 20.572040557861328,
      "learning_rate": 7.772357723577236e-06,
      "logits/chosen": -0.8169501423835754,
      "logits/rejected": -0.8277637958526611,
      "logps/chosen": -82.97859191894531,
      "logps/rejected": -80.77375793457031,
      "loss": 0.5455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08339409530162811,
      "rewards/margins": 0.3583889305591583,
      "rewards/rejected": -0.2749948501586914,
      "step": 960
    },
    {
      "epoch": 0.22326770052854736,
      "grad_norm": 23.13723373413086,
      "learning_rate": 7.770034843205574e-06,
      "logits/chosen": 0.07409423589706421,
      "logits/rejected": 0.07772661000490189,
      "logps/chosen": -81.03839111328125,
      "logps/rejected": -79.30525970458984,
      "loss": 0.6399,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006659321486949921,
      "rewards/margins": 0.24149900674819946,
      "rewards/rejected": -0.23483967781066895,
      "step": 961
    },
    {
      "epoch": 0.22350002904106406,
      "grad_norm": 20.93524742126465,
      "learning_rate": 7.767711962833915e-06,
      "logits/chosen": -0.29783573746681213,
      "logits/rejected": -0.2045671045780182,
      "logps/chosen": -85.12911224365234,
      "logps/rejected": -97.82340240478516,
      "loss": 0.4942,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3293475806713104,
      "rewards/margins": 0.5815902352333069,
      "rewards/rejected": -0.25224268436431885,
      "step": 962
    },
    {
      "epoch": 0.22373235755358076,
      "grad_norm": 19.918710708618164,
      "learning_rate": 7.765389082462254e-06,
      "logits/chosen": -0.7454972267150879,
      "logits/rejected": -0.8079276084899902,
      "logps/chosen": -64.41000366210938,
      "logps/rejected": -79.83732604980469,
      "loss": 0.5854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12253959476947784,
      "rewards/margins": 0.32206234335899353,
      "rewards/rejected": -0.19952274858951569,
      "step": 963
    },
    {
      "epoch": 0.22396468606609746,
      "grad_norm": 31.013607025146484,
      "learning_rate": 7.763066202090593e-06,
      "logits/chosen": -0.278067022562027,
      "logits/rejected": -0.1804206371307373,
      "logps/chosen": -64.53767395019531,
      "logps/rejected": -80.72480010986328,
      "loss": 0.7185,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.052373841404914856,
      "rewards/margins": 0.06405769288539886,
      "rewards/rejected": -0.11643153429031372,
      "step": 964
    },
    {
      "epoch": 0.22419701457861416,
      "grad_norm": 25.211631774902344,
      "learning_rate": 7.760743321718932e-06,
      "logits/chosen": -0.40385738015174866,
      "logits/rejected": -0.2490697205066681,
      "logps/chosen": -81.36909484863281,
      "logps/rejected": -83.21324920654297,
      "loss": 0.7351,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08447931706905365,
      "rewards/margins": 0.011880846694111824,
      "rewards/rejected": -0.09636016190052032,
      "step": 965
    },
    {
      "epoch": 0.22442934309113086,
      "grad_norm": 20.36042594909668,
      "learning_rate": 7.758420441347271e-06,
      "logits/chosen": -0.2547447681427002,
      "logits/rejected": -0.13913372159004211,
      "logps/chosen": -75.8065185546875,
      "logps/rejected": -81.22142791748047,
      "loss": 0.6055,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.022589491680264473,
      "rewards/margins": 0.21917222440242767,
      "rewards/rejected": -0.19658274948596954,
      "step": 966
    },
    {
      "epoch": 0.22466167160364756,
      "grad_norm": 25.791973114013672,
      "learning_rate": 7.75609756097561e-06,
      "logits/chosen": -0.39329326152801514,
      "logits/rejected": -0.46859443187713623,
      "logps/chosen": -79.22399139404297,
      "logps/rejected": -87.1568374633789,
      "loss": 0.4923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07639177143573761,
      "rewards/margins": 0.5270849466323853,
      "rewards/rejected": -0.45069319009780884,
      "step": 967
    },
    {
      "epoch": 0.22489400011616426,
      "grad_norm": 16.135879516601562,
      "learning_rate": 7.753774680603949e-06,
      "logits/chosen": -0.2888493835926056,
      "logits/rejected": -0.19525127112865448,
      "logps/chosen": -81.58549499511719,
      "logps/rejected": -96.6567153930664,
      "loss": 0.4017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3960426449775696,
      "rewards/margins": 0.9002395868301392,
      "rewards/rejected": -0.5041969418525696,
      "step": 968
    },
    {
      "epoch": 0.22512632862868096,
      "grad_norm": 85.86663055419922,
      "learning_rate": 7.75145180023229e-06,
      "logits/chosen": -0.49186721444129944,
      "logits/rejected": -0.42587974667549133,
      "logps/chosen": -130.14016723632812,
      "logps/rejected": -72.64012908935547,
      "loss": 1.4934,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.9113777875900269,
      "rewards/margins": -0.9256640672683716,
      "rewards/rejected": 0.014286253601312637,
      "step": 969
    },
    {
      "epoch": 0.22535865714119765,
      "grad_norm": 23.27623176574707,
      "learning_rate": 7.749128919860628e-06,
      "logits/chosen": -0.3100549876689911,
      "logits/rejected": -0.10946372151374817,
      "logps/chosen": -79.79769897460938,
      "logps/rejected": -71.1627426147461,
      "loss": 0.7101,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11862324178218842,
      "rewards/margins": 0.025723394006490707,
      "rewards/rejected": -0.14434662461280823,
      "step": 970
    },
    {
      "epoch": 0.22559098565371435,
      "grad_norm": 23.750391006469727,
      "learning_rate": 7.746806039488967e-06,
      "logits/chosen": 0.029304735362529755,
      "logits/rejected": 0.06583838164806366,
      "logps/chosen": -69.51856231689453,
      "logps/rejected": -92.08599853515625,
      "loss": 0.6671,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06327667832374573,
      "rewards/margins": 0.09570133686065674,
      "rewards/rejected": -0.15897804498672485,
      "step": 971
    },
    {
      "epoch": 0.22582331416623105,
      "grad_norm": 22.76723861694336,
      "learning_rate": 7.744483159117306e-06,
      "logits/chosen": -0.823974609375,
      "logits/rejected": -0.7642003893852234,
      "logps/chosen": -71.27806854248047,
      "logps/rejected": -81.6319580078125,
      "loss": 0.5711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1637469232082367,
      "rewards/margins": 0.36755192279815674,
      "rewards/rejected": -0.20380496978759766,
      "step": 972
    },
    {
      "epoch": 0.22605564267874775,
      "grad_norm": 23.31400489807129,
      "learning_rate": 7.742160278745645e-06,
      "logits/chosen": -0.36257630586624146,
      "logits/rejected": -0.3966262936592102,
      "logps/chosen": -77.23513793945312,
      "logps/rejected": -85.70529174804688,
      "loss": 0.6573,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07406061142683029,
      "rewards/margins": 0.1730629801750183,
      "rewards/rejected": -0.24712352454662323,
      "step": 973
    },
    {
      "epoch": 0.22628797119126445,
      "grad_norm": 19.802783966064453,
      "learning_rate": 7.739837398373984e-06,
      "logits/chosen": -0.4347826838493347,
      "logits/rejected": -0.5119542479515076,
      "logps/chosen": -66.4471435546875,
      "logps/rejected": -74.87068939208984,
      "loss": 0.5682,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15256445109844208,
      "rewards/margins": 0.3385355770587921,
      "rewards/rejected": -0.18597111105918884,
      "step": 974
    },
    {
      "epoch": 0.22652029970378115,
      "grad_norm": 20.638614654541016,
      "learning_rate": 7.737514518002323e-06,
      "logits/chosen": -0.7242615222930908,
      "logits/rejected": -0.6589169502258301,
      "logps/chosen": -68.9437255859375,
      "logps/rejected": -88.782958984375,
      "loss": 0.5355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0065749213099479675,
      "rewards/margins": 0.39484408497810364,
      "rewards/rejected": -0.401419073343277,
      "step": 975
    },
    {
      "epoch": 0.22675262821629785,
      "grad_norm": 17.92825698852539,
      "learning_rate": 7.735191637630662e-06,
      "logits/chosen": -1.1036667823791504,
      "logits/rejected": -1.15549635887146,
      "logps/chosen": -73.02422332763672,
      "logps/rejected": -94.3803482055664,
      "loss": 0.3793,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5760083198547363,
      "rewards/margins": 0.8788038492202759,
      "rewards/rejected": -0.3027954697608948,
      "step": 976
    },
    {
      "epoch": 0.22698495672881455,
      "grad_norm": 27.12857437133789,
      "learning_rate": 7.732868757259003e-06,
      "logits/chosen": -0.12834863364696503,
      "logits/rejected": -0.42521801590919495,
      "logps/chosen": -70.71821594238281,
      "logps/rejected": -95.41697692871094,
      "loss": 0.6391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0735660120844841,
      "rewards/margins": 0.1264568418264389,
      "rewards/rejected": -0.05289082229137421,
      "step": 977
    },
    {
      "epoch": 0.22721728524133125,
      "grad_norm": 22.523283004760742,
      "learning_rate": 7.730545876887341e-06,
      "logits/chosen": -0.6941235065460205,
      "logits/rejected": -0.7455169558525085,
      "logps/chosen": -75.225341796875,
      "logps/rejected": -82.04678344726562,
      "loss": 0.7038,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10398588329553604,
      "rewards/margins": 0.03448688983917236,
      "rewards/rejected": -0.1384727656841278,
      "step": 978
    },
    {
      "epoch": 0.22744961375384795,
      "grad_norm": 21.433334350585938,
      "learning_rate": 7.72822299651568e-06,
      "logits/chosen": 0.6198342442512512,
      "logits/rejected": 0.4161168932914734,
      "logps/chosen": -79.70622253417969,
      "logps/rejected": -95.1692123413086,
      "loss": 0.5703,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3033774197101593,
      "rewards/margins": 0.9381452798843384,
      "rewards/rejected": -0.6347679495811462,
      "step": 979
    },
    {
      "epoch": 0.22768194226636465,
      "grad_norm": 24.36786460876465,
      "learning_rate": 7.72590011614402e-06,
      "logits/chosen": -0.7118518948554993,
      "logits/rejected": -0.7578029036521912,
      "logps/chosen": -75.93482971191406,
      "logps/rejected": -76.39842224121094,
      "loss": 0.6336,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2103419303894043,
      "rewards/margins": 0.1345154047012329,
      "rewards/rejected": -0.3448573052883148,
      "step": 980
    },
    {
      "epoch": 0.22791427077888135,
      "grad_norm": 19.098834991455078,
      "learning_rate": 7.723577235772358e-06,
      "logits/chosen": -0.9381271600723267,
      "logits/rejected": -0.8263386487960815,
      "logps/chosen": -77.53291320800781,
      "logps/rejected": -85.60770416259766,
      "loss": 0.4865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2499588131904602,
      "rewards/margins": 0.6111293435096741,
      "rewards/rejected": -0.36117053031921387,
      "step": 981
    },
    {
      "epoch": 0.22814659929139805,
      "grad_norm": 19.607112884521484,
      "learning_rate": 7.721254355400697e-06,
      "logits/chosen": -0.185794398188591,
      "logits/rejected": -0.155105322599411,
      "logps/chosen": -75.92797088623047,
      "logps/rejected": -77.1618881225586,
      "loss": 0.5579,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.32095256447792053,
      "rewards/margins": 0.375236451625824,
      "rewards/rejected": -0.054283905774354935,
      "step": 982
    },
    {
      "epoch": 0.22837892780391472,
      "grad_norm": 23.200864791870117,
      "learning_rate": 7.718931475029036e-06,
      "logits/chosen": -1.0199787616729736,
      "logits/rejected": -0.9750707149505615,
      "logps/chosen": -77.42678833007812,
      "logps/rejected": -73.72653198242188,
      "loss": 0.6613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06483283638954163,
      "rewards/margins": 0.102065309882164,
      "rewards/rejected": -0.16689813137054443,
      "step": 983
    },
    {
      "epoch": 0.22861125631643142,
      "grad_norm": 25.192014694213867,
      "learning_rate": 7.716608594657375e-06,
      "logits/chosen": -0.9862290620803833,
      "logits/rejected": -0.8486899137496948,
      "logps/chosen": -90.62194061279297,
      "logps/rejected": -79.83747863769531,
      "loss": 0.7034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2158646434545517,
      "rewards/margins": 0.005800776183605194,
      "rewards/rejected": -0.2216654121875763,
      "step": 984
    },
    {
      "epoch": 0.22884358482894812,
      "grad_norm": 48.838706970214844,
      "learning_rate": 7.714285714285716e-06,
      "logits/chosen": -0.5440297722816467,
      "logits/rejected": -0.4976649880409241,
      "logps/chosen": -86.18816375732422,
      "logps/rejected": -96.02799224853516,
      "loss": 0.606,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07846520841121674,
      "rewards/margins": 0.23136919736862183,
      "rewards/rejected": -0.30983439087867737,
      "step": 985
    },
    {
      "epoch": 0.22907591334146482,
      "grad_norm": 23.82274627685547,
      "learning_rate": 7.711962833914055e-06,
      "logits/chosen": 0.4612060487270355,
      "logits/rejected": 0.5424727201461792,
      "logps/chosen": -70.63871765136719,
      "logps/rejected": -81.66793060302734,
      "loss": 0.6816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05753616988658905,
      "rewards/margins": 0.058662354946136475,
      "rewards/rejected": -0.0011261925101280212,
      "step": 986
    },
    {
      "epoch": 0.22930824185398152,
      "grad_norm": 25.577617645263672,
      "learning_rate": 7.709639953542392e-06,
      "logits/chosen": -0.5675795078277588,
      "logits/rejected": -0.34278130531311035,
      "logps/chosen": -95.87187194824219,
      "logps/rejected": -96.16112518310547,
      "loss": 0.6799,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22225236892700195,
      "rewards/margins": 0.10913001000881195,
      "rewards/rejected": -0.3313823342323303,
      "step": 987
    },
    {
      "epoch": 0.22954057036649822,
      "grad_norm": 22.313766479492188,
      "learning_rate": 7.707317073170732e-06,
      "logits/chosen": 0.0026833415031433105,
      "logits/rejected": 0.03268107771873474,
      "logps/chosen": -83.45039367675781,
      "logps/rejected": -70.58842468261719,
      "loss": 0.642,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017622828483581543,
      "rewards/margins": 0.15079054236412048,
      "rewards/rejected": -0.13316772878170013,
      "step": 988
    },
    {
      "epoch": 0.22977289887901492,
      "grad_norm": 27.169443130493164,
      "learning_rate": 7.704994192799071e-06,
      "logits/chosen": -0.06546244025230408,
      "logits/rejected": 0.23526160418987274,
      "logps/chosen": -75.83727264404297,
      "logps/rejected": -95.8851547241211,
      "loss": 0.7401,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14281602203845978,
      "rewards/margins": 0.02374676614999771,
      "rewards/rejected": -0.16656279563903809,
      "step": 989
    },
    {
      "epoch": 0.23000522739153162,
      "grad_norm": 24.264921188354492,
      "learning_rate": 7.70267131242741e-06,
      "logits/chosen": -0.9944357872009277,
      "logits/rejected": -0.9071022868156433,
      "logps/chosen": -75.08497619628906,
      "logps/rejected": -82.04243469238281,
      "loss": 0.6727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03542609512805939,
      "rewards/margins": 0.10419043898582458,
      "rewards/rejected": -0.13961653411388397,
      "step": 990
    },
    {
      "epoch": 0.23023755590404832,
      "grad_norm": 24.194538116455078,
      "learning_rate": 7.70034843205575e-06,
      "logits/chosen": -0.21319563686847687,
      "logits/rejected": -0.1894504874944687,
      "logps/chosen": -78.67818450927734,
      "logps/rejected": -86.79633331298828,
      "loss": 0.628,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04872152954339981,
      "rewards/margins": 0.46370017528533936,
      "rewards/rejected": -0.512421727180481,
      "step": 991
    },
    {
      "epoch": 0.23046988441656502,
      "grad_norm": 20.09741973876953,
      "learning_rate": 7.698025551684088e-06,
      "logits/chosen": -1.169605016708374,
      "logits/rejected": -1.0298149585723877,
      "logps/chosen": -83.5074691772461,
      "logps/rejected": -98.03545379638672,
      "loss": 0.521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02676365152001381,
      "rewards/margins": 0.4137677848339081,
      "rewards/rejected": -0.4405314326286316,
      "step": 992
    },
    {
      "epoch": 0.23070221292908172,
      "grad_norm": 21.97865104675293,
      "learning_rate": 7.695702671312429e-06,
      "logits/chosen": -0.5080382823944092,
      "logits/rejected": -0.7626125812530518,
      "logps/chosen": -80.4522705078125,
      "logps/rejected": -79.65426635742188,
      "loss": 0.567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16864962875843048,
      "rewards/margins": 0.3800222873687744,
      "rewards/rejected": -0.21137267351150513,
      "step": 993
    },
    {
      "epoch": 0.23093454144159842,
      "grad_norm": 24.516538619995117,
      "learning_rate": 7.693379790940768e-06,
      "logits/chosen": -0.44945400953292847,
      "logits/rejected": -0.5334354043006897,
      "logps/chosen": -87.0978775024414,
      "logps/rejected": -97.97447204589844,
      "loss": 0.6403,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1508675068616867,
      "rewards/margins": 0.5017802119255066,
      "rewards/rejected": -0.6526477336883545,
      "step": 994
    },
    {
      "epoch": 0.23116686995411512,
      "grad_norm": 23.027379989624023,
      "learning_rate": 7.691056910569107e-06,
      "logits/chosen": -0.5713979005813599,
      "logits/rejected": -0.5773361325263977,
      "logps/chosen": -77.44779205322266,
      "logps/rejected": -84.59324645996094,
      "loss": 0.6563,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06214618682861328,
      "rewards/margins": 0.11522373557090759,
      "rewards/rejected": -0.17736992239952087,
      "step": 995
    },
    {
      "epoch": 0.23139919846663182,
      "grad_norm": 27.94333839416504,
      "learning_rate": 7.688734030197446e-06,
      "logits/chosen": -1.1701810359954834,
      "logits/rejected": -1.1216394901275635,
      "logps/chosen": -85.9065933227539,
      "logps/rejected": -96.13313293457031,
      "loss": 0.7735,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16633331775665283,
      "rewards/margins": -0.11473886668682098,
      "rewards/rejected": -0.05159445106983185,
      "step": 996
    },
    {
      "epoch": 0.23163152697914852,
      "grad_norm": 24.667430877685547,
      "learning_rate": 7.686411149825785e-06,
      "logits/chosen": -0.8154934644699097,
      "logits/rejected": -0.8719544410705566,
      "logps/chosen": -73.39690399169922,
      "logps/rejected": -100.2962875366211,
      "loss": 0.63,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12089027464389801,
      "rewards/margins": 0.23819135129451752,
      "rewards/rejected": -0.35908159613609314,
      "step": 997
    },
    {
      "epoch": 0.23186385549166522,
      "grad_norm": 29.79874038696289,
      "learning_rate": 7.684088269454124e-06,
      "logits/chosen": 2.0483274459838867,
      "logits/rejected": 2.118321657180786,
      "logps/chosen": -91.49080657958984,
      "logps/rejected": -90.77949523925781,
      "loss": 0.6104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1833358258008957,
      "rewards/margins": 0.3458271622657776,
      "rewards/rejected": -0.5291629433631897,
      "step": 998
    },
    {
      "epoch": 0.23209618400418192,
      "grad_norm": 29.625221252441406,
      "learning_rate": 7.681765389082462e-06,
      "logits/chosen": -0.6370264291763306,
      "logits/rejected": -0.8474686145782471,
      "logps/chosen": -83.78926086425781,
      "logps/rejected": -97.52777099609375,
      "loss": 0.7978,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2320917248725891,
      "rewards/margins": -0.17101706564426422,
      "rewards/rejected": -0.0610746368765831,
      "step": 999
    },
    {
      "epoch": 0.23232851251669862,
      "grad_norm": 21.25395393371582,
      "learning_rate": 7.679442508710801e-06,
      "logits/chosen": -0.9953933358192444,
      "logits/rejected": -0.8965286612510681,
      "logps/chosen": -84.73737335205078,
      "logps/rejected": -80.92124938964844,
      "loss": 0.5608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.018526986241340637,
      "rewards/margins": 0.4161384701728821,
      "rewards/rejected": -0.4346655011177063,
      "step": 1000
    }
  ],
  "logging_steps": 1,
  "max_steps": 4305,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}