| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9989528795811519, |
| "eval_steps": 200, |
| "global_step": 477, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0020942408376963353, |
| "grad_norm": 26.356996536254883, |
| "learning_rate": 0.0, |
| "logits/chosen": -0.6103914976119995, |
| "logits/rejected": -0.6099507808685303, |
| "logps/chosen": -1.2526695728302002, |
| "logps/rejected": -1.3568028211593628, |
| "loss": 2.7368, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -2.5053391456604004, |
| "rewards/margins": 0.20826660096645355, |
| "rewards/rejected": -2.7136056423187256, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.020942408376963352, |
| "grad_norm": 17.384063720703125, |
| "learning_rate": 1.125e-07, |
| "logits/chosen": -0.6848690509796143, |
| "logits/rejected": -0.6671006679534912, |
| "logps/chosen": -1.1007671356201172, |
| "logps/rejected": -1.2118645906448364, |
| "loss": 2.6132, |
| "rewards/accuracies": 0.5416666865348816, |
| "rewards/chosen": -2.2015342712402344, |
| "rewards/margins": 0.2221948802471161, |
| "rewards/rejected": -2.423729181289673, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.041884816753926704, |
| "grad_norm": 25.446407318115234, |
| "learning_rate": 2.3749999999999998e-07, |
| "logits/chosen": -0.6405870318412781, |
| "logits/rejected": -0.6533440351486206, |
| "logps/chosen": -1.134265661239624, |
| "logps/rejected": -1.3199212551116943, |
| "loss": 2.5636, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -2.268531322479248, |
| "rewards/margins": 0.3713109493255615, |
| "rewards/rejected": -2.6398425102233887, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.06282722513089005, |
| "grad_norm": 22.045013427734375, |
| "learning_rate": 3.6249999999999997e-07, |
| "logits/chosen": -0.6656152009963989, |
| "logits/rejected": -0.67214435338974, |
| "logps/chosen": -1.082200288772583, |
| "logps/rejected": -1.2129806280136108, |
| "loss": 2.5356, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -2.164400577545166, |
| "rewards/margins": 0.26156044006347656, |
| "rewards/rejected": -2.4259612560272217, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.08376963350785341, |
| "grad_norm": 18.265399932861328, |
| "learning_rate": 4.875e-07, |
| "logits/chosen": -0.6319286227226257, |
| "logits/rejected": -0.6386948227882385, |
| "logps/chosen": -1.0040572881698608, |
| "logps/rejected": -1.1640336513519287, |
| "loss": 2.4952, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -2.0081145763397217, |
| "rewards/margins": 0.31995266675949097, |
| "rewards/rejected": -2.3280673027038574, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.10471204188481675, |
| "grad_norm": 13.381461143493652, |
| "learning_rate": 5.999919559552264e-07, |
| "logits/chosen": -0.6912713050842285, |
| "logits/rejected": -0.7003507018089294, |
| "logps/chosen": -1.003204584121704, |
| "logps/rejected": -1.0995913743972778, |
| "loss": 2.4474, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -2.006409168243408, |
| "rewards/margins": 0.1927735060453415, |
| "rewards/rejected": -2.1991827487945557, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.1256544502617801, |
| "grad_norm": 15.925363540649414, |
| "learning_rate": 5.99027192440263e-07, |
| "logits/chosen": -0.6539224982261658, |
| "logits/rejected": -0.6533476114273071, |
| "logps/chosen": -1.056113362312317, |
| "logps/rejected": -1.220874309539795, |
| "loss": 2.5074, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -2.112226724624634, |
| "rewards/margins": 0.32952195405960083, |
| "rewards/rejected": -2.44174861907959, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.14659685863874344, |
| "grad_norm": 18.052942276000977, |
| "learning_rate": 5.964595461857045e-07, |
| "logits/chosen": -0.670150876045227, |
| "logits/rejected": -0.6788632273674011, |
| "logps/chosen": -1.0223743915557861, |
| "logps/rejected": -1.2135789394378662, |
| "loss": 2.4297, |
| "rewards/accuracies": 0.581250011920929, |
| "rewards/chosen": -2.0447487831115723, |
| "rewards/margins": 0.3824090361595154, |
| "rewards/rejected": -2.4271578788757324, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16753926701570682, |
| "grad_norm": 15.041752815246582, |
| "learning_rate": 5.923027806082798e-07, |
| "logits/chosen": -0.6617791056632996, |
| "logits/rejected": -0.666069746017456, |
| "logps/chosen": -1.0529370307922363, |
| "logps/rejected": -1.243398666381836, |
| "loss": 2.4036, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -2.1058740615844727, |
| "rewards/margins": 0.38092344999313354, |
| "rewards/rejected": -2.486797332763672, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18848167539267016, |
| "grad_norm": 17.72820472717285, |
| "learning_rate": 5.865791773197119e-07, |
| "logits/chosen": -0.644040584564209, |
| "logits/rejected": -0.662616491317749, |
| "logps/chosen": -1.0962177515029907, |
| "logps/rejected": -1.333801507949829, |
| "loss": 2.3523, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -2.1924355030059814, |
| "rewards/margins": 0.47516757249832153, |
| "rewards/rejected": -2.667603015899658, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.2094240837696335, |
| "grad_norm": 18.668445587158203, |
| "learning_rate": 5.793194166900525e-07, |
| "logits/chosen": -0.6190682053565979, |
| "logits/rejected": -0.6458654403686523, |
| "logps/chosen": -1.0701862573623657, |
| "logps/rejected": -1.314082145690918, |
| "loss": 2.4183, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -2.1403725147247314, |
| "rewards/margins": 0.48779162764549255, |
| "rewards/rejected": -2.628164291381836, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.23036649214659685, |
| "grad_norm": 23.80174446105957, |
| "learning_rate": 5.705624133909468e-07, |
| "logits/chosen": -0.6541261672973633, |
| "logits/rejected": -0.6746215224266052, |
| "logps/chosen": -1.1125367879867554, |
| "logps/rejected": -1.4309237003326416, |
| "loss": 2.2568, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -2.2250735759735107, |
| "rewards/margins": 0.6367737054824829, |
| "rewards/rejected": -2.861847400665283, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.2513089005235602, |
| "grad_norm": 20.943939208984375, |
| "learning_rate": 5.603551078003725e-07, |
| "logits/chosen": -0.682715892791748, |
| "logits/rejected": -0.6975741386413574, |
| "logps/chosen": -1.1100105047225952, |
| "logps/rejected": -1.4905935525894165, |
| "loss": 2.3622, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -2.2200210094451904, |
| "rewards/margins": 0.7611659169197083, |
| "rewards/rejected": -2.981187105178833, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.27225130890052357, |
| "grad_norm": 15.774776458740234, |
| "learning_rate": 5.487522143869884e-07, |
| "logits/chosen": -0.6576386094093323, |
| "logits/rejected": -0.6527346968650818, |
| "logps/chosen": -1.1969980001449585, |
| "logps/rejected": -1.3874703645706177, |
| "loss": 2.3036, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -2.393996000289917, |
| "rewards/margins": 0.38094475865364075, |
| "rewards/rejected": -2.7749407291412354, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.2931937172774869, |
| "grad_norm": 22.345949172973633, |
| "learning_rate": 5.358159284228362e-07, |
| "logits/chosen": -0.6398555040359497, |
| "logits/rejected": -0.6468138694763184, |
| "logps/chosen": -1.1364694833755493, |
| "logps/rejected": -1.3303481340408325, |
| "loss": 2.4059, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -2.2729389667510986, |
| "rewards/margins": 0.38775745034217834, |
| "rewards/rejected": -2.660696268081665, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.31413612565445026, |
| "grad_norm": 16.85527801513672, |
| "learning_rate": 5.216155925965094e-07, |
| "logits/chosen": -0.673059344291687, |
| "logits/rejected": -0.6615468263626099, |
| "logps/chosen": -1.0914809703826904, |
| "logps/rejected": -1.4301069974899292, |
| "loss": 2.2954, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.182961940765381, |
| "rewards/margins": 0.6772524118423462, |
| "rewards/rejected": -2.8602139949798584, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33507853403141363, |
| "grad_norm": 21.642011642456055, |
| "learning_rate": 5.062273253138518e-07, |
| "logits/chosen": -0.6647015810012817, |
| "logits/rejected": -0.6685549020767212, |
| "logps/chosen": -1.1738396883010864, |
| "logps/rejected": -1.4873461723327637, |
| "loss": 2.2683, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -2.347679376602173, |
| "rewards/margins": 0.6270130276679993, |
| "rewards/rejected": -2.9746923446655273, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.35602094240837695, |
| "grad_norm": 28.569908142089844, |
| "learning_rate": 4.897336126786132e-07, |
| "logits/chosen": -0.6699012517929077, |
| "logits/rejected": -0.679189920425415, |
| "logps/chosen": -1.1100311279296875, |
| "logps/rejected": -1.4672223329544067, |
| "loss": 2.2618, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -2.220062255859375, |
| "rewards/margins": 0.7143827080726624, |
| "rewards/rejected": -2.9344446659088135, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.3769633507853403, |
| "grad_norm": 34.985107421875, |
| "learning_rate": 4.722228663401794e-07, |
| "logits/chosen": -0.6271170377731323, |
| "logits/rejected": -0.6412331461906433, |
| "logps/chosen": -1.1353423595428467, |
| "logps/rejected": -1.403346300125122, |
| "loss": 2.2445, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -2.2706847190856934, |
| "rewards/margins": 0.5360078811645508, |
| "rewards/rejected": -2.806692600250244, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39790575916230364, |
| "grad_norm": 20.208932876586914, |
| "learning_rate": 4.537889495784557e-07, |
| "logits/chosen": -0.6816627383232117, |
| "logits/rejected": -0.6941916942596436, |
| "logps/chosen": -1.2068521976470947, |
| "logps/rejected": -1.6221401691436768, |
| "loss": 2.1744, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.4137043952941895, |
| "rewards/margins": 0.8305760622024536, |
| "rewards/rejected": -3.2442803382873535, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.418848167539267, |
| "grad_norm": 20.75760841369629, |
| "learning_rate": 4.345306741662423e-07, |
| "logits/chosen": -0.6786922216415405, |
| "logits/rejected": -0.6947230100631714, |
| "logps/chosen": -1.2382934093475342, |
| "logps/rejected": -1.6950000524520874, |
| "loss": 2.1958, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -2.4765868186950684, |
| "rewards/margins": 0.9134136438369751, |
| "rewards/rejected": -3.390000104904175, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.418848167539267, |
| "eval_logits/chosen": -0.6647953987121582, |
| "eval_logits/rejected": -0.6653565764427185, |
| "eval_logps/chosen": -1.2070682048797607, |
| "eval_logps/rejected": -1.6066378355026245, |
| "eval_loss": 1.0817621946334839, |
| "eval_rewards/accuracies": 0.6935483813285828, |
| "eval_rewards/chosen": -2.4141364097595215, |
| "eval_rewards/margins": 0.7991395592689514, |
| "eval_rewards/rejected": -3.213275671005249, |
| "eval_runtime": 25.1854, |
| "eval_samples_per_second": 79.411, |
| "eval_steps_per_second": 1.271, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.4397905759162304, |
| "grad_norm": 20.93620491027832, |
| "learning_rate": 4.145512707060832e-07, |
| "logits/chosen": -0.6518492698669434, |
| "logits/rejected": -0.656699001789093, |
| "logps/chosen": -1.2251121997833252, |
| "logps/rejected": -1.6555721759796143, |
| "loss": 2.2519, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -2.4502243995666504, |
| "rewards/margins": 0.8609197735786438, |
| "rewards/rejected": -3.3111443519592285, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.4607329842931937, |
| "grad_norm": 25.06297492980957, |
| "learning_rate": 3.939578352807537e-07, |
| "logits/chosen": -0.6913427710533142, |
| "logits/rejected": -0.6714206337928772, |
| "logps/chosen": -1.3224003314971924, |
| "logps/rejected": -1.7003147602081299, |
| "loss": 2.2159, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -2.6448006629943848, |
| "rewards/margins": 0.7558287382125854, |
| "rewards/rejected": -3.4006295204162598, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.4816753926701571, |
| "grad_norm": 32.439979553222656, |
| "learning_rate": 3.7286075538352106e-07, |
| "logits/chosen": -0.686276376247406, |
| "logits/rejected": -0.6746798753738403, |
| "logps/chosen": -1.322486162185669, |
| "logps/rejected": -1.6524722576141357, |
| "loss": 2.164, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -2.644972324371338, |
| "rewards/margins": 0.6599723696708679, |
| "rewards/rejected": -3.3049445152282715, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5026178010471204, |
| "grad_norm": 21.72540855407715, |
| "learning_rate": 3.5137311820537683e-07, |
| "logits/chosen": -0.6239339113235474, |
| "logits/rejected": -0.6532629132270813, |
| "logps/chosen": -1.204216718673706, |
| "logps/rejected": -1.6880419254302979, |
| "loss": 2.1821, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -2.408433437347412, |
| "rewards/margins": 0.9676502346992493, |
| "rewards/rejected": -3.3760838508605957, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5235602094240838, |
| "grad_norm": 27.567710876464844, |
| "learning_rate": 3.296101044510136e-07, |
| "logits/chosen": -0.6909536123275757, |
| "logits/rejected": -0.6883527040481567, |
| "logps/chosen": -1.2906936407089233, |
| "logps/rejected": -1.750509262084961, |
| "loss": 2.1408, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -2.5813872814178467, |
| "rewards/margins": 0.9196311235427856, |
| "rewards/rejected": -3.501018524169922, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.5445026178010471, |
| "grad_norm": 32.868133544921875, |
| "learning_rate": 3.076883709328898e-07, |
| "logits/chosen": -0.6995635628700256, |
| "logits/rejected": -0.6956952214241028, |
| "logps/chosen": -1.3284438848495483, |
| "logps/rejected": -1.7075936794281006, |
| "loss": 2.2316, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -2.6568877696990967, |
| "rewards/margins": 0.758299708366394, |
| "rewards/rejected": -3.415187358856201, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5654450261780105, |
| "grad_norm": 62.45464324951172, |
| "learning_rate": 2.857254252528773e-07, |
| "logits/chosen": -0.691063404083252, |
| "logits/rejected": -0.6710867285728455, |
| "logps/chosen": -1.402073621749878, |
| "logps/rejected": -1.8776384592056274, |
| "loss": 2.1945, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.804147243499756, |
| "rewards/margins": 0.9511295557022095, |
| "rewards/rejected": -3.755276918411255, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5863874345549738, |
| "grad_norm": 38.48459243774414, |
| "learning_rate": 2.638389959234031e-07, |
| "logits/chosen": -0.6948752403259277, |
| "logits/rejected": -0.6988323330879211, |
| "logps/chosen": -1.332465648651123, |
| "logps/rejected": -2.0018393993377686, |
| "loss": 2.0703, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -2.664931297302246, |
| "rewards/margins": 1.3387477397918701, |
| "rewards/rejected": -4.003678798675537, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.6073298429319371, |
| "grad_norm": 56.83732604980469, |
| "learning_rate": 2.421464013044373e-07, |
| "logits/chosen": -0.7247036099433899, |
| "logits/rejected": -0.7209808230400085, |
| "logps/chosen": -1.400423526763916, |
| "logps/rejected": -1.9026374816894531, |
| "loss": 2.1931, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -2.800847053527832, |
| "rewards/margins": 1.0044279098510742, |
| "rewards/rejected": -3.8052749633789062, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6282722513089005, |
| "grad_norm": 31.81602668762207, |
| "learning_rate": 2.2076392073903244e-07, |
| "logits/chosen": -0.7202259302139282, |
| "logits/rejected": -0.7023282647132874, |
| "logps/chosen": -1.4328830242156982, |
| "logps/rejected": -1.7894738912582397, |
| "loss": 2.1941, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -2.8657660484313965, |
| "rewards/margins": 0.7131820321083069, |
| "rewards/rejected": -3.5789477825164795, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6492146596858639, |
| "grad_norm": 34.869110107421875, |
| "learning_rate": 1.9980617125832958e-07, |
| "logits/chosen": -0.7520186305046082, |
| "logits/rejected": -0.7353061437606812, |
| "logps/chosen": -1.3960636854171753, |
| "logps/rejected": -1.8475030660629272, |
| "loss": 2.0703, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.7921273708343506, |
| "rewards/margins": 0.9028788805007935, |
| "rewards/rejected": -3.6950061321258545, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.6701570680628273, |
| "grad_norm": 33.654422760009766, |
| "learning_rate": 1.7938549319709663e-07, |
| "logits/chosen": -0.7805129289627075, |
| "logits/rejected": -0.75370854139328, |
| "logps/chosen": -1.513319492340088, |
| "logps/rejected": -2.104207992553711, |
| "loss": 2.0994, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -3.026638984680176, |
| "rewards/margins": 1.181776762008667, |
| "rewards/rejected": -4.208415985107422, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6910994764397905, |
| "grad_norm": 34.889400482177734, |
| "learning_rate": 1.5961134801309614e-07, |
| "logits/chosen": -0.7453498840332031, |
| "logits/rejected": -0.7288404107093811, |
| "logps/chosen": -1.5413320064544678, |
| "logps/rejected": -2.1900875568389893, |
| "loss": 2.1361, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -3.0826640129089355, |
| "rewards/margins": 1.2975105047225952, |
| "rewards/rejected": -4.3801751136779785, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7120418848167539, |
| "grad_norm": 58.18461227416992, |
| "learning_rate": 1.4058973153816886e-07, |
| "logits/chosen": -0.7768866419792175, |
| "logits/rejected": -0.7656865119934082, |
| "logps/chosen": -1.6624263525009155, |
| "logps/rejected": -2.255127429962158, |
| "loss": 2.1658, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -3.324852705001831, |
| "rewards/margins": 1.1854020357131958, |
| "rewards/rejected": -4.510254859924316, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.7329842931937173, |
| "grad_norm": 55.78643798828125, |
| "learning_rate": 1.2242260580619538e-07, |
| "logits/chosen": -0.7563885450363159, |
| "logits/rejected": -0.7533862590789795, |
| "logps/chosen": -1.6073474884033203, |
| "logps/rejected": -2.1334166526794434, |
| "loss": 2.0949, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -3.2146949768066406, |
| "rewards/margins": 1.052138090133667, |
| "rewards/rejected": -4.266833305358887, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7539267015706806, |
| "grad_norm": 39.962528228759766, |
| "learning_rate": 1.0520735250352405e-07, |
| "logits/chosen": -0.703062891960144, |
| "logits/rejected": -0.702917218208313, |
| "logps/chosen": -1.4934108257293701, |
| "logps/rejected": -2.0725302696228027, |
| "loss": 2.1009, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -2.9868216514587402, |
| "rewards/margins": 1.1582386493682861, |
| "rewards/rejected": -4.1450605392456055, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.774869109947644, |
| "grad_norm": 36.544063568115234, |
| "learning_rate": 8.903625097154667e-08, |
| "logits/chosen": -0.7489289045333862, |
| "logits/rejected": -0.7575147747993469, |
| "logps/chosen": -1.3683464527130127, |
| "logps/rejected": -2.0208771228790283, |
| "loss": 1.9718, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.7366929054260254, |
| "rewards/margins": 1.3050611019134521, |
| "rewards/rejected": -4.041754245758057, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.7958115183246073, |
| "grad_norm": 53.06337356567383, |
| "learning_rate": 7.399598355949822e-08, |
| "logits/chosen": -0.7504714727401733, |
| "logits/rejected": -0.7333730459213257, |
| "logps/chosen": -1.5020231008529663, |
| "logps/rejected": -1.9611724615097046, |
| "loss": 2.1079, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -3.0040462017059326, |
| "rewards/margins": 0.9182990789413452, |
| "rewards/rejected": -3.922344923019409, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8167539267015707, |
| "grad_norm": 32.34917449951172, |
| "learning_rate": 6.01671709789497e-08, |
| "logits/chosen": -0.761965811252594, |
| "logits/rejected": -0.7571056485176086, |
| "logps/chosen": -1.473826289176941, |
| "logps/rejected": -1.9839645624160767, |
| "loss": 2.0849, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.947652578353882, |
| "rewards/margins": 1.020276427268982, |
| "rewards/rejected": -3.9679291248321533, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.837696335078534, |
| "grad_norm": 57.54490280151367, |
| "learning_rate": 4.76239401506456e-08, |
| "logits/chosen": -0.7651081681251526, |
| "logits/rejected": -0.7489873170852661, |
| "logps/chosen": -1.4767208099365234, |
| "logps/rejected": -1.997377634048462, |
| "loss": 2.1186, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.953441619873047, |
| "rewards/margins": 1.0413135290145874, |
| "rewards/rejected": -3.994755268096924, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.837696335078534, |
| "eval_logits/chosen": -0.75456303358078, |
| "eval_logits/rejected": -0.7413101196289062, |
| "eval_logps/chosen": -1.4940153360366821, |
| "eval_logps/rejected": -2.0286271572113037, |
| "eval_loss": 1.0269173383712769, |
| "eval_rewards/accuracies": 0.7379032373428345, |
| "eval_rewards/chosen": -2.9880306720733643, |
| "eval_rewards/margins": 1.0692235231399536, |
| "eval_rewards/rejected": -4.057254314422607, |
| "eval_runtime": 24.6202, |
| "eval_samples_per_second": 81.234, |
| "eval_steps_per_second": 1.3, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.8586387434554974, |
| "grad_norm": 47.848445892333984, |
| "learning_rate": 3.643352686016596e-08, |
| "logits/chosen": -0.790420651435852, |
| "logits/rejected": -0.7755736112594604, |
| "logps/chosen": -1.5548263788223267, |
| "logps/rejected": -2.1035819053649902, |
| "loss": 2.047, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -3.1096527576446533, |
| "rewards/margins": 1.0975114107131958, |
| "rewards/rejected": -4.2071638107299805, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8795811518324608, |
| "grad_norm": 35.658634185791016, |
| "learning_rate": 2.665591535230738e-08, |
| "logits/chosen": -0.7204955220222473, |
| "logits/rejected": -0.6889880895614624, |
| "logps/chosen": -1.4325242042541504, |
| "logps/rejected": -2.0371756553649902, |
| "loss": 2.0869, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -2.865048408508301, |
| "rewards/margins": 1.2093026638031006, |
| "rewards/rejected": -4.0743513107299805, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.900523560209424, |
| "grad_norm": 46.57322311401367, |
| "learning_rate": 1.834351679607603e-08, |
| "logits/chosen": -0.7658016681671143, |
| "logits/rejected": -0.7545614838600159, |
| "logps/chosen": -1.573411226272583, |
| "logps/rejected": -2.168039321899414, |
| "loss": 2.0035, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -3.146822452545166, |
| "rewards/margins": 1.1892563104629517, |
| "rewards/rejected": -4.336078643798828, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.9214659685863874, |
| "grad_norm": 49.09744644165039, |
| "learning_rate": 1.1540888343822164e-08, |
| "logits/chosen": -0.7819451093673706, |
| "logits/rejected": -0.7810377478599548, |
| "logps/chosen": -1.533359408378601, |
| "logps/rejected": -1.9900741577148438, |
| "loss": 2.0797, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -3.066718816757202, |
| "rewards/margins": 0.9134290814399719, |
| "rewards/rejected": -3.9801483154296875, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9424083769633508, |
| "grad_norm": 30.21855926513672, |
| "learning_rate": 6.284494290451603e-09, |
| "logits/chosen": -0.7444645762443542, |
| "logits/rejected": -0.7367347478866577, |
| "logps/chosen": -1.6120609045028687, |
| "logps/rejected": -2.1023104190826416, |
| "loss": 2.0586, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -3.2241218090057373, |
| "rewards/margins": 0.9804989099502563, |
| "rewards/rejected": -4.204620838165283, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9633507853403142, |
| "grad_norm": 48.88844680786133, |
| "learning_rate": 2.6025106129779263e-09, |
| "logits/chosen": -0.7713245153427124, |
| "logits/rejected": -0.763340950012207, |
| "logps/chosen": -1.6260992288589478, |
| "logps/rejected": -2.057934284210205, |
| "loss": 2.1323, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -3.2521984577178955, |
| "rewards/margins": 0.863669753074646, |
| "rewards/rejected": -4.11586856842041, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.9842931937172775, |
| "grad_norm": 30.39113426208496, |
| "learning_rate": 5.146739381471921e-10, |
| "logits/chosen": -0.742178201675415, |
| "logits/rejected": -0.7218775749206543, |
| "logps/chosen": -1.443922996520996, |
| "logps/rejected": -2.284044027328491, |
| "loss": 2.0688, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -2.887845993041992, |
| "rewards/margins": 1.6802421808242798, |
| "rewards/rejected": -4.568088054656982, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.9989528795811519, |
| "step": 477, |
| "total_flos": 0.0, |
| "train_loss": 2.230338767889411, |
| "train_runtime": 2802.8273, |
| "train_samples_per_second": 21.812, |
| "train_steps_per_second": 0.17 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 477, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": false, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
