| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.6423767941383117, |
| "eval_steps": 250, |
| "global_step": 800, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0, |
| "eval_logits/chosen": -7.611474514007568, |
| "eval_logits/rejected": -7.2800092697143555, |
| "eval_logps/chosen": -241.7949676513672, |
| "eval_logps/rejected": -221.7694854736328, |
| "eval_loss": 0.6931473016738892, |
| "eval_rewards/accuracies": 0.0, |
| "eval_rewards/chosen": 0.0, |
| "eval_rewards/margins": 0.0, |
| "eval_rewards/rejected": 0.0, |
| "eval_runtime": 713.9008, |
| "eval_samples_per_second": 49.62, |
| "eval_steps_per_second": 1.551, |
| "step": 0 |
| }, |
| { |
| "epoch": 0.0008029709926728897, |
| "grad_norm": 1.5940191745758057, |
| "learning_rate": 0.0, |
| "logits/chosen": -7.641777515411377, |
| "logits/rejected": -7.330385208129883, |
| "logps/chosen": -271.1852722167969, |
| "logps/rejected": -239.28961181640625, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.0016059419853457794, |
| "grad_norm": 1.4999037981033325, |
| "learning_rate": 6.000000000000001e-07, |
| "logits/chosen": -7.522189617156982, |
| "logits/rejected": -7.140087604522705, |
| "logps/chosen": -241.43341064453125, |
| "logps/rejected": -230.68113708496094, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.002408912978018669, |
| "grad_norm": 1.6589306592941284, |
| "learning_rate": 1.2000000000000002e-06, |
| "logits/chosen": -7.742486953735352, |
| "logits/rejected": -7.293023586273193, |
| "logps/chosen": -250.79940795898438, |
| "logps/rejected": -227.81385803222656, |
| "loss": 0.6925, |
| "rewards/accuracies": 0.51953125, |
| "rewards/chosen": 0.0036971382796764374, |
| "rewards/margins": 0.0017042233375832438, |
| "rewards/rejected": 0.001992915291339159, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.003211883970691559, |
| "grad_norm": 1.5587422847747803, |
| "learning_rate": 1.8e-06, |
| "logits/chosen": -7.553328514099121, |
| "logits/rejected": -7.231173992156982, |
| "logps/chosen": -243.49163818359375, |
| "logps/rejected": -228.23837280273438, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.484375, |
| "rewards/chosen": 0.0002594427205622196, |
| "rewards/margins": -8.5427425801754e-05, |
| "rewards/rejected": 0.0003448702918831259, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.004014854963364448, |
| "grad_norm": 1.5665065050125122, |
| "learning_rate": 2.4000000000000003e-06, |
| "logits/chosen": -7.631692409515381, |
| "logits/rejected": -7.383444786071777, |
| "logps/chosen": -244.651611328125, |
| "logps/rejected": -233.66732788085938, |
| "loss": 0.6919, |
| "rewards/accuracies": 0.5390625, |
| "rewards/chosen": 0.002985040657222271, |
| "rewards/margins": 0.0029604153241962194, |
| "rewards/rejected": 2.4625565856695175e-05, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.004817825956037338, |
| "grad_norm": 1.414748191833496, |
| "learning_rate": 3e-06, |
| "logits/chosen": -7.745750904083252, |
| "logits/rejected": -7.417798042297363, |
| "logps/chosen": -230.96176147460938, |
| "logps/rejected": -214.609619140625, |
| "loss": 0.6892, |
| "rewards/accuracies": 0.5390625, |
| "rewards/chosen": 0.0087535185739398, |
| "rewards/margins": 0.008384622633457184, |
| "rewards/rejected": 0.0003688961442094296, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.0056207969487102275, |
| "grad_norm": 1.4553792476654053, |
| "learning_rate": 3.6e-06, |
| "logits/chosen": -7.443927764892578, |
| "logits/rejected": -7.139378547668457, |
| "logps/chosen": -249.6968994140625, |
| "logps/rejected": -229.5579833984375, |
| "loss": 0.6907, |
| "rewards/accuracies": 0.57421875, |
| "rewards/chosen": 0.004703341517597437, |
| "rewards/margins": 0.005331031978130341, |
| "rewards/rejected": -0.0006276905187405646, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.006423767941383118, |
| "grad_norm": 1.5290026664733887, |
| "learning_rate": 4.2000000000000004e-06, |
| "logits/chosen": -7.673081398010254, |
| "logits/rejected": -7.266858100891113, |
| "logps/chosen": -244.72915649414062, |
| "logps/rejected": -252.00930786132812, |
| "loss": 0.6901, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.007142478134483099, |
| "rewards/margins": 0.006796684116125107, |
| "rewards/rejected": 0.0003457942511886358, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.007226738934056007, |
| "grad_norm": 1.5330169200897217, |
| "learning_rate": 4.800000000000001e-06, |
| "logits/chosen": -7.605165004730225, |
| "logits/rejected": -7.292326927185059, |
| "logps/chosen": -251.0775604248047, |
| "logps/rejected": -242.1954803466797, |
| "loss": 0.6855, |
| "rewards/accuracies": 0.66796875, |
| "rewards/chosen": 0.015020196326076984, |
| "rewards/margins": 0.015981096774339676, |
| "rewards/rejected": -0.000960900797508657, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.008029709926728896, |
| "grad_norm": 1.5272985696792603, |
| "learning_rate": 5.4e-06, |
| "logits/chosen": -7.418107986450195, |
| "logits/rejected": -7.083195686340332, |
| "logps/chosen": -257.3016357421875, |
| "logps/rejected": -222.3004913330078, |
| "loss": 0.6807, |
| "rewards/accuracies": 0.66796875, |
| "rewards/chosen": 0.02404569834470749, |
| "rewards/margins": 0.025940997526049614, |
| "rewards/rejected": -0.0018953010439872742, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.008832680919401786, |
| "grad_norm": 1.3459587097167969, |
| "learning_rate": 6e-06, |
| "logits/chosen": -7.64585542678833, |
| "logits/rejected": -7.496757984161377, |
| "logps/chosen": -263.0661315917969, |
| "logps/rejected": -235.56683349609375, |
| "loss": 0.6809, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": 0.026645416393876076, |
| "rewards/margins": 0.02555106207728386, |
| "rewards/rejected": 0.0010943511733785272, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.009635651912074676, |
| "grad_norm": 1.3396459817886353, |
| "learning_rate": 6.6e-06, |
| "logits/chosen": -7.640565872192383, |
| "logits/rejected": -7.290168285369873, |
| "logps/chosen": -208.63198852539062, |
| "logps/rejected": -186.49913024902344, |
| "loss": 0.6759, |
| "rewards/accuracies": 0.67578125, |
| "rewards/chosen": 0.0326472744345665, |
| "rewards/margins": 0.03588727116584778, |
| "rewards/rejected": -0.0032399988267570734, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.010438622904747567, |
| "grad_norm": 1.4921026229858398, |
| "learning_rate": 7.2e-06, |
| "logits/chosen": -7.823661804199219, |
| "logits/rejected": -7.595123291015625, |
| "logps/chosen": -232.67849731445312, |
| "logps/rejected": -217.09579467773438, |
| "loss": 0.6672, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.04918733611702919, |
| "rewards/margins": 0.05475744232535362, |
| "rewards/rejected": -0.005570105277001858, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.011241593897420455, |
| "grad_norm": 1.4722273349761963, |
| "learning_rate": 7.8e-06, |
| "logits/chosen": -7.585400581359863, |
| "logits/rejected": -7.18538236618042, |
| "logps/chosen": -242.79000854492188, |
| "logps/rejected": -205.89024353027344, |
| "loss": 0.6603, |
| "rewards/accuracies": 0.7578125, |
| "rewards/chosen": 0.06492122262716293, |
| "rewards/margins": 0.07029303163290024, |
| "rewards/rejected": -0.005371804814785719, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.012044564890093345, |
| "grad_norm": 1.2793220281600952, |
| "learning_rate": 8.400000000000001e-06, |
| "logits/chosen": -7.655429363250732, |
| "logits/rejected": -7.349386215209961, |
| "logps/chosen": -258.7111511230469, |
| "logps/rejected": -237.44857788085938, |
| "loss": 0.6562, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.07612358033657074, |
| "rewards/margins": 0.08108936250209808, |
| "rewards/rejected": -0.004965781234204769, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.012847535882766235, |
| "grad_norm": 1.1673407554626465, |
| "learning_rate": 9e-06, |
| "logits/chosen": -7.573231220245361, |
| "logits/rejected": -7.324514389038086, |
| "logps/chosen": -230.8852996826172, |
| "logps/rejected": -214.4945068359375, |
| "loss": 0.6527, |
| "rewards/accuracies": 0.76953125, |
| "rewards/chosen": 0.08438821882009506, |
| "rewards/margins": 0.08987204730510712, |
| "rewards/rejected": -0.005483835935592651, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.013650506875439125, |
| "grad_norm": 1.2871980667114258, |
| "learning_rate": 9.600000000000001e-06, |
| "logits/chosen": -7.563333988189697, |
| "logits/rejected": -7.220376491546631, |
| "logps/chosen": -259.0805358886719, |
| "logps/rejected": -230.6144561767578, |
| "loss": 0.6372, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": 0.12134990841150284, |
| "rewards/margins": 0.12648926675319672, |
| "rewards/rejected": -0.005139365792274475, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.014453477868112014, |
| "grad_norm": 1.1219907999038696, |
| "learning_rate": 1.02e-05, |
| "logits/chosen": -7.349454879760742, |
| "logits/rejected": -7.053747177124023, |
| "logps/chosen": -226.46055603027344, |
| "logps/rejected": -202.146728515625, |
| "loss": 0.6337, |
| "rewards/accuracies": 0.79296875, |
| "rewards/chosen": 0.1273726224899292, |
| "rewards/margins": 0.1391480714082718, |
| "rewards/rejected": -0.011775430291891098, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.015256448860784904, |
| "grad_norm": 1.093957781791687, |
| "learning_rate": 1.08e-05, |
| "logits/chosen": -7.864840030670166, |
| "logits/rejected": -7.660233020782471, |
| "logps/chosen": -239.39410400390625, |
| "logps/rejected": -211.85342407226562, |
| "loss": 0.6196, |
| "rewards/accuracies": 0.82421875, |
| "rewards/chosen": 0.15482831001281738, |
| "rewards/margins": 0.17550238966941833, |
| "rewards/rejected": -0.020674072206020355, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.016059419853457792, |
| "grad_norm": 1.1654642820358276, |
| "learning_rate": 1.1400000000000001e-05, |
| "logits/chosen": -7.522461414337158, |
| "logits/rejected": -7.271822452545166, |
| "logps/chosen": -253.74057006835938, |
| "logps/rejected": -233.15292358398438, |
| "loss": 0.5906, |
| "rewards/accuracies": 0.82421875, |
| "rewards/chosen": 0.2297699749469757, |
| "rewards/margins": 0.2600101828575134, |
| "rewards/rejected": -0.030240220949053764, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.016862390846130684, |
| "grad_norm": 1.058030128479004, |
| "learning_rate": 1.2e-05, |
| "logits/chosen": -7.475430011749268, |
| "logits/rejected": -7.113090991973877, |
| "logps/chosen": -257.91046142578125, |
| "logps/rejected": -232.12954711914062, |
| "loss": 0.5687, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": 0.30237430334091187, |
| "rewards/margins": 0.3408054709434509, |
| "rewards/rejected": -0.03843114897608757, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.017665361838803573, |
| "grad_norm": 0.9458898901939392, |
| "learning_rate": 1.26e-05, |
| "logits/chosen": -7.801554203033447, |
| "logits/rejected": -7.479979038238525, |
| "logps/chosen": -249.59722900390625, |
| "logps/rejected": -216.05242919921875, |
| "loss": 0.5716, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": 0.2715383768081665, |
| "rewards/margins": 0.32752737402915955, |
| "rewards/rejected": -0.05598897486925125, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.018468332831476465, |
| "grad_norm": 0.9516150951385498, |
| "learning_rate": 1.32e-05, |
| "logits/chosen": -7.534506320953369, |
| "logits/rejected": -7.214147090911865, |
| "logps/chosen": -258.27130126953125, |
| "logps/rejected": -241.6322479248047, |
| "loss": 0.5463, |
| "rewards/accuracies": 0.83984375, |
| "rewards/chosen": 0.3748800456523895, |
| "rewards/margins": 0.43512117862701416, |
| "rewards/rejected": -0.06024113669991493, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.019271303824149353, |
| "grad_norm": 0.873358964920044, |
| "learning_rate": 1.3800000000000002e-05, |
| "logits/chosen": -7.815151214599609, |
| "logits/rejected": -7.458766937255859, |
| "logps/chosen": -229.832763671875, |
| "logps/rejected": -211.19606018066406, |
| "loss": 0.5444, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.365281343460083, |
| "rewards/margins": 0.4309691786766052, |
| "rewards/rejected": -0.06568782031536102, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.02007427481682224, |
| "grad_norm": 0.8084851503372192, |
| "learning_rate": 1.44e-05, |
| "logits/chosen": -7.751460075378418, |
| "logits/rejected": -7.446683883666992, |
| "logps/chosen": -217.69430541992188, |
| "logps/rejected": -211.05780029296875, |
| "loss": 0.5316, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": 0.41402849555015564, |
| "rewards/margins": 0.5042967796325684, |
| "rewards/rejected": -0.09026830643415451, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.020877245809495133, |
| "grad_norm": 0.7675647139549255, |
| "learning_rate": 1.5e-05, |
| "logits/chosen": -7.7662153244018555, |
| "logits/rejected": -7.304013729095459, |
| "logps/chosen": -220.49008178710938, |
| "logps/rejected": -197.9363250732422, |
| "loss": 0.5047, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": 0.5476651191711426, |
| "rewards/margins": 0.6539051532745361, |
| "rewards/rejected": -0.10624003410339355, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.02168021680216802, |
| "grad_norm": 0.829423189163208, |
| "learning_rate": 1.56e-05, |
| "logits/chosen": -7.699043273925781, |
| "logits/rejected": -7.2956743240356445, |
| "logps/chosen": -224.11097717285156, |
| "logps/rejected": -225.8865203857422, |
| "loss": 0.532, |
| "rewards/accuracies": 0.82421875, |
| "rewards/chosen": 0.4515957236289978, |
| "rewards/margins": 0.5536822080612183, |
| "rewards/rejected": -0.10208651423454285, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.02248318779484091, |
| "grad_norm": 0.7462482452392578, |
| "learning_rate": 1.62e-05, |
| "logits/chosen": -7.571779251098633, |
| "logits/rejected": -7.2383713722229, |
| "logps/chosen": -226.74110412597656, |
| "logps/rejected": -207.83828735351562, |
| "loss": 0.4933, |
| "rewards/accuracies": 0.85546875, |
| "rewards/chosen": 0.6197806596755981, |
| "rewards/margins": 0.7652255892753601, |
| "rewards/rejected": -0.14544488489627838, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.023286158787513802, |
| "grad_norm": 0.7799257636070251, |
| "learning_rate": 1.6800000000000002e-05, |
| "logits/chosen": -7.747625350952148, |
| "logits/rejected": -7.5512471199035645, |
| "logps/chosen": -205.726806640625, |
| "logps/rejected": -205.65249633789062, |
| "loss": 0.5203, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.49129319190979004, |
| "rewards/margins": 0.6569412350654602, |
| "rewards/rejected": -0.16564807295799255, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.02408912978018669, |
| "grad_norm": 0.7830209732055664, |
| "learning_rate": 1.74e-05, |
| "logits/chosen": -7.636171340942383, |
| "logits/rejected": -7.259405136108398, |
| "logps/chosen": -231.21990966796875, |
| "logps/rejected": -201.7489013671875, |
| "loss": 0.4879, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": 0.6785256266593933, |
| "rewards/margins": 0.8415416479110718, |
| "rewards/rejected": -0.1630159318447113, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.02489210077285958, |
| "grad_norm": 0.7710943818092346, |
| "learning_rate": 1.8e-05, |
| "logits/chosen": -7.627671718597412, |
| "logits/rejected": -7.192222595214844, |
| "logps/chosen": -211.46731567382812, |
| "logps/rejected": -236.38140869140625, |
| "loss": 0.4783, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": 0.7061071395874023, |
| "rewards/margins": 0.9487599730491638, |
| "rewards/rejected": -0.2426527887582779, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.02569507176553247, |
| "grad_norm": 0.7015618085861206, |
| "learning_rate": 1.86e-05, |
| "logits/chosen": -7.748106956481934, |
| "logits/rejected": -7.282266139984131, |
| "logps/chosen": -237.6963653564453, |
| "logps/rejected": -244.8104248046875, |
| "loss": 0.4342, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.8935531377792358, |
| "rewards/margins": 1.2637848854064941, |
| "rewards/rejected": -0.3702317178249359, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.02649804275820536, |
| "grad_norm": 0.7551398873329163, |
| "learning_rate": 1.9200000000000003e-05, |
| "logits/chosen": -7.592668056488037, |
| "logits/rejected": -7.173820972442627, |
| "logps/chosen": -260.6484375, |
| "logps/rejected": -252.71807861328125, |
| "loss": 0.4272, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": 0.9243577718734741, |
| "rewards/margins": 1.316596269607544, |
| "rewards/rejected": -0.3922383785247803, |
| "step": 33 |
| }, |
| { |
| "epoch": 0.02730101375087825, |
| "grad_norm": 0.8450775146484375, |
| "learning_rate": 1.98e-05, |
| "logits/chosen": -7.547041416168213, |
| "logits/rejected": -7.2432355880737305, |
| "logps/chosen": -196.13365173339844, |
| "logps/rejected": -196.20367431640625, |
| "loss": 0.4628, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": 0.7197820544242859, |
| "rewards/margins": 1.0561505556106567, |
| "rewards/rejected": -0.33636847138404846, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.02810398474355114, |
| "grad_norm": 0.7595415115356445, |
| "learning_rate": 2.04e-05, |
| "logits/chosen": -7.627911567687988, |
| "logits/rejected": -7.362043857574463, |
| "logps/chosen": -226.6575469970703, |
| "logps/rejected": -220.4368896484375, |
| "loss": 0.4274, |
| "rewards/accuracies": 0.87890625, |
| "rewards/chosen": 0.9373118281364441, |
| "rewards/margins": 1.3558056354522705, |
| "rewards/rejected": -0.41849392652511597, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.028906955736224028, |
| "grad_norm": 0.7755140662193298, |
| "learning_rate": 2.1e-05, |
| "logits/chosen": -7.699193954467773, |
| "logits/rejected": -7.31709098815918, |
| "logps/chosen": -217.1302947998047, |
| "logps/rejected": -208.88174438476562, |
| "loss": 0.4018, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.9943079948425293, |
| "rewards/margins": 1.515647053718567, |
| "rewards/rejected": -0.5213391780853271, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.02970992672889692, |
| "grad_norm": 0.7642138004302979, |
| "learning_rate": 2.16e-05, |
| "logits/chosen": -7.753854274749756, |
| "logits/rejected": -7.458947658538818, |
| "logps/chosen": -225.11000061035156, |
| "logps/rejected": -212.3223876953125, |
| "loss": 0.4128, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.8810312747955322, |
| "rewards/margins": 1.417097806930542, |
| "rewards/rejected": -0.5360666513442993, |
| "step": 37 |
| }, |
| { |
| "epoch": 0.030512897721569808, |
| "grad_norm": 0.6594340205192566, |
| "learning_rate": 2.22e-05, |
| "logits/chosen": -7.718212127685547, |
| "logits/rejected": -7.422189235687256, |
| "logps/chosen": -205.40826416015625, |
| "logps/rejected": -208.5579071044922, |
| "loss": 0.3856, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": 1.0324681997299194, |
| "rewards/margins": 1.6548075675964355, |
| "rewards/rejected": -0.6223393082618713, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.031315868714242696, |
| "grad_norm": 0.7788670063018799, |
| "learning_rate": 2.2800000000000002e-05, |
| "logits/chosen": -7.611528396606445, |
| "logits/rejected": -7.383983135223389, |
| "logps/chosen": -232.39732360839844, |
| "logps/rejected": -231.44122314453125, |
| "loss": 0.4075, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": 0.9212095737457275, |
| "rewards/margins": 1.512988567352295, |
| "rewards/rejected": -0.5917789936065674, |
| "step": 39 |
| }, |
| { |
| "epoch": 0.032118839706915585, |
| "grad_norm": 0.7903236746788025, |
| "learning_rate": 2.3400000000000003e-05, |
| "logits/chosen": -7.531782150268555, |
| "logits/rejected": -7.156610488891602, |
| "logps/chosen": -246.94476318359375, |
| "logps/rejected": -237.928466796875, |
| "loss": 0.3882, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": 1.0078864097595215, |
| "rewards/margins": 1.6607829332351685, |
| "rewards/rejected": -0.6528965830802917, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.03292181069958848, |
| "grad_norm": 0.6888555884361267, |
| "learning_rate": 2.4e-05, |
| "logits/chosen": -7.657487869262695, |
| "logits/rejected": -7.385692596435547, |
| "logps/chosen": -227.40301513671875, |
| "logps/rejected": -222.26089477539062, |
| "loss": 0.3471, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 1.1523933410644531, |
| "rewards/margins": 2.0023767948150635, |
| "rewards/rejected": -0.8499833345413208, |
| "step": 41 |
| }, |
| { |
| "epoch": 0.03372478169226137, |
| "grad_norm": 0.7876406311988831, |
| "learning_rate": 2.4599999999999998e-05, |
| "logits/chosen": -7.733768939971924, |
| "logits/rejected": -7.40791654586792, |
| "logps/chosen": -246.4244842529297, |
| "logps/rejected": -238.3661651611328, |
| "loss": 0.3395, |
| "rewards/accuracies": 0.83984375, |
| "rewards/chosen": 1.164204478263855, |
| "rewards/margins": 2.143369674682617, |
| "rewards/rejected": -0.9791653156280518, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.03452775268493426, |
| "grad_norm": 0.7605945467948914, |
| "learning_rate": 2.52e-05, |
| "logits/chosen": -7.788512706756592, |
| "logits/rejected": -7.432503700256348, |
| "logps/chosen": -251.30767822265625, |
| "logps/rejected": -242.01046752929688, |
| "loss": 0.3185, |
| "rewards/accuracies": 0.88671875, |
| "rewards/chosen": 1.111971139907837, |
| "rewards/margins": 2.177443027496338, |
| "rewards/rejected": -1.065471887588501, |
| "step": 43 |
| }, |
| { |
| "epoch": 0.035330723677607145, |
| "grad_norm": 0.7297447919845581, |
| "learning_rate": 2.58e-05, |
| "logits/chosen": -7.6427412033081055, |
| "logits/rejected": -7.416354656219482, |
| "logps/chosen": -236.3736572265625, |
| "logps/rejected": -222.54000854492188, |
| "loss": 0.3097, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": 1.0835710763931274, |
| "rewards/margins": 2.250976324081421, |
| "rewards/rejected": -1.1674052476882935, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.036133694670280034, |
| "grad_norm": 0.7728644013404846, |
| "learning_rate": 2.64e-05, |
| "logits/chosen": -7.559998512268066, |
| "logits/rejected": -7.274555683135986, |
| "logps/chosen": -243.94595336914062, |
| "logps/rejected": -251.4634246826172, |
| "loss": 0.3012, |
| "rewards/accuracies": 0.90234375, |
| "rewards/chosen": 1.0970711708068848, |
| "rewards/margins": 2.390831470489502, |
| "rewards/rejected": -1.2937602996826172, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.03693666566295293, |
| "grad_norm": 0.7805203795433044, |
| "learning_rate": 2.7000000000000002e-05, |
| "logits/chosen": -7.4751410484313965, |
| "logits/rejected": -7.326271057128906, |
| "logps/chosen": -235.81393432617188, |
| "logps/rejected": -241.06796264648438, |
| "loss": 0.3237, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.8636063933372498, |
| "rewards/margins": 2.1940512657165527, |
| "rewards/rejected": -1.3304448127746582, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.03773963665562582, |
| "grad_norm": 0.8579618334770203, |
| "learning_rate": 2.7600000000000003e-05, |
| "logits/chosen": -7.427497863769531, |
| "logits/rejected": -7.067408561706543, |
| "logps/chosen": -237.98439025878906, |
| "logps/rejected": -240.469970703125, |
| "loss": 0.3037, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": 0.8533890843391418, |
| "rewards/margins": 2.392348289489746, |
| "rewards/rejected": -1.53895902633667, |
| "step": 47 |
| }, |
| { |
| "epoch": 0.038542607648298706, |
| "grad_norm": 0.8394532203674316, |
| "learning_rate": 2.8199999999999998e-05, |
| "logits/chosen": -7.540038585662842, |
| "logits/rejected": -7.090114593505859, |
| "logps/chosen": -231.59429931640625, |
| "logps/rejected": -250.90975952148438, |
| "loss": 0.3108, |
| "rewards/accuracies": 0.85546875, |
| "rewards/chosen": 0.9697648286819458, |
| "rewards/margins": 2.5256550312042236, |
| "rewards/rejected": -1.5558902025222778, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.039345578640971594, |
| "grad_norm": 0.7502954006195068, |
| "learning_rate": 2.88e-05, |
| "logits/chosen": -7.771185874938965, |
| "logits/rejected": -7.440066814422607, |
| "logps/chosen": -230.03700256347656, |
| "logps/rejected": -238.4860076904297, |
| "loss": 0.3043, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 1.0078611373901367, |
| "rewards/margins": 2.7364792823791504, |
| "rewards/rejected": -1.7286182641983032, |
| "step": 49 |
| }, |
| { |
| "epoch": 0.04014854963364448, |
| "grad_norm": 0.7674089670181274, |
| "learning_rate": 2.94e-05, |
| "logits/chosen": -7.606203556060791, |
| "logits/rejected": -7.219176292419434, |
| "logps/chosen": -239.03280639648438, |
| "logps/rejected": -247.16136169433594, |
| "loss": 0.2855, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": 0.9404242038726807, |
| "rewards/margins": 2.791240930557251, |
| "rewards/rejected": -1.8508167266845703, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.04095152062631737, |
| "grad_norm": 1.0955862998962402, |
| "learning_rate": 3e-05, |
| "logits/chosen": -7.6704816818237305, |
| "logits/rejected": -7.287598609924316, |
| "logps/chosen": -253.90805053710938, |
| "logps/rejected": -274.95892333984375, |
| "loss": 0.3324, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.7122849225997925, |
| "rewards/margins": 2.7398152351379395, |
| "rewards/rejected": -2.0275304317474365, |
| "step": 51 |
| }, |
| { |
| "epoch": 0.041754491618990267, |
| "grad_norm": 0.8694361448287964, |
| "learning_rate": 2.9999917981201608e-05, |
| "logits/chosen": -7.638002395629883, |
| "logits/rejected": -7.289333343505859, |
| "logps/chosen": -224.68020629882812, |
| "logps/rejected": -242.32745361328125, |
| "loss": 0.2923, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": 0.7295356392860413, |
| "rewards/margins": 2.949134349822998, |
| "rewards/rejected": -2.2195987701416016, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.042557462611663155, |
| "grad_norm": 0.8839179277420044, |
| "learning_rate": 2.9999671925703373e-05, |
| "logits/chosen": -7.649010181427002, |
| "logits/rejected": -7.402343273162842, |
| "logps/chosen": -244.00689697265625, |
| "logps/rejected": -259.0998229980469, |
| "loss": 0.2692, |
| "rewards/accuracies": 0.89453125, |
| "rewards/chosen": 0.6859833002090454, |
| "rewards/margins": 3.1641790866851807, |
| "rewards/rejected": -2.478195905685425, |
| "step": 53 |
| }, |
| { |
| "epoch": 0.04336043360433604, |
| "grad_norm": 0.884710431098938, |
| "learning_rate": 2.9999261836196112e-05, |
| "logits/chosen": -7.471078395843506, |
| "logits/rejected": -7.230543613433838, |
| "logps/chosen": -249.302490234375, |
| "logps/rejected": -256.478271484375, |
| "loss": 0.2735, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.6547988057136536, |
| "rewards/margins": 2.983334541320801, |
| "rewards/rejected": -2.328535556793213, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.04416340459700893, |
| "grad_norm": 0.6704531311988831, |
| "learning_rate": 2.999868771716451e-05, |
| "logits/chosen": -7.653019905090332, |
| "logits/rejected": -7.154124736785889, |
| "logps/chosen": -259.2315673828125, |
| "logps/rejected": -278.14678955078125, |
| "loss": 0.2465, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": 0.7018271684646606, |
| "rewards/margins": 3.3338613510131836, |
| "rewards/rejected": -2.6320340633392334, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.04496637558968182, |
| "grad_norm": 0.8747557401657104, |
| "learning_rate": 2.9997949574887035e-05, |
| "logits/chosen": -7.677107810974121, |
| "logits/rejected": -7.232270240783691, |
| "logps/chosen": -234.3196258544922, |
| "logps/rejected": -235.26495361328125, |
| "loss": 0.2461, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.9437451362609863, |
| "rewards/margins": 3.7134242057800293, |
| "rewards/rejected": -2.769679069519043, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.045769346582354715, |
| "grad_norm": 0.8970131278038025, |
| "learning_rate": 2.999704741743589e-05, |
| "logits/chosen": -7.7212443351745605, |
| "logits/rejected": -7.455051898956299, |
| "logps/chosen": -243.79678344726562, |
| "logps/rejected": -256.690673828125, |
| "loss": 0.2765, |
| "rewards/accuracies": 0.83984375, |
| "rewards/chosen": 0.7981311678886414, |
| "rewards/margins": 3.4718666076660156, |
| "rewards/rejected": -2.6737353801727295, |
| "step": 57 |
| }, |
| { |
| "epoch": 0.046572317575027604, |
| "grad_norm": 0.849990725517273, |
| "learning_rate": 2.9995981254676936e-05, |
| "logits/chosen": -7.872179985046387, |
| "logits/rejected": -7.472891807556152, |
| "logps/chosen": -214.06063842773438, |
| "logps/rejected": -227.86346435546875, |
| "loss": 0.2628, |
| "rewards/accuracies": 0.87890625, |
| "rewards/chosen": 0.7398649454116821, |
| "rewards/margins": 3.48549747467041, |
| "rewards/rejected": -2.7456324100494385, |
| "step": 58 |
| }, |
| { |
| "epoch": 0.04737528856770049, |
| "grad_norm": 0.765006422996521, |
| "learning_rate": 2.9994751098269543e-05, |
| "logits/chosen": -7.6032867431640625, |
| "logits/rejected": -7.1180291175842285, |
| "logps/chosen": -236.218994140625, |
| "logps/rejected": -229.9735565185547, |
| "loss": 0.2663, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": 0.847018837928772, |
| "rewards/margins": 3.556720733642578, |
| "rewards/rejected": -2.7097020149230957, |
| "step": 59 |
| }, |
| { |
| "epoch": 0.04817825956037338, |
| "grad_norm": 0.8039398193359375, |
| "learning_rate": 2.9993356961666506e-05, |
| "logits/chosen": -7.501357078552246, |
| "logits/rejected": -7.123836994171143, |
| "logps/chosen": -254.47991943359375, |
| "logps/rejected": -252.0731201171875, |
| "loss": 0.2066, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": 1.2554271221160889, |
| "rewards/margins": 4.119342803955078, |
| "rewards/rejected": -2.8639159202575684, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.04898123055304627, |
| "grad_norm": 0.8324689865112305, |
| "learning_rate": 2.999179886011389e-05, |
| "logits/chosen": -7.7756028175354, |
| "logits/rejected": -7.281389236450195, |
| "logps/chosen": -232.6284942626953, |
| "logps/rejected": -266.0207824707031, |
| "loss": 0.2285, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.980337381362915, |
| "rewards/margins": 3.8913097381591797, |
| "rewards/rejected": -2.9109721183776855, |
| "step": 61 |
| }, |
| { |
| "epoch": 0.04978420154571916, |
| "grad_norm": 0.859987199306488, |
| "learning_rate": 2.999007681065084e-05, |
| "logits/chosen": -7.397058963775635, |
| "logits/rejected": -7.019468307495117, |
| "logps/chosen": -237.9818878173828, |
| "logps/rejected": -254.8874969482422, |
| "loss": 0.2479, |
| "rewards/accuracies": 0.88671875, |
| "rewards/chosen": 0.9074105620384216, |
| "rewards/margins": 3.751347064971924, |
| "rewards/rejected": -2.8439364433288574, |
| "step": 62 |
| }, |
| { |
| "epoch": 0.05058717253839205, |
| "grad_norm": 0.7784184217453003, |
| "learning_rate": 2.998819083210941e-05, |
| "logits/chosen": -7.6179070472717285, |
| "logits/rejected": -7.210968494415283, |
| "logps/chosen": -230.21328735351562, |
| "logps/rejected": -245.97714233398438, |
| "loss": 0.2223, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.6296841502189636, |
| "rewards/margins": 3.8383357524871826, |
| "rewards/rejected": -3.208651542663574, |
| "step": 63 |
| }, |
| { |
| "epoch": 0.05139014353106494, |
| "grad_norm": 0.7579825520515442, |
| "learning_rate": 2.9986140945114355e-05, |
| "logits/chosen": -7.533819198608398, |
| "logits/rejected": -7.109480857849121, |
| "logps/chosen": -229.73736572265625, |
| "logps/rejected": -243.71713256835938, |
| "loss": 0.2209, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": 0.7137656211853027, |
| "rewards/margins": 4.006747722625732, |
| "rewards/rejected": -3.2929818630218506, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.05219311452373783, |
| "grad_norm": 0.8978877663612366, |
| "learning_rate": 2.9983927172082925e-05, |
| "logits/chosen": -7.481725692749023, |
| "logits/rejected": -7.121698379516602, |
| "logps/chosen": -249.92869567871094, |
| "logps/rejected": -242.99684143066406, |
| "loss": 0.2414, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": 0.653033971786499, |
| "rewards/margins": 3.833481788635254, |
| "rewards/rejected": -3.1804473400115967, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.05299608551641072, |
| "grad_norm": 1.0117154121398926, |
| "learning_rate": 2.9981549537224573e-05, |
| "logits/chosen": -7.345961093902588, |
| "logits/rejected": -7.043153762817383, |
| "logps/chosen": -237.4165496826172, |
| "logps/rejected": -273.90191650390625, |
| "loss": 0.2313, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": 0.5361462831497192, |
| "rewards/margins": 4.332768440246582, |
| "rewards/rejected": -3.7966220378875732, |
| "step": 66 |
| }, |
| { |
| "epoch": 0.053799056509083606, |
| "grad_norm": 0.9886789321899414, |
| "learning_rate": 2.9979008066540737e-05, |
| "logits/chosen": -7.555253982543945, |
| "logits/rejected": -7.195088863372803, |
| "logps/chosen": -207.80221557617188, |
| "logps/rejected": -225.96133422851562, |
| "loss": 0.2325, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.3758700489997864, |
| "rewards/margins": 4.130035400390625, |
| "rewards/rejected": -3.7541656494140625, |
| "step": 67 |
| }, |
| { |
| "epoch": 0.0546020275017565, |
| "grad_norm": 0.9568971991539001, |
| "learning_rate": 2.997630278782454e-05, |
| "logits/chosen": -7.5241193771362305, |
| "logits/rejected": -7.2070465087890625, |
| "logps/chosen": -220.0508270263672, |
| "logps/rejected": -249.30471801757812, |
| "loss": 0.2214, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": 0.35120701789855957, |
| "rewards/margins": 4.551080703735352, |
| "rewards/rejected": -4.199873924255371, |
| "step": 68 |
| }, |
| { |
| "epoch": 0.05540499849442939, |
| "grad_norm": 0.9649574160575867, |
| "learning_rate": 2.9973433730660467e-05, |
| "logits/chosen": -7.321793079376221, |
| "logits/rejected": -6.880770683288574, |
| "logps/chosen": -223.4376678466797, |
| "logps/rejected": -258.41552734375, |
| "loss": 0.2299, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": 0.47390297055244446, |
| "rewards/margins": 4.327374458312988, |
| "rewards/rejected": -3.853471279144287, |
| "step": 69 |
| }, |
| { |
| "epoch": 0.05620796948710228, |
| "grad_norm": 0.8487037420272827, |
| "learning_rate": 2.9970400926424075e-05, |
| "logits/chosen": -7.394587516784668, |
| "logits/rejected": -7.039050579071045, |
| "logps/chosen": -254.32894897460938, |
| "logps/rejected": -280.82098388671875, |
| "loss": 0.2053, |
| "rewards/accuracies": 0.89453125, |
| "rewards/chosen": 0.5118581056594849, |
| "rewards/margins": 4.636835098266602, |
| "rewards/rejected": -4.1249775886535645, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.05701094047977517, |
| "grad_norm": 1.1253381967544556, |
| "learning_rate": 2.9967204408281618e-05, |
| "logits/chosen": -7.397961139678955, |
| "logits/rejected": -7.079029083251953, |
| "logps/chosen": -253.62295532226562, |
| "logps/rejected": -265.20587158203125, |
| "loss": 0.2393, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": 0.33599036931991577, |
| "rewards/margins": 4.489375591278076, |
| "rewards/rejected": -4.153385639190674, |
| "step": 71 |
| }, |
| { |
| "epoch": 0.057813911472448055, |
| "grad_norm": 0.9146644473075867, |
| "learning_rate": 2.996384421118971e-05, |
| "logits/chosen": -7.437904357910156, |
| "logits/rejected": -7.092023849487305, |
| "logps/chosen": -237.00253295898438, |
| "logps/rejected": -258.56292724609375, |
| "loss": 0.1846, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": 0.23943190276622772, |
| "rewards/margins": 4.708088397979736, |
| "rewards/rejected": -4.468656063079834, |
| "step": 72 |
| }, |
| { |
| "epoch": 0.05861688246512095, |
| "grad_norm": 0.6899930834770203, |
| "learning_rate": 2.996032037189493e-05, |
| "logits/chosen": -7.428099632263184, |
| "logits/rejected": -7.031427383422852, |
| "logps/chosen": -214.56512451171875, |
| "logps/rejected": -257.9580078125, |
| "loss": 0.161, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.14567194879055023, |
| "rewards/margins": 4.756979942321777, |
| "rewards/rejected": -4.611307144165039, |
| "step": 73 |
| }, |
| { |
| "epoch": 0.05941985345779384, |
| "grad_norm": 0.7622628808021545, |
| "learning_rate": 2.995663292893342e-05, |
| "logits/chosen": -7.4583024978637695, |
| "logits/rejected": -7.0726399421691895, |
| "logps/chosen": -250.75961303710938, |
| "logps/rejected": -271.2342834472656, |
| "loss": 0.1704, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": 0.5154637098312378, |
| "rewards/margins": 5.119628429412842, |
| "rewards/rejected": -4.604165554046631, |
| "step": 74 |
| }, |
| { |
| "epoch": 0.06022282445046673, |
| "grad_norm": 0.9101508259773254, |
| "learning_rate": 2.9952781922630462e-05, |
| "logits/chosen": -7.416913986206055, |
| "logits/rejected": -7.01773738861084, |
| "logps/chosen": -224.97927856445312, |
| "logps/rejected": -297.1309509277344, |
| "loss": 0.1624, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.029226675629615784, |
| "rewards/margins": 5.056062698364258, |
| "rewards/rejected": -5.085289478302002, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.061025795443139616, |
| "grad_norm": 1.5278880596160889, |
| "learning_rate": 2.994876739510005e-05, |
| "logits/chosen": -7.504698276519775, |
| "logits/rejected": -7.072315692901611, |
| "logps/chosen": -243.32223510742188, |
| "logps/rejected": -278.40081787109375, |
| "loss": 0.2301, |
| "rewards/accuracies": 0.88671875, |
| "rewards/chosen": 0.049747683107852936, |
| "rewards/margins": 4.799650192260742, |
| "rewards/rejected": -4.749902248382568, |
| "step": 76 |
| }, |
| { |
| "epoch": 0.061828766435812504, |
| "grad_norm": 0.7334362864494324, |
| "learning_rate": 2.9944589390244404e-05, |
| "logits/chosen": -7.621161937713623, |
| "logits/rejected": -7.175778865814209, |
| "logps/chosen": -231.48342895507812, |
| "logps/rejected": -284.02545166015625, |
| "loss": 0.1595, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": 0.26768219470977783, |
| "rewards/margins": 5.426779270172119, |
| "rewards/rejected": -5.159097671508789, |
| "step": 77 |
| }, |
| { |
| "epoch": 0.06263173742848539, |
| "grad_norm": 0.8692532777786255, |
| "learning_rate": 2.9940247953753522e-05, |
| "logits/chosen": -7.524738788604736, |
| "logits/rejected": -7.211426258087158, |
| "logps/chosen": -231.1542205810547, |
| "logps/rejected": -249.01123046875, |
| "loss": 0.1862, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": 0.46067264676094055, |
| "rewards/margins": 5.12542200088501, |
| "rewards/rejected": -4.6647491455078125, |
| "step": 78 |
| }, |
| { |
| "epoch": 0.06343470842115828, |
| "grad_norm": 1.21908438205719, |
| "learning_rate": 2.9935743133104658e-05, |
| "logits/chosen": -7.440463066101074, |
| "logits/rejected": -7.008799076080322, |
| "logps/chosen": -239.59080505371094, |
| "logps/rejected": -270.9613342285156, |
| "loss": 0.2083, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": 0.3131418228149414, |
| "rewards/margins": 5.03348445892334, |
| "rewards/rejected": -4.720342636108398, |
| "step": 79 |
| }, |
| { |
| "epoch": 0.06423767941383117, |
| "grad_norm": 0.9510884881019592, |
| "learning_rate": 2.9931074977561806e-05, |
| "logits/chosen": -7.555513858795166, |
| "logits/rejected": -7.070905685424805, |
| "logps/chosen": -270.3348388671875, |
| "logps/rejected": -285.9028625488281, |
| "loss": 0.1994, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.4435756802558899, |
| "rewards/margins": 4.773502349853516, |
| "rewards/rejected": -4.329926490783691, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.06504065040650407, |
| "grad_norm": 0.8115398287773132, |
| "learning_rate": 2.9926243538175172e-05, |
| "logits/chosen": -7.228583812713623, |
| "logits/rejected": -6.860601425170898, |
| "logps/chosen": -234.41964721679688, |
| "logps/rejected": -287.138671875, |
| "loss": 0.1829, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": 0.23094916343688965, |
| "rewards/margins": 4.666954040527344, |
| "rewards/rejected": -4.436005115509033, |
| "step": 81 |
| }, |
| { |
| "epoch": 0.06584362139917696, |
| "grad_norm": 0.9240465760231018, |
| "learning_rate": 2.9921248867780598e-05, |
| "logits/chosen": -7.469148635864258, |
| "logits/rejected": -7.078195571899414, |
| "logps/chosen": -233.1526336669922, |
| "logps/rejected": -260.11846923828125, |
| "loss": 0.1895, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.5834409594535828, |
| "rewards/margins": 4.68798303604126, |
| "rewards/rejected": -4.104542255401611, |
| "step": 82 |
| }, |
| { |
| "epoch": 0.06664659239184985, |
| "grad_norm": 0.8426586389541626, |
| "learning_rate": 2.9916091020999002e-05, |
| "logits/chosen": -7.4082183837890625, |
| "logits/rejected": -7.06876277923584, |
| "logps/chosen": -213.0903778076172, |
| "logps/rejected": -253.0897674560547, |
| "loss": 0.1676, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.3614446520805359, |
| "rewards/margins": 4.84199857711792, |
| "rewards/rejected": -4.480554103851318, |
| "step": 83 |
| }, |
| { |
| "epoch": 0.06744956338452274, |
| "grad_norm": 0.9151988625526428, |
| "learning_rate": 2.991077005423577e-05, |
| "logits/chosen": -7.432582855224609, |
| "logits/rejected": -7.181061267852783, |
| "logps/chosen": -246.16668701171875, |
| "logps/rejected": -250.92820739746094, |
| "loss": 0.1893, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": 0.4403476417064667, |
| "rewards/margins": 4.7804670333862305, |
| "rewards/rejected": -4.340119361877441, |
| "step": 84 |
| }, |
| { |
| "epoch": 0.06825253437719563, |
| "grad_norm": 1.002474069595337, |
| "learning_rate": 2.9905286025680143e-05, |
| "logits/chosen": -7.533577919006348, |
| "logits/rejected": -6.993819713592529, |
| "logps/chosen": -229.14303588867188, |
| "logps/rejected": -255.38204956054688, |
| "loss": 0.181, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": 0.4913862347602844, |
| "rewards/margins": 4.918963432312012, |
| "rewards/rejected": -4.427577018737793, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.06905550536986851, |
| "grad_norm": 1.0264719724655151, |
| "learning_rate": 2.9899638995304575e-05, |
| "logits/chosen": -7.354506492614746, |
| "logits/rejected": -6.943487167358398, |
| "logps/chosen": -255.12295532226562, |
| "logps/rejected": -271.6700439453125, |
| "loss": 0.2005, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": 0.46227261424064636, |
| "rewards/margins": 4.829191207885742, |
| "rewards/rejected": -4.366919040679932, |
| "step": 86 |
| }, |
| { |
| "epoch": 0.0698584763625414, |
| "grad_norm": 0.767596423625946, |
| "learning_rate": 2.9893829024864087e-05, |
| "logits/chosen": -7.49196720123291, |
| "logits/rejected": -7.127873420715332, |
| "logps/chosen": -219.26620483398438, |
| "logps/rejected": -255.16810607910156, |
| "loss": 0.171, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": 0.6242790222167969, |
| "rewards/margins": 5.1640191078186035, |
| "rewards/rejected": -4.539740085601807, |
| "step": 87 |
| }, |
| { |
| "epoch": 0.07066144735521429, |
| "grad_norm": 1.38248610496521, |
| "learning_rate": 2.9887856177895586e-05, |
| "logits/chosen": -7.579738140106201, |
| "logits/rejected": -7.116502285003662, |
| "logps/chosen": -254.36306762695312, |
| "logps/rejected": -277.07550048828125, |
| "loss": 0.1994, |
| "rewards/accuracies": 0.90234375, |
| "rewards/chosen": 0.7055200934410095, |
| "rewards/margins": 5.129255294799805, |
| "rewards/rejected": -4.42373514175415, |
| "step": 88 |
| }, |
| { |
| "epoch": 0.07146441834788718, |
| "grad_norm": 0.886551022529602, |
| "learning_rate": 2.988172051971717e-05, |
| "logits/chosen": -7.44769811630249, |
| "logits/rejected": -7.047970294952393, |
| "logps/chosen": -245.8181915283203, |
| "logps/rejected": -277.3448181152344, |
| "loss": 0.1879, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": 0.6762005090713501, |
| "rewards/margins": 5.026655673980713, |
| "rewards/rejected": -4.350454807281494, |
| "step": 89 |
| }, |
| { |
| "epoch": 0.07226738934056007, |
| "grad_norm": 0.776666522026062, |
| "learning_rate": 2.987542211742741e-05, |
| "logits/chosen": -7.294948577880859, |
| "logits/rejected": -6.9330644607543945, |
| "logps/chosen": -211.06048583984375, |
| "logps/rejected": -245.34498596191406, |
| "loss": 0.1829, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": 0.6200848817825317, |
| "rewards/margins": 4.5923027992248535, |
| "rewards/rejected": -3.9722180366516113, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.07307036033323296, |
| "grad_norm": 0.876595675945282, |
| "learning_rate": 2.9868961039904628e-05, |
| "logits/chosen": -7.413234233856201, |
| "logits/rejected": -7.080507278442383, |
| "logps/chosen": -243.5048065185547, |
| "logps/rejected": -301.7332763671875, |
| "loss": 0.169, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.4787769317626953, |
| "rewards/margins": 4.9006733894348145, |
| "rewards/rejected": -4.421895980834961, |
| "step": 91 |
| }, |
| { |
| "epoch": 0.07387333132590586, |
| "grad_norm": 0.8127331137657166, |
| "learning_rate": 2.9862337357806133e-05, |
| "logits/chosen": -7.320777893066406, |
| "logits/rejected": -6.926668167114258, |
| "logps/chosen": -243.58323669433594, |
| "logps/rejected": -258.8891906738281, |
| "loss": 0.1587, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.6298668384552002, |
| "rewards/margins": 5.210723876953125, |
| "rewards/rejected": -4.580857276916504, |
| "step": 92 |
| }, |
| { |
| "epoch": 0.07467630231857875, |
| "grad_norm": 1.0447694063186646, |
| "learning_rate": 2.985555114356745e-05, |
| "logits/chosen": -7.212264060974121, |
| "logits/rejected": -6.825069427490234, |
| "logps/chosen": -236.60031127929688, |
| "logps/rejected": -256.35931396484375, |
| "loss": 0.2061, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": 0.25319987535476685, |
| "rewards/margins": 4.42329216003418, |
| "rewards/rejected": -4.1700921058654785, |
| "step": 93 |
| }, |
| { |
| "epoch": 0.07547927331125164, |
| "grad_norm": 1.1444194316864014, |
| "learning_rate": 2.9848602471401533e-05, |
| "logits/chosen": -7.287713050842285, |
| "logits/rejected": -6.999964714050293, |
| "logps/chosen": -261.9858703613281, |
| "logps/rejected": -292.1023254394531, |
| "loss": 0.1718, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.37012210488319397, |
| "rewards/margins": 5.104574680328369, |
| "rewards/rejected": -4.734452247619629, |
| "step": 94 |
| }, |
| { |
| "epoch": 0.07628224430392452, |
| "grad_norm": 0.8713824152946472, |
| "learning_rate": 2.9841491417297938e-05, |
| "logits/chosen": -7.299912452697754, |
| "logits/rejected": -7.046224117279053, |
| "logps/chosen": -245.82354736328125, |
| "logps/rejected": -257.1351318359375, |
| "loss": 0.1792, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.4084318280220032, |
| "rewards/margins": 5.178621292114258, |
| "rewards/rejected": -4.77018928527832, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.07708521529659741, |
| "grad_norm": 0.8346383571624756, |
| "learning_rate": 2.9834218059022027e-05, |
| "logits/chosen": -7.346923828125, |
| "logits/rejected": -6.863754749298096, |
| "logps/chosen": -257.537109375, |
| "logps/rejected": -286.3846130371094, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": 0.6534644961357117, |
| "rewards/margins": 5.407450199127197, |
| "rewards/rejected": -4.753986358642578, |
| "step": 96 |
| }, |
| { |
| "epoch": 0.0778881862892703, |
| "grad_norm": 0.8749032020568848, |
| "learning_rate": 2.9826782476114073e-05, |
| "logits/chosen": -7.4182634353637695, |
| "logits/rejected": -7.048748016357422, |
| "logps/chosen": -230.21066284179688, |
| "logps/rejected": -252.84852600097656, |
| "loss": 0.1911, |
| "rewards/accuracies": 0.89453125, |
| "rewards/chosen": 0.45283302664756775, |
| "rewards/margins": 5.344717025756836, |
| "rewards/rejected": -4.891884803771973, |
| "step": 97 |
| }, |
| { |
| "epoch": 0.07869115728194319, |
| "grad_norm": 1.1279855966567993, |
| "learning_rate": 2.9819184749888423e-05, |
| "logits/chosen": -7.311376571655273, |
| "logits/rejected": -6.940769195556641, |
| "logps/chosen": -247.23272705078125, |
| "logps/rejected": -270.7436828613281, |
| "loss": 0.1895, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.29446089267730713, |
| "rewards/margins": 5.132132053375244, |
| "rewards/rejected": -4.837671756744385, |
| "step": 98 |
| }, |
| { |
| "epoch": 0.07949412827461608, |
| "grad_norm": 0.8792976140975952, |
| "learning_rate": 2.9811424963432584e-05, |
| "logits/chosen": -7.296340465545654, |
| "logits/rejected": -6.8286943435668945, |
| "logps/chosen": -224.0811767578125, |
| "logps/rejected": -269.0909423828125, |
| "loss": 0.1784, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.09347806870937347, |
| "rewards/margins": 5.302676677703857, |
| "rewards/rejected": -5.209198474884033, |
| "step": 99 |
| }, |
| { |
| "epoch": 0.08029709926728897, |
| "grad_norm": 0.9837705492973328, |
| "learning_rate": 2.9803503201606352e-05, |
| "logits/chosen": -7.1162028312683105, |
| "logits/rejected": -6.659389495849609, |
| "logps/chosen": -260.27740478515625, |
| "logps/rejected": -307.0146179199219, |
| "loss": 0.1769, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": 0.3411463499069214, |
| "rewards/margins": 5.619727611541748, |
| "rewards/rejected": -5.278580665588379, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.08110007025996185, |
| "grad_norm": 1.0158653259277344, |
| "learning_rate": 2.9795419551040836e-05, |
| "logits/chosen": -7.451248645782471, |
| "logits/rejected": -6.968564510345459, |
| "logps/chosen": -235.38284301757812, |
| "logps/rejected": -299.03314208984375, |
| "loss": 0.1621, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": 0.3236550986766815, |
| "rewards/margins": 5.8568878173828125, |
| "rewards/rejected": -5.53323221206665, |
| "step": 101 |
| }, |
| { |
| "epoch": 0.08190304125263474, |
| "grad_norm": 1.0948830842971802, |
| "learning_rate": 2.9787174100137543e-05, |
| "logits/chosen": -7.306007385253906, |
| "logits/rejected": -6.985782623291016, |
| "logps/chosen": -219.64950561523438, |
| "logps/rejected": -250.8341522216797, |
| "loss": 0.1982, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": 0.049889594316482544, |
| "rewards/margins": 5.5454792976379395, |
| "rewards/rejected": -5.495589733123779, |
| "step": 102 |
| }, |
| { |
| "epoch": 0.08270601224530764, |
| "grad_norm": 0.8061056733131409, |
| "learning_rate": 2.977876693906741e-05, |
| "logits/chosen": -7.3995561599731445, |
| "logits/rejected": -7.059619903564453, |
| "logps/chosen": -199.7267303466797, |
| "logps/rejected": -234.407958984375, |
| "loss": 0.1132, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": 0.07142507284879684, |
| "rewards/margins": 5.5135321617126465, |
| "rewards/rejected": -5.442107677459717, |
| "step": 103 |
| }, |
| { |
| "epoch": 0.08350898323798053, |
| "grad_norm": 0.9276915788650513, |
| "learning_rate": 2.97701981597698e-05, |
| "logits/chosen": -7.331705570220947, |
| "logits/rejected": -6.7795305252075195, |
| "logps/chosen": -233.1027069091797, |
| "logps/rejected": -295.6918029785156, |
| "loss": 0.1453, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": 0.10915989428758621, |
| "rewards/margins": 5.937568187713623, |
| "rewards/rejected": -5.828408241271973, |
| "step": 104 |
| }, |
| { |
| "epoch": 0.08431195423065342, |
| "grad_norm": 0.9378758072853088, |
| "learning_rate": 2.976146785595151e-05, |
| "logits/chosen": -7.484736919403076, |
| "logits/rejected": -7.100762844085693, |
| "logps/chosen": -249.6339874267578, |
| "logps/rejected": -283.4140319824219, |
| "loss": 0.1505, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.21371471881866455, |
| "rewards/margins": 5.661227226257324, |
| "rewards/rejected": -5.447513580322266, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.08511492522332631, |
| "grad_norm": 1.0336297750473022, |
| "learning_rate": 2.9752576123085737e-05, |
| "logits/chosen": -7.155478477478027, |
| "logits/rejected": -6.616024494171143, |
| "logps/chosen": -259.01959228515625, |
| "logps/rejected": -294.64947509765625, |
| "loss": 0.1908, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.34220150113105774, |
| "rewards/margins": 5.335601806640625, |
| "rewards/rejected": -4.9934000968933105, |
| "step": 106 |
| }, |
| { |
| "epoch": 0.0859178962159992, |
| "grad_norm": 1.0831706523895264, |
| "learning_rate": 2.9743523058411057e-05, |
| "logits/chosen": -7.046363830566406, |
| "logits/rejected": -6.769369125366211, |
| "logps/chosen": -266.45904541015625, |
| "logps/rejected": -305.48663330078125, |
| "loss": 0.1972, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": -0.005778931081295013, |
| "rewards/margins": 5.336663722991943, |
| "rewards/rejected": -5.342442512512207, |
| "step": 107 |
| }, |
| { |
| "epoch": 0.08672086720867209, |
| "grad_norm": 0.8229761123657227, |
| "learning_rate": 2.9734308760930333e-05, |
| "logits/chosen": -7.5630292892456055, |
| "logits/rejected": -6.98720645904541, |
| "logps/chosen": -239.2142333984375, |
| "logps/rejected": -274.0261535644531, |
| "loss": 0.1085, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": 0.16363412141799927, |
| "rewards/margins": 6.146294593811035, |
| "rewards/rejected": -5.982660293579102, |
| "step": 108 |
| }, |
| { |
| "epoch": 0.08752383820134498, |
| "grad_norm": 0.8178077340126038, |
| "learning_rate": 2.9724933331409644e-05, |
| "logits/chosen": -7.169466972351074, |
| "logits/rejected": -6.720808506011963, |
| "logps/chosen": -261.9344177246094, |
| "logps/rejected": -280.03582763671875, |
| "loss": 0.1299, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": 0.2608301043510437, |
| "rewards/margins": 5.819622993469238, |
| "rewards/rejected": -5.5587921142578125, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.08832680919401786, |
| "grad_norm": 0.7964989542961121, |
| "learning_rate": 2.9715396872377182e-05, |
| "logits/chosen": -7.300821304321289, |
| "logits/rejected": -6.813967704772949, |
| "logps/chosen": -244.98574829101562, |
| "logps/rejected": -278.0948181152344, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": 0.0113583505153656, |
| "rewards/margins": 5.839400768280029, |
| "rewards/rejected": -5.828042984008789, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.08912978018669075, |
| "grad_norm": 0.7301980257034302, |
| "learning_rate": 2.970569948812214e-05, |
| "logits/chosen": -7.250796318054199, |
| "logits/rejected": -6.859468936920166, |
| "logps/chosen": -264.12640380859375, |
| "logps/rejected": -285.5746154785156, |
| "loss": 0.107, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": 0.5195602178573608, |
| "rewards/margins": 6.103140354156494, |
| "rewards/rejected": -5.5835795402526855, |
| "step": 111 |
| }, |
| { |
| "epoch": 0.08993275117936364, |
| "grad_norm": 0.8939338326454163, |
| "learning_rate": 2.969584128469356e-05, |
| "logits/chosen": -7.379607200622559, |
| "logits/rejected": -6.918723106384277, |
| "logps/chosen": -217.6083526611328, |
| "logps/rejected": -253.0339813232422, |
| "loss": 0.143, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.05075039714574814, |
| "rewards/margins": 5.584677219390869, |
| "rewards/rejected": -5.635426998138428, |
| "step": 112 |
| }, |
| { |
| "epoch": 0.09073572217203653, |
| "grad_norm": 1.0426470041275024, |
| "learning_rate": 2.968582236989917e-05, |
| "logits/chosen": -7.416778087615967, |
| "logits/rejected": -6.97111701965332, |
| "logps/chosen": -251.93753051757812, |
| "logps/rejected": -283.84295654296875, |
| "loss": 0.1435, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.013207152485847473, |
| "rewards/margins": 5.742201328277588, |
| "rewards/rejected": -5.75540828704834, |
| "step": 113 |
| }, |
| { |
| "epoch": 0.09153869316470943, |
| "grad_norm": 0.9616119861602783, |
| "learning_rate": 2.967564285330422e-05, |
| "logits/chosen": -7.213616847991943, |
| "logits/rejected": -6.93658447265625, |
| "logps/chosen": -247.92019653320312, |
| "logps/rejected": -305.8927307128906, |
| "loss": 0.1573, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.048170387744903564, |
| "rewards/margins": 5.841033458709717, |
| "rewards/rejected": -5.889204025268555, |
| "step": 114 |
| }, |
| { |
| "epoch": 0.09234166415738232, |
| "grad_norm": 1.0217643976211548, |
| "learning_rate": 2.9665302846230276e-05, |
| "logits/chosen": -7.087464809417725, |
| "logits/rejected": -6.612020492553711, |
| "logps/chosen": -236.9564208984375, |
| "logps/rejected": -267.678955078125, |
| "loss": 0.1474, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.05163656175136566, |
| "rewards/margins": 5.618931770324707, |
| "rewards/rejected": -5.670568466186523, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.09314463515005521, |
| "grad_norm": 1.0774489641189575, |
| "learning_rate": 2.9654802461753992e-05, |
| "logits/chosen": -7.269303798675537, |
| "logits/rejected": -6.815751075744629, |
| "logps/chosen": -217.4568634033203, |
| "logps/rejected": -291.1937561035156, |
| "loss": 0.1644, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.11190228164196014, |
| "rewards/margins": 5.5912652015686035, |
| "rewards/rejected": -5.703166961669922, |
| "step": 116 |
| }, |
| { |
| "epoch": 0.0939476061427281, |
| "grad_norm": 1.0120837688446045, |
| "learning_rate": 2.9644141814705893e-05, |
| "logits/chosen": -7.189496994018555, |
| "logits/rejected": -6.7153706550598145, |
| "logps/chosen": -276.7318115234375, |
| "logps/rejected": -329.1173095703125, |
| "loss": 0.1493, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.2905672490596771, |
| "rewards/margins": 5.610505104064941, |
| "rewards/rejected": -5.3199381828308105, |
| "step": 117 |
| }, |
| { |
| "epoch": 0.09475057713540098, |
| "grad_norm": 1.091394305229187, |
| "learning_rate": 2.9633321021669106e-05, |
| "logits/chosen": -7.381908416748047, |
| "logits/rejected": -6.880618572235107, |
| "logps/chosen": -207.2138671875, |
| "logps/rejected": -262.9717712402344, |
| "loss": 0.1918, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.14034339785575867, |
| "rewards/margins": 5.470845699310303, |
| "rewards/rejected": -5.330502033233643, |
| "step": 118 |
| }, |
| { |
| "epoch": 0.09555354812807387, |
| "grad_norm": 1.1229870319366455, |
| "learning_rate": 2.962234020097809e-05, |
| "logits/chosen": -7.243007659912109, |
| "logits/rejected": -6.823401927947998, |
| "logps/chosen": -234.16085815429688, |
| "logps/rejected": -253.6674346923828, |
| "loss": 0.1425, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.37414368987083435, |
| "rewards/margins": 5.3926544189453125, |
| "rewards/rejected": -5.0185112953186035, |
| "step": 119 |
| }, |
| { |
| "epoch": 0.09635651912074676, |
| "grad_norm": 0.9949536919593811, |
| "learning_rate": 2.9611199472717347e-05, |
| "logits/chosen": -7.415261745452881, |
| "logits/rejected": -7.056488513946533, |
| "logps/chosen": -232.06277465820312, |
| "logps/rejected": -275.40380859375, |
| "loss": 0.1399, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.007462687790393829, |
| "rewards/margins": 5.541759490966797, |
| "rewards/rejected": -5.549221992492676, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.09715949011341965, |
| "grad_norm": 0.9458962678909302, |
| "learning_rate": 2.9599898958720088e-05, |
| "logits/chosen": -7.401342391967773, |
| "logits/rejected": -7.082679748535156, |
| "logps/chosen": -262.08221435546875, |
| "logps/rejected": -274.79400634765625, |
| "loss": 0.1358, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": 0.1328846663236618, |
| "rewards/margins": 5.593487739562988, |
| "rewards/rejected": -5.460601806640625, |
| "step": 121 |
| }, |
| { |
| "epoch": 0.09796246110609254, |
| "grad_norm": 1.0495960712432861, |
| "learning_rate": 2.9588438782566927e-05, |
| "logits/chosen": -7.448173522949219, |
| "logits/rejected": -7.022300720214844, |
| "logps/chosen": -209.7882537841797, |
| "logps/rejected": -250.97549438476562, |
| "loss": 0.1405, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.1638614535331726, |
| "rewards/margins": 5.350771427154541, |
| "rewards/rejected": -5.514632701873779, |
| "step": 122 |
| }, |
| { |
| "epoch": 0.09876543209876543, |
| "grad_norm": 0.9879561066627502, |
| "learning_rate": 2.957681906958452e-05, |
| "logits/chosen": -7.191327095031738, |
| "logits/rejected": -6.719014644622803, |
| "logps/chosen": -249.91273498535156, |
| "logps/rejected": -273.9558410644531, |
| "loss": 0.1616, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.11608981341123581, |
| "rewards/margins": 5.338552474975586, |
| "rewards/rejected": -5.4546427726745605, |
| "step": 123 |
| }, |
| { |
| "epoch": 0.09956840309143832, |
| "grad_norm": 1.0466108322143555, |
| "learning_rate": 2.9565039946844175e-05, |
| "logits/chosen": -7.24060583114624, |
| "logits/rejected": -6.759690284729004, |
| "logps/chosen": -236.0293731689453, |
| "logps/rejected": -270.58203125, |
| "loss": 0.1483, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": 0.07284556329250336, |
| "rewards/margins": 5.684063911437988, |
| "rewards/rejected": -5.611218452453613, |
| "step": 124 |
| }, |
| { |
| "epoch": 0.10037137408411122, |
| "grad_norm": 1.0615557432174683, |
| "learning_rate": 2.9553101543160497e-05, |
| "logits/chosen": -7.343782424926758, |
| "logits/rejected": -7.055228233337402, |
| "logps/chosen": -244.26806640625, |
| "logps/rejected": -310.5832214355469, |
| "loss": 0.1568, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": 0.23528820276260376, |
| "rewards/margins": 5.8777570724487305, |
| "rewards/rejected": -5.6424689292907715, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.1011743450767841, |
| "grad_norm": 1.172917127609253, |
| "learning_rate": 2.9541003989089956e-05, |
| "logits/chosen": -7.364123344421387, |
| "logits/rejected": -6.928725719451904, |
| "logps/chosen": -245.62115478515625, |
| "logps/rejected": -277.0497131347656, |
| "loss": 0.1597, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.16948851943016052, |
| "rewards/margins": 5.416151523590088, |
| "rewards/rejected": -5.585639953613281, |
| "step": 126 |
| }, |
| { |
| "epoch": 0.101977316069457, |
| "grad_norm": 1.1584275960922241, |
| "learning_rate": 2.9528747416929467e-05, |
| "logits/chosen": -7.521541118621826, |
| "logits/rejected": -7.060096740722656, |
| "logps/chosen": -240.8060302734375, |
| "logps/rejected": -285.5921630859375, |
| "loss": 0.1433, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.14503498375415802, |
| "rewards/margins": 5.737887382507324, |
| "rewards/rejected": -5.882922649383545, |
| "step": 127 |
| }, |
| { |
| "epoch": 0.10278028706212988, |
| "grad_norm": 0.7190593481063843, |
| "learning_rate": 2.951633196071494e-05, |
| "logits/chosen": -7.460888385772705, |
| "logits/rejected": -7.112993240356445, |
| "logps/chosen": -219.41293334960938, |
| "logps/rejected": -258.94140625, |
| "loss": 0.1129, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": 0.12193179130554199, |
| "rewards/margins": 6.078322410583496, |
| "rewards/rejected": -5.956390380859375, |
| "step": 128 |
| }, |
| { |
| "epoch": 0.10358325805480277, |
| "grad_norm": 1.2085909843444824, |
| "learning_rate": 2.9503757756219807e-05, |
| "logits/chosen": -7.301778793334961, |
| "logits/rejected": -6.9096174240112305, |
| "logps/chosen": -255.9728240966797, |
| "logps/rejected": -275.04742431640625, |
| "loss": 0.1927, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.009759936481714249, |
| "rewards/margins": 5.584396839141846, |
| "rewards/rejected": -5.574637413024902, |
| "step": 129 |
| }, |
| { |
| "epoch": 0.10438622904747566, |
| "grad_norm": 0.8281040787696838, |
| "learning_rate": 2.949102494095356e-05, |
| "logits/chosen": -7.311952590942383, |
| "logits/rejected": -6.9732279777526855, |
| "logps/chosen": -244.96392822265625, |
| "logps/rejected": -283.3637390136719, |
| "loss": 0.1289, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.09688635170459747, |
| "rewards/margins": 6.069044589996338, |
| "rewards/rejected": -6.165931701660156, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.10518920004014855, |
| "grad_norm": 0.971869945526123, |
| "learning_rate": 2.947813365416023e-05, |
| "logits/chosen": -7.0070881843566895, |
| "logits/rejected": -6.782784938812256, |
| "logps/chosen": -271.96990966796875, |
| "logps/rejected": -279.790283203125, |
| "loss": 0.1566, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.32626500725746155, |
| "rewards/margins": 6.025716304779053, |
| "rewards/rejected": -6.3519816398620605, |
| "step": 131 |
| }, |
| { |
| "epoch": 0.10599217103282144, |
| "grad_norm": 0.8311020731925964, |
| "learning_rate": 2.946508403681686e-05, |
| "logits/chosen": -7.2097272872924805, |
| "logits/rejected": -6.746689796447754, |
| "logps/chosen": -243.11231994628906, |
| "logps/rejected": -285.3640441894531, |
| "loss": 0.1282, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.09736435115337372, |
| "rewards/margins": 6.236139297485352, |
| "rewards/rejected": -6.333502292633057, |
| "step": 132 |
| }, |
| { |
| "epoch": 0.10679514202549432, |
| "grad_norm": 1.148793339729309, |
| "learning_rate": 2.9451876231631974e-05, |
| "logits/chosen": -7.343828201293945, |
| "logits/rejected": -6.97832727432251, |
| "logps/chosen": -250.2877960205078, |
| "logps/rejected": -308.5293273925781, |
| "loss": 0.159, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.28806355595588684, |
| "rewards/margins": 6.598996162414551, |
| "rewards/rejected": -6.887059688568115, |
| "step": 133 |
| }, |
| { |
| "epoch": 0.10759811301816721, |
| "grad_norm": 0.9202746152877808, |
| "learning_rate": 2.943851038304401e-05, |
| "logits/chosen": -7.151074409484863, |
| "logits/rejected": -6.666351795196533, |
| "logps/chosen": -255.4661865234375, |
| "logps/rejected": -295.0926513671875, |
| "loss": 0.1382, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.35307544469833374, |
| "rewards/margins": 6.050268650054932, |
| "rewards/rejected": -6.403343677520752, |
| "step": 134 |
| }, |
| { |
| "epoch": 0.10840108401084012, |
| "grad_norm": 1.05068838596344, |
| "learning_rate": 2.9424986637219754e-05, |
| "logits/chosen": -7.4081196784973145, |
| "logits/rejected": -6.9919538497924805, |
| "logps/chosen": -252.04998779296875, |
| "logps/rejected": -292.7651672363281, |
| "loss": 0.1678, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.39531242847442627, |
| "rewards/margins": 6.699676036834717, |
| "rewards/rejected": -7.0949883460998535, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.109204055003513, |
| "grad_norm": 1.0534895658493042, |
| "learning_rate": 2.9411305142052725e-05, |
| "logits/chosen": -7.300457954406738, |
| "logits/rejected": -6.912496566772461, |
| "logps/chosen": -252.71786499023438, |
| "logps/rejected": -287.08203125, |
| "loss": 0.1451, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.5069544911384583, |
| "rewards/margins": 6.3179612159729, |
| "rewards/rejected": -6.824915409088135, |
| "step": 136 |
| }, |
| { |
| "epoch": 0.11000702599618589, |
| "grad_norm": 0.7271293997764587, |
| "learning_rate": 2.939746604716155e-05, |
| "logits/chosen": -7.228631973266602, |
| "logits/rejected": -6.793624401092529, |
| "logps/chosen": -237.30245971679688, |
| "logps/rejected": -308.9376220703125, |
| "loss": 0.0914, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.22811725735664368, |
| "rewards/margins": 7.030219078063965, |
| "rewards/rejected": -7.258335113525391, |
| "step": 137 |
| }, |
| { |
| "epoch": 0.11080999698885878, |
| "grad_norm": 0.8803237676620483, |
| "learning_rate": 2.9383469503888366e-05, |
| "logits/chosen": -7.334993839263916, |
| "logits/rejected": -6.979445457458496, |
| "logps/chosen": -291.6413879394531, |
| "logps/rejected": -321.457275390625, |
| "loss": 0.1232, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.49203962087631226, |
| "rewards/margins": 6.671972274780273, |
| "rewards/rejected": -7.164012908935547, |
| "step": 138 |
| }, |
| { |
| "epoch": 0.11161296798153167, |
| "grad_norm": 0.9887990951538086, |
| "learning_rate": 2.936931566529712e-05, |
| "logits/chosen": -7.093008041381836, |
| "logits/rejected": -6.688533782958984, |
| "logps/chosen": -260.06884765625, |
| "logps/rejected": -292.3977355957031, |
| "loss": 0.1335, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.5184158682823181, |
| "rewards/margins": 6.452533721923828, |
| "rewards/rejected": -6.970950126647949, |
| "step": 139 |
| }, |
| { |
| "epoch": 0.11241593897420456, |
| "grad_norm": 0.9897907376289368, |
| "learning_rate": 2.9355004686171928e-05, |
| "logits/chosen": -7.279977321624756, |
| "logits/rejected": -6.786205291748047, |
| "logps/chosen": -252.2016143798828, |
| "logps/rejected": -308.70404052734375, |
| "loss": 0.1273, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.2609643042087555, |
| "rewards/margins": 6.904458045959473, |
| "rewards/rejected": -7.165421962738037, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.11321890996687745, |
| "grad_norm": 0.9000648856163025, |
| "learning_rate": 2.9340536723015367e-05, |
| "logits/chosen": -7.231801986694336, |
| "logits/rejected": -6.835826396942139, |
| "logps/chosen": -231.9607391357422, |
| "logps/rejected": -273.9631652832031, |
| "loss": 0.1342, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.5190748572349548, |
| "rewards/margins": 6.149266719818115, |
| "rewards/rejected": -6.668341636657715, |
| "step": 141 |
| }, |
| { |
| "epoch": 0.11402188095955033, |
| "grad_norm": 0.8225992321968079, |
| "learning_rate": 2.9325911934046755e-05, |
| "logits/chosen": -7.216009616851807, |
| "logits/rejected": -6.761571884155273, |
| "logps/chosen": -244.92457580566406, |
| "logps/rejected": -290.5059814453125, |
| "loss": 0.1247, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.27341628074645996, |
| "rewards/margins": 6.44307804107666, |
| "rewards/rejected": -6.716493129730225, |
| "step": 142 |
| }, |
| { |
| "epoch": 0.11482485195222322, |
| "grad_norm": 1.0659303665161133, |
| "learning_rate": 2.9311130479200447e-05, |
| "logits/chosen": -7.258968830108643, |
| "logits/rejected": -6.733380317687988, |
| "logps/chosen": -232.01893615722656, |
| "logps/rejected": -275.0375061035156, |
| "loss": 0.1301, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.363068163394928, |
| "rewards/margins": 6.40583610534668, |
| "rewards/rejected": -6.768904685974121, |
| "step": 143 |
| }, |
| { |
| "epoch": 0.11562782294489611, |
| "grad_norm": 0.915408194065094, |
| "learning_rate": 2.9296192520124068e-05, |
| "logits/chosen": -7.205477714538574, |
| "logits/rejected": -6.765583038330078, |
| "logps/chosen": -232.42333984375, |
| "logps/rejected": -287.07965087890625, |
| "loss": 0.1724, |
| "rewards/accuracies": 0.90234375, |
| "rewards/chosen": -0.3488021790981293, |
| "rewards/margins": 5.909794330596924, |
| "rewards/rejected": -6.258595943450928, |
| "step": 144 |
| }, |
| { |
| "epoch": 0.116430793937569, |
| "grad_norm": 0.720211386680603, |
| "learning_rate": 2.9281098220176736e-05, |
| "logits/chosen": -7.251678943634033, |
| "logits/rejected": -6.812324523925781, |
| "logps/chosen": -230.3197784423828, |
| "logps/rejected": -308.15167236328125, |
| "loss": 0.1028, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.22217124700546265, |
| "rewards/margins": 6.7714033126831055, |
| "rewards/rejected": -6.9935736656188965, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.1172337649302419, |
| "grad_norm": 0.9661546349525452, |
| "learning_rate": 2.9265847744427305e-05, |
| "logits/chosen": -7.52396821975708, |
| "logits/rejected": -7.05275297164917, |
| "logps/chosen": -242.73240661621094, |
| "logps/rejected": -293.8949279785156, |
| "loss": 0.1274, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": 0.06816641986370087, |
| "rewards/margins": 6.6038360595703125, |
| "rewards/rejected": -6.535668849945068, |
| "step": 146 |
| }, |
| { |
| "epoch": 0.11803673592291479, |
| "grad_norm": 1.0441116094589233, |
| "learning_rate": 2.925044125965253e-05, |
| "logits/chosen": -7.1704864501953125, |
| "logits/rejected": -6.771600723266602, |
| "logps/chosen": -268.599609375, |
| "logps/rejected": -290.7333068847656, |
| "loss": 0.1396, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.013215314596891403, |
| "rewards/margins": 6.2258100509643555, |
| "rewards/rejected": -6.212594985961914, |
| "step": 147 |
| }, |
| { |
| "epoch": 0.11883970691558768, |
| "grad_norm": 0.9115970134735107, |
| "learning_rate": 2.9234878934335264e-05, |
| "logits/chosen": -7.338881492614746, |
| "logits/rejected": -6.984103679656982, |
| "logps/chosen": -221.7762451171875, |
| "logps/rejected": -258.9133605957031, |
| "loss": 0.1456, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.07353568077087402, |
| "rewards/margins": 6.218482971191406, |
| "rewards/rejected": -6.292018413543701, |
| "step": 148 |
| }, |
| { |
| "epoch": 0.11964267790826057, |
| "grad_norm": 0.8878861665725708, |
| "learning_rate": 2.9219160938662603e-05, |
| "logits/chosen": -7.360790252685547, |
| "logits/rejected": -6.8196940422058105, |
| "logps/chosen": -230.84762573242188, |
| "logps/rejected": -285.84051513671875, |
| "loss": 0.1375, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": 0.53909832239151, |
| "rewards/margins": 6.398922443389893, |
| "rewards/rejected": -5.859824180603027, |
| "step": 149 |
| }, |
| { |
| "epoch": 0.12044564890093346, |
| "grad_norm": 0.687070369720459, |
| "learning_rate": 2.9203287444524026e-05, |
| "logits/chosen": -7.5256476402282715, |
| "logits/rejected": -7.028613567352295, |
| "logps/chosen": -243.13427734375, |
| "logps/rejected": -272.699951171875, |
| "loss": 0.0989, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.4986129403114319, |
| "rewards/margins": 6.438756465911865, |
| "rewards/rejected": -5.94014310836792, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.12124861989360634, |
| "grad_norm": 0.8441542387008667, |
| "learning_rate": 2.9187258625509518e-05, |
| "logits/chosen": -7.3974151611328125, |
| "logits/rejected": -7.080513000488281, |
| "logps/chosen": -235.94296264648438, |
| "logps/rejected": -277.82720947265625, |
| "loss": 0.1229, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.4242815673351288, |
| "rewards/margins": 6.05021858215332, |
| "rewards/rejected": -5.625937461853027, |
| "step": 151 |
| }, |
| { |
| "epoch": 0.12205159088627923, |
| "grad_norm": 0.8054368495941162, |
| "learning_rate": 2.917107465690769e-05, |
| "logits/chosen": -7.258787155151367, |
| "logits/rejected": -6.907328128814697, |
| "logps/chosen": -231.28713989257812, |
| "logps/rejected": -284.2351989746094, |
| "loss": 0.1278, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": 0.37841394543647766, |
| "rewards/margins": 5.876884460449219, |
| "rewards/rejected": -5.498471260070801, |
| "step": 152 |
| }, |
| { |
| "epoch": 0.12285456187895212, |
| "grad_norm": 0.8753982186317444, |
| "learning_rate": 2.9154735715703816e-05, |
| "logits/chosen": -7.2926506996154785, |
| "logits/rejected": -6.812325954437256, |
| "logps/chosen": -228.2934112548828, |
| "logps/rejected": -277.3577575683594, |
| "loss": 0.1456, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.406047523021698, |
| "rewards/margins": 5.674933433532715, |
| "rewards/rejected": -5.268886089324951, |
| "step": 153 |
| }, |
| { |
| "epoch": 0.12365753287162501, |
| "grad_norm": 0.926150381565094, |
| "learning_rate": 2.9138241980577944e-05, |
| "logits/chosen": -7.347887992858887, |
| "logits/rejected": -6.956485271453857, |
| "logps/chosen": -217.6161651611328, |
| "logps/rejected": -258.41021728515625, |
| "loss": 0.1388, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.37315452098846436, |
| "rewards/margins": 5.76536750793457, |
| "rewards/rejected": -5.392213344573975, |
| "step": 154 |
| }, |
| { |
| "epoch": 0.1244605038642979, |
| "grad_norm": 0.8625725507736206, |
| "learning_rate": 2.9121593631902923e-05, |
| "logits/chosen": -7.2324090003967285, |
| "logits/rejected": -6.927871227264404, |
| "logps/chosen": -231.1959228515625, |
| "logps/rejected": -260.466064453125, |
| "loss": 0.1279, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.31107330322265625, |
| "rewards/margins": 5.7067084312438965, |
| "rewards/rejected": -5.39563512802124, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.12526347485697079, |
| "grad_norm": 0.9344547986984253, |
| "learning_rate": 2.910479085174242e-05, |
| "logits/chosen": -7.2075419425964355, |
| "logits/rejected": -6.906835079193115, |
| "logps/chosen": -266.5186767578125, |
| "logps/rejected": -290.01922607421875, |
| "loss": 0.14, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": 0.2683216333389282, |
| "rewards/margins": 5.790860176086426, |
| "rewards/rejected": -5.522538185119629, |
| "step": 156 |
| }, |
| { |
| "epoch": 0.1260664458496437, |
| "grad_norm": 0.9293583631515503, |
| "learning_rate": 2.9087833823848947e-05, |
| "logits/chosen": -7.124390602111816, |
| "logits/rejected": -6.594590663909912, |
| "logps/chosen": -223.9017333984375, |
| "logps/rejected": -273.6042175292969, |
| "loss": 0.1239, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": 0.1861969381570816, |
| "rewards/margins": 5.955235004425049, |
| "rewards/rejected": -5.769038200378418, |
| "step": 157 |
| }, |
| { |
| "epoch": 0.12686941684231656, |
| "grad_norm": 1.0380977392196655, |
| "learning_rate": 2.9070722733661856e-05, |
| "logits/chosen": -7.183553695678711, |
| "logits/rejected": -6.809614181518555, |
| "logps/chosen": -232.40655517578125, |
| "logps/rejected": -284.8603210449219, |
| "loss": 0.1294, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": 0.14984406530857086, |
| "rewards/margins": 6.0885725021362305, |
| "rewards/rejected": -5.938729286193848, |
| "step": 158 |
| }, |
| { |
| "epoch": 0.12767238783498946, |
| "grad_norm": 1.0368057489395142, |
| "learning_rate": 2.9053457768305268e-05, |
| "logits/chosen": -7.185681343078613, |
| "logits/rejected": -6.692957401275635, |
| "logps/chosen": -227.86204528808594, |
| "logps/rejected": -264.832275390625, |
| "loss": 0.1756, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.11191180348396301, |
| "rewards/margins": 5.628535747528076, |
| "rewards/rejected": -5.516623497009277, |
| "step": 159 |
| }, |
| { |
| "epoch": 0.12847535882766234, |
| "grad_norm": 1.232247233390808, |
| "learning_rate": 2.9036039116586097e-05, |
| "logits/chosen": -7.199278831481934, |
| "logits/rejected": -6.861912727355957, |
| "logps/chosen": -259.310546875, |
| "logps/rejected": -302.257080078125, |
| "loss": 0.1366, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.18357235193252563, |
| "rewards/margins": 6.2663469314575195, |
| "rewards/rejected": -6.449919700622559, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.12927832982033524, |
| "grad_norm": 0.8996439576148987, |
| "learning_rate": 2.9018466968991913e-05, |
| "logits/chosen": -7.069967269897461, |
| "logits/rejected": -6.671751022338867, |
| "logps/chosen": -248.725341796875, |
| "logps/rejected": -274.14599609375, |
| "loss": 0.1176, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.06392794847488403, |
| "rewards/margins": 6.06080961227417, |
| "rewards/rejected": -6.12473726272583, |
| "step": 161 |
| }, |
| { |
| "epoch": 0.13008130081300814, |
| "grad_norm": 1.0060917139053345, |
| "learning_rate": 2.9000741517688916e-05, |
| "logits/chosen": -7.322947025299072, |
| "logits/rejected": -6.945455551147461, |
| "logps/chosen": -252.95953369140625, |
| "logps/rejected": -295.2427673339844, |
| "loss": 0.1369, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.30330348014831543, |
| "rewards/margins": 6.233870029449463, |
| "rewards/rejected": -6.537173271179199, |
| "step": 162 |
| }, |
| { |
| "epoch": 0.13088427180568102, |
| "grad_norm": 0.916583776473999, |
| "learning_rate": 2.89828629565198e-05, |
| "logits/chosen": -6.946394443511963, |
| "logits/rejected": -6.567956447601318, |
| "logps/chosen": -263.36981201171875, |
| "logps/rejected": -288.17974853515625, |
| "loss": 0.1405, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.45375779271125793, |
| "rewards/margins": 5.8410773277282715, |
| "rewards/rejected": -6.294836044311523, |
| "step": 163 |
| }, |
| { |
| "epoch": 0.13168724279835392, |
| "grad_norm": 0.7520425319671631, |
| "learning_rate": 2.896483148100164e-05, |
| "logits/chosen": -7.239500045776367, |
| "logits/rejected": -6.808941841125488, |
| "logps/chosen": -253.20318603515625, |
| "logps/rejected": -315.1407165527344, |
| "loss": 0.1252, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": 0.05900028720498085, |
| "rewards/margins": 6.573638916015625, |
| "rewards/rejected": -6.514638900756836, |
| "step": 164 |
| }, |
| { |
| "epoch": 0.1324902137910268, |
| "grad_norm": 0.7469050288200378, |
| "learning_rate": 2.894664728832377e-05, |
| "logits/chosen": -7.288553237915039, |
| "logits/rejected": -6.83374547958374, |
| "logps/chosen": -228.03526306152344, |
| "logps/rejected": -268.7512512207031, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.01312924548983574, |
| "rewards/margins": 6.599348068237305, |
| "rewards/rejected": -6.612477779388428, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.1332931847836997, |
| "grad_norm": 1.0094337463378906, |
| "learning_rate": 2.8928310577345608e-05, |
| "logits/chosen": -7.2717509269714355, |
| "logits/rejected": -6.835561752319336, |
| "logps/chosen": -257.0513916015625, |
| "logps/rejected": -286.6746826171875, |
| "loss": 0.148, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.2640399634838104, |
| "rewards/margins": 6.239204406738281, |
| "rewards/rejected": -6.503244400024414, |
| "step": 166 |
| }, |
| { |
| "epoch": 0.13409615577637257, |
| "grad_norm": 0.8982539772987366, |
| "learning_rate": 2.890982154859448e-05, |
| "logits/chosen": -7.1839280128479, |
| "logits/rejected": -6.7328596115112305, |
| "logps/chosen": -272.18072509765625, |
| "logps/rejected": -306.2091369628906, |
| "loss": 0.1353, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.16021288931369781, |
| "rewards/margins": 6.352622032165527, |
| "rewards/rejected": -6.5128350257873535, |
| "step": 167 |
| }, |
| { |
| "epoch": 0.13489912676904547, |
| "grad_norm": 0.8648626208305359, |
| "learning_rate": 2.889118040426345e-05, |
| "logits/chosen": -6.902718544006348, |
| "logits/rejected": -6.55988883972168, |
| "logps/chosen": -233.5986328125, |
| "logps/rejected": -254.88931274414062, |
| "loss": 0.1534, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.33832401037216187, |
| "rewards/margins": 5.880598545074463, |
| "rewards/rejected": -6.2189226150512695, |
| "step": 168 |
| }, |
| { |
| "epoch": 0.13570209776171835, |
| "grad_norm": 0.6889446377754211, |
| "learning_rate": 2.8872387348209084e-05, |
| "logits/chosen": -7.308150291442871, |
| "logits/rejected": -6.833141326904297, |
| "logps/chosen": -232.03724670410156, |
| "logps/rejected": -288.5368957519531, |
| "loss": 0.114, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": 0.21436691284179688, |
| "rewards/margins": 6.878697395324707, |
| "rewards/rejected": -6.66433048248291, |
| "step": 169 |
| }, |
| { |
| "epoch": 0.13650506875439125, |
| "grad_norm": 1.1982077360153198, |
| "learning_rate": 2.885344258594923e-05, |
| "logits/chosen": -6.891546249389648, |
| "logits/rejected": -6.543699741363525, |
| "logps/chosen": -289.8294982910156, |
| "logps/rejected": -300.9779968261719, |
| "loss": 0.1618, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.014788907021284103, |
| "rewards/margins": 5.984076499938965, |
| "rewards/rejected": -5.998865604400635, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.13730803974706413, |
| "grad_norm": 1.0852102041244507, |
| "learning_rate": 2.883434632466077e-05, |
| "logits/chosen": -6.9026570320129395, |
| "logits/rejected": -6.523021697998047, |
| "logps/chosen": -239.19305419921875, |
| "logps/rejected": -261.19775390625, |
| "loss": 0.2007, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.13164648413658142, |
| "rewards/margins": 5.823849201202393, |
| "rewards/rejected": -5.955495834350586, |
| "step": 171 |
| }, |
| { |
| "epoch": 0.13811101073973703, |
| "grad_norm": 0.9284049868583679, |
| "learning_rate": 2.881509877317737e-05, |
| "logits/chosen": -7.169976234436035, |
| "logits/rejected": -6.791253089904785, |
| "logps/chosen": -260.9237060546875, |
| "logps/rejected": -306.494140625, |
| "loss": 0.1393, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": -0.10782945156097412, |
| "rewards/margins": 6.052058696746826, |
| "rewards/rejected": -6.159887790679932, |
| "step": 172 |
| }, |
| { |
| "epoch": 0.13891398173240993, |
| "grad_norm": 0.9177150130271912, |
| "learning_rate": 2.8795700141987153e-05, |
| "logits/chosen": -7.257651329040527, |
| "logits/rejected": -6.819592475891113, |
| "logps/chosen": -246.0475616455078, |
| "logps/rejected": -309.29974365234375, |
| "loss": 0.1522, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.5268869996070862, |
| "rewards/margins": 6.0183820724487305, |
| "rewards/rejected": -6.54526948928833, |
| "step": 173 |
| }, |
| { |
| "epoch": 0.1397169527250828, |
| "grad_norm": 0.9350268840789795, |
| "learning_rate": 2.877615064323045e-05, |
| "logits/chosen": -7.216957092285156, |
| "logits/rejected": -6.794558525085449, |
| "logps/chosen": -269.5796203613281, |
| "logps/rejected": -308.508056640625, |
| "loss": 0.1563, |
| "rewards/accuracies": 0.90234375, |
| "rewards/chosen": -0.2856195867061615, |
| "rewards/margins": 6.307086944580078, |
| "rewards/rejected": -6.59270715713501, |
| "step": 174 |
| }, |
| { |
| "epoch": 0.1405199237177557, |
| "grad_norm": 0.8900390863418579, |
| "learning_rate": 2.875645049069745e-05, |
| "logits/chosen": -7.07668399810791, |
| "logits/rejected": -6.801897048950195, |
| "logps/chosen": -245.149169921875, |
| "logps/rejected": -260.38372802734375, |
| "loss": 0.1452, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.38415631651878357, |
| "rewards/margins": 6.283596515655518, |
| "rewards/rejected": -6.667753219604492, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.14132289471042858, |
| "grad_norm": 0.6852567791938782, |
| "learning_rate": 2.873659989982586e-05, |
| "logits/chosen": -7.130537509918213, |
| "logits/rejected": -6.78173303604126, |
| "logps/chosen": -238.817626953125, |
| "logps/rejected": -289.56988525390625, |
| "loss": 0.0856, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.3728202283382416, |
| "rewards/margins": 6.958166599273682, |
| "rewards/rejected": -7.3309855461120605, |
| "step": 176 |
| }, |
| { |
| "epoch": 0.14212586570310148, |
| "grad_norm": 0.8934264779090881, |
| "learning_rate": 2.8716599087698565e-05, |
| "logits/chosen": -7.190873622894287, |
| "logits/rejected": -6.72062349319458, |
| "logps/chosen": -233.7152862548828, |
| "logps/rejected": -270.4144287109375, |
| "loss": 0.1213, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.5937744379043579, |
| "rewards/margins": 6.390446662902832, |
| "rewards/rejected": -6.984221458435059, |
| "step": 177 |
| }, |
| { |
| "epoch": 0.14292883669577436, |
| "grad_norm": 1.106290340423584, |
| "learning_rate": 2.8696448273041243e-05, |
| "logits/chosen": -7.174705982208252, |
| "logits/rejected": -6.707158088684082, |
| "logps/chosen": -228.9149169921875, |
| "logps/rejected": -277.5617980957031, |
| "loss": 0.1298, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6620899438858032, |
| "rewards/margins": 6.383820056915283, |
| "rewards/rejected": -7.045909881591797, |
| "step": 178 |
| }, |
| { |
| "epoch": 0.14373180768844726, |
| "grad_norm": 0.7993309497833252, |
| "learning_rate": 2.8676147676219968e-05, |
| "logits/chosen": -7.099809646606445, |
| "logits/rejected": -6.527121543884277, |
| "logps/chosen": -256.1249694824219, |
| "logps/rejected": -302.5453186035156, |
| "loss": 0.1107, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3866848945617676, |
| "rewards/margins": 6.834173202514648, |
| "rewards/rejected": -7.220857620239258, |
| "step": 179 |
| }, |
| { |
| "epoch": 0.14453477868112014, |
| "grad_norm": 0.697613000869751, |
| "learning_rate": 2.865569751923882e-05, |
| "logits/chosen": -7.188958168029785, |
| "logits/rejected": -6.680798053741455, |
| "logps/chosen": -245.09649658203125, |
| "logps/rejected": -289.06109619140625, |
| "loss": 0.0842, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.6699570417404175, |
| "rewards/margins": 6.983727931976318, |
| "rewards/rejected": -7.653684616088867, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.14533774967379304, |
| "grad_norm": 1.0621944665908813, |
| "learning_rate": 2.863509802573744e-05, |
| "logits/chosen": -7.026541709899902, |
| "logits/rejected": -6.508971214294434, |
| "logps/chosen": -251.75904846191406, |
| "logps/rejected": -300.0320129394531, |
| "loss": 0.1472, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7701475024223328, |
| "rewards/margins": 6.5906500816345215, |
| "rewards/rejected": -7.360796928405762, |
| "step": 181 |
| }, |
| { |
| "epoch": 0.1461407206664659, |
| "grad_norm": 1.0146931409835815, |
| "learning_rate": 2.8614349420988584e-05, |
| "logits/chosen": -7.0656962394714355, |
| "logits/rejected": -6.569565773010254, |
| "logps/chosen": -202.70034790039062, |
| "logps/rejected": -277.9149169921875, |
| "loss": 0.1248, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.455616295337677, |
| "rewards/margins": 6.851208686828613, |
| "rewards/rejected": -7.306825160980225, |
| "step": 182 |
| }, |
| { |
| "epoch": 0.14694369165913881, |
| "grad_norm": 0.9551364183425903, |
| "learning_rate": 2.859345193189567e-05, |
| "logits/chosen": -7.055583477020264, |
| "logits/rejected": -6.500993251800537, |
| "logps/chosen": -212.3512420654297, |
| "logps/rejected": -269.97259521484375, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.5044701099395752, |
| "rewards/margins": 6.887358665466309, |
| "rewards/rejected": -7.391829013824463, |
| "step": 183 |
| }, |
| { |
| "epoch": 0.14774666265181172, |
| "grad_norm": 1.0029313564300537, |
| "learning_rate": 2.8572405786990293e-05, |
| "logits/chosen": -7.026425838470459, |
| "logits/rejected": -6.601473331451416, |
| "logps/chosen": -215.61056518554688, |
| "logps/rejected": -268.7537536621094, |
| "loss": 0.1459, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.2453242540359497, |
| "rewards/margins": 6.522678852081299, |
| "rewards/rejected": -7.768002986907959, |
| "step": 184 |
| }, |
| { |
| "epoch": 0.1485496336444846, |
| "grad_norm": 1.4357188940048218, |
| "learning_rate": 2.8551211216429724e-05, |
| "logits/chosen": -7.05723237991333, |
| "logits/rejected": -6.613016128540039, |
| "logps/chosen": -269.4820251464844, |
| "logps/rejected": -308.440673828125, |
| "loss": 0.1744, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.7734962105751038, |
| "rewards/margins": 6.893777847290039, |
| "rewards/rejected": -7.6672749519348145, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.1493526046371575, |
| "grad_norm": 1.0982927083969116, |
| "learning_rate": 2.8529868451994387e-05, |
| "logits/chosen": -7.004734992980957, |
| "logits/rejected": -6.667719841003418, |
| "logps/chosen": -242.59979248046875, |
| "logps/rejected": -284.29925537109375, |
| "loss": 0.1608, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.6659376621246338, |
| "rewards/margins": 6.5710248947143555, |
| "rewards/rejected": -7.23696231842041, |
| "step": 186 |
| }, |
| { |
| "epoch": 0.15015557562983037, |
| "grad_norm": 1.0008646249771118, |
| "learning_rate": 2.8508377727085337e-05, |
| "logits/chosen": -6.979903221130371, |
| "logits/rejected": -6.540533065795898, |
| "logps/chosen": -235.71417236328125, |
| "logps/rejected": -294.48992919921875, |
| "loss": 0.1295, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5362163782119751, |
| "rewards/margins": 6.888940811157227, |
| "rewards/rejected": -7.425157070159912, |
| "step": 187 |
| }, |
| { |
| "epoch": 0.15095854662250327, |
| "grad_norm": 1.0796961784362793, |
| "learning_rate": 2.848673927672169e-05, |
| "logits/chosen": -6.95247745513916, |
| "logits/rejected": -6.567049980163574, |
| "logps/chosen": -253.25917053222656, |
| "logps/rejected": -278.66021728515625, |
| "loss": 0.1791, |
| "rewards/accuracies": 0.91015625, |
| "rewards/chosen": -0.09510260075330734, |
| "rewards/margins": 6.747255325317383, |
| "rewards/rejected": -6.842358589172363, |
| "step": 188 |
| }, |
| { |
| "epoch": 0.15176151761517614, |
| "grad_norm": 0.9111409783363342, |
| "learning_rate": 2.8464953337538083e-05, |
| "logits/chosen": -6.900506019592285, |
| "logits/rejected": -6.462779998779297, |
| "logps/chosen": -235.87026977539062, |
| "logps/rejected": -296.7842712402344, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.5601946711540222, |
| "rewards/margins": 6.615263938903809, |
| "rewards/rejected": -7.1754584312438965, |
| "step": 189 |
| }, |
| { |
| "epoch": 0.15256448860784905, |
| "grad_norm": 0.8946994543075562, |
| "learning_rate": 2.8443020147782055e-05, |
| "logits/chosen": -7.2119622230529785, |
| "logits/rejected": -6.72713565826416, |
| "logps/chosen": -213.63729858398438, |
| "logps/rejected": -278.2781982421875, |
| "loss": 0.1339, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.49036455154418945, |
| "rewards/margins": 6.954005718231201, |
| "rewards/rejected": -7.444370269775391, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.15336745960052192, |
| "grad_norm": 0.9741977453231812, |
| "learning_rate": 2.8420939947311454e-05, |
| "logits/chosen": -7.0920844078063965, |
| "logits/rejected": -6.579745769500732, |
| "logps/chosen": -259.325927734375, |
| "logps/rejected": -293.7746887207031, |
| "loss": 0.1344, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.32702019810676575, |
| "rewards/margins": 6.925469398498535, |
| "rewards/rejected": -7.25248908996582, |
| "step": 191 |
| }, |
| { |
| "epoch": 0.15417043059319482, |
| "grad_norm": 1.0351067781448364, |
| "learning_rate": 2.839871297759181e-05, |
| "logits/chosen": -7.353341102600098, |
| "logits/rejected": -6.8764519691467285, |
| "logps/chosen": -261.73248291015625, |
| "logps/rejected": -314.316162109375, |
| "loss": 0.1215, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.22120127081871033, |
| "rewards/margins": 7.455446243286133, |
| "rewards/rejected": -7.676647186279297, |
| "step": 192 |
| }, |
| { |
| "epoch": 0.1549734015858677, |
| "grad_norm": 1.0414035320281982, |
| "learning_rate": 2.837633948169371e-05, |
| "logits/chosen": -7.251976490020752, |
| "logits/rejected": -6.7385101318359375, |
| "logps/chosen": -237.1507568359375, |
| "logps/rejected": -280.85723876953125, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.456257164478302, |
| "rewards/margins": 6.9616312980651855, |
| "rewards/rejected": -7.417888641357422, |
| "step": 193 |
| }, |
| { |
| "epoch": 0.1557763725785406, |
| "grad_norm": 1.1375614404678345, |
| "learning_rate": 2.8353819704290113e-05, |
| "logits/chosen": -6.99810266494751, |
| "logits/rejected": -6.510506629943848, |
| "logps/chosen": -256.8636779785156, |
| "logps/rejected": -292.4454345703125, |
| "loss": 0.1653, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.40773043036460876, |
| "rewards/margins": 6.63405704498291, |
| "rewards/rejected": -7.041788101196289, |
| "step": 194 |
| }, |
| { |
| "epoch": 0.1565793435712135, |
| "grad_norm": 1.0164093971252441, |
| "learning_rate": 2.8331153891653707e-05, |
| "logits/chosen": -7.254598617553711, |
| "logits/rejected": -6.754511833190918, |
| "logps/chosen": -230.98089599609375, |
| "logps/rejected": -300.4831848144531, |
| "loss": 0.1192, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.4606966972351074, |
| "rewards/margins": 7.272715091705322, |
| "rewards/rejected": -7.7334113121032715, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.15738231456388638, |
| "grad_norm": 0.8370680809020996, |
| "learning_rate": 2.830834229165418e-05, |
| "logits/chosen": -7.034485816955566, |
| "logits/rejected": -6.5197224617004395, |
| "logps/chosen": -231.7747039794922, |
| "logps/rejected": -298.645263671875, |
| "loss": 0.1203, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.6550363302230835, |
| "rewards/margins": 6.6552653312683105, |
| "rewards/rejected": -7.310300827026367, |
| "step": 196 |
| }, |
| { |
| "epoch": 0.15818528555655928, |
| "grad_norm": 0.8957505822181702, |
| "learning_rate": 2.8285385153755532e-05, |
| "logits/chosen": -6.994017601013184, |
| "logits/rejected": -6.697011470794678, |
| "logps/chosen": -240.79922485351562, |
| "logps/rejected": -276.0355529785156, |
| "loss": 0.1242, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3661085367202759, |
| "rewards/margins": 6.949375152587891, |
| "rewards/rejected": -7.315483093261719, |
| "step": 197 |
| }, |
| { |
| "epoch": 0.15898825654923215, |
| "grad_norm": 1.0306806564331055, |
| "learning_rate": 2.8262282729013353e-05, |
| "logits/chosen": -7.204374313354492, |
| "logits/rejected": -6.730704307556152, |
| "logps/chosen": -244.46868896484375, |
| "logps/rejected": -284.58441162109375, |
| "loss": 0.1383, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.21752500534057617, |
| "rewards/margins": 7.016619682312012, |
| "rewards/rejected": -7.2341437339782715, |
| "step": 198 |
| }, |
| { |
| "epoch": 0.15979122754190506, |
| "grad_norm": 0.8039461970329285, |
| "learning_rate": 2.8239035270072055e-05, |
| "logits/chosen": -6.875657081604004, |
| "logits/rejected": -6.5066633224487305, |
| "logps/chosen": -257.2030029296875, |
| "logps/rejected": -282.41070556640625, |
| "loss": 0.1147, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.352853387594223, |
| "rewards/margins": 6.5738959312438965, |
| "rewards/rejected": -6.926748752593994, |
| "step": 199 |
| }, |
| { |
| "epoch": 0.16059419853457793, |
| "grad_norm": 0.908125638961792, |
| "learning_rate": 2.821564303116212e-05, |
| "logits/chosen": -7.159363269805908, |
| "logits/rejected": -6.701921463012695, |
| "logps/chosen": -241.97821044921875, |
| "logps/rejected": -287.5133361816406, |
| "loss": 0.1297, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.06099928170442581, |
| "rewards/margins": 6.793715000152588, |
| "rewards/rejected": -6.854714393615723, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.16139716952725083, |
| "grad_norm": 0.7971031665802002, |
| "learning_rate": 2.8192106268097336e-05, |
| "logits/chosen": -6.893752574920654, |
| "logits/rejected": -6.537377834320068, |
| "logps/chosen": -247.20901489257812, |
| "logps/rejected": -303.2108459472656, |
| "loss": 0.1179, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.5789100527763367, |
| "rewards/margins": 6.6839213371276855, |
| "rewards/rejected": -7.26283073425293, |
| "step": 201 |
| }, |
| { |
| "epoch": 0.1622001405199237, |
| "grad_norm": 0.7816908359527588, |
| "learning_rate": 2.8168425238271965e-05, |
| "logits/chosen": -7.062497138977051, |
| "logits/rejected": -6.738613605499268, |
| "logps/chosen": -247.6879119873047, |
| "logps/rejected": -271.6099548339844, |
| "loss": 0.1138, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.4790666103363037, |
| "rewards/margins": 6.366783142089844, |
| "rewards/rejected": -6.845849514007568, |
| "step": 202 |
| }, |
| { |
| "epoch": 0.1630031115125966, |
| "grad_norm": 0.8900995254516602, |
| "learning_rate": 2.8144600200657953e-05, |
| "logits/chosen": -7.072137355804443, |
| "logits/rejected": -6.589000225067139, |
| "logps/chosen": -239.6106414794922, |
| "logps/rejected": -280.016357421875, |
| "loss": 0.1179, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.3842998147010803, |
| "rewards/margins": 6.371336460113525, |
| "rewards/rejected": -6.755636215209961, |
| "step": 203 |
| }, |
| { |
| "epoch": 0.16380608250526948, |
| "grad_norm": 0.7133951783180237, |
| "learning_rate": 2.8120631415802102e-05, |
| "logits/chosen": -7.400625228881836, |
| "logits/rejected": -6.904072284698486, |
| "logps/chosen": -227.05860900878906, |
| "logps/rejected": -276.07427978515625, |
| "loss": 0.0967, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.20231446623802185, |
| "rewards/margins": 6.981462478637695, |
| "rewards/rejected": -7.183776378631592, |
| "step": 204 |
| }, |
| { |
| "epoch": 0.1646090534979424, |
| "grad_norm": 0.751691997051239, |
| "learning_rate": 2.8096519145823196e-05, |
| "logits/chosen": -7.229235649108887, |
| "logits/rejected": -6.738106727600098, |
| "logps/chosen": -250.77410888671875, |
| "logps/rejected": -291.5039367675781, |
| "loss": 0.0999, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": 0.34943392872810364, |
| "rewards/margins": 7.135815620422363, |
| "rewards/rejected": -6.78638219833374, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.1654120244906153, |
| "grad_norm": 0.7939348220825195, |
| "learning_rate": 2.807226365440916e-05, |
| "logits/chosen": -7.132170677185059, |
| "logits/rejected": -6.7212748527526855, |
| "logps/chosen": -225.26710510253906, |
| "logps/rejected": -283.0187072753906, |
| "loss": 0.0987, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8082407712936401, |
| "rewards/margins": 6.615687370300293, |
| "rewards/rejected": -7.4239277839660645, |
| "step": 206 |
| }, |
| { |
| "epoch": 0.16621499548328816, |
| "grad_norm": 0.9501697421073914, |
| "learning_rate": 2.8047865206814164e-05, |
| "logits/chosen": -7.10981559753418, |
| "logits/rejected": -6.643800735473633, |
| "logps/chosen": -260.46063232421875, |
| "logps/rejected": -310.0489501953125, |
| "loss": 0.1284, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.18858715891838074, |
| "rewards/margins": 6.7222442626953125, |
| "rewards/rejected": -6.910831928253174, |
| "step": 207 |
| }, |
| { |
| "epoch": 0.16701796647596107, |
| "grad_norm": 0.7089926600456238, |
| "learning_rate": 2.8023324069855714e-05, |
| "logits/chosen": -7.279808521270752, |
| "logits/rejected": -6.794492244720459, |
| "logps/chosen": -242.9237060546875, |
| "logps/rejected": -310.3043212890625, |
| "loss": 0.0945, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.3004932403564453, |
| "rewards/margins": 6.618015766143799, |
| "rewards/rejected": -6.918508529663086, |
| "step": 208 |
| }, |
| { |
| "epoch": 0.16782093746863394, |
| "grad_norm": 1.1816922426223755, |
| "learning_rate": 2.7998640511911757e-05, |
| "logits/chosen": -7.191790580749512, |
| "logits/rejected": -6.732461452484131, |
| "logps/chosen": -240.1959686279297, |
| "logps/rejected": -282.6694641113281, |
| "loss": 0.1685, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -1.074135184288025, |
| "rewards/margins": 6.0143632888793945, |
| "rewards/rejected": -7.088499069213867, |
| "step": 209 |
| }, |
| { |
| "epoch": 0.16862390846130684, |
| "grad_norm": 0.8570040464401245, |
| "learning_rate": 2.797381480291773e-05, |
| "logits/chosen": -7.424015998840332, |
| "logits/rejected": -6.978085994720459, |
| "logps/chosen": -222.77342224121094, |
| "logps/rejected": -278.3456115722656, |
| "loss": 0.0885, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.6906812191009521, |
| "rewards/margins": 6.762721061706543, |
| "rewards/rejected": -7.453402042388916, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.16942687945397972, |
| "grad_norm": 0.8010284900665283, |
| "learning_rate": 2.794884721436361e-05, |
| "logits/chosen": -7.10498571395874, |
| "logits/rejected": -6.7436418533325195, |
| "logps/chosen": -219.07510375976562, |
| "logps/rejected": -286.34906005859375, |
| "loss": 0.094, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.40968257188796997, |
| "rewards/margins": 6.485257148742676, |
| "rewards/rejected": -6.894938945770264, |
| "step": 211 |
| }, |
| { |
| "epoch": 0.17022985044665262, |
| "grad_norm": 1.0010613203048706, |
| "learning_rate": 2.792373801929094e-05, |
| "logits/chosen": -7.173435211181641, |
| "logits/rejected": -6.907317638397217, |
| "logps/chosen": -260.1988220214844, |
| "logps/rejected": -279.89752197265625, |
| "loss": 0.1255, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.3175370693206787, |
| "rewards/margins": 6.2941389083862305, |
| "rewards/rejected": -6.61167573928833, |
| "step": 212 |
| }, |
| { |
| "epoch": 0.1710328214393255, |
| "grad_norm": 1.0070956945419312, |
| "learning_rate": 2.7898487492289855e-05, |
| "logits/chosen": -7.24013614654541, |
| "logits/rejected": -6.8287482261657715, |
| "logps/chosen": -253.9784698486328, |
| "logps/rejected": -276.7708740234375, |
| "loss": 0.1465, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.18907347321510315, |
| "rewards/margins": 6.2533979415893555, |
| "rewards/rejected": -6.442471504211426, |
| "step": 213 |
| }, |
| { |
| "epoch": 0.1718357924319984, |
| "grad_norm": 0.9078125953674316, |
| "learning_rate": 2.7873095909496076e-05, |
| "logits/chosen": -6.793828010559082, |
| "logits/rejected": -6.4473395347595215, |
| "logps/chosen": -255.51339721679688, |
| "logps/rejected": -288.9300842285156, |
| "loss": 0.1327, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.2808770537376404, |
| "rewards/margins": 5.637259483337402, |
| "rewards/rejected": -5.918137073516846, |
| "step": 214 |
| }, |
| { |
| "epoch": 0.17263876342467127, |
| "grad_norm": 0.9293401837348938, |
| "learning_rate": 2.7847563548587888e-05, |
| "logits/chosen": -6.958491325378418, |
| "logits/rejected": -6.688619136810303, |
| "logps/chosen": -272.62164306640625, |
| "logps/rejected": -290.7449035644531, |
| "loss": 0.1526, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": 0.23415488004684448, |
| "rewards/margins": 5.801517009735107, |
| "rewards/rejected": -5.567361831665039, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.17344173441734417, |
| "grad_norm": 0.9865174889564514, |
| "learning_rate": 2.7821890688783088e-05, |
| "logits/chosen": -7.204586982727051, |
| "logits/rejected": -6.801849842071533, |
| "logps/chosen": -221.61651611328125, |
| "logps/rejected": -277.6392822265625, |
| "loss": 0.1131, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": 0.09113295376300812, |
| "rewards/margins": 6.1976237297058105, |
| "rewards/rejected": -6.1064910888671875, |
| "step": 216 |
| }, |
| { |
| "epoch": 0.17424470541001708, |
| "grad_norm": 0.6169539093971252, |
| "learning_rate": 2.779607761083596e-05, |
| "logits/chosen": -7.304375648498535, |
| "logits/rejected": -6.889275550842285, |
| "logps/chosen": -248.2451171875, |
| "logps/rejected": -295.1099548339844, |
| "loss": 0.0881, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.08813758194446564, |
| "rewards/margins": 6.1621413230896, |
| "rewards/rejected": -6.250278472900391, |
| "step": 217 |
| }, |
| { |
| "epoch": 0.17504767640268995, |
| "grad_norm": 1.0091732740402222, |
| "learning_rate": 2.7770124597034205e-05, |
| "logits/chosen": -7.377964973449707, |
| "logits/rejected": -6.849767684936523, |
| "logps/chosen": -254.886474609375, |
| "logps/rejected": -298.0146484375, |
| "loss": 0.1377, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.09490342438220978, |
| "rewards/margins": 6.492976188659668, |
| "rewards/rejected": -6.398072242736816, |
| "step": 218 |
| }, |
| { |
| "epoch": 0.17585064739536285, |
| "grad_norm": 0.8362334966659546, |
| "learning_rate": 2.7744031931195803e-05, |
| "logits/chosen": -7.011676788330078, |
| "logits/rejected": -6.587613105773926, |
| "logps/chosen": -241.74887084960938, |
| "logps/rejected": -269.30548095703125, |
| "loss": 0.1154, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2750447392463684, |
| "rewards/margins": 5.792877674102783, |
| "rewards/rejected": -6.067922115325928, |
| "step": 219 |
| }, |
| { |
| "epoch": 0.17665361838803573, |
| "grad_norm": 0.7466740608215332, |
| "learning_rate": 2.7717799898665977e-05, |
| "logits/chosen": -7.112973690032959, |
| "logits/rejected": -6.596190452575684, |
| "logps/chosen": -258.7005920410156, |
| "logps/rejected": -314.06597900390625, |
| "loss": 0.091, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": 0.06373302638530731, |
| "rewards/margins": 6.695507049560547, |
| "rewards/rejected": -6.631773948669434, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.17745658938070863, |
| "grad_norm": 0.680537223815918, |
| "learning_rate": 2.769142878631403e-05, |
| "logits/chosen": -7.230843544006348, |
| "logits/rejected": -6.803351402282715, |
| "logps/chosen": -239.3734588623047, |
| "logps/rejected": -267.59039306640625, |
| "loss": 0.0881, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": 0.0525919571518898, |
| "rewards/margins": 6.6245198249816895, |
| "rewards/rejected": -6.571927547454834, |
| "step": 221 |
| }, |
| { |
| "epoch": 0.1782595603733815, |
| "grad_norm": 0.8267503976821899, |
| "learning_rate": 2.7664918882530227e-05, |
| "logits/chosen": -7.266627311706543, |
| "logits/rejected": -6.75255012512207, |
| "logps/chosen": -230.12396240234375, |
| "logps/rejected": -270.2829895019531, |
| "loss": 0.1088, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.11840138584375381, |
| "rewards/margins": 6.679051399230957, |
| "rewards/rejected": -6.797453880310059, |
| "step": 222 |
| }, |
| { |
| "epoch": 0.1790625313660544, |
| "grad_norm": 0.7312098741531372, |
| "learning_rate": 2.763827047722262e-05, |
| "logits/chosen": -7.379392623901367, |
| "logits/rejected": -6.913332462310791, |
| "logps/chosen": -252.22634887695312, |
| "logps/rejected": -290.7511901855469, |
| "loss": 0.0922, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": 0.08495733141899109, |
| "rewards/margins": 6.907980918884277, |
| "rewards/rejected": -6.823024272918701, |
| "step": 223 |
| }, |
| { |
| "epoch": 0.17986550235872728, |
| "grad_norm": 0.9363484978675842, |
| "learning_rate": 2.7611483861813908e-05, |
| "logits/chosen": -7.2217912673950195, |
| "logits/rejected": -6.755009651184082, |
| "logps/chosen": -226.36795043945312, |
| "logps/rejected": -257.615478515625, |
| "loss": 0.1248, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.24932131171226501, |
| "rewards/margins": 6.638134002685547, |
| "rewards/rejected": -6.887454986572266, |
| "step": 224 |
| }, |
| { |
| "epoch": 0.18066847335140018, |
| "grad_norm": 1.009516954421997, |
| "learning_rate": 2.7584559329238218e-05, |
| "logits/chosen": -7.129251480102539, |
| "logits/rejected": -6.652354717254639, |
| "logps/chosen": -270.228515625, |
| "logps/rejected": -312.0833435058594, |
| "loss": 0.1079, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.17091360688209534, |
| "rewards/margins": 6.849265098571777, |
| "rewards/rejected": -7.02017879486084, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.18147144434407306, |
| "grad_norm": 3.592782497406006, |
| "learning_rate": 2.7557497173937928e-05, |
| "logits/chosen": -7.227541923522949, |
| "logits/rejected": -6.778803825378418, |
| "logps/chosen": -239.37460327148438, |
| "logps/rejected": -296.29266357421875, |
| "loss": 0.1214, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.3931058943271637, |
| "rewards/margins": 7.016083717346191, |
| "rewards/rejected": -7.409189701080322, |
| "step": 226 |
| }, |
| { |
| "epoch": 0.18227441533674596, |
| "grad_norm": 0.8727150559425354, |
| "learning_rate": 2.7530297691860436e-05, |
| "logits/chosen": -7.089144229888916, |
| "logits/rejected": -6.657008171081543, |
| "logps/chosen": -262.0313415527344, |
| "logps/rejected": -307.26007080078125, |
| "loss": 0.104, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.04190448671579361, |
| "rewards/margins": 7.205683708190918, |
| "rewards/rejected": -7.247588634490967, |
| "step": 227 |
| }, |
| { |
| "epoch": 0.18307738632941886, |
| "grad_norm": 0.8800702691078186, |
| "learning_rate": 2.7502961180454915e-05, |
| "logits/chosen": -7.192566394805908, |
| "logits/rejected": -6.720447063446045, |
| "logps/chosen": -224.3740692138672, |
| "logps/rejected": -281.92840576171875, |
| "loss": 0.1074, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.09098710119724274, |
| "rewards/margins": 7.396082401275635, |
| "rewards/rejected": -7.487069129943848, |
| "step": 228 |
| }, |
| { |
| "epoch": 0.18388035732209174, |
| "grad_norm": 1.1270753145217896, |
| "learning_rate": 2.7475487938669074e-05, |
| "logits/chosen": -7.34289026260376, |
| "logits/rejected": -6.791628837585449, |
| "logps/chosen": -224.40528869628906, |
| "logps/rejected": -293.8467712402344, |
| "loss": 0.1195, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4163200557231903, |
| "rewards/margins": 7.307151794433594, |
| "rewards/rejected": -7.723472595214844, |
| "step": 229 |
| }, |
| { |
| "epoch": 0.18468332831476464, |
| "grad_norm": 1.0317673683166504, |
| "learning_rate": 2.744787826694589e-05, |
| "logits/chosen": -6.952282428741455, |
| "logits/rejected": -6.5210113525390625, |
| "logps/chosen": -242.7403564453125, |
| "logps/rejected": -295.57147216796875, |
| "loss": 0.1286, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6410664319992065, |
| "rewards/margins": 7.14946985244751, |
| "rewards/rejected": -7.790535926818848, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.1854862993074375, |
| "grad_norm": 1.1216742992401123, |
| "learning_rate": 2.74201324672203e-05, |
| "logits/chosen": -7.154869556427002, |
| "logits/rejected": -6.6431450843811035, |
| "logps/chosen": -232.35134887695312, |
| "logps/rejected": -286.8792419433594, |
| "loss": 0.1367, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.13038557767868042, |
| "rewards/margins": 7.466836452484131, |
| "rewards/rejected": -7.597221851348877, |
| "step": 231 |
| }, |
| { |
| "epoch": 0.18628927030011042, |
| "grad_norm": 0.9936255216598511, |
| "learning_rate": 2.7392250842915932e-05, |
| "logits/chosen": -7.1509690284729, |
| "logits/rejected": -6.624875545501709, |
| "logps/chosen": -238.4701690673828, |
| "logps/rejected": -280.0693359375, |
| "loss": 0.109, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.02964412420988083, |
| "rewards/margins": 7.326846122741699, |
| "rewards/rejected": -7.297203063964844, |
| "step": 232 |
| }, |
| { |
| "epoch": 0.1870922412927833, |
| "grad_norm": 0.9139237403869629, |
| "learning_rate": 2.7364233698941764e-05, |
| "logits/chosen": -7.23187780380249, |
| "logits/rejected": -6.709383010864258, |
| "logps/chosen": -239.5867919921875, |
| "logps/rejected": -297.4087829589844, |
| "loss": 0.0907, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.2775725722312927, |
| "rewards/margins": 7.577857494354248, |
| "rewards/rejected": -7.855430603027344, |
| "step": 233 |
| }, |
| { |
| "epoch": 0.1878952122854562, |
| "grad_norm": 0.8945658206939697, |
| "learning_rate": 2.7336081341688787e-05, |
| "logits/chosen": -7.148744583129883, |
| "logits/rejected": -6.629730224609375, |
| "logps/chosen": -230.7365264892578, |
| "logps/rejected": -286.87060546875, |
| "loss": 0.123, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.02580508589744568, |
| "rewards/margins": 7.404598712921143, |
| "rewards/rejected": -7.430404186248779, |
| "step": 234 |
| }, |
| { |
| "epoch": 0.18869818327812907, |
| "grad_norm": 1.081912875175476, |
| "learning_rate": 2.7307794079026666e-05, |
| "logits/chosen": -7.0780348777771, |
| "logits/rejected": -6.735588073730469, |
| "logps/chosen": -261.2464599609375, |
| "logps/rejected": -304.53204345703125, |
| "loss": 0.1023, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.43904542922973633, |
| "rewards/margins": 7.305832862854004, |
| "rewards/rejected": -7.744877815246582, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.18950115427080197, |
| "grad_norm": 0.8479633331298828, |
| "learning_rate": 2.727937222030039e-05, |
| "logits/chosen": -7.009301662445068, |
| "logits/rejected": -6.526599407196045, |
| "logps/chosen": -242.895751953125, |
| "logps/rejected": -272.1319580078125, |
| "loss": 0.1099, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.28827884793281555, |
| "rewards/margins": 6.9105024337768555, |
| "rewards/rejected": -7.198781490325928, |
| "step": 236 |
| }, |
| { |
| "epoch": 0.19030412526347484, |
| "grad_norm": 0.9074047207832336, |
| "learning_rate": 2.7250816076326834e-05, |
| "logits/chosen": -7.058507919311523, |
| "logits/rejected": -6.660304069519043, |
| "logps/chosen": -213.474853515625, |
| "logps/rejected": -260.62884521484375, |
| "loss": 0.1151, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.3388282358646393, |
| "rewards/margins": 7.26230001449585, |
| "rewards/rejected": -7.601129055023193, |
| "step": 237 |
| }, |
| { |
| "epoch": 0.19110709625614775, |
| "grad_norm": 1.1081340312957764, |
| "learning_rate": 2.7222125959391424e-05, |
| "logits/chosen": -6.873591423034668, |
| "logits/rejected": -6.451449394226074, |
| "logps/chosen": -239.97592163085938, |
| "logps/rejected": -310.7208251953125, |
| "loss": 0.1321, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.668763279914856, |
| "rewards/margins": 6.451139450073242, |
| "rewards/rejected": -7.119902610778809, |
| "step": 238 |
| }, |
| { |
| "epoch": 0.19191006724882065, |
| "grad_norm": 0.9222313165664673, |
| "learning_rate": 2.7193302183244683e-05, |
| "logits/chosen": -7.040648460388184, |
| "logits/rejected": -6.590151786804199, |
| "logps/chosen": -263.19293212890625, |
| "logps/rejected": -304.15557861328125, |
| "loss": 0.1119, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.16399967670440674, |
| "rewards/margins": 6.784195423126221, |
| "rewards/rejected": -6.9481940269470215, |
| "step": 239 |
| }, |
| { |
| "epoch": 0.19271303824149352, |
| "grad_norm": 1.1215649843215942, |
| "learning_rate": 2.71643450630988e-05, |
| "logits/chosen": -7.121896266937256, |
| "logits/rejected": -6.6946210861206055, |
| "logps/chosen": -219.42283630371094, |
| "logps/rejected": -274.3426818847656, |
| "loss": 0.1361, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.3100290894508362, |
| "rewards/margins": 6.942667007446289, |
| "rewards/rejected": -7.252695560455322, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.19351600923416643, |
| "grad_norm": 0.7611123919487, |
| "learning_rate": 2.7135254915624213e-05, |
| "logits/chosen": -7.409577369689941, |
| "logits/rejected": -6.924159049987793, |
| "logps/chosen": -266.3190002441406, |
| "logps/rejected": -304.4206848144531, |
| "loss": 0.0928, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.15145006775856018, |
| "rewards/margins": 6.891002655029297, |
| "rewards/rejected": -7.042452812194824, |
| "step": 241 |
| }, |
| { |
| "epoch": 0.1943189802268393, |
| "grad_norm": 0.7684003114700317, |
| "learning_rate": 2.7106032058946106e-05, |
| "logits/chosen": -7.19840669631958, |
| "logits/rejected": -6.804621696472168, |
| "logps/chosen": -233.7779541015625, |
| "logps/rejected": -272.0475158691406, |
| "loss": 0.0886, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.18616220355033875, |
| "rewards/margins": 6.849691390991211, |
| "rewards/rejected": -7.035853862762451, |
| "step": 242 |
| }, |
| { |
| "epoch": 0.1951219512195122, |
| "grad_norm": 0.8755264282226562, |
| "learning_rate": 2.7076676812640962e-05, |
| "logits/chosen": -7.150163173675537, |
| "logits/rejected": -6.628775119781494, |
| "logps/chosen": -247.1013946533203, |
| "logps/rejected": -296.917724609375, |
| "loss": 0.1224, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.30714353919029236, |
| "rewards/margins": 6.4777679443359375, |
| "rewards/rejected": -6.784911155700684, |
| "step": 243 |
| }, |
| { |
| "epoch": 0.19592492221218508, |
| "grad_norm": 0.9280303716659546, |
| "learning_rate": 2.7047189497733044e-05, |
| "logits/chosen": -7.000631332397461, |
| "logits/rejected": -6.55134391784668, |
| "logps/chosen": -259.8746643066406, |
| "logps/rejected": -305.20855712890625, |
| "loss": 0.1258, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.3451167643070221, |
| "rewards/margins": 6.241878986358643, |
| "rewards/rejected": -6.586996078491211, |
| "step": 244 |
| }, |
| { |
| "epoch": 0.19672789320485798, |
| "grad_norm": 0.8134123682975769, |
| "learning_rate": 2.7017570436690914e-05, |
| "logits/chosen": -7.173943996429443, |
| "logits/rejected": -6.731693744659424, |
| "logps/chosen": -237.5293426513672, |
| "logps/rejected": -271.93115234375, |
| "loss": 0.107, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.29382652044296265, |
| "rewards/margins": 6.51574182510376, |
| "rewards/rejected": -6.809567451477051, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.19753086419753085, |
| "grad_norm": 0.7581705451011658, |
| "learning_rate": 2.698781995342387e-05, |
| "logits/chosen": -7.3205718994140625, |
| "logits/rejected": -6.8700408935546875, |
| "logps/chosen": -244.8485870361328, |
| "logps/rejected": -312.4197082519531, |
| "loss": 0.0925, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.13184109330177307, |
| "rewards/margins": 7.397711753845215, |
| "rewards/rejected": -7.529553413391113, |
| "step": 246 |
| }, |
| { |
| "epoch": 0.19833383519020376, |
| "grad_norm": 0.8754745721817017, |
| "learning_rate": 2.695793837327844e-05, |
| "logits/chosen": -7.149688720703125, |
| "logits/rejected": -6.677689552307129, |
| "logps/chosen": -260.34014892578125, |
| "logps/rejected": -310.9524230957031, |
| "loss": 0.1153, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2624453902244568, |
| "rewards/margins": 7.07906436920166, |
| "rewards/rejected": -7.3415093421936035, |
| "step": 247 |
| }, |
| { |
| "epoch": 0.19913680618287663, |
| "grad_norm": 0.9530984163284302, |
| "learning_rate": 2.6927926023034784e-05, |
| "logits/chosen": -7.071310043334961, |
| "logits/rejected": -6.585930347442627, |
| "logps/chosen": -245.0313720703125, |
| "logps/rejected": -315.0196533203125, |
| "loss": 0.1014, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.6259324550628662, |
| "rewards/margins": 7.058736801147461, |
| "rewards/rejected": -7.684670448303223, |
| "step": 248 |
| }, |
| { |
| "epoch": 0.19993977717554953, |
| "grad_norm": 0.6453978419303894, |
| "learning_rate": 2.689778323090317e-05, |
| "logits/chosen": -7.1092987060546875, |
| "logits/rejected": -6.585870265960693, |
| "logps/chosen": -267.34722900390625, |
| "logps/rejected": -333.858154296875, |
| "loss": 0.0706, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.47900617122650146, |
| "rewards/margins": 7.373687744140625, |
| "rewards/rejected": -7.852694034576416, |
| "step": 249 |
| }, |
| { |
| "epoch": 0.20074274816822243, |
| "grad_norm": 1.1177862882614136, |
| "learning_rate": 2.686751032652033e-05, |
| "logits/chosen": -7.066874027252197, |
| "logits/rejected": -6.593591213226318, |
| "logps/chosen": -246.0029296875, |
| "logps/rejected": -297.5874938964844, |
| "loss": 0.136, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.9040495157241821, |
| "rewards/margins": 6.5052900314331055, |
| "rewards/rejected": -7.40933895111084, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.20074274816822243, |
| "eval_logits/chosen": -7.049773216247559, |
| "eval_logits/rejected": -6.588283061981201, |
| "eval_logps/chosen": -249.5520782470703, |
| "eval_logps/rejected": -299.0399169921875, |
| "eval_loss": 0.11633451282978058, |
| "eval_rewards/accuracies": 0.9434846639633179, |
| "eval_rewards/chosen": -0.7757118344306946, |
| "eval_rewards/margins": 6.951329231262207, |
| "eval_rewards/rejected": -7.727040767669678, |
| "eval_runtime": 714.4468, |
| "eval_samples_per_second": 49.582, |
| "eval_steps_per_second": 1.549, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.2015457191608953, |
| "grad_norm": 0.7219516634941101, |
| "learning_rate": 2.6837107640945904e-05, |
| "logits/chosen": -7.067862510681152, |
| "logits/rejected": -6.528360843658447, |
| "logps/chosen": -225.75608825683594, |
| "logps/rejected": -292.418701171875, |
| "loss": 0.1001, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.7914926409721375, |
| "rewards/margins": 7.444153785705566, |
| "rewards/rejected": -8.23564624786377, |
| "step": 251 |
| }, |
| { |
| "epoch": 0.2023486901535682, |
| "grad_norm": 0.9089459776878357, |
| "learning_rate": 2.680657550665879e-05, |
| "logits/chosen": -6.910080909729004, |
| "logits/rejected": -6.54853630065918, |
| "logps/chosen": -264.0087585449219, |
| "logps/rejected": -294.68719482421875, |
| "loss": 0.1115, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.7029778361320496, |
| "rewards/margins": 6.817586898803711, |
| "rewards/rejected": -7.520565032958984, |
| "step": 252 |
| }, |
| { |
| "epoch": 0.20315166114624109, |
| "grad_norm": 0.8644925951957703, |
| "learning_rate": 2.6775914257553517e-05, |
| "logits/chosen": -6.9857683181762695, |
| "logits/rejected": -6.492401123046875, |
| "logps/chosen": -276.776611328125, |
| "logps/rejected": -310.47760009765625, |
| "loss": 0.129, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.14865434169769287, |
| "rewards/margins": 7.059405326843262, |
| "rewards/rejected": -7.2080607414245605, |
| "step": 253 |
| }, |
| { |
| "epoch": 0.203954632138914, |
| "grad_norm": 1.2983856201171875, |
| "learning_rate": 2.674512422893659e-05, |
| "logits/chosen": -7.262933731079102, |
| "logits/rejected": -6.705418109893799, |
| "logps/chosen": -273.8624572753906, |
| "logps/rejected": -341.263916015625, |
| "loss": 0.1378, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.43835729360580444, |
| "rewards/margins": 7.259766101837158, |
| "rewards/rejected": -7.698123455047607, |
| "step": 254 |
| }, |
| { |
| "epoch": 0.20475760313158686, |
| "grad_norm": 0.9966720938682556, |
| "learning_rate": 2.6714205757522828e-05, |
| "logits/chosen": -7.04391622543335, |
| "logits/rejected": -6.5014801025390625, |
| "logps/chosen": -232.0685577392578, |
| "logps/rejected": -283.8289489746094, |
| "loss": 0.1312, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.700526237487793, |
| "rewards/margins": 7.055577754974365, |
| "rewards/rejected": -7.756103515625, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.20556057412425977, |
| "grad_norm": 0.7574974894523621, |
| "learning_rate": 2.668315918143169e-05, |
| "logits/chosen": -7.052381992340088, |
| "logits/rejected": -6.607940673828125, |
| "logps/chosen": -265.0798645019531, |
| "logps/rejected": -319.6061706542969, |
| "loss": 0.1, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.5573646426200867, |
| "rewards/margins": 7.151032447814941, |
| "rewards/rejected": -7.708396911621094, |
| "step": 256 |
| }, |
| { |
| "epoch": 0.20636354511693264, |
| "grad_norm": 0.8103229403495789, |
| "learning_rate": 2.6651984840183545e-05, |
| "logits/chosen": -6.817974090576172, |
| "logits/rejected": -6.325244903564453, |
| "logps/chosen": -262.6474609375, |
| "logps/rejected": -299.1306457519531, |
| "loss": 0.1127, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.5646272897720337, |
| "rewards/margins": 7.011369705200195, |
| "rewards/rejected": -7.575996398925781, |
| "step": 257 |
| }, |
| { |
| "epoch": 0.20716651610960554, |
| "grad_norm": 0.6573091149330139, |
| "learning_rate": 2.6620683074696e-05, |
| "logits/chosen": -7.120821475982666, |
| "logits/rejected": -6.629361152648926, |
| "logps/chosen": -242.10784912109375, |
| "logps/rejected": -298.9933776855469, |
| "loss": 0.0864, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.8143295049667358, |
| "rewards/margins": 7.263848304748535, |
| "rewards/rejected": -8.078177452087402, |
| "step": 258 |
| }, |
| { |
| "epoch": 0.20796948710227842, |
| "grad_norm": 1.0727308988571167, |
| "learning_rate": 2.6589254227280153e-05, |
| "logits/chosen": -7.1261725425720215, |
| "logits/rejected": -6.705826759338379, |
| "logps/chosen": -254.943115234375, |
| "logps/rejected": -301.5133056640625, |
| "loss": 0.1534, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.6636333465576172, |
| "rewards/margins": 6.903466701507568, |
| "rewards/rejected": -7.567100524902344, |
| "step": 259 |
| }, |
| { |
| "epoch": 0.20877245809495132, |
| "grad_norm": 0.9400967955589294, |
| "learning_rate": 2.655769864163684e-05, |
| "logits/chosen": -7.019414901733398, |
| "logits/rejected": -6.612437725067139, |
| "logps/chosen": -250.4803466796875, |
| "logps/rejected": -304.8699951171875, |
| "loss": 0.1463, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.4649803936481476, |
| "rewards/margins": 6.883397579193115, |
| "rewards/rejected": -7.3483781814575195, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.20957542908762422, |
| "grad_norm": 0.9623787999153137, |
| "learning_rate": 2.6526016662852887e-05, |
| "logits/chosen": -7.093018531799316, |
| "logits/rejected": -6.609579086303711, |
| "logps/chosen": -259.5165710449219, |
| "logps/rejected": -299.682373046875, |
| "loss": 0.1214, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.5595082640647888, |
| "rewards/margins": 6.968295097351074, |
| "rewards/rejected": -7.527803421020508, |
| "step": 261 |
| }, |
| { |
| "epoch": 0.2103784000802971, |
| "grad_norm": 0.9380446672439575, |
| "learning_rate": 2.6494208637397337e-05, |
| "logits/chosen": -7.131163597106934, |
| "logits/rejected": -6.57238245010376, |
| "logps/chosen": -240.54371643066406, |
| "logps/rejected": -296.6920471191406, |
| "loss": 0.1454, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7259230017662048, |
| "rewards/margins": 6.411162853240967, |
| "rewards/rejected": -7.137085437774658, |
| "step": 262 |
| }, |
| { |
| "epoch": 0.21118137107297, |
| "grad_norm": 0.7752313613891602, |
| "learning_rate": 2.6462274913117666e-05, |
| "logits/chosen": -7.228878498077393, |
| "logits/rejected": -6.697650909423828, |
| "logps/chosen": -240.008056640625, |
| "logps/rejected": -293.8602294921875, |
| "loss": 0.1309, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.39782315492630005, |
| "rewards/margins": 6.976722240447998, |
| "rewards/rejected": -7.374545097351074, |
| "step": 263 |
| }, |
| { |
| "epoch": 0.21198434206564287, |
| "grad_norm": 0.9595717787742615, |
| "learning_rate": 2.6430215839235966e-05, |
| "logits/chosen": -6.837466716766357, |
| "logits/rejected": -6.465234279632568, |
| "logps/chosen": -246.7811737060547, |
| "logps/rejected": -276.33697509765625, |
| "loss": 0.1233, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5251038074493408, |
| "rewards/margins": 6.526254653930664, |
| "rewards/rejected": -7.051359176635742, |
| "step": 264 |
| }, |
| { |
| "epoch": 0.21278731305831577, |
| "grad_norm": 0.7899120450019836, |
| "learning_rate": 2.6398031766345133e-05, |
| "logits/chosen": -7.406018257141113, |
| "logits/rejected": -6.871142387390137, |
| "logps/chosen": -244.82952880859375, |
| "logps/rejected": -302.47003173828125, |
| "loss": 0.0777, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.47826892137527466, |
| "rewards/margins": 7.0176801681518555, |
| "rewards/rejected": -7.495948791503906, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.21359028405098865, |
| "grad_norm": 0.8629099130630493, |
| "learning_rate": 2.6365723046405022e-05, |
| "logits/chosen": -7.01243782043457, |
| "logits/rejected": -6.679934978485107, |
| "logps/chosen": -232.23947143554688, |
| "logps/rejected": -275.5145263671875, |
| "loss": 0.1392, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.1589539647102356, |
| "rewards/margins": 6.676060676574707, |
| "rewards/rejected": -6.835014343261719, |
| "step": 266 |
| }, |
| { |
| "epoch": 0.21439325504366155, |
| "grad_norm": 0.572425365447998, |
| "learning_rate": 2.6333290032738626e-05, |
| "logits/chosen": -7.2883758544921875, |
| "logits/rejected": -6.740113258361816, |
| "logps/chosen": -246.82115173339844, |
| "logps/rejected": -279.5056457519531, |
| "loss": 0.0865, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.5019100308418274, |
| "rewards/margins": 6.780235290527344, |
| "rewards/rejected": -7.2821455001831055, |
| "step": 267 |
| }, |
| { |
| "epoch": 0.21519622603633443, |
| "grad_norm": 0.8958895802497864, |
| "learning_rate": 2.630073308002819e-05, |
| "logits/chosen": -7.007678508758545, |
| "logits/rejected": -6.616161823272705, |
| "logps/chosen": -272.045654296875, |
| "logps/rejected": -301.95159912109375, |
| "loss": 0.1145, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.440017968416214, |
| "rewards/margins": 6.732750415802002, |
| "rewards/rejected": -7.172768592834473, |
| "step": 268 |
| }, |
| { |
| "epoch": 0.21599919702900733, |
| "grad_norm": 0.9391863942146301, |
| "learning_rate": 2.6268052544311326e-05, |
| "logits/chosen": -6.958611488342285, |
| "logits/rejected": -6.63724422454834, |
| "logps/chosen": -228.4644012451172, |
| "logps/rejected": -277.26531982421875, |
| "loss": 0.1205, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.6924605369567871, |
| "rewards/margins": 6.6420416831970215, |
| "rewards/rejected": -7.334502220153809, |
| "step": 269 |
| }, |
| { |
| "epoch": 0.21680216802168023, |
| "grad_norm": 0.7772032618522644, |
| "learning_rate": 2.623524878297714e-05, |
| "logits/chosen": -7.030828475952148, |
| "logits/rejected": -6.470731258392334, |
| "logps/chosen": -215.94664001464844, |
| "logps/rejected": -271.24786376953125, |
| "loss": 0.1161, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.4272594749927521, |
| "rewards/margins": 6.838250160217285, |
| "rewards/rejected": -7.265509605407715, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.2176051390143531, |
| "grad_norm": 0.7274638414382935, |
| "learning_rate": 2.620232215476231e-05, |
| "logits/chosen": -7.200814247131348, |
| "logits/rejected": -6.667116165161133, |
| "logps/chosen": -258.4487609863281, |
| "logps/rejected": -300.7549743652344, |
| "loss": 0.0852, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.16493308544158936, |
| "rewards/margins": 7.0916643142700195, |
| "rewards/rejected": -7.256597518920898, |
| "step": 271 |
| }, |
| { |
| "epoch": 0.218408110007026, |
| "grad_norm": 0.8859208226203918, |
| "learning_rate": 2.6169273019747167e-05, |
| "logits/chosen": -7.033838748931885, |
| "logits/rejected": -6.68560266494751, |
| "logps/chosen": -272.8867492675781, |
| "logps/rejected": -300.18798828125, |
| "loss": 0.1143, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6055153012275696, |
| "rewards/margins": 6.835446357727051, |
| "rewards/rejected": -7.440960884094238, |
| "step": 272 |
| }, |
| { |
| "epoch": 0.21921108099969888, |
| "grad_norm": 0.5519632697105408, |
| "learning_rate": 2.613610173935175e-05, |
| "logits/chosen": -7.190359592437744, |
| "logits/rejected": -6.635400295257568, |
| "logps/chosen": -245.51806640625, |
| "logps/rejected": -302.7086486816406, |
| "loss": 0.0657, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.47917255759239197, |
| "rewards/margins": 7.531355381011963, |
| "rewards/rejected": -8.010528564453125, |
| "step": 273 |
| }, |
| { |
| "epoch": 0.22001405199237178, |
| "grad_norm": 0.9850606322288513, |
| "learning_rate": 2.6102808676331877e-05, |
| "logits/chosen": -7.097142219543457, |
| "logits/rejected": -6.683685302734375, |
| "logps/chosen": -260.854248046875, |
| "logps/rejected": -313.39361572265625, |
| "loss": 0.1149, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.7430561184883118, |
| "rewards/margins": 6.890620708465576, |
| "rewards/rejected": -7.6336774826049805, |
| "step": 274 |
| }, |
| { |
| "epoch": 0.22081702298504466, |
| "grad_norm": 1.1795189380645752, |
| "learning_rate": 2.6069394194775147e-05, |
| "logits/chosen": -7.186300754547119, |
| "logits/rejected": -6.738768577575684, |
| "logps/chosen": -257.1663818359375, |
| "logps/rejected": -309.90887451171875, |
| "loss": 0.115, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.5854865312576294, |
| "rewards/margins": 7.3305511474609375, |
| "rewards/rejected": -7.916037559509277, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.22161999397771756, |
| "grad_norm": 0.9400411248207092, |
| "learning_rate": 2.6035858660096975e-05, |
| "logits/chosen": -6.97589111328125, |
| "logits/rejected": -6.516839027404785, |
| "logps/chosen": -253.39967346191406, |
| "logps/rejected": -292.86212158203125, |
| "loss": 0.1198, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.8843185901641846, |
| "rewards/margins": 6.837711334228516, |
| "rewards/rejected": -7.722029685974121, |
| "step": 276 |
| }, |
| { |
| "epoch": 0.22242296497039044, |
| "grad_norm": 0.6403718590736389, |
| "learning_rate": 2.60022024390366e-05, |
| "logits/chosen": -7.046829700469971, |
| "logits/rejected": -6.580545425415039, |
| "logps/chosen": -233.23544311523438, |
| "logps/rejected": -282.2193603515625, |
| "loss": 0.0836, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8401697278022766, |
| "rewards/margins": 7.291221618652344, |
| "rewards/rejected": -8.131390571594238, |
| "step": 277 |
| }, |
| { |
| "epoch": 0.22322593596306334, |
| "grad_norm": 1.0084043741226196, |
| "learning_rate": 2.5968425899653055e-05, |
| "logits/chosen": -6.9526848793029785, |
| "logits/rejected": -6.513876914978027, |
| "logps/chosen": -246.982421875, |
| "logps/rejected": -298.6771240234375, |
| "loss": 0.117, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.0391035079956055, |
| "rewards/margins": 7.317519664764404, |
| "rewards/rejected": -8.356622695922852, |
| "step": 278 |
| }, |
| { |
| "epoch": 0.2240289069557362, |
| "grad_norm": 0.5959298014640808, |
| "learning_rate": 2.5934529411321174e-05, |
| "logits/chosen": -7.220340728759766, |
| "logits/rejected": -6.698398590087891, |
| "logps/chosen": -266.0975646972656, |
| "logps/rejected": -333.0186767578125, |
| "loss": 0.0588, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.5001435279846191, |
| "rewards/margins": 7.762099266052246, |
| "rewards/rejected": -8.262243270874023, |
| "step": 279 |
| }, |
| { |
| "epoch": 0.22483187794840911, |
| "grad_norm": 0.7655050158500671, |
| "learning_rate": 2.590051334472751e-05, |
| "logits/chosen": -7.2204718589782715, |
| "logits/rejected": -6.635551929473877, |
| "logps/chosen": -248.95925903320312, |
| "logps/rejected": -296.4516906738281, |
| "loss": 0.0863, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.8780251145362854, |
| "rewards/margins": 7.1864705085754395, |
| "rewards/rejected": -8.064496040344238, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.22563484894108202, |
| "grad_norm": 1.1339064836502075, |
| "learning_rate": 2.5866378071866338e-05, |
| "logits/chosen": -7.2281646728515625, |
| "logits/rejected": -6.7415080070495605, |
| "logps/chosen": -256.5589599609375, |
| "logps/rejected": -307.8895263671875, |
| "loss": 0.1337, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.9373075366020203, |
| "rewards/margins": 7.127184867858887, |
| "rewards/rejected": -8.064493179321289, |
| "step": 281 |
| }, |
| { |
| "epoch": 0.2264378199337549, |
| "grad_norm": 1.13326895236969, |
| "learning_rate": 2.583212396603551e-05, |
| "logits/chosen": -7.03151273727417, |
| "logits/rejected": -6.534742832183838, |
| "logps/chosen": -238.08740234375, |
| "logps/rejected": -294.03021240234375, |
| "loss": 0.0968, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6004853844642639, |
| "rewards/margins": 7.366070747375488, |
| "rewards/rejected": -7.966556549072266, |
| "step": 282 |
| }, |
| { |
| "epoch": 0.2272407909264278, |
| "grad_norm": 0.5554599165916443, |
| "learning_rate": 2.579775140183245e-05, |
| "logits/chosen": -7.068852424621582, |
| "logits/rejected": -6.522769451141357, |
| "logps/chosen": -262.4130554199219, |
| "logps/rejected": -320.53277587890625, |
| "loss": 0.0586, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.4434080719947815, |
| "rewards/margins": 7.625604629516602, |
| "rewards/rejected": -8.069012641906738, |
| "step": 283 |
| }, |
| { |
| "epoch": 0.22804376191910067, |
| "grad_norm": 0.7181034684181213, |
| "learning_rate": 2.5763260755150013e-05, |
| "logits/chosen": -7.026515960693359, |
| "logits/rejected": -6.682558059692383, |
| "logps/chosen": -260.947021484375, |
| "logps/rejected": -305.22283935546875, |
| "loss": 0.0923, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.8288884162902832, |
| "rewards/margins": 7.025484085083008, |
| "rewards/rejected": -7.854372024536133, |
| "step": 284 |
| }, |
| { |
| "epoch": 0.22884673291177357, |
| "grad_norm": 0.9962047934532166, |
| "learning_rate": 2.5728652403172384e-05, |
| "logits/chosen": -7.102780818939209, |
| "logits/rejected": -6.564681529998779, |
| "logps/chosen": -253.6741180419922, |
| "logps/rejected": -320.7935485839844, |
| "loss": 0.124, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3143165707588196, |
| "rewards/margins": 6.892307281494141, |
| "rewards/rejected": -7.2066240310668945, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.22964970390444644, |
| "grad_norm": 0.8722308874130249, |
| "learning_rate": 2.5693926724370958e-05, |
| "logits/chosen": -7.249421119689941, |
| "logits/rejected": -6.792860984802246, |
| "logps/chosen": -246.83181762695312, |
| "logps/rejected": -284.97906494140625, |
| "loss": 0.1057, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.039907556027173996, |
| "rewards/margins": 7.040358543395996, |
| "rewards/rejected": -7.080265998840332, |
| "step": 286 |
| }, |
| { |
| "epoch": 0.23045267489711935, |
| "grad_norm": 0.8343302607536316, |
| "learning_rate": 2.565908409850019e-05, |
| "logits/chosen": -7.208025932312012, |
| "logits/rejected": -6.787450790405273, |
| "logps/chosen": -254.50367736816406, |
| "logps/rejected": -297.1814880371094, |
| "loss": 0.0907, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.07483983039855957, |
| "rewards/margins": 7.235809803009033, |
| "rewards/rejected": -7.310649871826172, |
| "step": 287 |
| }, |
| { |
| "epoch": 0.23125564588979222, |
| "grad_norm": 0.7909135818481445, |
| "learning_rate": 2.5624124906593458e-05, |
| "logits/chosen": -7.015960216522217, |
| "logits/rejected": -6.693521976470947, |
| "logps/chosen": -241.93495178222656, |
| "logps/rejected": -302.6620788574219, |
| "loss": 0.104, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.6034737229347229, |
| "rewards/margins": 6.832521915435791, |
| "rewards/rejected": -7.435996055603027, |
| "step": 288 |
| }, |
| { |
| "epoch": 0.23205861688246512, |
| "grad_norm": 0.9027450084686279, |
| "learning_rate": 2.5589049530958885e-05, |
| "logits/chosen": -7.181616306304932, |
| "logits/rejected": -6.640651226043701, |
| "logps/chosen": -258.97747802734375, |
| "logps/rejected": -313.831787109375, |
| "loss": 0.1311, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8030258417129517, |
| "rewards/margins": 6.819934368133545, |
| "rewards/rejected": -7.622961044311523, |
| "step": 289 |
| }, |
| { |
| "epoch": 0.232861587875138, |
| "grad_norm": 0.7640787363052368, |
| "learning_rate": 2.5553858355175156e-05, |
| "logits/chosen": -7.018610000610352, |
| "logits/rejected": -6.727492332458496, |
| "logps/chosen": -287.6529541015625, |
| "logps/rejected": -315.2305908203125, |
| "loss": 0.1285, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.5612961053848267, |
| "rewards/margins": 6.882949352264404, |
| "rewards/rejected": -7.444245338439941, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.2336645588678109, |
| "grad_norm": 0.8705466985702515, |
| "learning_rate": 2.5518551764087326e-05, |
| "logits/chosen": -7.365253925323486, |
| "logits/rejected": -6.764880657196045, |
| "logps/chosen": -230.2239227294922, |
| "logps/rejected": -304.6335754394531, |
| "loss": 0.1214, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.4071289300918579, |
| "rewards/margins": 7.063294887542725, |
| "rewards/rejected": -7.470423221588135, |
| "step": 291 |
| }, |
| { |
| "epoch": 0.2344675298604838, |
| "grad_norm": 0.9589917063713074, |
| "learning_rate": 2.5483130143802632e-05, |
| "logits/chosen": -7.022066593170166, |
| "logits/rejected": -6.467358112335205, |
| "logps/chosen": -240.3975372314453, |
| "logps/rejected": -300.68084716796875, |
| "loss": 0.1405, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.6492630243301392, |
| "rewards/margins": 6.790482997894287, |
| "rewards/rejected": -7.439745903015137, |
| "step": 292 |
| }, |
| { |
| "epoch": 0.23527050085315668, |
| "grad_norm": 0.7496669888496399, |
| "learning_rate": 2.5447593881686235e-05, |
| "logits/chosen": -7.1047563552856445, |
| "logits/rejected": -6.666567325592041, |
| "logps/chosen": -251.41395568847656, |
| "logps/rejected": -324.03973388671875, |
| "loss": 0.1161, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4329589009284973, |
| "rewards/margins": 7.043006420135498, |
| "rewards/rejected": -7.47596549987793, |
| "step": 293 |
| }, |
| { |
| "epoch": 0.23607347184582958, |
| "grad_norm": 1.0116653442382812, |
| "learning_rate": 2.5411943366356994e-05, |
| "logits/chosen": -7.196882247924805, |
| "logits/rejected": -6.742383003234863, |
| "logps/chosen": -276.7082824707031, |
| "logps/rejected": -313.4183044433594, |
| "loss": 0.1143, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.087460458278656, |
| "rewards/margins": 7.186445713043213, |
| "rewards/rejected": -7.273906707763672, |
| "step": 294 |
| }, |
| { |
| "epoch": 0.23687644283850245, |
| "grad_norm": 0.8734626770019531, |
| "learning_rate": 2.537617898768324e-05, |
| "logits/chosen": -6.950597763061523, |
| "logits/rejected": -6.39291524887085, |
| "logps/chosen": -253.1763153076172, |
| "logps/rejected": -318.060302734375, |
| "loss": 0.1182, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.25249069929122925, |
| "rewards/margins": 6.689419746398926, |
| "rewards/rejected": -6.941910266876221, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.23767941383117536, |
| "grad_norm": 0.7076531052589417, |
| "learning_rate": 2.534030113677849e-05, |
| "logits/chosen": -7.224164009094238, |
| "logits/rejected": -6.800457000732422, |
| "logps/chosen": -229.44015502929688, |
| "logps/rejected": -282.5396728515625, |
| "loss": 0.0862, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.44332632422447205, |
| "rewards/margins": 7.231780529022217, |
| "rewards/rejected": -7.675107479095459, |
| "step": 296 |
| }, |
| { |
| "epoch": 0.23848238482384823, |
| "grad_norm": 0.9676220417022705, |
| "learning_rate": 2.5304310205997168e-05, |
| "logits/chosen": -7.126812934875488, |
| "logits/rejected": -6.740604400634766, |
| "logps/chosen": -281.7055358886719, |
| "logps/rejected": -324.7397766113281, |
| "loss": 0.1124, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.45793235301971436, |
| "rewards/margins": 7.008180141448975, |
| "rewards/rejected": -7.46611213684082, |
| "step": 297 |
| }, |
| { |
| "epoch": 0.23928535581652113, |
| "grad_norm": 0.8058264255523682, |
| "learning_rate": 2.5268206588930332e-05, |
| "logits/chosen": -7.145132541656494, |
| "logits/rejected": -6.706300258636475, |
| "logps/chosen": -253.83493041992188, |
| "logps/rejected": -306.24072265625, |
| "loss": 0.0915, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.3610036373138428, |
| "rewards/margins": 7.147863864898682, |
| "rewards/rejected": -7.508867263793945, |
| "step": 298 |
| }, |
| { |
| "epoch": 0.240088326809194, |
| "grad_norm": 0.7874707579612732, |
| "learning_rate": 2.523199068040135e-05, |
| "logits/chosen": -7.1350555419921875, |
| "logits/rejected": -6.818406105041504, |
| "logps/chosen": -258.18115234375, |
| "logps/rejected": -293.64471435546875, |
| "loss": 0.0912, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.15227629244327545, |
| "rewards/margins": 7.395872592926025, |
| "rewards/rejected": -7.548149108886719, |
| "step": 299 |
| }, |
| { |
| "epoch": 0.2408912978018669, |
| "grad_norm": 0.7579779028892517, |
| "learning_rate": 2.51956628764616e-05, |
| "logits/chosen": -7.205224990844727, |
| "logits/rejected": -6.645421504974365, |
| "logps/chosen": -234.98594665527344, |
| "logps/rejected": -285.3193359375, |
| "loss": 0.099, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.1901416778564453, |
| "rewards/margins": 7.560548782348633, |
| "rewards/rejected": -7.75068998336792, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.24169426879453978, |
| "grad_norm": 0.6849891543388367, |
| "learning_rate": 2.5159223574386117e-05, |
| "logits/chosen": -6.903081893920898, |
| "logits/rejected": -6.467089653015137, |
| "logps/chosen": -246.24853515625, |
| "logps/rejected": -284.72564697265625, |
| "loss": 0.085, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.40131676197052, |
| "rewards/margins": 7.419934272766113, |
| "rewards/rejected": -7.821250915527344, |
| "step": 301 |
| }, |
| { |
| "epoch": 0.2424972397872127, |
| "grad_norm": 0.827204167842865, |
| "learning_rate": 2.5122673172669278e-05, |
| "logits/chosen": -7.16361141204834, |
| "logits/rejected": -6.721137523651123, |
| "logps/chosen": -231.84799194335938, |
| "logps/rejected": -282.978759765625, |
| "loss": 0.0877, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6805617809295654, |
| "rewards/margins": 7.346146583557129, |
| "rewards/rejected": -8.026708602905273, |
| "step": 302 |
| }, |
| { |
| "epoch": 0.2433002107798856, |
| "grad_norm": 0.9220207333564758, |
| "learning_rate": 2.508601207102042e-05, |
| "logits/chosen": -7.09849214553833, |
| "logits/rejected": -6.624366760253906, |
| "logps/chosen": -249.54847717285156, |
| "logps/rejected": -317.1817932128906, |
| "loss": 0.1066, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.7175077795982361, |
| "rewards/margins": 7.394064426422119, |
| "rewards/rejected": -8.111572265625, |
| "step": 303 |
| }, |
| { |
| "epoch": 0.24410318177255846, |
| "grad_norm": 1.0148415565490723, |
| "learning_rate": 2.5049240670359476e-05, |
| "logits/chosen": -7.04465389251709, |
| "logits/rejected": -6.602618217468262, |
| "logps/chosen": -214.12237548828125, |
| "logps/rejected": -270.5555725097656, |
| "loss": 0.1306, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.3492436110973358, |
| "rewards/margins": 6.99552059173584, |
| "rewards/rejected": -7.344764232635498, |
| "step": 304 |
| }, |
| { |
| "epoch": 0.24490615276523137, |
| "grad_norm": 0.7954779863357544, |
| "learning_rate": 2.501235937281259e-05, |
| "logits/chosen": -7.143594741821289, |
| "logits/rejected": -6.606503486633301, |
| "logps/chosen": -239.0674285888672, |
| "logps/rejected": -305.9482116699219, |
| "loss": 0.0947, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.0706334114074707, |
| "rewards/margins": 7.059913635253906, |
| "rewards/rejected": -8.130546569824219, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.24570912375790424, |
| "grad_norm": 1.0818005800247192, |
| "learning_rate": 2.4975368581707724e-05, |
| "logits/chosen": -7.120449066162109, |
| "logits/rejected": -6.668785095214844, |
| "logps/chosen": -254.53501892089844, |
| "logps/rejected": -307.1061706542969, |
| "loss": 0.1456, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.27733150124549866, |
| "rewards/margins": 7.069423198699951, |
| "rewards/rejected": -7.346755027770996, |
| "step": 306 |
| }, |
| { |
| "epoch": 0.24651209475057714, |
| "grad_norm": 0.8592018485069275, |
| "learning_rate": 2.4938268701570245e-05, |
| "logits/chosen": -6.886562824249268, |
| "logits/rejected": -6.416623115539551, |
| "logps/chosen": -261.11273193359375, |
| "logps/rejected": -307.96600341796875, |
| "loss": 0.0994, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.367803156375885, |
| "rewards/margins": 7.248906135559082, |
| "rewards/rejected": -7.6167097091674805, |
| "step": 307 |
| }, |
| { |
| "epoch": 0.24731506574325002, |
| "grad_norm": 1.0856597423553467, |
| "learning_rate": 2.4901060138118502e-05, |
| "logits/chosen": -6.883216381072998, |
| "logits/rejected": -6.515850067138672, |
| "logps/chosen": -267.6678466796875, |
| "logps/rejected": -305.791259765625, |
| "loss": 0.1579, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.6215819120407104, |
| "rewards/margins": 6.756573677062988, |
| "rewards/rejected": -7.37815523147583, |
| "step": 308 |
| }, |
| { |
| "epoch": 0.24811803673592292, |
| "grad_norm": 2.6424474716186523, |
| "learning_rate": 2.486374329825937e-05, |
| "logits/chosen": -7.339808464050293, |
| "logits/rejected": -6.784490585327148, |
| "logps/chosen": -238.132080078125, |
| "logps/rejected": -292.2194519042969, |
| "loss": 0.1354, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.3008340895175934, |
| "rewards/margins": 7.674061298370361, |
| "rewards/rejected": -7.97489595413208, |
| "step": 309 |
| }, |
| { |
| "epoch": 0.2489210077285958, |
| "grad_norm": 1.027953863143921, |
| "learning_rate": 2.482631859008384e-05, |
| "logits/chosen": -7.216251373291016, |
| "logits/rejected": -6.64981746673584, |
| "logps/chosen": -266.1959228515625, |
| "logps/rejected": -304.2769470214844, |
| "loss": 0.1137, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.35820472240448, |
| "rewards/margins": 6.899946212768555, |
| "rewards/rejected": -7.258150100708008, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.2497239787212687, |
| "grad_norm": 0.6347997188568115, |
| "learning_rate": 2.478878642286253e-05, |
| "logits/chosen": -7.188714981079102, |
| "logits/rejected": -6.616113662719727, |
| "logps/chosen": -213.85801696777344, |
| "logps/rejected": -266.7655334472656, |
| "loss": 0.0916, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.07190874218940735, |
| "rewards/margins": 6.639386177062988, |
| "rewards/rejected": -6.711293697357178, |
| "step": 311 |
| }, |
| { |
| "epoch": 0.25052694971394157, |
| "grad_norm": 0.9738149642944336, |
| "learning_rate": 2.4751147207041194e-05, |
| "logits/chosen": -7.021158218383789, |
| "logits/rejected": -6.658725738525391, |
| "logps/chosen": -244.0069122314453, |
| "logps/rejected": -291.40521240234375, |
| "loss": 0.1125, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.11363965272903442, |
| "rewards/margins": 6.79201602935791, |
| "rewards/rejected": -6.905656337738037, |
| "step": 312 |
| }, |
| { |
| "epoch": 0.2513299207066145, |
| "grad_norm": 0.8075509667396545, |
| "learning_rate": 2.4713401354236282e-05, |
| "logits/chosen": -7.065709590911865, |
| "logits/rejected": -6.517449378967285, |
| "logps/chosen": -233.1536102294922, |
| "logps/rejected": -291.3530578613281, |
| "loss": 0.1306, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": 0.2563735544681549, |
| "rewards/margins": 6.326435089111328, |
| "rewards/rejected": -6.070061206817627, |
| "step": 313 |
| }, |
| { |
| "epoch": 0.2521328916992874, |
| "grad_norm": 0.9227325916290283, |
| "learning_rate": 2.4675549277230382e-05, |
| "logits/chosen": -7.401317596435547, |
| "logits/rejected": -7.012799263000488, |
| "logps/chosen": -223.1483154296875, |
| "logps/rejected": -287.06982421875, |
| "loss": 0.1262, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.029965095221996307, |
| "rewards/margins": 6.710182189941406, |
| "rewards/rejected": -6.740147590637207, |
| "step": 314 |
| }, |
| { |
| "epoch": 0.2529358626919603, |
| "grad_norm": 0.7337055802345276, |
| "learning_rate": 2.463759138996775e-05, |
| "logits/chosen": -7.104825496673584, |
| "logits/rejected": -6.664860725402832, |
| "logps/chosen": -238.87930297851562, |
| "logps/rejected": -260.3129577636719, |
| "loss": 0.1022, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": 0.46236589550971985, |
| "rewards/margins": 6.5408759117126465, |
| "rewards/rejected": -6.07850980758667, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.2537388336846331, |
| "grad_norm": 0.8037142157554626, |
| "learning_rate": 2.459952810754975e-05, |
| "logits/chosen": -7.411218643188477, |
| "logits/rejected": -6.833322525024414, |
| "logps/chosen": -242.4769744873047, |
| "logps/rejected": -299.2050476074219, |
| "loss": 0.1177, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": 0.5150518417358398, |
| "rewards/margins": 6.743181228637695, |
| "rewards/rejected": -6.228128910064697, |
| "step": 316 |
| }, |
| { |
| "epoch": 0.254541804677306, |
| "grad_norm": 0.6726399064064026, |
| "learning_rate": 2.4561359846230346e-05, |
| "logits/chosen": -7.36439323425293, |
| "logits/rejected": -6.893357276916504, |
| "logps/chosen": -211.25872802734375, |
| "logps/rejected": -264.9126892089844, |
| "loss": 0.0815, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.6482738852500916, |
| "rewards/margins": 7.223107814788818, |
| "rewards/rejected": -6.574834823608398, |
| "step": 317 |
| }, |
| { |
| "epoch": 0.25534477566997893, |
| "grad_norm": 0.9088069796562195, |
| "learning_rate": 2.452308702341153e-05, |
| "logits/chosen": -7.144931793212891, |
| "logits/rejected": -6.675646781921387, |
| "logps/chosen": -263.61651611328125, |
| "logps/rejected": -300.9358825683594, |
| "loss": 0.1206, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": 0.2616302967071533, |
| "rewards/margins": 6.753662109375, |
| "rewards/rejected": -6.492032051086426, |
| "step": 318 |
| }, |
| { |
| "epoch": 0.25614774666265183, |
| "grad_norm": 0.9113364219665527, |
| "learning_rate": 2.4484710057638763e-05, |
| "logits/chosen": -7.347232818603516, |
| "logits/rejected": -6.714489936828613, |
| "logps/chosen": -219.88046264648438, |
| "logps/rejected": -271.4178161621094, |
| "loss": 0.1407, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.1510578691959381, |
| "rewards/margins": 6.8534111976623535, |
| "rewards/rejected": -7.004469871520996, |
| "step": 319 |
| }, |
| { |
| "epoch": 0.2569507176553247, |
| "grad_norm": 0.7765845060348511, |
| "learning_rate": 2.4446229368596388e-05, |
| "logits/chosen": -7.194561958312988, |
| "logits/rejected": -6.709545612335205, |
| "logps/chosen": -221.9159698486328, |
| "logps/rejected": -292.2004089355469, |
| "loss": 0.1085, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.4721475839614868, |
| "rewards/margins": 7.099097728729248, |
| "rewards/rejected": -7.571245193481445, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.2577536886479976, |
| "grad_norm": 0.8845237493515015, |
| "learning_rate": 2.4407645377103056e-05, |
| "logits/chosen": -7.145198822021484, |
| "logits/rejected": -6.638523578643799, |
| "logps/chosen": -271.4456481933594, |
| "logps/rejected": -309.3156433105469, |
| "loss": 0.1234, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": 0.14606264233589172, |
| "rewards/margins": 7.39940881729126, |
| "rewards/rejected": -7.2533464431762695, |
| "step": 321 |
| }, |
| { |
| "epoch": 0.2585566596406705, |
| "grad_norm": 0.9140969514846802, |
| "learning_rate": 2.436895850510712e-05, |
| "logits/chosen": -7.073066711425781, |
| "logits/rejected": -6.4587721824646, |
| "logps/chosen": -243.04190063476562, |
| "logps/rejected": -307.6632385253906, |
| "loss": 0.1159, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.5952476263046265, |
| "rewards/margins": 7.134223461151123, |
| "rewards/rejected": -7.729471206665039, |
| "step": 322 |
| }, |
| { |
| "epoch": 0.2593596306333434, |
| "grad_norm": 0.7821839451789856, |
| "learning_rate": 2.433016917568201e-05, |
| "logits/chosen": -7.053502559661865, |
| "logits/rejected": -6.518143177032471, |
| "logps/chosen": -235.94712829589844, |
| "logps/rejected": -300.259765625, |
| "loss": 0.0845, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.26944661140441895, |
| "rewards/margins": 8.15035629272461, |
| "rewards/rejected": -8.41980266571045, |
| "step": 323 |
| }, |
| { |
| "epoch": 0.2601626016260163, |
| "grad_norm": 1.0952534675598145, |
| "learning_rate": 2.4291277813021623e-05, |
| "logits/chosen": -7.311715126037598, |
| "logits/rejected": -6.812038421630859, |
| "logps/chosen": -243.85963439941406, |
| "logps/rejected": -288.942626953125, |
| "loss": 0.0981, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.3787408769130707, |
| "rewards/margins": 7.947113990783691, |
| "rewards/rejected": -8.325854301452637, |
| "step": 324 |
| }, |
| { |
| "epoch": 0.26096557261868913, |
| "grad_norm": 0.9332665801048279, |
| "learning_rate": 2.4252284842435667e-05, |
| "logits/chosen": -7.043581485748291, |
| "logits/rejected": -6.566289901733398, |
| "logps/chosen": -267.8500061035156, |
| "logps/rejected": -310.16558837890625, |
| "loss": 0.1187, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.7956825494766235, |
| "rewards/margins": 7.570168972015381, |
| "rewards/rejected": -8.365851402282715, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.26176854361136204, |
| "grad_norm": 0.6914594173431396, |
| "learning_rate": 2.4213190690345018e-05, |
| "logits/chosen": -6.971610069274902, |
| "logits/rejected": -6.4836907386779785, |
| "logps/chosen": -267.9631042480469, |
| "logps/rejected": -335.6561584472656, |
| "loss": 0.0864, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5961199402809143, |
| "rewards/margins": 7.8604912757873535, |
| "rewards/rejected": -8.456611633300781, |
| "step": 326 |
| }, |
| { |
| "epoch": 0.26257151460403494, |
| "grad_norm": 0.7942874431610107, |
| "learning_rate": 2.4173995784277065e-05, |
| "logits/chosen": -7.303437232971191, |
| "logits/rejected": -6.799282073974609, |
| "logps/chosen": -251.81719970703125, |
| "logps/rejected": -323.23504638671875, |
| "loss": 0.0892, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.5071293711662292, |
| "rewards/margins": 8.140687942504883, |
| "rewards/rejected": -8.647817611694336, |
| "step": 327 |
| }, |
| { |
| "epoch": 0.26337448559670784, |
| "grad_norm": 0.8098292946815491, |
| "learning_rate": 2.4134700552861018e-05, |
| "logits/chosen": -6.969442367553711, |
| "logits/rejected": -6.468677520751953, |
| "logps/chosen": -269.0477600097656, |
| "logps/rejected": -317.16021728515625, |
| "loss": 0.0914, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.6387518048286438, |
| "rewards/margins": 7.709242820739746, |
| "rewards/rejected": -8.347993850708008, |
| "step": 328 |
| }, |
| { |
| "epoch": 0.2641774565893807, |
| "grad_norm": 0.9219384789466858, |
| "learning_rate": 2.409530542582324e-05, |
| "logits/chosen": -6.882040977478027, |
| "logits/rejected": -6.479324817657471, |
| "logps/chosen": -260.9018859863281, |
| "logps/rejected": -329.5838317871094, |
| "loss": 0.1243, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.9928497076034546, |
| "rewards/margins": 7.579207897186279, |
| "rewards/rejected": -8.572057723999023, |
| "step": 329 |
| }, |
| { |
| "epoch": 0.2649804275820536, |
| "grad_norm": 1.0734751224517822, |
| "learning_rate": 2.4055810833982512e-05, |
| "logits/chosen": -6.916300296783447, |
| "logits/rejected": -6.422715663909912, |
| "logps/chosen": -239.25523376464844, |
| "logps/rejected": -315.0804748535156, |
| "loss": 0.116, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.0065722465515137, |
| "rewards/margins": 7.732917785644531, |
| "rewards/rejected": -8.739490509033203, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.2657833985747265, |
| "grad_norm": 1.0518746376037598, |
| "learning_rate": 2.4016217209245377e-05, |
| "logits/chosen": -6.968029975891113, |
| "logits/rejected": -6.501012802124023, |
| "logps/chosen": -256.6072998046875, |
| "logps/rejected": -322.8544616699219, |
| "loss": 0.1385, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.6141036152839661, |
| "rewards/margins": 7.661413669586182, |
| "rewards/rejected": -8.275516510009766, |
| "step": 331 |
| }, |
| { |
| "epoch": 0.2665863695673994, |
| "grad_norm": 0.8632018566131592, |
| "learning_rate": 2.397652498460137e-05, |
| "logits/chosen": -6.940664768218994, |
| "logits/rejected": -6.4828596115112305, |
| "logps/chosen": -248.01382446289062, |
| "logps/rejected": -318.27886962890625, |
| "loss": 0.1152, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.565380334854126, |
| "rewards/margins": 7.727349758148193, |
| "rewards/rejected": -8.292730331420898, |
| "step": 332 |
| }, |
| { |
| "epoch": 0.26738934056007224, |
| "grad_norm": 0.9606359601020813, |
| "learning_rate": 2.3936734594118295e-05, |
| "logits/chosen": -7.193376064300537, |
| "logits/rejected": -6.714990615844727, |
| "logps/chosen": -230.35427856445312, |
| "logps/rejected": -294.9207458496094, |
| "loss": 0.1145, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.0302073955535889, |
| "rewards/margins": 7.435544013977051, |
| "rewards/rejected": -8.465751647949219, |
| "step": 333 |
| }, |
| { |
| "epoch": 0.26819231155274514, |
| "grad_norm": 1.0204445123672485, |
| "learning_rate": 2.389684647293749e-05, |
| "logits/chosen": -6.831692218780518, |
| "logits/rejected": -6.416192054748535, |
| "logps/chosen": -269.37335205078125, |
| "logps/rejected": -322.0726013183594, |
| "loss": 0.1579, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.5350735783576965, |
| "rewards/margins": 7.279733180999756, |
| "rewards/rejected": -7.814807415008545, |
| "step": 334 |
| }, |
| { |
| "epoch": 0.26899528254541805, |
| "grad_norm": 0.7498471736907959, |
| "learning_rate": 2.3856861057269058e-05, |
| "logits/chosen": -6.918461322784424, |
| "logits/rejected": -6.515640735626221, |
| "logps/chosen": -238.66897583007812, |
| "logps/rejected": -289.28887939453125, |
| "loss": 0.0854, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.363850474357605, |
| "rewards/margins": 7.716858863830566, |
| "rewards/rejected": -8.080709457397461, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.26979825353809095, |
| "grad_norm": 0.9493563175201416, |
| "learning_rate": 2.3816778784387097e-05, |
| "logits/chosen": -7.102866172790527, |
| "logits/rejected": -6.650218963623047, |
| "logps/chosen": -250.93487548828125, |
| "logps/rejected": -301.5310974121094, |
| "loss": 0.1168, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.6025190353393555, |
| "rewards/margins": 7.428962230682373, |
| "rewards/rejected": -8.03148078918457, |
| "step": 336 |
| }, |
| { |
| "epoch": 0.27060122453076385, |
| "grad_norm": 0.7263461947441101, |
| "learning_rate": 2.3776600092624925e-05, |
| "logits/chosen": -7.151240825653076, |
| "logits/rejected": -6.6239824295043945, |
| "logps/chosen": -226.57876586914062, |
| "logps/rejected": -293.99786376953125, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.5080627202987671, |
| "rewards/margins": 7.369439125061035, |
| "rewards/rejected": -7.877501964569092, |
| "step": 337 |
| }, |
| { |
| "epoch": 0.2714041955234367, |
| "grad_norm": 1.0387061834335327, |
| "learning_rate": 2.3736325421370275e-05, |
| "logits/chosen": -7.194113731384277, |
| "logits/rejected": -6.6587815284729, |
| "logps/chosen": -229.46530151367188, |
| "logps/rejected": -280.972412109375, |
| "loss": 0.1376, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.16640064120292664, |
| "rewards/margins": 7.648268222808838, |
| "rewards/rejected": -7.814669609069824, |
| "step": 338 |
| }, |
| { |
| "epoch": 0.2722071665161096, |
| "grad_norm": 0.9479448199272156, |
| "learning_rate": 2.3695955211060497e-05, |
| "logits/chosen": -7.099273681640625, |
| "logits/rejected": -6.549319744110107, |
| "logps/chosen": -214.00790405273438, |
| "logps/rejected": -270.494384765625, |
| "loss": 0.1595, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.658450186252594, |
| "rewards/margins": 6.866494655609131, |
| "rewards/rejected": -7.5249457359313965, |
| "step": 339 |
| }, |
| { |
| "epoch": 0.2730101375087825, |
| "grad_norm": 0.9099304676055908, |
| "learning_rate": 2.365548990317775e-05, |
| "logits/chosen": -7.184257984161377, |
| "logits/rejected": -6.746098041534424, |
| "logps/chosen": -240.0023651123047, |
| "logps/rejected": -287.3996887207031, |
| "loss": 0.1509, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.2059761881828308, |
| "rewards/margins": 7.4958391189575195, |
| "rewards/rejected": -7.701815128326416, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.2738131085014554, |
| "grad_norm": 0.7760254740715027, |
| "learning_rate": 2.3614929940244155e-05, |
| "logits/chosen": -7.242477893829346, |
| "logits/rejected": -6.7018866539001465, |
| "logps/chosen": -242.55686950683594, |
| "logps/rejected": -298.6614990234375, |
| "loss": 0.1041, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.26043128967285156, |
| "rewards/margins": 7.581421375274658, |
| "rewards/rejected": -7.841852188110352, |
| "step": 341 |
| }, |
| { |
| "epoch": 0.27461607949412825, |
| "grad_norm": 0.6187599301338196, |
| "learning_rate": 2.3574275765816962e-05, |
| "logits/chosen": -7.039289474487305, |
| "logits/rejected": -6.553256988525391, |
| "logps/chosen": -269.30230712890625, |
| "logps/rejected": -307.9279479980469, |
| "loss": 0.0739, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.2465018332004547, |
| "rewards/margins": 7.528380870819092, |
| "rewards/rejected": -7.774882793426514, |
| "step": 342 |
| }, |
| { |
| "epoch": 0.27541905048680115, |
| "grad_norm": 0.7153140306472778, |
| "learning_rate": 2.3533527824483722e-05, |
| "logits/chosen": -7.251349925994873, |
| "logits/rejected": -6.7530012130737305, |
| "logps/chosen": -227.3306121826172, |
| "logps/rejected": -279.5009765625, |
| "loss": 0.0797, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.3342316150665283, |
| "rewards/margins": 7.868841171264648, |
| "rewards/rejected": -8.203073501586914, |
| "step": 343 |
| }, |
| { |
| "epoch": 0.27622202147947406, |
| "grad_norm": 0.8673416376113892, |
| "learning_rate": 2.3492686561857382e-05, |
| "logits/chosen": -6.96228551864624, |
| "logits/rejected": -6.441510200500488, |
| "logps/chosen": -229.01654052734375, |
| "logps/rejected": -303.3737487792969, |
| "loss": 0.1209, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.715925931930542, |
| "rewards/margins": 7.289467811584473, |
| "rewards/rejected": -8.005393981933594, |
| "step": 344 |
| }, |
| { |
| "epoch": 0.27702499247214696, |
| "grad_norm": 0.7761366963386536, |
| "learning_rate": 2.345175242457145e-05, |
| "logits/chosen": -7.239856243133545, |
| "logits/rejected": -6.700331211090088, |
| "logps/chosen": -256.78070068359375, |
| "logps/rejected": -327.82708740234375, |
| "loss": 0.0832, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.017277270555496216, |
| "rewards/margins": 8.008116722106934, |
| "rewards/rejected": -8.02539348602295, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.27782796346481986, |
| "grad_norm": 0.7174155116081238, |
| "learning_rate": 2.3410725860275092e-05, |
| "logits/chosen": -6.968440055847168, |
| "logits/rejected": -6.455938339233398, |
| "logps/chosen": -251.1138153076172, |
| "logps/rejected": -322.4520263671875, |
| "loss": 0.0846, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.1589069664478302, |
| "rewards/margins": 7.908819675445557, |
| "rewards/rejected": -8.067726135253906, |
| "step": 346 |
| }, |
| { |
| "epoch": 0.2786309344574927, |
| "grad_norm": 1.164890170097351, |
| "learning_rate": 2.3369607317628244e-05, |
| "logits/chosen": -6.969342231750488, |
| "logits/rejected": -6.404745578765869, |
| "logps/chosen": -259.37518310546875, |
| "logps/rejected": -312.76214599609375, |
| "loss": 0.1203, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4851428270339966, |
| "rewards/margins": 7.197408199310303, |
| "rewards/rejected": -7.682551860809326, |
| "step": 347 |
| }, |
| { |
| "epoch": 0.2794339054501656, |
| "grad_norm": 0.9198136925697327, |
| "learning_rate": 2.33283972462967e-05, |
| "logits/chosen": -7.175347805023193, |
| "logits/rejected": -6.670289516448975, |
| "logps/chosen": -264.0846252441406, |
| "logps/rejected": -308.40692138671875, |
| "loss": 0.0927, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.3511177897453308, |
| "rewards/margins": 7.53209114074707, |
| "rewards/rejected": -7.883209705352783, |
| "step": 348 |
| }, |
| { |
| "epoch": 0.2802368764428385, |
| "grad_norm": 0.9923511147499084, |
| "learning_rate": 2.3287096096947202e-05, |
| "logits/chosen": -7.192119598388672, |
| "logits/rejected": -6.688043594360352, |
| "logps/chosen": -232.76608276367188, |
| "logps/rejected": -274.8439025878906, |
| "loss": 0.0985, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.47156834602355957, |
| "rewards/margins": 7.682293891906738, |
| "rewards/rejected": -8.153861999511719, |
| "step": 349 |
| }, |
| { |
| "epoch": 0.2810398474355114, |
| "grad_norm": 1.0304067134857178, |
| "learning_rate": 2.3245704321242494e-05, |
| "logits/chosen": -7.0880208015441895, |
| "logits/rejected": -6.622589111328125, |
| "logps/chosen": -242.23458862304688, |
| "logps/rejected": -294.5273132324219, |
| "loss": 0.1369, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.26982617378234863, |
| "rewards/margins": 7.148758888244629, |
| "rewards/rejected": -7.418584823608398, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.28184281842818426, |
| "grad_norm": 0.8729216456413269, |
| "learning_rate": 2.320422237183641e-05, |
| "logits/chosen": -7.2745208740234375, |
| "logits/rejected": -6.736506938934326, |
| "logps/chosen": -235.64813232421875, |
| "logps/rejected": -284.6999816894531, |
| "loss": 0.0999, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.025683067739009857, |
| "rewards/margins": 7.5068135261535645, |
| "rewards/rejected": -7.532495498657227, |
| "step": 351 |
| }, |
| { |
| "epoch": 0.28264578942085716, |
| "grad_norm": 0.9139456152915955, |
| "learning_rate": 2.3162650702368896e-05, |
| "logits/chosen": -7.155797481536865, |
| "logits/rejected": -6.7269134521484375, |
| "logps/chosen": -239.5316162109375, |
| "logps/rejected": -275.61083984375, |
| "loss": 0.1138, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.32349392771720886, |
| "rewards/margins": 7.031448841094971, |
| "rewards/rejected": -7.354942798614502, |
| "step": 352 |
| }, |
| { |
| "epoch": 0.28344876041353007, |
| "grad_norm": 1.0829644203186035, |
| "learning_rate": 2.312098976746107e-05, |
| "logits/chosen": -7.061025619506836, |
| "logits/rejected": -6.5226731300354, |
| "logps/chosen": -261.513671875, |
| "logps/rejected": -312.84649658203125, |
| "loss": 0.1305, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.09789466857910156, |
| "rewards/margins": 7.543811321258545, |
| "rewards/rejected": -7.641705513000488, |
| "step": 353 |
| }, |
| { |
| "epoch": 0.28425173140620297, |
| "grad_norm": 0.8415951132774353, |
| "learning_rate": 2.307924002271025e-05, |
| "logits/chosen": -7.136366844177246, |
| "logits/rejected": -6.655204772949219, |
| "logps/chosen": -240.50953674316406, |
| "logps/rejected": -288.30145263671875, |
| "loss": 0.1126, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.4409840703010559, |
| "rewards/margins": 7.2280778884887695, |
| "rewards/rejected": -7.669061183929443, |
| "step": 354 |
| }, |
| { |
| "epoch": 0.2850547023988758, |
| "grad_norm": 0.744634747505188, |
| "learning_rate": 2.303740192468495e-05, |
| "logits/chosen": -6.914259910583496, |
| "logits/rejected": -6.460628032684326, |
| "logps/chosen": -246.8975372314453, |
| "logps/rejected": -314.4864501953125, |
| "loss": 0.1008, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.41177719831466675, |
| "rewards/margins": 7.658634185791016, |
| "rewards/rejected": -8.070411682128906, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.2858576733915487, |
| "grad_norm": 0.7998935580253601, |
| "learning_rate": 2.2995475930919907e-05, |
| "logits/chosen": -7.118217468261719, |
| "logits/rejected": -6.548300266265869, |
| "logps/chosen": -239.43505859375, |
| "logps/rejected": -301.7339782714844, |
| "loss": 0.0874, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.26125428080558777, |
| "rewards/margins": 8.033935546875, |
| "rewards/rejected": -8.295188903808594, |
| "step": 356 |
| }, |
| { |
| "epoch": 0.2866606443842216, |
| "grad_norm": 0.8120717406272888, |
| "learning_rate": 2.2953462499911072e-05, |
| "logits/chosen": -6.872447967529297, |
| "logits/rejected": -6.4354658126831055, |
| "logps/chosen": -232.77626037597656, |
| "logps/rejected": -295.0360107421875, |
| "loss": 0.1042, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5053384304046631, |
| "rewards/margins": 7.561707496643066, |
| "rewards/rejected": -8.067046165466309, |
| "step": 357 |
| }, |
| { |
| "epoch": 0.2874636153768945, |
| "grad_norm": 1.0435761213302612, |
| "learning_rate": 2.2911362091110596e-05, |
| "logits/chosen": -7.07387113571167, |
| "logits/rejected": -6.536746501922607, |
| "logps/chosen": -261.0136413574219, |
| "logps/rejected": -332.11956787109375, |
| "loss": 0.1207, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.6103707551956177, |
| "rewards/margins": 7.593594551086426, |
| "rewards/rejected": -8.203965187072754, |
| "step": 358 |
| }, |
| { |
| "epoch": 0.2882665863695674, |
| "grad_norm": 0.7408345937728882, |
| "learning_rate": 2.2869175164921807e-05, |
| "logits/chosen": -6.716737270355225, |
| "logits/rejected": -6.186054706573486, |
| "logps/chosen": -229.17645263671875, |
| "logps/rejected": -293.9906921386719, |
| "loss": 0.1051, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.8713277578353882, |
| "rewards/margins": 7.128081321716309, |
| "rewards/rejected": -7.999408721923828, |
| "step": 359 |
| }, |
| { |
| "epoch": 0.28906955736224027, |
| "grad_norm": 0.9802799820899963, |
| "learning_rate": 2.282690218269416e-05, |
| "logits/chosen": -7.144526958465576, |
| "logits/rejected": -6.505621910095215, |
| "logps/chosen": -222.1000518798828, |
| "logps/rejected": -297.21795654296875, |
| "loss": 0.1201, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.7401584386825562, |
| "rewards/margins": 7.86820125579834, |
| "rewards/rejected": -8.608359336853027, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.2898725283549132, |
| "grad_norm": 0.7505455613136292, |
| "learning_rate": 2.2784543606718227e-05, |
| "logits/chosen": -7.154356002807617, |
| "logits/rejected": -6.596198081970215, |
| "logps/chosen": -218.4374237060547, |
| "logps/rejected": -276.716064453125, |
| "loss": 0.0812, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.6324419379234314, |
| "rewards/margins": 7.558305740356445, |
| "rewards/rejected": -8.190747261047363, |
| "step": 361 |
| }, |
| { |
| "epoch": 0.2906754993475861, |
| "grad_norm": 0.906490683555603, |
| "learning_rate": 2.2742099900220603e-05, |
| "logits/chosen": -7.131973743438721, |
| "logits/rejected": -6.576871395111084, |
| "logps/chosen": -249.00521850585938, |
| "logps/rejected": -304.16387939453125, |
| "loss": 0.1165, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.23080775141716003, |
| "rewards/margins": 7.49576997756958, |
| "rewards/rejected": -7.726577281951904, |
| "step": 362 |
| }, |
| { |
| "epoch": 0.291478470340259, |
| "grad_norm": 0.8336560726165771, |
| "learning_rate": 2.269957152735887e-05, |
| "logits/chosen": -7.0923895835876465, |
| "logits/rejected": -6.573888778686523, |
| "logps/chosen": -277.31182861328125, |
| "logps/rejected": -336.8761291503906, |
| "loss": 0.1049, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.5974608063697815, |
| "rewards/margins": 7.692646026611328, |
| "rewards/rejected": -8.290107727050781, |
| "step": 363 |
| }, |
| { |
| "epoch": 0.2922814413329318, |
| "grad_norm": 0.799720048904419, |
| "learning_rate": 2.265695895321649e-05, |
| "logits/chosen": -7.24672794342041, |
| "logits/rejected": -6.603967189788818, |
| "logps/chosen": -234.96090698242188, |
| "logps/rejected": -299.73089599609375, |
| "loss": 0.0891, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.5872701406478882, |
| "rewards/margins": 7.929731369018555, |
| "rewards/rejected": -8.51700210571289, |
| "step": 364 |
| }, |
| { |
| "epoch": 0.2930844123256047, |
| "grad_norm": 0.5944221615791321, |
| "learning_rate": 2.2614262643797757e-05, |
| "logits/chosen": -7.174671649932861, |
| "logits/rejected": -6.628643035888672, |
| "logps/chosen": -242.10484313964844, |
| "logps/rejected": -278.5067138671875, |
| "loss": 0.0763, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.6918413043022156, |
| "rewards/margins": 7.971635818481445, |
| "rewards/rejected": -8.663476943969727, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.29388738331827763, |
| "grad_norm": 0.827883243560791, |
| "learning_rate": 2.257148306602266e-05, |
| "logits/chosen": -7.06397008895874, |
| "logits/rejected": -6.457691669464111, |
| "logps/chosen": -238.92132568359375, |
| "logps/rejected": -308.870361328125, |
| "loss": 0.0837, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5876378417015076, |
| "rewards/margins": 7.6011762619018555, |
| "rewards/rejected": -8.188814163208008, |
| "step": 366 |
| }, |
| { |
| "epoch": 0.29469035431095053, |
| "grad_norm": 0.9409468173980713, |
| "learning_rate": 2.2528620687721802e-05, |
| "logits/chosen": -6.917649269104004, |
| "logits/rejected": -6.455188274383545, |
| "logps/chosen": -232.50828552246094, |
| "logps/rejected": -300.833984375, |
| "loss": 0.094, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.6666216254234314, |
| "rewards/margins": 7.9217352867126465, |
| "rewards/rejected": -8.588356971740723, |
| "step": 367 |
| }, |
| { |
| "epoch": 0.29549332530362343, |
| "grad_norm": 0.8579176068305969, |
| "learning_rate": 2.2485675977631308e-05, |
| "logits/chosen": -7.081417560577393, |
| "logits/rejected": -6.553350448608398, |
| "logps/chosen": -233.04917907714844, |
| "logps/rejected": -270.5080871582031, |
| "loss": 0.1057, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9565406441688538, |
| "rewards/margins": 7.806694507598877, |
| "rewards/rejected": -8.763235092163086, |
| "step": 368 |
| }, |
| { |
| "epoch": 0.2962962962962963, |
| "grad_norm": 0.8730171918869019, |
| "learning_rate": 2.2442649405387632e-05, |
| "logits/chosen": -7.203672885894775, |
| "logits/rejected": -6.67544412612915, |
| "logps/chosen": -250.46131896972656, |
| "logps/rejected": -313.3692321777344, |
| "loss": 0.0929, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.2980150580406189, |
| "rewards/margins": 8.31704330444336, |
| "rewards/rejected": -8.615058898925781, |
| "step": 369 |
| }, |
| { |
| "epoch": 0.2970992672889692, |
| "grad_norm": 0.5971737504005432, |
| "learning_rate": 2.2399541441522476e-05, |
| "logits/chosen": -6.995357990264893, |
| "logits/rejected": -6.462562561035156, |
| "logps/chosen": -237.61221313476562, |
| "logps/rejected": -288.45635986328125, |
| "loss": 0.0627, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.6406427621841431, |
| "rewards/margins": 8.05375862121582, |
| "rewards/rejected": -8.694402694702148, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.2979022382816421, |
| "grad_norm": 0.7195870280265808, |
| "learning_rate": 2.2356352557457624e-05, |
| "logits/chosen": -6.849624156951904, |
| "logits/rejected": -6.437941551208496, |
| "logps/chosen": -250.61135864257812, |
| "logps/rejected": -292.5483703613281, |
| "loss": 0.0945, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.5779178738594055, |
| "rewards/margins": 7.955745220184326, |
| "rewards/rejected": -8.533662796020508, |
| "step": 371 |
| }, |
| { |
| "epoch": 0.298705209274315, |
| "grad_norm": 0.9270005226135254, |
| "learning_rate": 2.2313083225499796e-05, |
| "logits/chosen": -6.909468650817871, |
| "logits/rejected": -6.309047698974609, |
| "logps/chosen": -229.16146850585938, |
| "logps/rejected": -305.1193542480469, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.2821406126022339, |
| "rewards/margins": 7.8656721115112305, |
| "rewards/rejected": -9.14781379699707, |
| "step": 372 |
| }, |
| { |
| "epoch": 0.29950818026698783, |
| "grad_norm": 0.8219568133354187, |
| "learning_rate": 2.2269733918835478e-05, |
| "logits/chosen": -6.842496871948242, |
| "logits/rejected": -6.3901286125183105, |
| "logps/chosen": -230.1271514892578, |
| "logps/rejected": -287.2200012207031, |
| "loss": 0.0789, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.1203663349151611, |
| "rewards/margins": 8.134592056274414, |
| "rewards/rejected": -9.254958152770996, |
| "step": 373 |
| }, |
| { |
| "epoch": 0.30031115125966074, |
| "grad_norm": 0.8218191862106323, |
| "learning_rate": 2.222630511152573e-05, |
| "logits/chosen": -6.966914653778076, |
| "logits/rejected": -6.487677097320557, |
| "logps/chosen": -246.5464324951172, |
| "logps/rejected": -305.7148132324219, |
| "loss": 0.0699, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.197892189025879, |
| "rewards/margins": 7.9742865562438965, |
| "rewards/rejected": -9.172179222106934, |
| "step": 374 |
| }, |
| { |
| "epoch": 0.30111412225233364, |
| "grad_norm": 0.8649128079414368, |
| "learning_rate": 2.218279727850104e-05, |
| "logits/chosen": -6.956799030303955, |
| "logits/rejected": -6.466164588928223, |
| "logps/chosen": -257.6927185058594, |
| "logps/rejected": -320.65093994140625, |
| "loss": 0.0969, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9542887806892395, |
| "rewards/margins": 7.865854740142822, |
| "rewards/rejected": -8.82014274597168, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.30191709324500654, |
| "grad_norm": 0.7877534031867981, |
| "learning_rate": 2.2139210895556104e-05, |
| "logits/chosen": -7.133412837982178, |
| "logits/rejected": -6.522618293762207, |
| "logps/chosen": -234.63067626953125, |
| "logps/rejected": -285.729248046875, |
| "loss": 0.0957, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.8910524845123291, |
| "rewards/margins": 7.85192346572876, |
| "rewards/rejected": -8.742977142333984, |
| "step": 376 |
| }, |
| { |
| "epoch": 0.3027200642376794, |
| "grad_norm": 0.9421678185462952, |
| "learning_rate": 2.2095546439344614e-05, |
| "logits/chosen": -7.198690891265869, |
| "logits/rejected": -6.629060745239258, |
| "logps/chosen": -234.28802490234375, |
| "logps/rejected": -295.1626892089844, |
| "loss": 0.1127, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.4040381908416748, |
| "rewards/margins": 8.069625854492188, |
| "rewards/rejected": -9.473664283752441, |
| "step": 377 |
| }, |
| { |
| "epoch": 0.3035230352303523, |
| "grad_norm": 1.0842405557632446, |
| "learning_rate": 2.205180438737407e-05, |
| "logits/chosen": -6.771623134613037, |
| "logits/rejected": -6.313328266143799, |
| "logps/chosen": -273.8349304199219, |
| "logps/rejected": -313.0619812011719, |
| "loss": 0.1234, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9841660857200623, |
| "rewards/margins": 7.717597484588623, |
| "rewards/rejected": -8.701764106750488, |
| "step": 378 |
| }, |
| { |
| "epoch": 0.3043260062230252, |
| "grad_norm": 0.8916366100311279, |
| "learning_rate": 2.2007985218000543e-05, |
| "logits/chosen": -7.0051140785217285, |
| "logits/rejected": -6.463620662689209, |
| "logps/chosen": -235.1730499267578, |
| "logps/rejected": -315.97894287109375, |
| "loss": 0.1098, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.3058133125305176, |
| "rewards/margins": 7.760996341705322, |
| "rewards/rejected": -9.066808700561523, |
| "step": 379 |
| }, |
| { |
| "epoch": 0.3051289772156981, |
| "grad_norm": 0.6778852343559265, |
| "learning_rate": 2.1964089410423457e-05, |
| "logits/chosen": -7.102459907531738, |
| "logits/rejected": -6.511131286621094, |
| "logps/chosen": -233.19422912597656, |
| "logps/rejected": -313.62774658203125, |
| "loss": 0.0759, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -1.2374646663665771, |
| "rewards/margins": 8.028099060058594, |
| "rewards/rejected": -9.26556396484375, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.305931948208371, |
| "grad_norm": 0.880423367023468, |
| "learning_rate": 2.1920117444680317e-05, |
| "logits/chosen": -7.077634811401367, |
| "logits/rejected": -6.550751686096191, |
| "logps/chosen": -264.34375, |
| "logps/rejected": -314.9694519042969, |
| "loss": 0.1115, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.7476577758789062, |
| "rewards/margins": 8.020127296447754, |
| "rewards/rejected": -8.767784118652344, |
| "step": 381 |
| }, |
| { |
| "epoch": 0.30673491920104384, |
| "grad_norm": 0.9886122345924377, |
| "learning_rate": 2.1876069801641506e-05, |
| "logits/chosen": -7.031638145446777, |
| "logits/rejected": -6.4737372398376465, |
| "logps/chosen": -248.13861083984375, |
| "logps/rejected": -333.40655517578125, |
| "loss": 0.1322, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.808936357498169, |
| "rewards/margins": 8.263845443725586, |
| "rewards/rejected": -9.072782516479492, |
| "step": 382 |
| }, |
| { |
| "epoch": 0.30753789019371675, |
| "grad_norm": 0.9590216279029846, |
| "learning_rate": 2.1831946963004988e-05, |
| "logits/chosen": -7.160924434661865, |
| "logits/rejected": -6.505220413208008, |
| "logps/chosen": -243.02020263671875, |
| "logps/rejected": -314.9360046386719, |
| "loss": 0.1236, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.0556272268295288, |
| "rewards/margins": 8.110542297363281, |
| "rewards/rejected": -9.166169166564941, |
| "step": 383 |
| }, |
| { |
| "epoch": 0.30834086118638965, |
| "grad_norm": 0.8976796269416809, |
| "learning_rate": 2.1787749411291056e-05, |
| "logits/chosen": -7.002971172332764, |
| "logits/rejected": -6.574981689453125, |
| "logps/chosen": -272.0745849609375, |
| "logps/rejected": -338.3102111816406, |
| "loss": 0.1297, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9497753977775574, |
| "rewards/margins": 7.699694633483887, |
| "rewards/rejected": -8.649470329284668, |
| "step": 384 |
| }, |
| { |
| "epoch": 0.30914383217906255, |
| "grad_norm": 0.8060115575790405, |
| "learning_rate": 2.1743477629837057e-05, |
| "logits/chosen": -6.959688186645508, |
| "logits/rejected": -6.544022560119629, |
| "logps/chosen": -256.5281677246094, |
| "logps/rejected": -301.2608642578125, |
| "loss": 0.0918, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7253895998001099, |
| "rewards/margins": 7.547745704650879, |
| "rewards/rejected": -8.273136138916016, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.3099468031717354, |
| "grad_norm": 0.6023092865943909, |
| "learning_rate": 2.16991321027921e-05, |
| "logits/chosen": -7.165547847747803, |
| "logits/rejected": -6.591835975646973, |
| "logps/chosen": -270.77001953125, |
| "logps/rejected": -331.0622863769531, |
| "loss": 0.0698, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.6521594524383545, |
| "rewards/margins": 8.221850395202637, |
| "rewards/rejected": -8.874011039733887, |
| "step": 386 |
| }, |
| { |
| "epoch": 0.3107497741644083, |
| "grad_norm": 0.8687335252761841, |
| "learning_rate": 2.165471331511176e-05, |
| "logits/chosen": -7.113095283508301, |
| "logits/rejected": -6.675935745239258, |
| "logps/chosen": -230.63902282714844, |
| "logps/rejected": -277.78790283203125, |
| "loss": 0.0987, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6908210515975952, |
| "rewards/margins": 7.717652797698975, |
| "rewards/rejected": -8.40847396850586, |
| "step": 387 |
| }, |
| { |
| "epoch": 0.3115527451570812, |
| "grad_norm": 0.6948081851005554, |
| "learning_rate": 2.1610221752552784e-05, |
| "logits/chosen": -7.056583881378174, |
| "logits/rejected": -6.613354682922363, |
| "logps/chosen": -242.65614318847656, |
| "logps/rejected": -286.830322265625, |
| "loss": 0.0992, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.4980300962924957, |
| "rewards/margins": 7.864409923553467, |
| "rewards/rejected": -8.36244010925293, |
| "step": 388 |
| }, |
| { |
| "epoch": 0.3123557161497541, |
| "grad_norm": 1.068792700767517, |
| "learning_rate": 2.1565657901667777e-05, |
| "logits/chosen": -6.986828804016113, |
| "logits/rejected": -6.458230018615723, |
| "logps/chosen": -266.3401794433594, |
| "logps/rejected": -305.89398193359375, |
| "loss": 0.1627, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.8312156796455383, |
| "rewards/margins": 7.1317572593688965, |
| "rewards/rejected": -7.962973594665527, |
| "step": 389 |
| }, |
| { |
| "epoch": 0.313158687142427, |
| "grad_norm": 0.825005292892456, |
| "learning_rate": 2.1521022249799872e-05, |
| "logits/chosen": -7.132630348205566, |
| "logits/rejected": -6.718094348907471, |
| "logps/chosen": -250.86947631835938, |
| "logps/rejected": -309.60186767578125, |
| "loss": 0.1279, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.5285347104072571, |
| "rewards/margins": 7.658768653869629, |
| "rewards/rejected": -8.187302589416504, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.31396165813509985, |
| "grad_norm": 0.7309999465942383, |
| "learning_rate": 2.1476315285077393e-05, |
| "logits/chosen": -7.206225395202637, |
| "logits/rejected": -6.7328290939331055, |
| "logps/chosen": -239.95883178710938, |
| "logps/rejected": -308.23980712890625, |
| "loss": 0.0773, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7804017066955566, |
| "rewards/margins": 7.880527973175049, |
| "rewards/rejected": -8.660928726196289, |
| "step": 391 |
| }, |
| { |
| "epoch": 0.31476462912777275, |
| "grad_norm": 0.7790502905845642, |
| "learning_rate": 2.1431537496408562e-05, |
| "logits/chosen": -7.341536045074463, |
| "logits/rejected": -6.817572593688965, |
| "logps/chosen": -223.16049194335938, |
| "logps/rejected": -276.67822265625, |
| "loss": 0.0839, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7653395533561707, |
| "rewards/margins": 7.790850639343262, |
| "rewards/rejected": -8.55618953704834, |
| "step": 392 |
| }, |
| { |
| "epoch": 0.31556760012044566, |
| "grad_norm": 0.8895344734191895, |
| "learning_rate": 2.138668937347609e-05, |
| "logits/chosen": -7.218590259552002, |
| "logits/rejected": -6.678465366363525, |
| "logps/chosen": -220.08065795898438, |
| "logps/rejected": -271.2347412109375, |
| "loss": 0.1167, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9858065247535706, |
| "rewards/margins": 7.239531993865967, |
| "rewards/rejected": -8.22533893585205, |
| "step": 393 |
| }, |
| { |
| "epoch": 0.31637057111311856, |
| "grad_norm": 0.641010046005249, |
| "learning_rate": 2.134177140673187e-05, |
| "logits/chosen": -7.150949954986572, |
| "logits/rejected": -6.707058906555176, |
| "logps/chosen": -276.6315002441406, |
| "logps/rejected": -321.0830383300781, |
| "loss": 0.0726, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.17668873071670532, |
| "rewards/margins": 8.063828468322754, |
| "rewards/rejected": -8.240516662597656, |
| "step": 394 |
| }, |
| { |
| "epoch": 0.3171735421057914, |
| "grad_norm": 0.632405161857605, |
| "learning_rate": 2.1296784087391586e-05, |
| "logits/chosen": -6.947802543640137, |
| "logits/rejected": -6.424903869628906, |
| "logps/chosen": -268.1100769042969, |
| "logps/rejected": -314.6433410644531, |
| "loss": 0.0819, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.3643196225166321, |
| "rewards/margins": 7.584046840667725, |
| "rewards/rejected": -7.948367118835449, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.3179765130984643, |
| "grad_norm": 0.7696956992149353, |
| "learning_rate": 2.1251727907429357e-05, |
| "logits/chosen": -7.333001136779785, |
| "logits/rejected": -6.813477516174316, |
| "logps/chosen": -232.9025421142578, |
| "logps/rejected": -295.028564453125, |
| "loss": 0.0919, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7348997592926025, |
| "rewards/margins": 7.67524528503418, |
| "rewards/rejected": -8.410144805908203, |
| "step": 396 |
| }, |
| { |
| "epoch": 0.3187794840911372, |
| "grad_norm": 0.9200910925865173, |
| "learning_rate": 2.1206603359572346e-05, |
| "logits/chosen": -7.141473293304443, |
| "logits/rejected": -6.698199272155762, |
| "logps/chosen": -241.26214599609375, |
| "logps/rejected": -291.30511474609375, |
| "loss": 0.1197, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.595912516117096, |
| "rewards/margins": 7.802241802215576, |
| "rewards/rejected": -8.398154258728027, |
| "step": 397 |
| }, |
| { |
| "epoch": 0.3195824550838101, |
| "grad_norm": 0.8468185663223267, |
| "learning_rate": 2.1161410937295385e-05, |
| "logits/chosen": -7.159466743469238, |
| "logits/rejected": -6.61655330657959, |
| "logps/chosen": -257.2116394042969, |
| "logps/rejected": -303.525634765625, |
| "loss": 0.1106, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.41002053022384644, |
| "rewards/margins": 7.774020671844482, |
| "rewards/rejected": -8.184040069580078, |
| "step": 398 |
| }, |
| { |
| "epoch": 0.32038542607648296, |
| "grad_norm": 0.8794032335281372, |
| "learning_rate": 2.1116151134815555e-05, |
| "logits/chosen": -7.306539058685303, |
| "logits/rejected": -6.777565002441406, |
| "logps/chosen": -228.7540740966797, |
| "logps/rejected": -272.5121765136719, |
| "loss": 0.1216, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6913648843765259, |
| "rewards/margins": 7.405577182769775, |
| "rewards/rejected": -8.096941947937012, |
| "step": 399 |
| }, |
| { |
| "epoch": 0.32118839706915586, |
| "grad_norm": 0.9035705924034119, |
| "learning_rate": 2.107082444708681e-05, |
| "logits/chosen": -7.068881034851074, |
| "logits/rejected": -6.750105381011963, |
| "logps/chosen": -262.3298034667969, |
| "logps/rejected": -328.3765869140625, |
| "loss": 0.1254, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.9687249660491943, |
| "rewards/margins": 7.2472004890441895, |
| "rewards/rejected": -8.215925216674805, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.32199136806182876, |
| "grad_norm": 0.9415023326873779, |
| "learning_rate": 2.1025431369794546e-05, |
| "logits/chosen": -7.0469818115234375, |
| "logits/rejected": -6.488866329193115, |
| "logps/chosen": -242.50955200195312, |
| "logps/rejected": -295.12255859375, |
| "loss": 0.1258, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.2747727334499359, |
| "rewards/margins": 7.701444625854492, |
| "rewards/rejected": -7.976217269897461, |
| "step": 401 |
| }, |
| { |
| "epoch": 0.32279433905450167, |
| "grad_norm": 0.638993501663208, |
| "learning_rate": 2.0979972399350176e-05, |
| "logits/chosen": -7.037402629852295, |
| "logits/rejected": -6.588835716247559, |
| "logps/chosen": -261.5014953613281, |
| "logps/rejected": -287.1429443359375, |
| "loss": 0.0856, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.43414613604545593, |
| "rewards/margins": 7.771677017211914, |
| "rewards/rejected": -8.20582389831543, |
| "step": 402 |
| }, |
| { |
| "epoch": 0.32359731004717457, |
| "grad_norm": 0.8404697179794312, |
| "learning_rate": 2.0934448032885724e-05, |
| "logits/chosen": -6.856971740722656, |
| "logits/rejected": -6.379488468170166, |
| "logps/chosen": -240.2373809814453, |
| "logps/rejected": -290.4339294433594, |
| "loss": 0.1275, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.7628170847892761, |
| "rewards/margins": 7.089502334594727, |
| "rewards/rejected": -7.852319717407227, |
| "step": 403 |
| }, |
| { |
| "epoch": 0.3244002810398474, |
| "grad_norm": 0.66420578956604, |
| "learning_rate": 2.088885876824837e-05, |
| "logits/chosen": -7.085887908935547, |
| "logits/rejected": -6.55605936050415, |
| "logps/chosen": -246.12681579589844, |
| "logps/rejected": -311.8985595703125, |
| "loss": 0.094, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.5131916999816895, |
| "rewards/margins": 8.20376205444336, |
| "rewards/rejected": -8.71695327758789, |
| "step": 404 |
| }, |
| { |
| "epoch": 0.3252032520325203, |
| "grad_norm": 0.9190989136695862, |
| "learning_rate": 2.0843205103995003e-05, |
| "logits/chosen": -7.0275044441223145, |
| "logits/rejected": -6.629448890686035, |
| "logps/chosen": -257.7423400878906, |
| "logps/rejected": -320.6184997558594, |
| "loss": 0.102, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.9183521866798401, |
| "rewards/margins": 7.740780830383301, |
| "rewards/rejected": -8.65913200378418, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.3260062230251932, |
| "grad_norm": 0.7964940071105957, |
| "learning_rate": 2.0797487539386782e-05, |
| "logits/chosen": -7.193318843841553, |
| "logits/rejected": -6.612943649291992, |
| "logps/chosen": -255.85467529296875, |
| "logps/rejected": -322.6330871582031, |
| "loss": 0.0961, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.22339469194412231, |
| "rewards/margins": 8.518314361572266, |
| "rewards/rejected": -8.741708755493164, |
| "step": 406 |
| }, |
| { |
| "epoch": 0.3268091940178661, |
| "grad_norm": 0.8375403881072998, |
| "learning_rate": 2.0751706574383676e-05, |
| "logits/chosen": -7.1184210777282715, |
| "logits/rejected": -6.658360958099365, |
| "logps/chosen": -247.8142547607422, |
| "logps/rejected": -325.388671875, |
| "loss": 0.0875, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7249178886413574, |
| "rewards/margins": 8.412040710449219, |
| "rewards/rejected": -9.136958122253418, |
| "step": 407 |
| }, |
| { |
| "epoch": 0.32761216501053897, |
| "grad_norm": 0.7731269598007202, |
| "learning_rate": 2.0705862709638972e-05, |
| "logits/chosen": -6.815896987915039, |
| "logits/rejected": -6.3233184814453125, |
| "logps/chosen": -227.92572021484375, |
| "logps/rejected": -294.21929931640625, |
| "loss": 0.099, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.924447774887085, |
| "rewards/margins": 7.62613582611084, |
| "rewards/rejected": -8.550582885742188, |
| "step": 408 |
| }, |
| { |
| "epoch": 0.32841513600321187, |
| "grad_norm": 1.0949829816818237, |
| "learning_rate": 2.065995644649384e-05, |
| "logits/chosen": -6.891806602478027, |
| "logits/rejected": -6.483293533325195, |
| "logps/chosen": -253.94378662109375, |
| "logps/rejected": -296.117919921875, |
| "loss": 0.1169, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9100632071495056, |
| "rewards/margins": 8.106210708618164, |
| "rewards/rejected": -9.016273498535156, |
| "step": 409 |
| }, |
| { |
| "epoch": 0.3292181069958848, |
| "grad_norm": 0.9440847039222717, |
| "learning_rate": 2.0613988286971805e-05, |
| "logits/chosen": -6.792534828186035, |
| "logits/rejected": -6.426342487335205, |
| "logps/chosen": -234.8439178466797, |
| "logps/rejected": -299.595458984375, |
| "loss": 0.1352, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9257516860961914, |
| "rewards/margins": 7.82083797454834, |
| "rewards/rejected": -8.746590614318848, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.3300210779885577, |
| "grad_norm": 0.7644413113594055, |
| "learning_rate": 2.056795873377331e-05, |
| "logits/chosen": -6.811699867248535, |
| "logits/rejected": -6.318783760070801, |
| "logps/chosen": -265.04638671875, |
| "logps/rejected": -323.1253662109375, |
| "loss": 0.1062, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7678769826889038, |
| "rewards/margins": 8.006941795349121, |
| "rewards/rejected": -8.774818420410156, |
| "step": 411 |
| }, |
| { |
| "epoch": 0.3308240489812306, |
| "grad_norm": 0.8954298496246338, |
| "learning_rate": 2.052186829027017e-05, |
| "logits/chosen": -7.012080669403076, |
| "logits/rejected": -6.497799873352051, |
| "logps/chosen": -255.0225067138672, |
| "logps/rejected": -309.0657958984375, |
| "loss": 0.0966, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7220298647880554, |
| "rewards/margins": 8.231367111206055, |
| "rewards/rejected": -8.953397750854492, |
| "step": 412 |
| }, |
| { |
| "epoch": 0.3316270199739034, |
| "grad_norm": 0.8641648888587952, |
| "learning_rate": 2.0475717460500088e-05, |
| "logits/chosen": -6.974394798278809, |
| "logits/rejected": -6.439783573150635, |
| "logps/chosen": -267.2938232421875, |
| "logps/rejected": -330.1260986328125, |
| "loss": 0.1176, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9153153896331787, |
| "rewards/margins": 7.763271331787109, |
| "rewards/rejected": -8.678586959838867, |
| "step": 413 |
| }, |
| { |
| "epoch": 0.3324299909665763, |
| "grad_norm": 0.6658833026885986, |
| "learning_rate": 2.0429506749161144e-05, |
| "logits/chosen": -6.871219635009766, |
| "logits/rejected": -6.437784671783447, |
| "logps/chosen": -238.15701293945312, |
| "logps/rejected": -305.763916015625, |
| "loss": 0.0959, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.091659426689148, |
| "rewards/margins": 8.114177703857422, |
| "rewards/rejected": -9.20583724975586, |
| "step": 414 |
| }, |
| { |
| "epoch": 0.33323296195924923, |
| "grad_norm": 0.8514337539672852, |
| "learning_rate": 2.0383236661606275e-05, |
| "logits/chosen": -7.03883695602417, |
| "logits/rejected": -6.4415602684021, |
| "logps/chosen": -242.29815673828125, |
| "logps/rejected": -303.9954833984375, |
| "loss": 0.113, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.4168682098388672, |
| "rewards/margins": 7.7852020263671875, |
| "rewards/rejected": -9.202070236206055, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.33403593295192213, |
| "grad_norm": 0.882880687713623, |
| "learning_rate": 2.033690770383775e-05, |
| "logits/chosen": -6.9290361404418945, |
| "logits/rejected": -6.581358432769775, |
| "logps/chosen": -268.37200927734375, |
| "logps/rejected": -313.39111328125, |
| "loss": 0.1094, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.6618314385414124, |
| "rewards/margins": 8.264297485351562, |
| "rewards/rejected": -8.926129341125488, |
| "step": 416 |
| }, |
| { |
| "epoch": 0.334838903944595, |
| "grad_norm": 0.8963824510574341, |
| "learning_rate": 2.029052038250162e-05, |
| "logits/chosen": -6.987563610076904, |
| "logits/rejected": -6.373250484466553, |
| "logps/chosen": -264.83673095703125, |
| "logps/rejected": -333.64080810546875, |
| "loss": 0.0848, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.9685074687004089, |
| "rewards/margins": 8.535039901733398, |
| "rewards/rejected": -9.503547668457031, |
| "step": 417 |
| }, |
| { |
| "epoch": 0.3356418749372679, |
| "grad_norm": 1.0316649675369263, |
| "learning_rate": 2.02440752048822e-05, |
| "logits/chosen": -6.865973949432373, |
| "logits/rejected": -6.438227653503418, |
| "logps/chosen": -248.80157470703125, |
| "logps/rejected": -309.55865478515625, |
| "loss": 0.128, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.060164451599121, |
| "rewards/margins": 8.215744018554688, |
| "rewards/rejected": -9.275907516479492, |
| "step": 418 |
| }, |
| { |
| "epoch": 0.3364448459299408, |
| "grad_norm": 0.9749038219451904, |
| "learning_rate": 2.0197572678896522e-05, |
| "logits/chosen": -6.89204216003418, |
| "logits/rejected": -6.381191730499268, |
| "logps/chosen": -242.6867218017578, |
| "logps/rejected": -290.550048828125, |
| "loss": 0.1155, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8523032069206238, |
| "rewards/margins": 8.118548393249512, |
| "rewards/rejected": -8.97085189819336, |
| "step": 419 |
| }, |
| { |
| "epoch": 0.3372478169226137, |
| "grad_norm": 0.7400681376457214, |
| "learning_rate": 2.0151013313088747e-05, |
| "logits/chosen": -7.0694146156311035, |
| "logits/rejected": -6.446078777313232, |
| "logps/chosen": -246.19581604003906, |
| "logps/rejected": -312.5214538574219, |
| "loss": 0.0998, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3486824035644531, |
| "rewards/margins": 8.512846946716309, |
| "rewards/rejected": -8.861530303955078, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.3380507879152866, |
| "grad_norm": 0.8385043144226074, |
| "learning_rate": 2.0104397616624646e-05, |
| "logits/chosen": -6.892091751098633, |
| "logits/rejected": -6.384653568267822, |
| "logps/chosen": -242.2142333984375, |
| "logps/rejected": -310.5517578125, |
| "loss": 0.109, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.1313475370407104, |
| "rewards/margins": 8.243256568908691, |
| "rewards/rejected": -9.374604225158691, |
| "step": 421 |
| }, |
| { |
| "epoch": 0.33885375890795943, |
| "grad_norm": 0.8268947601318359, |
| "learning_rate": 2.0057726099286008e-05, |
| "logits/chosen": -6.945862770080566, |
| "logits/rejected": -6.528287887573242, |
| "logps/chosen": -247.18701171875, |
| "logps/rejected": -316.63433837890625, |
| "loss": 0.0768, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.9762799143791199, |
| "rewards/margins": 8.285195350646973, |
| "rewards/rejected": -9.261475563049316, |
| "step": 422 |
| }, |
| { |
| "epoch": 0.33965672990063234, |
| "grad_norm": 1.1565417051315308, |
| "learning_rate": 2.001099927146507e-05, |
| "logits/chosen": -7.151336669921875, |
| "logits/rejected": -6.546234130859375, |
| "logps/chosen": -257.0443420410156, |
| "logps/rejected": -330.1152038574219, |
| "loss": 0.1193, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7729849219322205, |
| "rewards/margins": 8.298803329467773, |
| "rewards/rejected": -9.071788787841797, |
| "step": 423 |
| }, |
| { |
| "epoch": 0.34045970089330524, |
| "grad_norm": 0.9829962253570557, |
| "learning_rate": 1.9964217644158925e-05, |
| "logits/chosen": -6.715579986572266, |
| "logits/rejected": -6.337420463562012, |
| "logps/chosen": -252.19891357421875, |
| "logps/rejected": -293.76824951171875, |
| "loss": 0.1197, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.7915411591529846, |
| "rewards/margins": 7.679810523986816, |
| "rewards/rejected": -8.471352577209473, |
| "step": 424 |
| }, |
| { |
| "epoch": 0.34126267188597814, |
| "grad_norm": 0.860735297203064, |
| "learning_rate": 1.9917381728963962e-05, |
| "logits/chosen": -7.0221452713012695, |
| "logits/rejected": -6.559934616088867, |
| "logps/chosen": -265.17010498046875, |
| "logps/rejected": -320.07171630859375, |
| "loss": 0.1014, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8033758401870728, |
| "rewards/margins": 8.149457931518555, |
| "rewards/rejected": -8.95283317565918, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.342065642878651, |
| "grad_norm": 0.9274251461029053, |
| "learning_rate": 1.9870492038070255e-05, |
| "logits/chosen": -7.103310585021973, |
| "logits/rejected": -6.496764659881592, |
| "logps/chosen": -241.57737731933594, |
| "logps/rejected": -326.06500244140625, |
| "loss": 0.0995, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.8578530550003052, |
| "rewards/margins": 8.227887153625488, |
| "rewards/rejected": -9.085738182067871, |
| "step": 426 |
| }, |
| { |
| "epoch": 0.3428686138713239, |
| "grad_norm": 0.7809159159660339, |
| "learning_rate": 1.982354908425593e-05, |
| "logits/chosen": -6.862217903137207, |
| "logits/rejected": -6.398541450500488, |
| "logps/chosen": -251.714111328125, |
| "logps/rejected": -327.6371154785156, |
| "loss": 0.0773, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.2999022006988525, |
| "rewards/margins": 7.8834733963012695, |
| "rewards/rejected": -9.183375358581543, |
| "step": 427 |
| }, |
| { |
| "epoch": 0.3436715848639968, |
| "grad_norm": 0.6849416494369507, |
| "learning_rate": 1.9776553380881634e-05, |
| "logits/chosen": -7.111811637878418, |
| "logits/rejected": -6.653296947479248, |
| "logps/chosen": -235.7373504638672, |
| "logps/rejected": -306.1861267089844, |
| "loss": 0.0783, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.7285397052764893, |
| "rewards/margins": 8.227411270141602, |
| "rewards/rejected": -8.955950736999512, |
| "step": 428 |
| }, |
| { |
| "epoch": 0.3444745558566697, |
| "grad_norm": 0.676033616065979, |
| "learning_rate": 1.9729505441884825e-05, |
| "logits/chosen": -6.959185600280762, |
| "logits/rejected": -6.434902191162109, |
| "logps/chosen": -220.4851837158203, |
| "logps/rejected": -291.1605224609375, |
| "loss": 0.086, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6876360177993774, |
| "rewards/margins": 7.821449279785156, |
| "rewards/rejected": -8.509085655212402, |
| "step": 429 |
| }, |
| { |
| "epoch": 0.34527752684934254, |
| "grad_norm": 0.6505718231201172, |
| "learning_rate": 1.968240578177424e-05, |
| "logits/chosen": -7.163081645965576, |
| "logits/rejected": -6.539896011352539, |
| "logps/chosen": -272.4314880371094, |
| "logps/rejected": -322.9618835449219, |
| "loss": 0.0722, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.21793456375598907, |
| "rewards/margins": 8.098798751831055, |
| "rewards/rejected": -8.316733360290527, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.34608049784201544, |
| "grad_norm": 0.6922266483306885, |
| "learning_rate": 1.963525491562421e-05, |
| "logits/chosen": -6.962629318237305, |
| "logits/rejected": -6.342557907104492, |
| "logps/chosen": -254.84959411621094, |
| "logps/rejected": -312.2868957519531, |
| "loss": 0.078, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5237269997596741, |
| "rewards/margins": 8.112176895141602, |
| "rewards/rejected": -8.635904312133789, |
| "step": 431 |
| }, |
| { |
| "epoch": 0.34688346883468835, |
| "grad_norm": 0.7436343431472778, |
| "learning_rate": 1.958805335906906e-05, |
| "logits/chosen": -6.9938530921936035, |
| "logits/rejected": -6.491478443145752, |
| "logps/chosen": -230.0971221923828, |
| "logps/rejected": -283.6400451660156, |
| "loss": 0.0853, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.5516283512115479, |
| "rewards/margins": 7.826017379760742, |
| "rewards/rejected": -8.377646446228027, |
| "step": 432 |
| }, |
| { |
| "epoch": 0.34768643982736125, |
| "grad_norm": 0.6518259644508362, |
| "learning_rate": 1.954080162829745e-05, |
| "logits/chosen": -7.160483360290527, |
| "logits/rejected": -6.722951412200928, |
| "logps/chosen": -238.1177978515625, |
| "logps/rejected": -290.6171569824219, |
| "loss": 0.076, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.5515754222869873, |
| "rewards/margins": 7.898458480834961, |
| "rewards/rejected": -8.450034141540527, |
| "step": 433 |
| }, |
| { |
| "epoch": 0.34848941082003415, |
| "grad_norm": 1.1883739233016968, |
| "learning_rate": 1.9493500240046727e-05, |
| "logits/chosen": -6.876124858856201, |
| "logits/rejected": -6.366808891296387, |
| "logps/chosen": -254.67916870117188, |
| "logps/rejected": -297.6768798828125, |
| "loss": 0.1287, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7257499694824219, |
| "rewards/margins": 7.452016830444336, |
| "rewards/rejected": -8.177765846252441, |
| "step": 434 |
| }, |
| { |
| "epoch": 0.349292381812707, |
| "grad_norm": 0.9302873611450195, |
| "learning_rate": 1.9446149711597306e-05, |
| "logits/chosen": -7.10894775390625, |
| "logits/rejected": -6.65434455871582, |
| "logps/chosen": -216.32443237304688, |
| "logps/rejected": -285.4888916015625, |
| "loss": 0.1066, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.4297732412815094, |
| "rewards/margins": 7.821617603302002, |
| "rewards/rejected": -8.251391410827637, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.3500953528053799, |
| "grad_norm": 0.7004202008247375, |
| "learning_rate": 1.9398750560766973e-05, |
| "logits/chosen": -7.1696319580078125, |
| "logits/rejected": -6.4806623458862305, |
| "logps/chosen": -263.01177978515625, |
| "logps/rejected": -329.2699279785156, |
| "loss": 0.072, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.37684571743011475, |
| "rewards/margins": 8.405688285827637, |
| "rewards/rejected": -8.7825345993042, |
| "step": 436 |
| }, |
| { |
| "epoch": 0.3508983237980528, |
| "grad_norm": 1.016250729560852, |
| "learning_rate": 1.935130330590525e-05, |
| "logits/chosen": -6.9120073318481445, |
| "logits/rejected": -6.451547622680664, |
| "logps/chosen": -268.41717529296875, |
| "logps/rejected": -315.1661071777344, |
| "loss": 0.126, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.5277736783027649, |
| "rewards/margins": 7.664824962615967, |
| "rewards/rejected": -8.192598342895508, |
| "step": 437 |
| }, |
| { |
| "epoch": 0.3517012947907257, |
| "grad_norm": 0.9289959669113159, |
| "learning_rate": 1.9303808465887713e-05, |
| "logits/chosen": -6.8682732582092285, |
| "logits/rejected": -6.374983310699463, |
| "logps/chosen": -274.0198669433594, |
| "logps/rejected": -311.4768371582031, |
| "loss": 0.1071, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4386255741119385, |
| "rewards/margins": 7.977336406707764, |
| "rewards/rejected": -8.415962219238281, |
| "step": 438 |
| }, |
| { |
| "epoch": 0.35250426578339855, |
| "grad_norm": 0.8622022867202759, |
| "learning_rate": 1.9256266560110322e-05, |
| "logits/chosen": -6.887612819671631, |
| "logits/rejected": -6.3301262855529785, |
| "logps/chosen": -246.86647033691406, |
| "logps/rejected": -318.3326416015625, |
| "loss": 0.1198, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.9959688782691956, |
| "rewards/margins": 7.922917366027832, |
| "rewards/rejected": -8.918886184692383, |
| "step": 439 |
| }, |
| { |
| "epoch": 0.35330723677607145, |
| "grad_norm": 0.8112181425094604, |
| "learning_rate": 1.920867810848375e-05, |
| "logits/chosen": -6.993706226348877, |
| "logits/rejected": -6.513548374176025, |
| "logps/chosen": -244.98104858398438, |
| "logps/rejected": -310.7253112792969, |
| "loss": 0.0818, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.7140471935272217, |
| "rewards/margins": 8.296806335449219, |
| "rewards/rejected": -9.010852813720703, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.35411020776874436, |
| "grad_norm": 0.7768401503562927, |
| "learning_rate": 1.916104363142767e-05, |
| "logits/chosen": -6.8701066970825195, |
| "logits/rejected": -6.43572473526001, |
| "logps/chosen": -282.9633483886719, |
| "logps/rejected": -342.3634338378906, |
| "loss": 0.102, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6696599125862122, |
| "rewards/margins": 7.9216718673706055, |
| "rewards/rejected": -8.591331481933594, |
| "step": 441 |
| }, |
| { |
| "epoch": 0.35491317876141726, |
| "grad_norm": 0.8104515671730042, |
| "learning_rate": 1.9113363649865095e-05, |
| "logits/chosen": -6.884217262268066, |
| "logits/rejected": -6.41135835647583, |
| "logps/chosen": -260.64031982421875, |
| "logps/rejected": -340.097412109375, |
| "loss": 0.1032, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.42694711685180664, |
| "rewards/margins": 7.777458190917969, |
| "rewards/rejected": -8.204404830932617, |
| "step": 442 |
| }, |
| { |
| "epoch": 0.35571614975409016, |
| "grad_norm": 1.0887058973312378, |
| "learning_rate": 1.9065638685216667e-05, |
| "logits/chosen": -6.96589469909668, |
| "logits/rejected": -6.389214038848877, |
| "logps/chosen": -289.8725280761719, |
| "logps/rejected": -327.43341064453125, |
| "loss": 0.0836, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.692720890045166, |
| "rewards/margins": 8.126760482788086, |
| "rewards/rejected": -8.819480895996094, |
| "step": 443 |
| }, |
| { |
| "epoch": 0.356519120746763, |
| "grad_norm": 0.8155497312545776, |
| "learning_rate": 1.9017869259394944e-05, |
| "logits/chosen": -6.873871326446533, |
| "logits/rejected": -6.252667427062988, |
| "logps/chosen": -241.41015625, |
| "logps/rejected": -322.0431213378906, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9360982775688171, |
| "rewards/margins": 7.587653160095215, |
| "rewards/rejected": -8.523751258850098, |
| "step": 444 |
| }, |
| { |
| "epoch": 0.3573220917394359, |
| "grad_norm": 1.1729787588119507, |
| "learning_rate": 1.8970055894798724e-05, |
| "logits/chosen": -6.988368988037109, |
| "logits/rejected": -6.565577030181885, |
| "logps/chosen": -231.1584014892578, |
| "logps/rejected": -287.90264892578125, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8417105674743652, |
| "rewards/margins": 7.8905534744262695, |
| "rewards/rejected": -8.732263565063477, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.3581250627321088, |
| "grad_norm": 0.8921499848365784, |
| "learning_rate": 1.8922199114307297e-05, |
| "logits/chosen": -6.944387435913086, |
| "logits/rejected": -6.433530330657959, |
| "logps/chosen": -236.83779907226562, |
| "logps/rejected": -279.7940673828125, |
| "loss": 0.1316, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.5465189218521118, |
| "rewards/margins": 6.912130355834961, |
| "rewards/rejected": -7.458649158477783, |
| "step": 446 |
| }, |
| { |
| "epoch": 0.3589280337247817, |
| "grad_norm": 0.7855164408683777, |
| "learning_rate": 1.887429944127475e-05, |
| "logits/chosen": -7.334751605987549, |
| "logits/rejected": -6.575271129608154, |
| "logps/chosen": -248.7917022705078, |
| "logps/rejected": -304.7749938964844, |
| "loss": 0.1068, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.31790250539779663, |
| "rewards/margins": 7.679190635681152, |
| "rewards/rejected": -7.997092247009277, |
| "step": 447 |
| }, |
| { |
| "epoch": 0.35973100471745456, |
| "grad_norm": 0.59900963306427, |
| "learning_rate": 1.8826357399524227e-05, |
| "logits/chosen": -7.1491570472717285, |
| "logits/rejected": -6.588574409484863, |
| "logps/chosen": -272.842529296875, |
| "logps/rejected": -327.59503173828125, |
| "loss": 0.0639, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.2245875597000122, |
| "rewards/margins": 8.238356590270996, |
| "rewards/rejected": -8.013769149780273, |
| "step": 448 |
| }, |
| { |
| "epoch": 0.36053397571012746, |
| "grad_norm": 0.8680687546730042, |
| "learning_rate": 1.8778373513342223e-05, |
| "logits/chosen": -7.1484808921813965, |
| "logits/rejected": -6.534069538116455, |
| "logps/chosen": -261.27630615234375, |
| "logps/rejected": -331.3484191894531, |
| "loss": 0.109, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.5502551794052124, |
| "rewards/margins": 7.472077369689941, |
| "rewards/rejected": -8.022333145141602, |
| "step": 449 |
| }, |
| { |
| "epoch": 0.36133694670280037, |
| "grad_norm": 0.8833668828010559, |
| "learning_rate": 1.8730348307472828e-05, |
| "logits/chosen": -6.946518898010254, |
| "logits/rejected": -6.448605537414551, |
| "logps/chosen": -262.5815124511719, |
| "logps/rejected": -345.7423400878906, |
| "loss": 0.126, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.35796454548835754, |
| "rewards/margins": 7.355600833892822, |
| "rewards/rejected": -7.713565826416016, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.36213991769547327, |
| "grad_norm": 0.7468612194061279, |
| "learning_rate": 1.8682282307111988e-05, |
| "logits/chosen": -6.876276016235352, |
| "logits/rejected": -6.433337211608887, |
| "logps/chosen": -247.5784149169922, |
| "logps/rejected": -295.5265197753906, |
| "loss": 0.1161, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.46249234676361084, |
| "rewards/margins": 7.425567626953125, |
| "rewards/rejected": -7.888059616088867, |
| "step": 451 |
| }, |
| { |
| "epoch": 0.3629428886881461, |
| "grad_norm": 0.9048655033111572, |
| "learning_rate": 1.8634176037901797e-05, |
| "logits/chosen": -6.833037853240967, |
| "logits/rejected": -6.416132926940918, |
| "logps/chosen": -282.77691650390625, |
| "logps/rejected": -320.2112121582031, |
| "loss": 0.1048, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.060280993580818176, |
| "rewards/margins": 7.361087322235107, |
| "rewards/rejected": -7.421369552612305, |
| "step": 452 |
| }, |
| { |
| "epoch": 0.363745859680819, |
| "grad_norm": 0.772041380405426, |
| "learning_rate": 1.8586030025924697e-05, |
| "logits/chosen": -7.157096862792969, |
| "logits/rejected": -6.583315849304199, |
| "logps/chosen": -247.8695068359375, |
| "logps/rejected": -313.20526123046875, |
| "loss": 0.0985, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.42252907156944275, |
| "rewards/margins": 7.520169734954834, |
| "rewards/rejected": -7.9426984786987305, |
| "step": 453 |
| }, |
| { |
| "epoch": 0.3645488306734919, |
| "grad_norm": 0.8485280275344849, |
| "learning_rate": 1.8537844797697775e-05, |
| "logits/chosen": -7.057374954223633, |
| "logits/rejected": -6.617366790771484, |
| "logps/chosen": -252.18325805664062, |
| "logps/rejected": -303.9050598144531, |
| "loss": 0.1022, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.4023701846599579, |
| "rewards/margins": 7.37721061706543, |
| "rewards/rejected": -7.779581069946289, |
| "step": 454 |
| }, |
| { |
| "epoch": 0.3653518016661648, |
| "grad_norm": 1.1745760440826416, |
| "learning_rate": 1.8489620880166956e-05, |
| "logits/chosen": -7.157766819000244, |
| "logits/rejected": -6.593669414520264, |
| "logps/chosen": -237.5233154296875, |
| "logps/rejected": -289.4632263183594, |
| "loss": 0.1095, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.4018235504627228, |
| "rewards/margins": 7.7774128913879395, |
| "rewards/rejected": -8.17923641204834, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.3661547726588377, |
| "grad_norm": 1.0049073696136475, |
| "learning_rate": 1.8441358800701276e-05, |
| "logits/chosen": -7.0141801834106445, |
| "logits/rejected": -6.492361545562744, |
| "logps/chosen": -251.62545776367188, |
| "logps/rejected": -297.1331787109375, |
| "loss": 0.1112, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.973258376121521, |
| "rewards/margins": 7.299931526184082, |
| "rewards/rejected": -8.273189544677734, |
| "step": 456 |
| }, |
| { |
| "epoch": 0.36695774365151057, |
| "grad_norm": 0.8520278334617615, |
| "learning_rate": 1.8393059087087106e-05, |
| "logits/chosen": -7.1340107917785645, |
| "logits/rejected": -6.518468379974365, |
| "logps/chosen": -267.3978271484375, |
| "logps/rejected": -335.26251220703125, |
| "loss": 0.0988, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.22372883558273315, |
| "rewards/margins": 7.526930809020996, |
| "rewards/rejected": -7.750659942626953, |
| "step": 457 |
| }, |
| { |
| "epoch": 0.3677607146441835, |
| "grad_norm": 0.5203014016151428, |
| "learning_rate": 1.8344722267522377e-05, |
| "logits/chosen": -7.169615745544434, |
| "logits/rejected": -6.580011367797852, |
| "logps/chosen": -236.51620483398438, |
| "logps/rejected": -280.47247314453125, |
| "loss": 0.046, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.4789540767669678, |
| "rewards/margins": 8.536890983581543, |
| "rewards/rejected": -9.015844345092773, |
| "step": 458 |
| }, |
| { |
| "epoch": 0.3685636856368564, |
| "grad_norm": 0.7870261669158936, |
| "learning_rate": 1.8296348870610798e-05, |
| "logits/chosen": -7.025090217590332, |
| "logits/rejected": -6.497011184692383, |
| "logps/chosen": -228.1007080078125, |
| "logps/rejected": -294.1159362792969, |
| "loss": 0.1075, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7050939202308655, |
| "rewards/margins": 7.689603805541992, |
| "rewards/rejected": -8.394698143005371, |
| "step": 459 |
| }, |
| { |
| "epoch": 0.3693666566295293, |
| "grad_norm": 0.6552004218101501, |
| "learning_rate": 1.82479394253561e-05, |
| "logits/chosen": -6.886502265930176, |
| "logits/rejected": -6.4395599365234375, |
| "logps/chosen": -266.8811340332031, |
| "logps/rejected": -324.7569580078125, |
| "loss": 0.0872, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.18474626541137695, |
| "rewards/margins": 7.91121768951416, |
| "rewards/rejected": -8.095963478088379, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.3701696276222021, |
| "grad_norm": 0.6798030734062195, |
| "learning_rate": 1.8199494461156203e-05, |
| "logits/chosen": -7.284564018249512, |
| "logits/rejected": -6.579012393951416, |
| "logps/chosen": -229.57550048828125, |
| "logps/rejected": -293.93389892578125, |
| "loss": 0.0622, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.2847965359687805, |
| "rewards/margins": 8.45785903930664, |
| "rewards/rejected": -8.742656707763672, |
| "step": 461 |
| }, |
| { |
| "epoch": 0.370972598614875, |
| "grad_norm": 1.2206978797912598, |
| "learning_rate": 1.815101450779749e-05, |
| "logits/chosen": -7.076620101928711, |
| "logits/rejected": -6.548724174499512, |
| "logps/chosen": -235.90797424316406, |
| "logps/rejected": -281.5258483886719, |
| "loss": 0.1162, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.6784594058990479, |
| "rewards/margins": 7.812564849853516, |
| "rewards/rejected": -8.491024017333984, |
| "step": 462 |
| }, |
| { |
| "epoch": 0.37177556960754793, |
| "grad_norm": 1.0261527299880981, |
| "learning_rate": 1.810250009544895e-05, |
| "logits/chosen": -7.115738868713379, |
| "logits/rejected": -6.70600700378418, |
| "logps/chosen": -228.22215270996094, |
| "logps/rejected": -301.8640441894531, |
| "loss": 0.1012, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9627201557159424, |
| "rewards/margins": 7.9360270500183105, |
| "rewards/rejected": -8.898747444152832, |
| "step": 463 |
| }, |
| { |
| "epoch": 0.37257854060022083, |
| "grad_norm": 1.0820735692977905, |
| "learning_rate": 1.8053951754656438e-05, |
| "logits/chosen": -6.868494510650635, |
| "logits/rejected": -6.386383056640625, |
| "logps/chosen": -269.4851989746094, |
| "logps/rejected": -320.227783203125, |
| "loss": 0.1319, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.6601322889328003, |
| "rewards/margins": 7.531393051147461, |
| "rewards/rejected": -8.191524505615234, |
| "step": 464 |
| }, |
| { |
| "epoch": 0.37338151159289373, |
| "grad_norm": 0.9172157049179077, |
| "learning_rate": 1.800537001633682e-05, |
| "logits/chosen": -7.038186073303223, |
| "logits/rejected": -6.52276086807251, |
| "logps/chosen": -231.00648498535156, |
| "logps/rejected": -298.9999084472656, |
| "loss": 0.1036, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.6557649374008179, |
| "rewards/margins": 7.962772846221924, |
| "rewards/rejected": -8.618537902832031, |
| "step": 465 |
| }, |
| { |
| "epoch": 0.3741844825855666, |
| "grad_norm": 0.7982479929924011, |
| "learning_rate": 1.7956755411772203e-05, |
| "logits/chosen": -7.06585168838501, |
| "logits/rejected": -6.610898971557617, |
| "logps/chosen": -257.69012451171875, |
| "logps/rejected": -301.1226806640625, |
| "loss": 0.0955, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.6350902318954468, |
| "rewards/margins": 7.995002746582031, |
| "rewards/rejected": -8.63009262084961, |
| "step": 466 |
| }, |
| { |
| "epoch": 0.3749874535782395, |
| "grad_norm": 0.8446144461631775, |
| "learning_rate": 1.7908108472604124e-05, |
| "logits/chosen": -7.202521800994873, |
| "logits/rejected": -6.632724761962891, |
| "logps/chosen": -238.05712890625, |
| "logps/rejected": -315.97869873046875, |
| "loss": 0.0933, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9967688322067261, |
| "rewards/margins": 7.882102966308594, |
| "rewards/rejected": -8.878870964050293, |
| "step": 467 |
| }, |
| { |
| "epoch": 0.3757904245709124, |
| "grad_norm": 1.2383058071136475, |
| "learning_rate": 1.7859429730827704e-05, |
| "logits/chosen": -7.020427703857422, |
| "logits/rejected": -6.427013874053955, |
| "logps/chosen": -241.44422912597656, |
| "logps/rejected": -300.1913146972656, |
| "loss": 0.1186, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8708614706993103, |
| "rewards/margins": 7.704665184020996, |
| "rewards/rejected": -8.57552719116211, |
| "step": 468 |
| }, |
| { |
| "epoch": 0.3765933955635853, |
| "grad_norm": 1.08328378200531, |
| "learning_rate": 1.781071971878587e-05, |
| "logits/chosen": -7.016770362854004, |
| "logits/rejected": -6.475790977478027, |
| "logps/chosen": -247.5828399658203, |
| "logps/rejected": -300.3699645996094, |
| "loss": 0.117, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.6060992479324341, |
| "rewards/margins": 8.000041961669922, |
| "rewards/rejected": -8.60614013671875, |
| "step": 469 |
| }, |
| { |
| "epoch": 0.37739636655625813, |
| "grad_norm": 0.8031619787216187, |
| "learning_rate": 1.7761978969163508e-05, |
| "logits/chosen": -7.066303253173828, |
| "logits/rejected": -6.505469799041748, |
| "logps/chosen": -226.7652130126953, |
| "logps/rejected": -299.9187316894531, |
| "loss": 0.0947, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.47484496235847473, |
| "rewards/margins": 8.053740501403809, |
| "rewards/rejected": -8.528585433959961, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.37819933754893104, |
| "grad_norm": 0.9957223534584045, |
| "learning_rate": 1.771320801498165e-05, |
| "logits/chosen": -6.949533939361572, |
| "logits/rejected": -6.455217361450195, |
| "logps/chosen": -234.5625762939453, |
| "logps/rejected": -271.65594482421875, |
| "loss": 0.1141, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.7566280961036682, |
| "rewards/margins": 7.591424465179443, |
| "rewards/rejected": -8.348053932189941, |
| "step": 471 |
| }, |
| { |
| "epoch": 0.37900230854160394, |
| "grad_norm": 0.8835733532905579, |
| "learning_rate": 1.766440738959163e-05, |
| "logits/chosen": -7.201594352722168, |
| "logits/rejected": -6.566706657409668, |
| "logps/chosen": -230.23959350585938, |
| "logps/rejected": -284.4130859375, |
| "loss": 0.1103, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.4764450788497925, |
| "rewards/margins": 7.908781051635742, |
| "rewards/rejected": -8.385226249694824, |
| "step": 472 |
| }, |
| { |
| "epoch": 0.37980527953427684, |
| "grad_norm": 0.7579358220100403, |
| "learning_rate": 1.7615577626669274e-05, |
| "logits/chosen": -6.889339447021484, |
| "logits/rejected": -6.314489364624023, |
| "logps/chosen": -228.36241149902344, |
| "logps/rejected": -296.3380432128906, |
| "loss": 0.1145, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8669180870056152, |
| "rewards/margins": 7.24848747253418, |
| "rewards/rejected": -8.115405082702637, |
| "step": 473 |
| }, |
| { |
| "epoch": 0.3806082505269497, |
| "grad_norm": 1.0094982385635376, |
| "learning_rate": 1.756671926020905e-05, |
| "logits/chosen": -6.904953956604004, |
| "logits/rejected": -6.439263820648193, |
| "logps/chosen": -257.5345153808594, |
| "logps/rejected": -313.7396240234375, |
| "loss": 0.1196, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.14854465425014496, |
| "rewards/margins": 7.966803073883057, |
| "rewards/rejected": -8.115347862243652, |
| "step": 474 |
| }, |
| { |
| "epoch": 0.3814112215196226, |
| "grad_norm": 0.803789496421814, |
| "learning_rate": 1.7517832824518216e-05, |
| "logits/chosen": -7.293473243713379, |
| "logits/rejected": -6.769782543182373, |
| "logps/chosen": -209.95570373535156, |
| "logps/rejected": -292.32110595703125, |
| "loss": 0.1011, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.4576115012168884, |
| "rewards/margins": 8.011465072631836, |
| "rewards/rejected": -8.469078063964844, |
| "step": 475 |
| }, |
| { |
| "epoch": 0.3822141925122955, |
| "grad_norm": 0.8738084435462952, |
| "learning_rate": 1.746891885421101e-05, |
| "logits/chosen": -7.064651966094971, |
| "logits/rejected": -6.599350452423096, |
| "logps/chosen": -247.2891845703125, |
| "logps/rejected": -299.1720275878906, |
| "loss": 0.0959, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7152971029281616, |
| "rewards/margins": 7.362991809844971, |
| "rewards/rejected": -8.078289031982422, |
| "step": 476 |
| }, |
| { |
| "epoch": 0.3830171635049684, |
| "grad_norm": 0.7580540180206299, |
| "learning_rate": 1.7419977884202765e-05, |
| "logits/chosen": -7.023658275604248, |
| "logits/rejected": -6.508403301239014, |
| "logps/chosen": -227.27915954589844, |
| "logps/rejected": -294.82855224609375, |
| "loss": 0.0793, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.5473260879516602, |
| "rewards/margins": 7.520500183105469, |
| "rewards/rejected": -8.067825317382812, |
| "step": 477 |
| }, |
| { |
| "epoch": 0.3838201344976413, |
| "grad_norm": 0.5841532349586487, |
| "learning_rate": 1.7371010449704097e-05, |
| "logits/chosen": -7.330380439758301, |
| "logits/rejected": -6.7224812507629395, |
| "logps/chosen": -234.01498413085938, |
| "logps/rejected": -310.41204833984375, |
| "loss": 0.0656, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.3101215958595276, |
| "rewards/margins": 8.111172676086426, |
| "rewards/rejected": -8.421293258666992, |
| "step": 478 |
| }, |
| { |
| "epoch": 0.38462310549031414, |
| "grad_norm": 0.6563722491264343, |
| "learning_rate": 1.7322017086215023e-05, |
| "logits/chosen": -7.259189605712891, |
| "logits/rejected": -6.768599510192871, |
| "logps/chosen": -245.41262817382812, |
| "logps/rejected": -282.0882568359375, |
| "loss": 0.078, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.3107805848121643, |
| "rewards/margins": 7.831932067871094, |
| "rewards/rejected": -8.142712593078613, |
| "step": 479 |
| }, |
| { |
| "epoch": 0.38542607648298705, |
| "grad_norm": 0.8157557249069214, |
| "learning_rate": 1.7272998329519106e-05, |
| "logits/chosen": -7.113288879394531, |
| "logits/rejected": -6.5889434814453125, |
| "logps/chosen": -255.74407958984375, |
| "logps/rejected": -308.52691650390625, |
| "loss": 0.1101, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.23130030930042267, |
| "rewards/margins": 7.521341800689697, |
| "rewards/rejected": -7.752641677856445, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.38622904747565995, |
| "grad_norm": 0.7508776187896729, |
| "learning_rate": 1.722395471567763e-05, |
| "logits/chosen": -7.144013404846191, |
| "logits/rejected": -6.601110458374023, |
| "logps/chosen": -236.16802978515625, |
| "logps/rejected": -288.15179443359375, |
| "loss": 0.0837, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.4036630094051361, |
| "rewards/margins": 7.805217742919922, |
| "rewards/rejected": -8.208881378173828, |
| "step": 481 |
| }, |
| { |
| "epoch": 0.38703201846833285, |
| "grad_norm": 0.9542524218559265, |
| "learning_rate": 1.717488678102369e-05, |
| "logits/chosen": -7.03851318359375, |
| "logits/rejected": -6.465890407562256, |
| "logps/chosen": -259.56109619140625, |
| "logps/rejected": -312.1416931152344, |
| "loss": 0.1065, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5327616333961487, |
| "rewards/margins": 7.554989337921143, |
| "rewards/rejected": -8.087750434875488, |
| "step": 482 |
| }, |
| { |
| "epoch": 0.3878349894610057, |
| "grad_norm": 0.842280387878418, |
| "learning_rate": 1.7125795062156363e-05, |
| "logits/chosen": -7.07091760635376, |
| "logits/rejected": -6.529902458190918, |
| "logps/chosen": -232.80889892578125, |
| "logps/rejected": -287.49285888671875, |
| "loss": 0.0951, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.19765782356262207, |
| "rewards/margins": 7.831417560577393, |
| "rewards/rejected": -8.029075622558594, |
| "step": 483 |
| }, |
| { |
| "epoch": 0.3886379604536786, |
| "grad_norm": 0.8058556318283081, |
| "learning_rate": 1.7076680095934813e-05, |
| "logits/chosen": -6.862829208374023, |
| "logits/rejected": -6.381878852844238, |
| "logps/chosen": -232.39813232421875, |
| "logps/rejected": -285.0888366699219, |
| "loss": 0.1213, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.6812834143638611, |
| "rewards/margins": 7.22131872177124, |
| "rewards/rejected": -7.902602672576904, |
| "step": 484 |
| }, |
| { |
| "epoch": 0.3894409314463515, |
| "grad_norm": 0.7739570140838623, |
| "learning_rate": 1.7027542419472454e-05, |
| "logits/chosen": -7.133261203765869, |
| "logits/rejected": -6.770265579223633, |
| "logps/chosen": -256.95648193359375, |
| "logps/rejected": -298.272216796875, |
| "loss": 0.1088, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.2508806884288788, |
| "rewards/margins": 7.246029376983643, |
| "rewards/rejected": -7.4969096183776855, |
| "step": 485 |
| }, |
| { |
| "epoch": 0.3902439024390244, |
| "grad_norm": 0.619405210018158, |
| "learning_rate": 1.6978382570131037e-05, |
| "logits/chosen": -7.279349327087402, |
| "logits/rejected": -6.692509651184082, |
| "logps/chosen": -279.9124755859375, |
| "logps/rejected": -338.10186767578125, |
| "loss": 0.0672, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.2043972760438919, |
| "rewards/margins": 7.895685195922852, |
| "rewards/rejected": -8.100082397460938, |
| "step": 486 |
| }, |
| { |
| "epoch": 0.3910468734316973, |
| "grad_norm": 0.7752019166946411, |
| "learning_rate": 1.6929201085514793e-05, |
| "logits/chosen": -7.04326868057251, |
| "logits/rejected": -6.589953422546387, |
| "logps/chosen": -229.86892700195312, |
| "logps/rejected": -283.73406982421875, |
| "loss": 0.1112, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.6013769507408142, |
| "rewards/margins": 7.5253753662109375, |
| "rewards/rejected": -8.126752853393555, |
| "step": 487 |
| }, |
| { |
| "epoch": 0.39184984442437015, |
| "grad_norm": 0.8248339295387268, |
| "learning_rate": 1.6879998503464565e-05, |
| "logits/chosen": -6.900211811065674, |
| "logits/rejected": -6.4468092918396, |
| "logps/chosen": -225.80978393554688, |
| "logps/rejected": -277.3603820800781, |
| "loss": 0.1004, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7860382199287415, |
| "rewards/margins": 7.686061859130859, |
| "rewards/rejected": -8.472100257873535, |
| "step": 488 |
| }, |
| { |
| "epoch": 0.39265281541704306, |
| "grad_norm": 0.8041821122169495, |
| "learning_rate": 1.6830775362051904e-05, |
| "logits/chosen": -7.124450206756592, |
| "logits/rejected": -6.4981794357299805, |
| "logps/chosen": -229.47723388671875, |
| "logps/rejected": -311.91204833984375, |
| "loss": 0.0954, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.3344499170780182, |
| "rewards/margins": 7.895968437194824, |
| "rewards/rejected": -8.23041820526123, |
| "step": 489 |
| }, |
| { |
| "epoch": 0.39345578640971596, |
| "grad_norm": 0.8334702253341675, |
| "learning_rate": 1.6781532199573203e-05, |
| "logits/chosen": -6.6048784255981445, |
| "logits/rejected": -6.339022636413574, |
| "logps/chosen": -275.9911193847656, |
| "logps/rejected": -318.113037109375, |
| "loss": 0.1067, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8185275793075562, |
| "rewards/margins": 7.372434616088867, |
| "rewards/rejected": -8.190962791442871, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.39425875740238886, |
| "grad_norm": 0.6908960342407227, |
| "learning_rate": 1.6732269554543794e-05, |
| "logits/chosen": -6.906381607055664, |
| "logits/rejected": -6.3860907554626465, |
| "logps/chosen": -240.2025604248047, |
| "logps/rejected": -309.26690673828125, |
| "loss": 0.0911, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5807979702949524, |
| "rewards/margins": 8.098869323730469, |
| "rewards/rejected": -8.679668426513672, |
| "step": 491 |
| }, |
| { |
| "epoch": 0.3950617283950617, |
| "grad_norm": 0.9036932587623596, |
| "learning_rate": 1.668298796569207e-05, |
| "logits/chosen": -6.930481433868408, |
| "logits/rejected": -6.444879055023193, |
| "logps/chosen": -251.92010498046875, |
| "logps/rejected": -314.49676513671875, |
| "loss": 0.1135, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.4067593216896057, |
| "rewards/margins": 7.779881000518799, |
| "rewards/rejected": -8.186639785766602, |
| "step": 492 |
| }, |
| { |
| "epoch": 0.3958646993877346, |
| "grad_norm": 1.1294622421264648, |
| "learning_rate": 1.663368797195359e-05, |
| "logits/chosen": -6.961066722869873, |
| "logits/rejected": -6.44651985168457, |
| "logps/chosen": -219.45565795898438, |
| "logps/rejected": -297.49517822265625, |
| "loss": 0.1483, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -1.011552333831787, |
| "rewards/margins": 7.6737799644470215, |
| "rewards/rejected": -8.685332298278809, |
| "step": 493 |
| }, |
| { |
| "epoch": 0.3966676703804075, |
| "grad_norm": 0.9393503069877625, |
| "learning_rate": 1.6584370112465187e-05, |
| "logits/chosen": -7.141911029815674, |
| "logits/rejected": -6.6896843910217285, |
| "logps/chosen": -226.10821533203125, |
| "logps/rejected": -290.251708984375, |
| "loss": 0.0898, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.7292478084564209, |
| "rewards/margins": 7.684271335601807, |
| "rewards/rejected": -8.413518905639648, |
| "step": 494 |
| }, |
| { |
| "epoch": 0.3974706413730804, |
| "grad_norm": 0.7560012340545654, |
| "learning_rate": 1.6535034926559075e-05, |
| "logits/chosen": -7.154983997344971, |
| "logits/rejected": -6.501282215118408, |
| "logps/chosen": -232.07040405273438, |
| "logps/rejected": -298.7330017089844, |
| "loss": 0.0793, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.896610677242279, |
| "rewards/margins": 7.9785542488098145, |
| "rewards/rejected": -8.875164031982422, |
| "step": 495 |
| }, |
| { |
| "epoch": 0.39827361236575326, |
| "grad_norm": 0.8159294128417969, |
| "learning_rate": 1.6485682953756945e-05, |
| "logits/chosen": -6.862372875213623, |
| "logits/rejected": -6.352356433868408, |
| "logps/chosen": -241.40794372558594, |
| "logps/rejected": -305.573486328125, |
| "loss": 0.1058, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8417636752128601, |
| "rewards/margins": 7.507452487945557, |
| "rewards/rejected": -8.34921646118164, |
| "step": 496 |
| }, |
| { |
| "epoch": 0.39907658335842616, |
| "grad_norm": 0.8525014519691467, |
| "learning_rate": 1.643631473376405e-05, |
| "logits/chosen": -7.264664173126221, |
| "logits/rejected": -6.706014156341553, |
| "logps/chosen": -238.5486602783203, |
| "logps/rejected": -314.1079406738281, |
| "loss": 0.0818, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.5642629861831665, |
| "rewards/margins": 8.082209587097168, |
| "rewards/rejected": -8.646471977233887, |
| "step": 497 |
| }, |
| { |
| "epoch": 0.39987955435109906, |
| "grad_norm": 0.7408416867256165, |
| "learning_rate": 1.6386930806463355e-05, |
| "logits/chosen": -7.087009906768799, |
| "logits/rejected": -6.519339084625244, |
| "logps/chosen": -239.04034423828125, |
| "logps/rejected": -304.6636047363281, |
| "loss": 0.0807, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.8081749081611633, |
| "rewards/margins": 7.79912805557251, |
| "rewards/rejected": -8.60730266571045, |
| "step": 498 |
| }, |
| { |
| "epoch": 0.40068252534377197, |
| "grad_norm": 0.7809262275695801, |
| "learning_rate": 1.633753171190956e-05, |
| "logits/chosen": -7.199151039123535, |
| "logits/rejected": -6.584648132324219, |
| "logps/chosen": -245.2847137451172, |
| "logps/rejected": -318.604248046875, |
| "loss": 0.0883, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.4906904101371765, |
| "rewards/margins": 8.184606552124023, |
| "rewards/rejected": -8.675296783447266, |
| "step": 499 |
| }, |
| { |
| "epoch": 0.40148549633644487, |
| "grad_norm": 1.1680251359939575, |
| "learning_rate": 1.628811799032326e-05, |
| "logits/chosen": -6.977715492248535, |
| "logits/rejected": -6.524835109710693, |
| "logps/chosen": -264.2124938964844, |
| "logps/rejected": -305.6918029785156, |
| "loss": 0.1243, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.7463261485099792, |
| "rewards/margins": 7.711738586425781, |
| "rewards/rejected": -8.458065032958984, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.40148549633644487, |
| "eval_logits/chosen": -7.019352436065674, |
| "eval_logits/rejected": -6.507395267486572, |
| "eval_logps/chosen": -249.57070922851562, |
| "eval_logps/rejected": -308.8583984375, |
| "eval_loss": 0.09778793156147003, |
| "eval_rewards/accuracies": 0.9526874423027039, |
| "eval_rewards/chosen": -0.7775754928588867, |
| "eval_rewards/margins": 7.931314945220947, |
| "eval_rewards/rejected": -8.708890914916992, |
| "eval_runtime": 714.2962, |
| "eval_samples_per_second": 49.593, |
| "eval_steps_per_second": 1.55, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.4022884673291177, |
| "grad_norm": 0.9121918678283691, |
| "learning_rate": 1.623869018208499e-05, |
| "logits/chosen": -6.932864189147949, |
| "logits/rejected": -6.380678176879883, |
| "logps/chosen": -231.28111267089844, |
| "logps/rejected": -292.5357666015625, |
| "loss": 0.1064, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.8121476769447327, |
| "rewards/margins": 7.75382661819458, |
| "rewards/rejected": -8.565973281860352, |
| "step": 501 |
| }, |
| { |
| "epoch": 0.4030914383217906, |
| "grad_norm": 0.8205099701881409, |
| "learning_rate": 1.6189248827729336e-05, |
| "logits/chosen": -7.108974456787109, |
| "logits/rejected": -6.7085862159729, |
| "logps/chosen": -253.17938232421875, |
| "logps/rejected": -320.60931396484375, |
| "loss": 0.1121, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.0115556716918945, |
| "rewards/margins": 7.573432922363281, |
| "rewards/rejected": -8.584988594055176, |
| "step": 502 |
| }, |
| { |
| "epoch": 0.4038944093144635, |
| "grad_norm": 0.8929060697555542, |
| "learning_rate": 1.6139794467939047e-05, |
| "logits/chosen": -6.961796283721924, |
| "logits/rejected": -6.451146125793457, |
| "logps/chosen": -250.86392211914062, |
| "logps/rejected": -294.2071838378906, |
| "loss": 0.0963, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6852113008499146, |
| "rewards/margins": 8.070762634277344, |
| "rewards/rejected": -8.755974769592285, |
| "step": 503 |
| }, |
| { |
| "epoch": 0.4046973803071364, |
| "grad_norm": 0.8446370363235474, |
| "learning_rate": 1.6090327643539067e-05, |
| "logits/chosen": -7.023614883422852, |
| "logits/rejected": -6.418012619018555, |
| "logps/chosen": -247.16030883789062, |
| "logps/rejected": -291.47821044921875, |
| "loss": 0.0899, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.2261275053024292, |
| "rewards/margins": 8.314494132995605, |
| "rewards/rejected": -9.540621757507324, |
| "step": 504 |
| }, |
| { |
| "epoch": 0.40550035129980927, |
| "grad_norm": 0.7505039572715759, |
| "learning_rate": 1.604084889549066e-05, |
| "logits/chosen": -7.045934200286865, |
| "logits/rejected": -6.434757709503174, |
| "logps/chosen": -259.7769775390625, |
| "logps/rejected": -332.60467529296875, |
| "loss": 0.0866, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.6253436207771301, |
| "rewards/margins": 8.152265548706055, |
| "rewards/rejected": -8.777608871459961, |
| "step": 505 |
| }, |
| { |
| "epoch": 0.40630332229248217, |
| "grad_norm": 0.6741944551467896, |
| "learning_rate": 1.5991358764885492e-05, |
| "logits/chosen": -7.2066240310668945, |
| "logits/rejected": -6.641116142272949, |
| "logps/chosen": -249.11111450195312, |
| "logps/rejected": -329.6019592285156, |
| "loss": 0.0653, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.9750706553459167, |
| "rewards/margins": 8.132675170898438, |
| "rewards/rejected": -9.107746124267578, |
| "step": 506 |
| }, |
| { |
| "epoch": 0.4071062932851551, |
| "grad_norm": 0.7370848655700684, |
| "learning_rate": 1.5941857792939702e-05, |
| "logits/chosen": -7.030134201049805, |
| "logits/rejected": -6.405796527862549, |
| "logps/chosen": -242.0516357421875, |
| "logps/rejected": -320.4596252441406, |
| "loss": 0.0856, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.0916215181350708, |
| "rewards/margins": 8.084473609924316, |
| "rewards/rejected": -9.176095962524414, |
| "step": 507 |
| }, |
| { |
| "epoch": 0.407909264277828, |
| "grad_norm": 0.9687086343765259, |
| "learning_rate": 1.5892346520987986e-05, |
| "logits/chosen": -6.836935520172119, |
| "logits/rejected": -6.283965587615967, |
| "logps/chosen": -264.1382751464844, |
| "logps/rejected": -317.37298583984375, |
| "loss": 0.1144, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9645687341690063, |
| "rewards/margins": 7.4496917724609375, |
| "rewards/rejected": -8.414260864257812, |
| "step": 508 |
| }, |
| { |
| "epoch": 0.4087122352705009, |
| "grad_norm": 0.6565617918968201, |
| "learning_rate": 1.5842825490477683e-05, |
| "logits/chosen": -7.075970649719238, |
| "logits/rejected": -6.418222427368164, |
| "logps/chosen": -249.136962890625, |
| "logps/rejected": -307.1061096191406, |
| "loss": 0.0853, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8276813626289368, |
| "rewards/margins": 8.052398681640625, |
| "rewards/rejected": -8.88007926940918, |
| "step": 509 |
| }, |
| { |
| "epoch": 0.4095152062631737, |
| "grad_norm": 1.0568733215332031, |
| "learning_rate": 1.5793295242962852e-05, |
| "logits/chosen": -6.872131824493408, |
| "logits/rejected": -6.450734615325928, |
| "logps/chosen": -267.2333068847656, |
| "logps/rejected": -330.80877685546875, |
| "loss": 0.0849, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6713874936103821, |
| "rewards/margins": 8.265071868896484, |
| "rewards/rejected": -8.936458587646484, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.41031817725584663, |
| "grad_norm": 0.9189753532409668, |
| "learning_rate": 1.5743756320098334e-05, |
| "logits/chosen": -7.1187849044799805, |
| "logits/rejected": -6.542022228240967, |
| "logps/chosen": -237.70709228515625, |
| "logps/rejected": -313.7509460449219, |
| "loss": 0.0954, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.3060979843139648, |
| "rewards/margins": 8.040251731872559, |
| "rewards/rejected": -9.346349716186523, |
| "step": 511 |
| }, |
| { |
| "epoch": 0.41112114824851953, |
| "grad_norm": 0.6864936351776123, |
| "learning_rate": 1.5694209263633866e-05, |
| "logits/chosen": -7.121218204498291, |
| "logits/rejected": -6.675223350524902, |
| "logps/chosen": -243.2194061279297, |
| "logps/rejected": -309.6638488769531, |
| "loss": 0.0911, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.0186771154403687, |
| "rewards/margins": 8.004389762878418, |
| "rewards/rejected": -9.023067474365234, |
| "step": 512 |
| }, |
| { |
| "epoch": 0.41192411924119243, |
| "grad_norm": 0.5543720126152039, |
| "learning_rate": 1.564465461540811e-05, |
| "logits/chosen": -7.166328430175781, |
| "logits/rejected": -6.513040542602539, |
| "logps/chosen": -220.5634765625, |
| "logps/rejected": -282.79010009765625, |
| "loss": 0.089, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.4594486653804779, |
| "rewards/margins": 8.53774642944336, |
| "rewards/rejected": -8.99719524383545, |
| "step": 513 |
| }, |
| { |
| "epoch": 0.4127270902338653, |
| "grad_norm": 1.1004250049591064, |
| "learning_rate": 1.5595092917342764e-05, |
| "logits/chosen": -7.02945613861084, |
| "logits/rejected": -6.4935302734375, |
| "logps/chosen": -257.0346984863281, |
| "logps/rejected": -317.5794372558594, |
| "loss": 0.1257, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.8619599938392639, |
| "rewards/margins": 7.793645858764648, |
| "rewards/rejected": -8.655606269836426, |
| "step": 514 |
| }, |
| { |
| "epoch": 0.4135300612265382, |
| "grad_norm": 0.7745633721351624, |
| "learning_rate": 1.5545524711436618e-05, |
| "logits/chosen": -7.072223663330078, |
| "logits/rejected": -6.585541725158691, |
| "logps/chosen": -259.9875183105469, |
| "logps/rejected": -315.0348205566406, |
| "loss": 0.1049, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.1875803470611572, |
| "rewards/margins": 7.880129814147949, |
| "rewards/rejected": -9.067710876464844, |
| "step": 515 |
| }, |
| { |
| "epoch": 0.4143330322192111, |
| "grad_norm": 1.0902003049850464, |
| "learning_rate": 1.549595053975962e-05, |
| "logits/chosen": -6.937098503112793, |
| "logits/rejected": -6.343303680419922, |
| "logps/chosen": -236.2198944091797, |
| "logps/rejected": -289.6206970214844, |
| "loss": 0.1339, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.9511594176292419, |
| "rewards/margins": 7.451602935791016, |
| "rewards/rejected": -8.402763366699219, |
| "step": 516 |
| }, |
| { |
| "epoch": 0.415136003211884, |
| "grad_norm": 0.8388290405273438, |
| "learning_rate": 1.5446370944446987e-05, |
| "logits/chosen": -7.06181526184082, |
| "logits/rejected": -6.648491859436035, |
| "logps/chosen": -249.7332000732422, |
| "logps/rejected": -288.31207275390625, |
| "loss": 0.0978, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6662516593933105, |
| "rewards/margins": 7.627220630645752, |
| "rewards/rejected": -8.293472290039062, |
| "step": 517 |
| }, |
| { |
| "epoch": 0.41593897420455683, |
| "grad_norm": 0.704113245010376, |
| "learning_rate": 1.5396786467693216e-05, |
| "logits/chosen": -6.95879602432251, |
| "logits/rejected": -6.577679634094238, |
| "logps/chosen": -278.2839660644531, |
| "logps/rejected": -326.628662109375, |
| "loss": 0.0814, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.5986055135726929, |
| "rewards/margins": 7.629505634307861, |
| "rewards/rejected": -8.228111267089844, |
| "step": 518 |
| }, |
| { |
| "epoch": 0.41674194519722974, |
| "grad_norm": 0.8245015144348145, |
| "learning_rate": 1.5347197651746207e-05, |
| "logits/chosen": -7.004845142364502, |
| "logits/rejected": -6.598676681518555, |
| "logps/chosen": -237.98019409179688, |
| "logps/rejected": -304.2162780761719, |
| "loss": 0.0788, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.4867791533470154, |
| "rewards/margins": 7.876059055328369, |
| "rewards/rejected": -8.362837791442871, |
| "step": 519 |
| }, |
| { |
| "epoch": 0.41754491618990264, |
| "grad_norm": 0.9469345211982727, |
| "learning_rate": 1.5297605038901304e-05, |
| "logits/chosen": -7.09865665435791, |
| "logits/rejected": -6.6095452308654785, |
| "logps/chosen": -243.17507934570312, |
| "logps/rejected": -287.5968322753906, |
| "loss": 0.0868, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.6636908650398254, |
| "rewards/margins": 7.974848747253418, |
| "rewards/rejected": -8.63853931427002, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.41834788718257554, |
| "grad_norm": 0.8036234974861145, |
| "learning_rate": 1.524800917149538e-05, |
| "logits/chosen": -7.062855243682861, |
| "logits/rejected": -6.552639961242676, |
| "logps/chosen": -254.9627227783203, |
| "logps/rejected": -308.19927978515625, |
| "loss": 0.112, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.0428905487060547, |
| "rewards/margins": 7.334146499633789, |
| "rewards/rejected": -8.377037048339844, |
| "step": 521 |
| }, |
| { |
| "epoch": 0.41915085817524844, |
| "grad_norm": 1.1863948106765747, |
| "learning_rate": 1.5198410591900888e-05, |
| "logits/chosen": -7.0133256912231445, |
| "logits/rejected": -6.5594072341918945, |
| "logps/chosen": -261.4333190917969, |
| "logps/rejected": -299.3178405761719, |
| "loss": 0.1206, |
| "rewards/accuracies": 0.92578125, |
| "rewards/chosen": -0.8112399578094482, |
| "rewards/margins": 7.998819351196289, |
| "rewards/rejected": -8.810059547424316, |
| "step": 522 |
| }, |
| { |
| "epoch": 0.4199538291679213, |
| "grad_norm": 0.8004628419876099, |
| "learning_rate": 1.5148809842519945e-05, |
| "logits/chosen": -6.889098167419434, |
| "logits/rejected": -6.492946147918701, |
| "logps/chosen": -265.1706237792969, |
| "logps/rejected": -326.752685546875, |
| "loss": 0.0985, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -1.1163023710250854, |
| "rewards/margins": 7.857896327972412, |
| "rewards/rejected": -8.974197387695312, |
| "step": 523 |
| }, |
| { |
| "epoch": 0.4207568001605942, |
| "grad_norm": 0.7822092175483704, |
| "learning_rate": 1.5099207465778414e-05, |
| "logits/chosen": -7.119641304016113, |
| "logits/rejected": -6.641016960144043, |
| "logps/chosen": -255.9061279296875, |
| "logps/rejected": -306.5825500488281, |
| "loss": 0.0974, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.2355962991714478, |
| "rewards/margins": 7.409433841705322, |
| "rewards/rejected": -8.645030975341797, |
| "step": 524 |
| }, |
| { |
| "epoch": 0.4215597711532671, |
| "grad_norm": 1.0118803977966309, |
| "learning_rate": 1.5049604004119928e-05, |
| "logits/chosen": -7.191835880279541, |
| "logits/rejected": -6.66995906829834, |
| "logps/chosen": -239.56710815429688, |
| "logps/rejected": -292.98101806640625, |
| "loss": 0.1023, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.9885956048965454, |
| "rewards/margins": 8.0877103805542, |
| "rewards/rejected": -9.076306343078613, |
| "step": 525 |
| }, |
| { |
| "epoch": 0.42236274214594, |
| "grad_norm": 0.7418387532234192, |
| "learning_rate": 1.5e-05, |
| "logits/chosen": -7.13426399230957, |
| "logits/rejected": -6.501273155212402, |
| "logps/chosen": -252.71278381347656, |
| "logps/rejected": -323.33489990234375, |
| "loss": 0.0873, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3328896760940552, |
| "rewards/margins": 7.765531539916992, |
| "rewards/rejected": -9.098421096801758, |
| "step": 526 |
| }, |
| { |
| "epoch": 0.42316571313861284, |
| "grad_norm": 0.7180081009864807, |
| "learning_rate": 1.4950395995880073e-05, |
| "logits/chosen": -7.138564586639404, |
| "logits/rejected": -6.6011128425598145, |
| "logps/chosen": -235.04037475585938, |
| "logps/rejected": -318.1728820800781, |
| "loss": 0.0807, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.6578997373580933, |
| "rewards/margins": 8.294034957885742, |
| "rewards/rejected": -9.951934814453125, |
| "step": 527 |
| }, |
| { |
| "epoch": 0.42396868413128574, |
| "grad_norm": 1.031200647354126, |
| "learning_rate": 1.4900792534221589e-05, |
| "logits/chosen": -6.809314250946045, |
| "logits/rejected": -6.288721084594727, |
| "logps/chosen": -232.68869018554688, |
| "logps/rejected": -291.77581787109375, |
| "loss": 0.1075, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.6751749515533447, |
| "rewards/margins": 7.8925909996032715, |
| "rewards/rejected": -9.567766189575195, |
| "step": 528 |
| }, |
| { |
| "epoch": 0.42477165512395865, |
| "grad_norm": 0.7010549306869507, |
| "learning_rate": 1.4851190157480054e-05, |
| "logits/chosen": -7.156745910644531, |
| "logits/rejected": -6.580558776855469, |
| "logps/chosen": -246.65606689453125, |
| "logps/rejected": -326.1361999511719, |
| "loss": 0.0971, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9826784133911133, |
| "rewards/margins": 8.223730087280273, |
| "rewards/rejected": -9.206408500671387, |
| "step": 529 |
| }, |
| { |
| "epoch": 0.42557462611663155, |
| "grad_norm": 0.6139161586761475, |
| "learning_rate": 1.4801589408099118e-05, |
| "logits/chosen": -6.824873924255371, |
| "logits/rejected": -6.466273307800293, |
| "logps/chosen": -289.3915100097656, |
| "logps/rejected": -338.7675476074219, |
| "loss": 0.0778, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.9523260593414307, |
| "rewards/margins": 8.030743598937988, |
| "rewards/rejected": -8.98306941986084, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.42637759710930445, |
| "grad_norm": 0.7861096262931824, |
| "learning_rate": 1.4751990828504623e-05, |
| "logits/chosen": -7.236479759216309, |
| "logits/rejected": -6.629884719848633, |
| "logps/chosen": -268.0399475097656, |
| "logps/rejected": -338.933837890625, |
| "loss": 0.0916, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6251427531242371, |
| "rewards/margins": 8.31301212310791, |
| "rewards/rejected": -8.938154220581055, |
| "step": 531 |
| }, |
| { |
| "epoch": 0.4271805681019773, |
| "grad_norm": 0.7177116274833679, |
| "learning_rate": 1.4702394961098699e-05, |
| "logits/chosen": -6.945029258728027, |
| "logits/rejected": -6.425079345703125, |
| "logps/chosen": -266.2700500488281, |
| "logps/rejected": -318.0312805175781, |
| "loss": 0.0843, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.0344312191009521, |
| "rewards/margins": 8.214509010314941, |
| "rewards/rejected": -9.248941421508789, |
| "step": 532 |
| }, |
| { |
| "epoch": 0.4279835390946502, |
| "grad_norm": 0.8623508810997009, |
| "learning_rate": 1.4652802348253793e-05, |
| "logits/chosen": -6.953853607177734, |
| "logits/rejected": -6.401447296142578, |
| "logps/chosen": -252.7352294921875, |
| "logps/rejected": -313.76409912109375, |
| "loss": 0.1089, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.1851781606674194, |
| "rewards/margins": 7.919229984283447, |
| "rewards/rejected": -9.10440731048584, |
| "step": 533 |
| }, |
| { |
| "epoch": 0.4287865100873231, |
| "grad_norm": 0.9705774188041687, |
| "learning_rate": 1.4603213532306782e-05, |
| "logits/chosen": -6.996805191040039, |
| "logits/rejected": -6.506040096282959, |
| "logps/chosen": -222.56333923339844, |
| "logps/rejected": -295.5704345703125, |
| "loss": 0.0844, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.4789319038391113, |
| "rewards/margins": 8.398710250854492, |
| "rewards/rejected": -9.877641677856445, |
| "step": 534 |
| }, |
| { |
| "epoch": 0.429589481079996, |
| "grad_norm": 0.9695616960525513, |
| "learning_rate": 1.4553629055553013e-05, |
| "logits/chosen": -7.009624481201172, |
| "logits/rejected": -6.58917760848999, |
| "logps/chosen": -238.0480499267578, |
| "logps/rejected": -302.17626953125, |
| "loss": 0.1159, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.863762617111206, |
| "rewards/margins": 8.272629737854004, |
| "rewards/rejected": -9.136391639709473, |
| "step": 535 |
| }, |
| { |
| "epoch": 0.43039245207266885, |
| "grad_norm": 0.7318723797798157, |
| "learning_rate": 1.4504049460240376e-05, |
| "logits/chosen": -6.966203212738037, |
| "logits/rejected": -6.557267665863037, |
| "logps/chosen": -241.29205322265625, |
| "logps/rejected": -318.2552490234375, |
| "loss": 0.0774, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.2327791452407837, |
| "rewards/margins": 8.223346710205078, |
| "rewards/rejected": -9.456127166748047, |
| "step": 536 |
| }, |
| { |
| "epoch": 0.43119542306534175, |
| "grad_norm": 0.8294877409934998, |
| "learning_rate": 1.4454475288563387e-05, |
| "logits/chosen": -6.851326942443848, |
| "logits/rejected": -6.348145484924316, |
| "logps/chosen": -256.45257568359375, |
| "logps/rejected": -330.8794860839844, |
| "loss": 0.1008, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.2834573984146118, |
| "rewards/margins": 8.003429412841797, |
| "rewards/rejected": -9.286886215209961, |
| "step": 537 |
| }, |
| { |
| "epoch": 0.43199839405801466, |
| "grad_norm": 1.0455961227416992, |
| "learning_rate": 1.4404907082657238e-05, |
| "logits/chosen": -6.904001235961914, |
| "logits/rejected": -6.350428581237793, |
| "logps/chosen": -289.5302734375, |
| "logps/rejected": -352.3595886230469, |
| "loss": 0.1292, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.6992630362510681, |
| "rewards/margins": 7.7425103187561035, |
| "rewards/rejected": -8.4417724609375, |
| "step": 538 |
| }, |
| { |
| "epoch": 0.43280136505068756, |
| "grad_norm": 0.8768832087516785, |
| "learning_rate": 1.4355345384591894e-05, |
| "logits/chosen": -7.1678242683410645, |
| "logits/rejected": -6.523069858551025, |
| "logps/chosen": -247.74880981445312, |
| "logps/rejected": -310.9901123046875, |
| "loss": 0.099, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.2284319400787354, |
| "rewards/margins": 8.398492813110352, |
| "rewards/rejected": -9.626924514770508, |
| "step": 539 |
| }, |
| { |
| "epoch": 0.43360433604336046, |
| "grad_norm": 1.002357840538025, |
| "learning_rate": 1.4305790736366135e-05, |
| "logits/chosen": -6.913356781005859, |
| "logits/rejected": -6.435115337371826, |
| "logps/chosen": -233.6885528564453, |
| "logps/rejected": -325.83001708984375, |
| "loss": 0.1203, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.1142196655273438, |
| "rewards/margins": 7.761749267578125, |
| "rewards/rejected": -8.875968933105469, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.4344073070360333, |
| "grad_norm": 0.8878774046897888, |
| "learning_rate": 1.4256243679901665e-05, |
| "logits/chosen": -6.871967792510986, |
| "logits/rejected": -6.3378448486328125, |
| "logps/chosen": -241.61708068847656, |
| "logps/rejected": -320.18890380859375, |
| "loss": 0.0952, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8749432563781738, |
| "rewards/margins": 8.508054733276367, |
| "rewards/rejected": -9.382997512817383, |
| "step": 541 |
| }, |
| { |
| "epoch": 0.4352102780287062, |
| "grad_norm": 0.723964273929596, |
| "learning_rate": 1.4206704757037153e-05, |
| "logits/chosen": -6.980746269226074, |
| "logits/rejected": -6.327948093414307, |
| "logps/chosen": -247.17889404296875, |
| "logps/rejected": -306.51043701171875, |
| "loss": 0.0896, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.47397708892822266, |
| "rewards/margins": 8.306581497192383, |
| "rewards/rejected": -8.780558586120605, |
| "step": 542 |
| }, |
| { |
| "epoch": 0.4360132490213791, |
| "grad_norm": 0.7974479794502258, |
| "learning_rate": 1.4157174509522318e-05, |
| "logits/chosen": -6.89686393737793, |
| "logits/rejected": -6.43813419342041, |
| "logps/chosen": -263.5614013671875, |
| "logps/rejected": -304.0465087890625, |
| "loss": 0.1238, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.816796600818634, |
| "rewards/margins": 7.840972900390625, |
| "rewards/rejected": -8.657769203186035, |
| "step": 543 |
| }, |
| { |
| "epoch": 0.436816220014052, |
| "grad_norm": 0.9344457983970642, |
| "learning_rate": 1.4107653479012016e-05, |
| "logits/chosen": -7.028533458709717, |
| "logits/rejected": -6.562801837921143, |
| "logps/chosen": -250.69248962402344, |
| "logps/rejected": -309.6461486816406, |
| "loss": 0.1042, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.7335835099220276, |
| "rewards/margins": 8.033063888549805, |
| "rewards/rejected": -8.766647338867188, |
| "step": 544 |
| }, |
| { |
| "epoch": 0.43761919100672486, |
| "grad_norm": 1.0400757789611816, |
| "learning_rate": 1.40581422070603e-05, |
| "logits/chosen": -6.921531677246094, |
| "logits/rejected": -6.468945026397705, |
| "logps/chosen": -254.36656188964844, |
| "logps/rejected": -327.0810546875, |
| "loss": 0.118, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.8909276723861694, |
| "rewards/margins": 8.345698356628418, |
| "rewards/rejected": -9.236625671386719, |
| "step": 545 |
| }, |
| { |
| "epoch": 0.43842216199939776, |
| "grad_norm": 0.9862475395202637, |
| "learning_rate": 1.400864123511451e-05, |
| "logits/chosen": -7.042816638946533, |
| "logits/rejected": -6.507579803466797, |
| "logps/chosen": -262.513671875, |
| "logps/rejected": -299.19281005859375, |
| "loss": 0.0941, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.0383877754211426, |
| "rewards/margins": 8.257445335388184, |
| "rewards/rejected": -9.295833587646484, |
| "step": 546 |
| }, |
| { |
| "epoch": 0.43922513299207067, |
| "grad_norm": 0.8215150237083435, |
| "learning_rate": 1.395915110450934e-05, |
| "logits/chosen": -6.825544834136963, |
| "logits/rejected": -6.310960292816162, |
| "logps/chosen": -251.5355987548828, |
| "logps/rejected": -315.09423828125, |
| "loss": 0.1042, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8086391091346741, |
| "rewards/margins": 7.977949142456055, |
| "rewards/rejected": -8.786588668823242, |
| "step": 547 |
| }, |
| { |
| "epoch": 0.44002810398474357, |
| "grad_norm": 0.7922818064689636, |
| "learning_rate": 1.3909672356460934e-05, |
| "logits/chosen": -7.228545188903809, |
| "logits/rejected": -6.575509548187256, |
| "logps/chosen": -229.340087890625, |
| "logps/rejected": -302.9287109375, |
| "loss": 0.0953, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6116417646408081, |
| "rewards/margins": 8.525511741638184, |
| "rewards/rejected": -9.137153625488281, |
| "step": 548 |
| }, |
| { |
| "epoch": 0.4408310749774164, |
| "grad_norm": 0.6565718054771423, |
| "learning_rate": 1.3860205532060953e-05, |
| "logits/chosen": -6.941289901733398, |
| "logits/rejected": -6.440770626068115, |
| "logps/chosen": -268.0898132324219, |
| "logps/rejected": -312.2543640136719, |
| "loss": 0.0802, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.4033500552177429, |
| "rewards/margins": 8.241222381591797, |
| "rewards/rejected": -8.644571304321289, |
| "step": 549 |
| }, |
| { |
| "epoch": 0.4416340459700893, |
| "grad_norm": 0.8805581331253052, |
| "learning_rate": 1.381075117227066e-05, |
| "logits/chosen": -6.918531894683838, |
| "logits/rejected": -6.356103420257568, |
| "logps/chosen": -249.268310546875, |
| "logps/rejected": -285.7378845214844, |
| "loss": 0.108, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.645024836063385, |
| "rewards/margins": 7.661341667175293, |
| "rewards/rejected": -8.306366920471191, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.4424370169627622, |
| "grad_norm": 0.6661244034767151, |
| "learning_rate": 1.3761309817915017e-05, |
| "logits/chosen": -6.953717231750488, |
| "logits/rejected": -6.374336242675781, |
| "logps/chosen": -252.18385314941406, |
| "logps/rejected": -320.710205078125, |
| "loss": 0.0727, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.3206496238708496, |
| "rewards/margins": 8.491475105285645, |
| "rewards/rejected": -8.812124252319336, |
| "step": 551 |
| }, |
| { |
| "epoch": 0.4432399879554351, |
| "grad_norm": 0.5906367301940918, |
| "learning_rate": 1.3711882009676744e-05, |
| "logits/chosen": -7.03802490234375, |
| "logits/rejected": -6.483231067657471, |
| "logps/chosen": -265.9961853027344, |
| "logps/rejected": -318.4466247558594, |
| "loss": 0.0588, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.5948689579963684, |
| "rewards/margins": 8.481563568115234, |
| "rewards/rejected": -9.076432228088379, |
| "step": 552 |
| }, |
| { |
| "epoch": 0.444042958948108, |
| "grad_norm": 0.5871016383171082, |
| "learning_rate": 1.3662468288090446e-05, |
| "logits/chosen": -6.954483509063721, |
| "logits/rejected": -6.536128044128418, |
| "logps/chosen": -277.76123046875, |
| "logps/rejected": -351.3174133300781, |
| "loss": 0.0667, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.3535917401313782, |
| "rewards/margins": 8.249122619628906, |
| "rewards/rejected": -8.602714538574219, |
| "step": 553 |
| }, |
| { |
| "epoch": 0.44484592994078087, |
| "grad_norm": 0.9364761710166931, |
| "learning_rate": 1.361306919353665e-05, |
| "logits/chosen": -7.246333599090576, |
| "logits/rejected": -6.6546173095703125, |
| "logps/chosen": -230.2370147705078, |
| "logps/rejected": -308.80816650390625, |
| "loss": 0.0947, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.668330729007721, |
| "rewards/margins": 8.044631958007812, |
| "rewards/rejected": -8.712963104248047, |
| "step": 554 |
| }, |
| { |
| "epoch": 0.4456489009334538, |
| "grad_norm": 0.856311559677124, |
| "learning_rate": 1.3563685266235948e-05, |
| "logits/chosen": -7.095914840698242, |
| "logits/rejected": -6.722827911376953, |
| "logps/chosen": -216.9248809814453, |
| "logps/rejected": -281.245849609375, |
| "loss": 0.0997, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.46450161933898926, |
| "rewards/margins": 8.14419174194336, |
| "rewards/rejected": -8.608694076538086, |
| "step": 555 |
| }, |
| { |
| "epoch": 0.4464518719261267, |
| "grad_norm": 0.9367163181304932, |
| "learning_rate": 1.351431704624306e-05, |
| "logits/chosen": -6.870026588439941, |
| "logits/rejected": -6.508073329925537, |
| "logps/chosen": -274.8689880371094, |
| "logps/rejected": -310.7038269042969, |
| "loss": 0.106, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.9549875259399414, |
| "rewards/margins": 7.395700454711914, |
| "rewards/rejected": -8.350687980651855, |
| "step": 556 |
| }, |
| { |
| "epoch": 0.4472548429187996, |
| "grad_norm": 0.6092349886894226, |
| "learning_rate": 1.3464965073440924e-05, |
| "logits/chosen": -7.089807510375977, |
| "logits/rejected": -6.587115287780762, |
| "logps/chosen": -255.1664581298828, |
| "logps/rejected": -312.0454406738281, |
| "loss": 0.0891, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.5835572481155396, |
| "rewards/margins": 7.626180648803711, |
| "rewards/rejected": -8.209737777709961, |
| "step": 557 |
| }, |
| { |
| "epoch": 0.4480578139114724, |
| "grad_norm": 0.7615747451782227, |
| "learning_rate": 1.3415629887534817e-05, |
| "logits/chosen": -7.0474748611450195, |
| "logits/rejected": -6.580780982971191, |
| "logps/chosen": -240.4439697265625, |
| "logps/rejected": -293.221435546875, |
| "loss": 0.1029, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6962559819221497, |
| "rewards/margins": 8.086463928222656, |
| "rewards/rejected": -8.782719612121582, |
| "step": 558 |
| }, |
| { |
| "epoch": 0.4488607849041453, |
| "grad_norm": 0.6443572044372559, |
| "learning_rate": 1.3366312028046412e-05, |
| "logits/chosen": -7.109622955322266, |
| "logits/rejected": -6.544712066650391, |
| "logps/chosen": -279.27484130859375, |
| "logps/rejected": -312.99847412109375, |
| "loss": 0.0752, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.44775474071502686, |
| "rewards/margins": 8.2374906539917, |
| "rewards/rejected": -8.685245513916016, |
| "step": 559 |
| }, |
| { |
| "epoch": 0.44966375589681823, |
| "grad_norm": 0.5533212423324585, |
| "learning_rate": 1.3317012034307937e-05, |
| "logits/chosen": -7.13261604309082, |
| "logits/rejected": -6.569200038909912, |
| "logps/chosen": -242.31390380859375, |
| "logps/rejected": -314.0146484375, |
| "loss": 0.0657, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.5953407287597656, |
| "rewards/margins": 8.502303123474121, |
| "rewards/rejected": -9.097643852233887, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.45046672688949113, |
| "grad_norm": 0.9312994480133057, |
| "learning_rate": 1.3267730445456208e-05, |
| "logits/chosen": -7.001958847045898, |
| "logits/rejected": -6.497946739196777, |
| "logps/chosen": -252.1693878173828, |
| "logps/rejected": -309.2901611328125, |
| "loss": 0.107, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.508165717124939, |
| "rewards/margins": 7.937198162078857, |
| "rewards/rejected": -8.44536304473877, |
| "step": 561 |
| }, |
| { |
| "epoch": 0.45126969788216403, |
| "grad_norm": 0.9563602805137634, |
| "learning_rate": 1.3218467800426798e-05, |
| "logits/chosen": -6.968316078186035, |
| "logits/rejected": -6.477238655090332, |
| "logps/chosen": -245.49798583984375, |
| "logps/rejected": -292.4193115234375, |
| "loss": 0.1193, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.5243959426879883, |
| "rewards/margins": 7.789290428161621, |
| "rewards/rejected": -9.31368637084961, |
| "step": 562 |
| }, |
| { |
| "epoch": 0.4520726688748369, |
| "grad_norm": 0.6081119775772095, |
| "learning_rate": 1.3169224637948099e-05, |
| "logits/chosen": -7.040828227996826, |
| "logits/rejected": -6.440508842468262, |
| "logps/chosen": -240.68287658691406, |
| "logps/rejected": -305.806884765625, |
| "loss": 0.0624, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.5017206072807312, |
| "rewards/margins": 8.3018798828125, |
| "rewards/rejected": -8.80359935760498, |
| "step": 563 |
| }, |
| { |
| "epoch": 0.4528756398675098, |
| "grad_norm": 0.6449005603790283, |
| "learning_rate": 1.3120001496535434e-05, |
| "logits/chosen": -6.827923774719238, |
| "logits/rejected": -6.373751640319824, |
| "logps/chosen": -274.67242431640625, |
| "logps/rejected": -349.6941833496094, |
| "loss": 0.0855, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.9361369609832764, |
| "rewards/margins": 8.233305931091309, |
| "rewards/rejected": -9.169443130493164, |
| "step": 564 |
| }, |
| { |
| "epoch": 0.4536786108601827, |
| "grad_norm": 0.691442608833313, |
| "learning_rate": 1.3070798914485211e-05, |
| "logits/chosen": -7.009668350219727, |
| "logits/rejected": -6.602151870727539, |
| "logps/chosen": -248.281005859375, |
| "logps/rejected": -295.61737060546875, |
| "loss": 0.0955, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.1644386053085327, |
| "rewards/margins": 7.625636100769043, |
| "rewards/rejected": -8.790074348449707, |
| "step": 565 |
| }, |
| { |
| "epoch": 0.4544815818528556, |
| "grad_norm": 0.7413257956504822, |
| "learning_rate": 1.3021617429868966e-05, |
| "logits/chosen": -6.870475769042969, |
| "logits/rejected": -6.392442226409912, |
| "logps/chosen": -267.52203369140625, |
| "logps/rejected": -322.917236328125, |
| "loss": 0.086, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8330942392349243, |
| "rewards/margins": 7.9583892822265625, |
| "rewards/rejected": -8.791483879089355, |
| "step": 566 |
| }, |
| { |
| "epoch": 0.45528455284552843, |
| "grad_norm": 0.5746920108795166, |
| "learning_rate": 1.2972457580527551e-05, |
| "logits/chosen": -7.036513328552246, |
| "logits/rejected": -6.51309871673584, |
| "logps/chosen": -224.05552673339844, |
| "logps/rejected": -289.66552734375, |
| "loss": 0.0604, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -1.2872111797332764, |
| "rewards/margins": 8.115041732788086, |
| "rewards/rejected": -9.402253150939941, |
| "step": 567 |
| }, |
| { |
| "epoch": 0.45608752383820134, |
| "grad_norm": 0.661065399646759, |
| "learning_rate": 1.2923319904065187e-05, |
| "logits/chosen": -6.970797061920166, |
| "logits/rejected": -6.4415082931518555, |
| "logps/chosen": -270.30511474609375, |
| "logps/rejected": -342.4693603515625, |
| "loss": 0.0658, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -0.9600474834442139, |
| "rewards/margins": 8.636031150817871, |
| "rewards/rejected": -9.59607982635498, |
| "step": 568 |
| }, |
| { |
| "epoch": 0.45689049483087424, |
| "grad_norm": 0.9690901041030884, |
| "learning_rate": 1.2874204937843636e-05, |
| "logits/chosen": -6.993248462677002, |
| "logits/rejected": -6.522023677825928, |
| "logps/chosen": -243.56011962890625, |
| "logps/rejected": -311.65667724609375, |
| "loss": 0.0813, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.477194905281067, |
| "rewards/margins": 7.909328460693359, |
| "rewards/rejected": -9.386524200439453, |
| "step": 569 |
| }, |
| { |
| "epoch": 0.45769346582354714, |
| "grad_norm": 1.0371527671813965, |
| "learning_rate": 1.2825113218976312e-05, |
| "logits/chosen": -6.750287055969238, |
| "logits/rejected": -6.255166530609131, |
| "logps/chosen": -245.60777282714844, |
| "logps/rejected": -324.57379150390625, |
| "loss": 0.1119, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.5608936548233032, |
| "rewards/margins": 8.427144050598145, |
| "rewards/rejected": -9.988038063049316, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.45849643681622, |
| "grad_norm": 0.9875096678733826, |
| "learning_rate": 1.277604528432237e-05, |
| "logits/chosen": -6.978541374206543, |
| "logits/rejected": -6.450058937072754, |
| "logps/chosen": -229.01805114746094, |
| "logps/rejected": -294.82281494140625, |
| "loss": 0.0794, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -1.1487610340118408, |
| "rewards/margins": 8.559868812561035, |
| "rewards/rejected": -9.708629608154297, |
| "step": 571 |
| }, |
| { |
| "epoch": 0.4592994078088929, |
| "grad_norm": 1.0726308822631836, |
| "learning_rate": 1.2727001670480896e-05, |
| "logits/chosen": -6.875815391540527, |
| "logits/rejected": -6.2449750900268555, |
| "logps/chosen": -255.47357177734375, |
| "logps/rejected": -306.7621154785156, |
| "loss": 0.1139, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.183271050453186, |
| "rewards/margins": 7.907237529754639, |
| "rewards/rejected": -9.090508460998535, |
| "step": 572 |
| }, |
| { |
| "epoch": 0.4601023788015658, |
| "grad_norm": 0.9184864163398743, |
| "learning_rate": 1.267798291378498e-05, |
| "logits/chosen": -6.771671295166016, |
| "logits/rejected": -6.127359390258789, |
| "logps/chosen": -239.16357421875, |
| "logps/rejected": -312.2468566894531, |
| "loss": 0.1256, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -1.19147527217865, |
| "rewards/margins": 8.142675399780273, |
| "rewards/rejected": -9.334151268005371, |
| "step": 573 |
| }, |
| { |
| "epoch": 0.4609053497942387, |
| "grad_norm": 0.7804308533668518, |
| "learning_rate": 1.2628989550295907e-05, |
| "logits/chosen": -6.9797821044921875, |
| "logits/rejected": -6.4109907150268555, |
| "logps/chosen": -259.9932861328125, |
| "logps/rejected": -342.84368896484375, |
| "loss": 0.0935, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.057735800743103, |
| "rewards/margins": 8.690528869628906, |
| "rewards/rejected": -9.74826431274414, |
| "step": 574 |
| }, |
| { |
| "epoch": 0.4617083207869116, |
| "grad_norm": 1.0066481828689575, |
| "learning_rate": 1.2580022115797237e-05, |
| "logits/chosen": -7.015582084655762, |
| "logits/rejected": -6.438292026519775, |
| "logps/chosen": -245.5735626220703, |
| "logps/rejected": -295.16497802734375, |
| "loss": 0.0885, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.3438326120376587, |
| "rewards/margins": 8.883735656738281, |
| "rewards/rejected": -10.227567672729492, |
| "step": 575 |
| }, |
| { |
| "epoch": 0.46251129177958444, |
| "grad_norm": 0.7363642454147339, |
| "learning_rate": 1.2531081145788989e-05, |
| "logits/chosen": -6.800288200378418, |
| "logits/rejected": -6.36141300201416, |
| "logps/chosen": -255.8954315185547, |
| "logps/rejected": -318.6156311035156, |
| "loss": 0.0837, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8195849657058716, |
| "rewards/margins": 8.455997467041016, |
| "rewards/rejected": -9.275582313537598, |
| "step": 576 |
| }, |
| { |
| "epoch": 0.46331426277225735, |
| "grad_norm": 0.9252678751945496, |
| "learning_rate": 1.2482167175481786e-05, |
| "logits/chosen": -6.961581230163574, |
| "logits/rejected": -6.471203804016113, |
| "logps/chosen": -235.9860076904297, |
| "logps/rejected": -310.0054931640625, |
| "loss": 0.0792, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -1.3675531148910522, |
| "rewards/margins": 8.560356140136719, |
| "rewards/rejected": -9.927909851074219, |
| "step": 577 |
| }, |
| { |
| "epoch": 0.46411723376493025, |
| "grad_norm": 1.2859218120574951, |
| "learning_rate": 1.243328073979095e-05, |
| "logits/chosen": -6.9278974533081055, |
| "logits/rejected": -6.324116230010986, |
| "logps/chosen": -272.5286560058594, |
| "logps/rejected": -352.1285705566406, |
| "loss": 0.1106, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9970148205757141, |
| "rewards/margins": 8.906410217285156, |
| "rewards/rejected": -9.903425216674805, |
| "step": 578 |
| }, |
| { |
| "epoch": 0.46492020475760315, |
| "grad_norm": 1.2158070802688599, |
| "learning_rate": 1.2384422373330728e-05, |
| "logits/chosen": -7.097422122955322, |
| "logits/rejected": -6.525650501251221, |
| "logps/chosen": -239.6403350830078, |
| "logps/rejected": -324.9807434082031, |
| "loss": 0.0905, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.2991890907287598, |
| "rewards/margins": 9.010651588439941, |
| "rewards/rejected": -10.309842109680176, |
| "step": 579 |
| }, |
| { |
| "epoch": 0.465723175750276, |
| "grad_norm": 0.7207929491996765, |
| "learning_rate": 1.2335592610408372e-05, |
| "logits/chosen": -6.894137859344482, |
| "logits/rejected": -6.361636161804199, |
| "logps/chosen": -251.25201416015625, |
| "logps/rejected": -329.07452392578125, |
| "loss": 0.1094, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.8159255385398865, |
| "rewards/margins": 8.669663429260254, |
| "rewards/rejected": -9.485589027404785, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.4665261467429489, |
| "grad_norm": 0.8102943301200867, |
| "learning_rate": 1.2286791985018356e-05, |
| "logits/chosen": -6.719347953796387, |
| "logits/rejected": -6.232395648956299, |
| "logps/chosen": -290.72259521484375, |
| "logps/rejected": -339.76239013671875, |
| "loss": 0.0939, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8780815005302429, |
| "rewards/margins": 8.487323760986328, |
| "rewards/rejected": -9.36540412902832, |
| "step": 581 |
| }, |
| { |
| "epoch": 0.4673291177356218, |
| "grad_norm": 0.9166164398193359, |
| "learning_rate": 1.2238021030836493e-05, |
| "logits/chosen": -6.824174880981445, |
| "logits/rejected": -6.386189937591553, |
| "logps/chosen": -258.6803894042969, |
| "logps/rejected": -318.63916015625, |
| "loss": 0.1126, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.2175662517547607, |
| "rewards/margins": 8.276704788208008, |
| "rewards/rejected": -9.494272232055664, |
| "step": 582 |
| }, |
| { |
| "epoch": 0.4681320887282947, |
| "grad_norm": 1.1293888092041016, |
| "learning_rate": 1.2189280281214128e-05, |
| "logits/chosen": -7.027843952178955, |
| "logits/rejected": -6.373488426208496, |
| "logps/chosen": -236.0067138671875, |
| "logps/rejected": -298.4288024902344, |
| "loss": 0.0795, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.3197840452194214, |
| "rewards/margins": 8.296957015991211, |
| "rewards/rejected": -9.616740226745605, |
| "step": 583 |
| }, |
| { |
| "epoch": 0.4689350597209676, |
| "grad_norm": 0.9828873872756958, |
| "learning_rate": 1.21405702691723e-05, |
| "logits/chosen": -6.643815994262695, |
| "logits/rejected": -6.147243976593018, |
| "logps/chosen": -290.41650390625, |
| "logps/rejected": -336.6736755371094, |
| "loss": 0.0979, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9446360468864441, |
| "rewards/margins": 8.31712532043457, |
| "rewards/rejected": -9.261760711669922, |
| "step": 584 |
| }, |
| { |
| "epoch": 0.46973803071364045, |
| "grad_norm": 0.8115407228469849, |
| "learning_rate": 1.2091891527395878e-05, |
| "logits/chosen": -7.017604351043701, |
| "logits/rejected": -6.572064399719238, |
| "logps/chosen": -240.04266357421875, |
| "logps/rejected": -325.9983825683594, |
| "loss": 0.0908, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.1076565980911255, |
| "rewards/margins": 8.58370590209961, |
| "rewards/rejected": -9.691362380981445, |
| "step": 585 |
| }, |
| { |
| "epoch": 0.47054100170631336, |
| "grad_norm": 0.8567003607749939, |
| "learning_rate": 1.2043244588227797e-05, |
| "logits/chosen": -7.049973487854004, |
| "logits/rejected": -6.527139186859131, |
| "logps/chosen": -264.3322448730469, |
| "logps/rejected": -325.392333984375, |
| "loss": 0.1012, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7964427471160889, |
| "rewards/margins": 7.991184711456299, |
| "rewards/rejected": -8.787627220153809, |
| "step": 586 |
| }, |
| { |
| "epoch": 0.47134397269898626, |
| "grad_norm": 0.8607868552207947, |
| "learning_rate": 1.1994629983663183e-05, |
| "logits/chosen": -6.890265464782715, |
| "logits/rejected": -6.362273216247559, |
| "logps/chosen": -234.0949249267578, |
| "logps/rejected": -310.505615234375, |
| "loss": 0.0872, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.1343491077423096, |
| "rewards/margins": 8.408004760742188, |
| "rewards/rejected": -9.542353630065918, |
| "step": 587 |
| }, |
| { |
| "epoch": 0.47214694369165916, |
| "grad_norm": 0.9671961665153503, |
| "learning_rate": 1.1946048245343568e-05, |
| "logits/chosen": -7.0048112869262695, |
| "logits/rejected": -6.563193321228027, |
| "logps/chosen": -253.8204345703125, |
| "logps/rejected": -293.33685302734375, |
| "loss": 0.0909, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.5214956402778625, |
| "rewards/margins": 8.46933364868164, |
| "rewards/rejected": -8.990830421447754, |
| "step": 588 |
| }, |
| { |
| "epoch": 0.472949914684332, |
| "grad_norm": 0.8027867674827576, |
| "learning_rate": 1.189749990455105e-05, |
| "logits/chosen": -6.9679765701293945, |
| "logits/rejected": -6.41765832901001, |
| "logps/chosen": -252.14437866210938, |
| "logps/rejected": -317.990478515625, |
| "loss": 0.1124, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.3911430537700653, |
| "rewards/margins": 8.306731224060059, |
| "rewards/rejected": -8.697875022888184, |
| "step": 589 |
| }, |
| { |
| "epoch": 0.4737528856770049, |
| "grad_norm": 0.8122281432151794, |
| "learning_rate": 1.1848985492202514e-05, |
| "logits/chosen": -6.941572666168213, |
| "logits/rejected": -6.424032688140869, |
| "logps/chosen": -248.44970703125, |
| "logps/rejected": -301.4393310546875, |
| "loss": 0.0845, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.712451696395874, |
| "rewards/margins": 8.438041687011719, |
| "rewards/rejected": -9.150492668151855, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.4745558566696778, |
| "grad_norm": 0.753387451171875, |
| "learning_rate": 1.18005055388438e-05, |
| "logits/chosen": -6.9822211265563965, |
| "logits/rejected": -6.555486679077148, |
| "logps/chosen": -259.5833740234375, |
| "logps/rejected": -315.9266052246094, |
| "loss": 0.0854, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.4612896740436554, |
| "rewards/margins": 8.548084259033203, |
| "rewards/rejected": -9.009374618530273, |
| "step": 591 |
| }, |
| { |
| "epoch": 0.4753588276623507, |
| "grad_norm": 1.02212655544281, |
| "learning_rate": 1.1752060574643905e-05, |
| "logits/chosen": -7.119367599487305, |
| "logits/rejected": -6.631015777587891, |
| "logps/chosen": -254.99252319335938, |
| "logps/rejected": -326.20599365234375, |
| "loss": 0.1022, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.8184077143669128, |
| "rewards/margins": 8.63607406616211, |
| "rewards/rejected": -9.454482078552246, |
| "step": 592 |
| }, |
| { |
| "epoch": 0.47616179865502356, |
| "grad_norm": 0.8852695226669312, |
| "learning_rate": 1.1703651129389203e-05, |
| "logits/chosen": -6.798479080200195, |
| "logits/rejected": -6.4815473556518555, |
| "logps/chosen": -240.85276794433594, |
| "logps/rejected": -310.73773193359375, |
| "loss": 0.0988, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.5627226829528809, |
| "rewards/margins": 8.517350196838379, |
| "rewards/rejected": -9.080072402954102, |
| "step": 593 |
| }, |
| { |
| "epoch": 0.47696476964769646, |
| "grad_norm": 0.5402423739433289, |
| "learning_rate": 1.1655277732477627e-05, |
| "logits/chosen": -6.874414443969727, |
| "logits/rejected": -6.4372334480285645, |
| "logps/chosen": -265.0986633300781, |
| "logps/rejected": -303.206298828125, |
| "loss": 0.0604, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -0.46068108081817627, |
| "rewards/margins": 8.35940170288086, |
| "rewards/rejected": -8.820082664489746, |
| "step": 594 |
| }, |
| { |
| "epoch": 0.47776774064036936, |
| "grad_norm": 0.9045907855033875, |
| "learning_rate": 1.16069409129129e-05, |
| "logits/chosen": -7.0202741622924805, |
| "logits/rejected": -6.443297863006592, |
| "logps/chosen": -253.13330078125, |
| "logps/rejected": -310.3221740722656, |
| "loss": 0.0914, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6723207235336304, |
| "rewards/margins": 8.523781776428223, |
| "rewards/rejected": -9.1961030960083, |
| "step": 595 |
| }, |
| { |
| "epoch": 0.47857071163304227, |
| "grad_norm": 0.7604978680610657, |
| "learning_rate": 1.1558641199298728e-05, |
| "logits/chosen": -6.982942581176758, |
| "logits/rejected": -6.513798713684082, |
| "logps/chosen": -233.39749145507812, |
| "logps/rejected": -282.1934814453125, |
| "loss": 0.0855, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.0295100212097168, |
| "rewards/margins": 8.517677307128906, |
| "rewards/rejected": -9.547187805175781, |
| "step": 596 |
| }, |
| { |
| "epoch": 0.47937368262571517, |
| "grad_norm": 0.8567836284637451, |
| "learning_rate": 1.1510379119833048e-05, |
| "logits/chosen": -6.807964324951172, |
| "logits/rejected": -6.411593437194824, |
| "logps/chosen": -264.2077941894531, |
| "logps/rejected": -320.9981689453125, |
| "loss": 0.0968, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.8917703628540039, |
| "rewards/margins": 7.984936237335205, |
| "rewards/rejected": -8.87670612335205, |
| "step": 597 |
| }, |
| { |
| "epoch": 0.480176653618388, |
| "grad_norm": 1.063594937324524, |
| "learning_rate": 1.1462155202302227e-05, |
| "logits/chosen": -6.887166500091553, |
| "logits/rejected": -6.273863315582275, |
| "logps/chosen": -283.5629577636719, |
| "logps/rejected": -343.53729248046875, |
| "loss": 0.0886, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.4634888470172882, |
| "rewards/margins": 8.231271743774414, |
| "rewards/rejected": -8.694759368896484, |
| "step": 598 |
| }, |
| { |
| "epoch": 0.4809796246110609, |
| "grad_norm": 0.9595116972923279, |
| "learning_rate": 1.1413969974075299e-05, |
| "logits/chosen": -6.922754764556885, |
| "logits/rejected": -6.350415229797363, |
| "logps/chosen": -252.357666015625, |
| "logps/rejected": -327.85943603515625, |
| "loss": 0.1002, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.136838674545288, |
| "rewards/margins": 8.123347282409668, |
| "rewards/rejected": -9.260186195373535, |
| "step": 599 |
| }, |
| { |
| "epoch": 0.4817825956037338, |
| "grad_norm": 0.9518589377403259, |
| "learning_rate": 1.1365823962098208e-05, |
| "logits/chosen": -6.8367486000061035, |
| "logits/rejected": -6.343166351318359, |
| "logps/chosen": -267.0815124511719, |
| "logps/rejected": -305.98211669921875, |
| "loss": 0.098, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7901928424835205, |
| "rewards/margins": 8.472061157226562, |
| "rewards/rejected": -9.262253761291504, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.4825855665964067, |
| "grad_norm": 0.8955071568489075, |
| "learning_rate": 1.1317717692888014e-05, |
| "logits/chosen": -6.818015098571777, |
| "logits/rejected": -6.390987396240234, |
| "logps/chosen": -238.9673309326172, |
| "logps/rejected": -293.705322265625, |
| "loss": 0.0991, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.4080883264541626, |
| "rewards/margins": 7.741964340209961, |
| "rewards/rejected": -9.150053024291992, |
| "step": 601 |
| }, |
| { |
| "epoch": 0.48338853758907957, |
| "grad_norm": 0.7909305691719055, |
| "learning_rate": 1.1269651692527181e-05, |
| "logits/chosen": -7.193325519561768, |
| "logits/rejected": -6.597301483154297, |
| "logps/chosen": -254.6690673828125, |
| "logps/rejected": -328.458984375, |
| "loss": 0.1076, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7603957653045654, |
| "rewards/margins": 8.167791366577148, |
| "rewards/rejected": -8.928186416625977, |
| "step": 602 |
| }, |
| { |
| "epoch": 0.48419150858175247, |
| "grad_norm": 0.5836922526359558, |
| "learning_rate": 1.1221626486657781e-05, |
| "logits/chosen": -7.055166244506836, |
| "logits/rejected": -6.497473239898682, |
| "logps/chosen": -256.06414794921875, |
| "logps/rejected": -304.9680480957031, |
| "loss": 0.0664, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6376128196716309, |
| "rewards/margins": 8.625146865844727, |
| "rewards/rejected": -9.262760162353516, |
| "step": 603 |
| }, |
| { |
| "epoch": 0.4849944795744254, |
| "grad_norm": 0.8678056597709656, |
| "learning_rate": 1.1173642600475774e-05, |
| "logits/chosen": -6.936151027679443, |
| "logits/rejected": -6.3123087882995605, |
| "logps/chosen": -234.0343475341797, |
| "logps/rejected": -305.862060546875, |
| "loss": 0.1076, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.1707110404968262, |
| "rewards/margins": 8.353006362915039, |
| "rewards/rejected": -9.523716926574707, |
| "step": 604 |
| }, |
| { |
| "epoch": 0.4857974505670983, |
| "grad_norm": 1.0789023637771606, |
| "learning_rate": 1.1125700558725254e-05, |
| "logits/chosen": -6.810320854187012, |
| "logits/rejected": -6.470407485961914, |
| "logps/chosen": -275.76324462890625, |
| "logps/rejected": -319.7564392089844, |
| "loss": 0.117, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.239778995513916, |
| "rewards/margins": 8.004508018493652, |
| "rewards/rejected": -9.24428653717041, |
| "step": 605 |
| }, |
| { |
| "epoch": 0.4866004215597712, |
| "grad_norm": 0.6227245926856995, |
| "learning_rate": 1.1077800885692704e-05, |
| "logits/chosen": -7.131650447845459, |
| "logits/rejected": -6.660788536071777, |
| "logps/chosen": -224.27906799316406, |
| "logps/rejected": -283.93328857421875, |
| "loss": 0.0626, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6879215240478516, |
| "rewards/margins": 9.256577491760254, |
| "rewards/rejected": -9.944498062133789, |
| "step": 606 |
| }, |
| { |
| "epoch": 0.487403392552444, |
| "grad_norm": 1.0200092792510986, |
| "learning_rate": 1.1029944105201278e-05, |
| "logits/chosen": -6.932712078094482, |
| "logits/rejected": -6.4876790046691895, |
| "logps/chosen": -252.79795837402344, |
| "logps/rejected": -312.1822814941406, |
| "loss": 0.1095, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.8475680947303772, |
| "rewards/margins": 8.368267059326172, |
| "rewards/rejected": -9.215835571289062, |
| "step": 607 |
| }, |
| { |
| "epoch": 0.48820636354511693, |
| "grad_norm": 0.8927953243255615, |
| "learning_rate": 1.0982130740605056e-05, |
| "logits/chosen": -6.884553909301758, |
| "logits/rejected": -6.480442523956299, |
| "logps/chosen": -295.8332214355469, |
| "logps/rejected": -331.70416259765625, |
| "loss": 0.1149, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9821190237998962, |
| "rewards/margins": 8.047282218933105, |
| "rewards/rejected": -9.029401779174805, |
| "step": 608 |
| }, |
| { |
| "epoch": 0.48900933453778983, |
| "grad_norm": 0.764371395111084, |
| "learning_rate": 1.0934361314783339e-05, |
| "logits/chosen": -7.003166198730469, |
| "logits/rejected": -6.6338887214660645, |
| "logps/chosen": -282.10223388671875, |
| "logps/rejected": -320.5048522949219, |
| "loss": 0.0749, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.6313217878341675, |
| "rewards/margins": 8.83208179473877, |
| "rewards/rejected": -9.463401794433594, |
| "step": 609 |
| }, |
| { |
| "epoch": 0.48981230553046273, |
| "grad_norm": 0.60218745470047, |
| "learning_rate": 1.0886636350134906e-05, |
| "logits/chosen": -6.8953857421875, |
| "logits/rejected": -6.431609153747559, |
| "logps/chosen": -256.8603210449219, |
| "logps/rejected": -316.01904296875, |
| "loss": 0.079, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.1930614709854126, |
| "rewards/margins": 8.342509269714355, |
| "rewards/rejected": -9.535571098327637, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.4906152765231356, |
| "grad_norm": 0.8572884798049927, |
| "learning_rate": 1.0838956368572335e-05, |
| "logits/chosen": -7.1111016273498535, |
| "logits/rejected": -6.646880626678467, |
| "logps/chosen": -251.05149841308594, |
| "logps/rejected": -293.66448974609375, |
| "loss": 0.0769, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -1.3064361810684204, |
| "rewards/margins": 8.250782012939453, |
| "rewards/rejected": -9.557218551635742, |
| "step": 611 |
| }, |
| { |
| "epoch": 0.4914182475158085, |
| "grad_norm": 0.6160441040992737, |
| "learning_rate": 1.0791321891516253e-05, |
| "logits/chosen": -7.144175052642822, |
| "logits/rejected": -6.5556864738464355, |
| "logps/chosen": -233.29107666015625, |
| "logps/rejected": -308.48748779296875, |
| "loss": 0.0717, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.217383623123169, |
| "rewards/margins": 8.634063720703125, |
| "rewards/rejected": -9.851447105407715, |
| "step": 612 |
| }, |
| { |
| "epoch": 0.4922212185084814, |
| "grad_norm": 1.038957118988037, |
| "learning_rate": 1.0743733439889675e-05, |
| "logits/chosen": -6.967656135559082, |
| "logits/rejected": -6.420195579528809, |
| "logps/chosen": -215.63052368164062, |
| "logps/rejected": -299.3995361328125, |
| "loss": 0.0881, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.3918496370315552, |
| "rewards/margins": 8.823095321655273, |
| "rewards/rejected": -10.214944839477539, |
| "step": 613 |
| }, |
| { |
| "epoch": 0.4930241895011543, |
| "grad_norm": 0.8946200609207153, |
| "learning_rate": 1.0696191534112288e-05, |
| "logits/chosen": -6.941075325012207, |
| "logits/rejected": -6.3346333503723145, |
| "logps/chosen": -231.2095489501953, |
| "logps/rejected": -293.3424072265625, |
| "loss": 0.0996, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.865527868270874, |
| "rewards/margins": 8.286029815673828, |
| "rewards/rejected": -10.151557922363281, |
| "step": 614 |
| }, |
| { |
| "epoch": 0.49382716049382713, |
| "grad_norm": 0.9254114031791687, |
| "learning_rate": 1.0648696694094751e-05, |
| "logits/chosen": -7.09945011138916, |
| "logits/rejected": -6.551877021789551, |
| "logps/chosen": -230.9491424560547, |
| "logps/rejected": -294.6939697265625, |
| "loss": 0.1002, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.182483434677124, |
| "rewards/margins": 8.509724617004395, |
| "rewards/rejected": -9.692207336425781, |
| "step": 615 |
| }, |
| { |
| "epoch": 0.49463013148650004, |
| "grad_norm": 1.0513269901275635, |
| "learning_rate": 1.0601249439233031e-05, |
| "logits/chosen": -6.873156547546387, |
| "logits/rejected": -6.313059329986572, |
| "logps/chosen": -266.9542236328125, |
| "logps/rejected": -348.78765869140625, |
| "loss": 0.0909, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.2137614488601685, |
| "rewards/margins": 9.071240425109863, |
| "rewards/rejected": -10.285001754760742, |
| "step": 616 |
| }, |
| { |
| "epoch": 0.49543310247917294, |
| "grad_norm": 1.0390101671218872, |
| "learning_rate": 1.0553850288402696e-05, |
| "logits/chosen": -6.893197536468506, |
| "logits/rejected": -6.3730058670043945, |
| "logps/chosen": -268.0085144042969, |
| "logps/rejected": -332.54864501953125, |
| "loss": 0.1023, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.723333477973938, |
| "rewards/margins": 8.314759254455566, |
| "rewards/rejected": -10.038092613220215, |
| "step": 617 |
| }, |
| { |
| "epoch": 0.49623607347184584, |
| "grad_norm": 0.8784691691398621, |
| "learning_rate": 1.0506499759953277e-05, |
| "logits/chosen": -6.901166915893555, |
| "logits/rejected": -6.376160621643066, |
| "logps/chosen": -280.9024353027344, |
| "logps/rejected": -321.51788330078125, |
| "loss": 0.1234, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -1.1872642040252686, |
| "rewards/margins": 8.314569473266602, |
| "rewards/rejected": -9.501834869384766, |
| "step": 618 |
| }, |
| { |
| "epoch": 0.49703904446451874, |
| "grad_norm": 0.9463022947311401, |
| "learning_rate": 1.0459198371702553e-05, |
| "logits/chosen": -6.945871829986572, |
| "logits/rejected": -6.461452484130859, |
| "logps/chosen": -240.98464965820312, |
| "logps/rejected": -320.8720397949219, |
| "loss": 0.0982, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.322672963142395, |
| "rewards/margins": 8.465933799743652, |
| "rewards/rejected": -9.788606643676758, |
| "step": 619 |
| }, |
| { |
| "epoch": 0.4978420154571916, |
| "grad_norm": 1.1833386421203613, |
| "learning_rate": 1.041194664093094e-05, |
| "logits/chosen": -7.004557132720947, |
| "logits/rejected": -6.421897888183594, |
| "logps/chosen": -246.5269012451172, |
| "logps/rejected": -321.2958068847656, |
| "loss": 0.0879, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.4977498054504395, |
| "rewards/margins": 8.484550476074219, |
| "rewards/rejected": -9.9822998046875, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.4986449864498645, |
| "grad_norm": 0.7722090482711792, |
| "learning_rate": 1.036474508437579e-05, |
| "logits/chosen": -7.111397743225098, |
| "logits/rejected": -6.560418605804443, |
| "logps/chosen": -247.43873596191406, |
| "logps/rejected": -319.1822204589844, |
| "loss": 0.0917, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.9242525100708008, |
| "rewards/margins": 9.24601936340332, |
| "rewards/rejected": -10.170270919799805, |
| "step": 621 |
| }, |
| { |
| "epoch": 0.4994479574425374, |
| "grad_norm": 0.7703889012336731, |
| "learning_rate": 1.0317594218225761e-05, |
| "logits/chosen": -7.024586200714111, |
| "logits/rejected": -6.4166741371154785, |
| "logps/chosen": -259.1438903808594, |
| "logps/rejected": -334.1586608886719, |
| "loss": 0.0852, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -1.761697769165039, |
| "rewards/margins": 8.708311080932617, |
| "rewards/rejected": -10.470008850097656, |
| "step": 622 |
| }, |
| { |
| "epoch": 0.5002509284352102, |
| "grad_norm": 1.0063753128051758, |
| "learning_rate": 1.0270494558115179e-05, |
| "logits/chosen": -7.084958076477051, |
| "logits/rejected": -6.604973316192627, |
| "logps/chosen": -261.8873596191406, |
| "logps/rejected": -322.689453125, |
| "loss": 0.0923, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.5070250034332275, |
| "rewards/margins": 8.447161674499512, |
| "rewards/rejected": -9.954187393188477, |
| "step": 623 |
| }, |
| { |
| "epoch": 0.5010538994278831, |
| "grad_norm": 0.904191255569458, |
| "learning_rate": 1.0223446619118372e-05, |
| "logits/chosen": -7.0453057289123535, |
| "logits/rejected": -6.451320171356201, |
| "logps/chosen": -252.38577270507812, |
| "logps/rejected": -308.7978515625, |
| "loss": 0.1053, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.407565951347351, |
| "rewards/margins": 8.363129615783691, |
| "rewards/rejected": -9.770697593688965, |
| "step": 624 |
| }, |
| { |
| "epoch": 0.501856870420556, |
| "grad_norm": 0.6791244745254517, |
| "learning_rate": 1.0176450915744072e-05, |
| "logits/chosen": -6.896179676055908, |
| "logits/rejected": -6.254065036773682, |
| "logps/chosen": -253.58091735839844, |
| "logps/rejected": -315.45599365234375, |
| "loss": 0.0761, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.9789646863937378, |
| "rewards/margins": 8.9932222366333, |
| "rewards/rejected": -9.972186088562012, |
| "step": 625 |
| }, |
| { |
| "epoch": 0.502659841413229, |
| "grad_norm": 0.9272138476371765, |
| "learning_rate": 1.0129507961929749e-05, |
| "logits/chosen": -6.885366439819336, |
| "logits/rejected": -6.377744197845459, |
| "logps/chosen": -304.37371826171875, |
| "logps/rejected": -344.8218994140625, |
| "loss": 0.0978, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5300726890563965, |
| "rewards/margins": 8.253254890441895, |
| "rewards/rejected": -8.783327102661133, |
| "step": 626 |
| }, |
| { |
| "epoch": 0.5034628124059018, |
| "grad_norm": 0.8518726825714111, |
| "learning_rate": 1.0082618271036033e-05, |
| "logits/chosen": -7.19764518737793, |
| "logits/rejected": -6.676663398742676, |
| "logps/chosen": -241.10374450683594, |
| "logps/rejected": -312.06939697265625, |
| "loss": 0.1006, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.244410514831543, |
| "rewards/margins": 8.715635299682617, |
| "rewards/rejected": -9.96004581451416, |
| "step": 627 |
| }, |
| { |
| "epoch": 0.5042657833985748, |
| "grad_norm": 0.8423402309417725, |
| "learning_rate": 1.0035782355841076e-05, |
| "logits/chosen": -7.016829967498779, |
| "logits/rejected": -6.36716365814209, |
| "logps/chosen": -261.99139404296875, |
| "logps/rejected": -320.7078857421875, |
| "loss": 0.0861, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8085633516311646, |
| "rewards/margins": 8.671195030212402, |
| "rewards/rejected": -9.479758262634277, |
| "step": 628 |
| }, |
| { |
| "epoch": 0.5050687543912477, |
| "grad_norm": 0.7682253122329712, |
| "learning_rate": 9.989000728534936e-06, |
| "logits/chosen": -7.066242218017578, |
| "logits/rejected": -6.440845012664795, |
| "logps/chosen": -261.39202880859375, |
| "logps/rejected": -334.09466552734375, |
| "loss": 0.0861, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.1554303169250488, |
| "rewards/margins": 8.356002807617188, |
| "rewards/rejected": -9.511432647705078, |
| "step": 629 |
| }, |
| { |
| "epoch": 0.5058717253839206, |
| "grad_norm": 1.1299771070480347, |
| "learning_rate": 9.942273900713997e-06, |
| "logits/chosen": -6.973911762237549, |
| "logits/rejected": -6.415992736816406, |
| "logps/chosen": -261.88623046875, |
| "logps/rejected": -330.87744140625, |
| "loss": 0.1442, |
| "rewards/accuracies": 0.91796875, |
| "rewards/chosen": -0.9479069709777832, |
| "rewards/margins": 8.328766822814941, |
| "rewards/rejected": -9.276673316955566, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.5066746963765933, |
| "grad_norm": 0.9987703561782837, |
| "learning_rate": 9.895602383375355e-06, |
| "logits/chosen": -6.868826866149902, |
| "logits/rejected": -6.448574542999268, |
| "logps/chosen": -287.3743896484375, |
| "logps/rejected": -321.5795593261719, |
| "loss": 0.1008, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.2393484115600586, |
| "rewards/margins": 8.14437198638916, |
| "rewards/rejected": -9.383720397949219, |
| "step": 631 |
| }, |
| { |
| "epoch": 0.5074776673692662, |
| "grad_norm": 0.7289369106292725, |
| "learning_rate": 9.84898668691126e-06, |
| "logits/chosen": -6.918392181396484, |
| "logits/rejected": -6.527281761169434, |
| "logps/chosen": -254.09738159179688, |
| "logps/rejected": -332.33416748046875, |
| "loss": 0.0838, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.3122237920761108, |
| "rewards/margins": 8.838020324707031, |
| "rewards/rejected": -10.150243759155273, |
| "step": 632 |
| }, |
| { |
| "epoch": 0.5082806383619392, |
| "grad_norm": 1.0582826137542725, |
| "learning_rate": 9.802427321103482e-06, |
| "logits/chosen": -6.993238925933838, |
| "logits/rejected": -6.4911041259765625, |
| "logps/chosen": -246.20431518554688, |
| "logps/rejected": -299.8311767578125, |
| "loss": 0.0809, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.8941358923912048, |
| "rewards/margins": 8.3082857131958, |
| "rewards/rejected": -9.202422142028809, |
| "step": 633 |
| }, |
| { |
| "epoch": 0.509083609354612, |
| "grad_norm": 0.6557310223579407, |
| "learning_rate": 9.755924795117798e-06, |
| "logits/chosen": -7.158468723297119, |
| "logits/rejected": -6.440249443054199, |
| "logps/chosen": -240.5247802734375, |
| "logps/rejected": -299.4957275390625, |
| "loss": 0.0826, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.9043979048728943, |
| "rewards/margins": 8.525805473327637, |
| "rewards/rejected": -9.43020248413086, |
| "step": 634 |
| }, |
| { |
| "epoch": 0.509886580347285, |
| "grad_norm": 0.6114540696144104, |
| "learning_rate": 9.709479617498384e-06, |
| "logits/chosen": -6.854625701904297, |
| "logits/rejected": -6.321982383728027, |
| "logps/chosen": -243.19720458984375, |
| "logps/rejected": -306.2549133300781, |
| "loss": 0.0654, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -0.9180538058280945, |
| "rewards/margins": 8.60292911529541, |
| "rewards/rejected": -9.52098274230957, |
| "step": 635 |
| }, |
| { |
| "epoch": 0.5106895513399579, |
| "grad_norm": 0.6771186590194702, |
| "learning_rate": 9.663092296162252e-06, |
| "logits/chosen": -7.165428161621094, |
| "logits/rejected": -6.5121846199035645, |
| "logps/chosen": -229.4063720703125, |
| "logps/rejected": -305.2828063964844, |
| "loss": 0.0888, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.898493230342865, |
| "rewards/margins": 8.729482650756836, |
| "rewards/rejected": -9.627975463867188, |
| "step": 636 |
| }, |
| { |
| "epoch": 0.5114925223326308, |
| "grad_norm": 0.8891004323959351, |
| "learning_rate": 9.616763338393728e-06, |
| "logits/chosen": -7.080987453460693, |
| "logits/rejected": -6.546280384063721, |
| "logps/chosen": -245.97142028808594, |
| "logps/rejected": -303.46435546875, |
| "loss": 0.1034, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.7168923616409302, |
| "rewards/margins": 8.908449172973633, |
| "rewards/rejected": -9.625341415405273, |
| "step": 637 |
| }, |
| { |
| "epoch": 0.5122954933253037, |
| "grad_norm": 0.6347550749778748, |
| "learning_rate": 9.570493250838855e-06, |
| "logits/chosen": -7.039628505706787, |
| "logits/rejected": -6.483885765075684, |
| "logps/chosen": -248.46563720703125, |
| "logps/rejected": -316.7767639160156, |
| "loss": 0.0681, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.7220199704170227, |
| "rewards/margins": 8.580655097961426, |
| "rewards/rejected": -9.302675247192383, |
| "step": 638 |
| }, |
| { |
| "epoch": 0.5130984643179766, |
| "grad_norm": 0.7126450538635254, |
| "learning_rate": 9.524282539499916e-06, |
| "logits/chosen": -6.900891304016113, |
| "logits/rejected": -6.556222915649414, |
| "logps/chosen": -240.21435546875, |
| "logps/rejected": -306.2138977050781, |
| "loss": 0.0977, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.4539770185947418, |
| "rewards/margins": 8.481987953186035, |
| "rewards/rejected": -8.935965538024902, |
| "step": 639 |
| }, |
| { |
| "epoch": 0.5139014353106494, |
| "grad_norm": 0.7878826260566711, |
| "learning_rate": 9.478131709729831e-06, |
| "logits/chosen": -7.056272506713867, |
| "logits/rejected": -6.4692912101745605, |
| "logps/chosen": -250.43165588378906, |
| "logps/rejected": -318.7052917480469, |
| "loss": 0.0907, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.6150429844856262, |
| "rewards/margins": 8.44477367401123, |
| "rewards/rejected": -9.059816360473633, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.5147044063033223, |
| "grad_norm": 0.9534127116203308, |
| "learning_rate": 9.432041266226686e-06, |
| "logits/chosen": -7.197908878326416, |
| "logits/rejected": -6.733092784881592, |
| "logps/chosen": -254.27825927734375, |
| "logps/rejected": -306.39739990234375, |
| "loss": 0.107, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7424541711807251, |
| "rewards/margins": 8.183919906616211, |
| "rewards/rejected": -8.926374435424805, |
| "step": 641 |
| }, |
| { |
| "epoch": 0.5155073772959952, |
| "grad_norm": 0.6675994992256165, |
| "learning_rate": 9.386011713028196e-06, |
| "logits/chosen": -7.223260402679443, |
| "logits/rejected": -6.6209187507629395, |
| "logps/chosen": -268.07470703125, |
| "logps/rejected": -314.8468322753906, |
| "loss": 0.074, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.3342647850513458, |
| "rewards/margins": 8.46653938293457, |
| "rewards/rejected": -8.800804138183594, |
| "step": 642 |
| }, |
| { |
| "epoch": 0.5163103482886681, |
| "grad_norm": 0.7144777774810791, |
| "learning_rate": 9.340043553506164e-06, |
| "logits/chosen": -7.054248332977295, |
| "logits/rejected": -6.507267475128174, |
| "logps/chosen": -263.5543518066406, |
| "logps/rejected": -338.1730651855469, |
| "loss": 0.0846, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.6972871422767639, |
| "rewards/margins": 8.201329231262207, |
| "rewards/rejected": -8.898616790771484, |
| "step": 643 |
| }, |
| { |
| "epoch": 0.517113319281341, |
| "grad_norm": 0.76971435546875, |
| "learning_rate": 9.294137290361032e-06, |
| "logits/chosen": -6.898094177246094, |
| "logits/rejected": -6.619019985198975, |
| "logps/chosen": -250.62872314453125, |
| "logps/rejected": -291.21990966796875, |
| "loss": 0.0959, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7997147440910339, |
| "rewards/margins": 8.04134750366211, |
| "rewards/rejected": -8.84106159210205, |
| "step": 644 |
| }, |
| { |
| "epoch": 0.5179162902740139, |
| "grad_norm": 0.8303004503250122, |
| "learning_rate": 9.248293425616326e-06, |
| "logits/chosen": -6.9606194496154785, |
| "logits/rejected": -6.533857822418213, |
| "logps/chosen": -241.5054473876953, |
| "logps/rejected": -310.2570495605469, |
| "loss": 0.0939, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7664681673049927, |
| "rewards/margins": 8.095749855041504, |
| "rewards/rejected": -8.862218856811523, |
| "step": 645 |
| }, |
| { |
| "epoch": 0.5187192612666868, |
| "grad_norm": 1.0154165029525757, |
| "learning_rate": 9.20251246061322e-06, |
| "logits/chosen": -7.091289043426514, |
| "logits/rejected": -6.550615310668945, |
| "logps/chosen": -250.82862854003906, |
| "logps/rejected": -316.5213317871094, |
| "loss": 0.1062, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.7024128437042236, |
| "rewards/margins": 8.27251148223877, |
| "rewards/rejected": -8.974923133850098, |
| "step": 646 |
| }, |
| { |
| "epoch": 0.5195222322593597, |
| "grad_norm": 0.7569310665130615, |
| "learning_rate": 9.156794896005e-06, |
| "logits/chosen": -6.986323356628418, |
| "logits/rejected": -6.346773147583008, |
| "logps/chosen": -233.66961669921875, |
| "logps/rejected": -302.9237976074219, |
| "loss": 0.0762, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.7576214075088501, |
| "rewards/margins": 8.237204551696777, |
| "rewards/rejected": -8.99482536315918, |
| "step": 647 |
| }, |
| { |
| "epoch": 0.5203252032520326, |
| "grad_norm": 0.7330907583236694, |
| "learning_rate": 9.11114123175163e-06, |
| "logits/chosen": -6.869880676269531, |
| "logits/rejected": -6.373012542724609, |
| "logps/chosen": -245.85057067871094, |
| "logps/rejected": -318.125244140625, |
| "loss": 0.087, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.6066738963127136, |
| "rewards/margins": 8.254448890686035, |
| "rewards/rejected": -8.861123085021973, |
| "step": 648 |
| }, |
| { |
| "epoch": 0.5211281742447054, |
| "grad_norm": 0.7225344777107239, |
| "learning_rate": 9.06555196711428e-06, |
| "logits/chosen": -6.869635581970215, |
| "logits/rejected": -6.38453483581543, |
| "logps/chosen": -265.4332275390625, |
| "logps/rejected": -325.72454833984375, |
| "loss": 0.0976, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8228551149368286, |
| "rewards/margins": 7.929225444793701, |
| "rewards/rejected": -8.752079963684082, |
| "step": 649 |
| }, |
| { |
| "epoch": 0.5219311452373783, |
| "grad_norm": 0.6116411089897156, |
| "learning_rate": 9.020027600649825e-06, |
| "logits/chosen": -6.932907581329346, |
| "logits/rejected": -6.399709701538086, |
| "logps/chosen": -248.69854736328125, |
| "logps/rejected": -293.49249267578125, |
| "loss": 0.0683, |
| "rewards/accuracies": 0.984375, |
| "rewards/chosen": -0.2522527575492859, |
| "rewards/margins": 8.582403182983398, |
| "rewards/rejected": -8.83465576171875, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.5227341162300512, |
| "grad_norm": 0.568399965763092, |
| "learning_rate": 8.974568630205462e-06, |
| "logits/chosen": -6.996361255645752, |
| "logits/rejected": -6.476181507110596, |
| "logps/chosen": -246.54360961914062, |
| "logps/rejected": -323.1460266113281, |
| "loss": 0.0631, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7161757349967957, |
| "rewards/margins": 8.6231689453125, |
| "rewards/rejected": -9.339344024658203, |
| "step": 651 |
| }, |
| { |
| "epoch": 0.5235370872227241, |
| "grad_norm": 0.9051775932312012, |
| "learning_rate": 8.929175552913195e-06, |
| "logits/chosen": -6.789648532867432, |
| "logits/rejected": -6.280582427978516, |
| "logps/chosen": -254.869873046875, |
| "logps/rejected": -330.73870849609375, |
| "loss": 0.1288, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.8961936235427856, |
| "rewards/margins": 7.921999931335449, |
| "rewards/rejected": -8.818193435668945, |
| "step": 652 |
| }, |
| { |
| "epoch": 0.524340058215397, |
| "grad_norm": 0.7827865481376648, |
| "learning_rate": 8.88384886518445e-06, |
| "logits/chosen": -7.064261436462402, |
| "logits/rejected": -6.528904914855957, |
| "logps/chosen": -258.16156005859375, |
| "logps/rejected": -312.7088317871094, |
| "loss": 0.0585, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5941786766052246, |
| "rewards/margins": 8.80756950378418, |
| "rewards/rejected": -9.401748657226562, |
| "step": 653 |
| }, |
| { |
| "epoch": 0.5251430292080699, |
| "grad_norm": 0.780322253704071, |
| "learning_rate": 8.838589062704621e-06, |
| "logits/chosen": -6.990790843963623, |
| "logits/rejected": -6.4251251220703125, |
| "logps/chosen": -236.61672973632812, |
| "logps/rejected": -307.07550048828125, |
| "loss": 0.0928, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9267867803573608, |
| "rewards/margins": 8.471660614013672, |
| "rewards/rejected": -9.398448944091797, |
| "step": 654 |
| }, |
| { |
| "epoch": 0.5259460002007428, |
| "grad_norm": 1.1125768423080444, |
| "learning_rate": 8.793396640427651e-06, |
| "logits/chosen": -6.973318576812744, |
| "logits/rejected": -6.467411041259766, |
| "logps/chosen": -260.439697265625, |
| "logps/rejected": -346.6363220214844, |
| "loss": 0.1233, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.185038447380066, |
| "rewards/margins": 8.275997161865234, |
| "rewards/rejected": -9.46103572845459, |
| "step": 655 |
| }, |
| { |
| "epoch": 0.5267489711934157, |
| "grad_norm": 0.9295575618743896, |
| "learning_rate": 8.748272092570648e-06, |
| "logits/chosen": -6.9483442306518555, |
| "logits/rejected": -6.496287822723389, |
| "logps/chosen": -254.5398406982422, |
| "logps/rejected": -312.6842346191406, |
| "loss": 0.0912, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.0137869119644165, |
| "rewards/margins": 8.425420761108398, |
| "rewards/rejected": -9.439207077026367, |
| "step": 656 |
| }, |
| { |
| "epoch": 0.5275519421860885, |
| "grad_norm": 0.9135299324989319, |
| "learning_rate": 8.703215912608416e-06, |
| "logits/chosen": -6.986708641052246, |
| "logits/rejected": -6.496175765991211, |
| "logps/chosen": -273.2483825683594, |
| "logps/rejected": -335.2017822265625, |
| "loss": 0.0961, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.9520747065544128, |
| "rewards/margins": 8.357061386108398, |
| "rewards/rejected": -9.309135437011719, |
| "step": 657 |
| }, |
| { |
| "epoch": 0.5283549131787614, |
| "grad_norm": 0.8569380044937134, |
| "learning_rate": 8.65822859326813e-06, |
| "logits/chosen": -6.964521408081055, |
| "logits/rejected": -6.4368133544921875, |
| "logps/chosen": -259.2574768066406, |
| "logps/rejected": -307.3487243652344, |
| "loss": 0.1175, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.0825127363204956, |
| "rewards/margins": 7.958081245422363, |
| "rewards/rejected": -9.040594100952148, |
| "step": 658 |
| }, |
| { |
| "epoch": 0.5291578841714343, |
| "grad_norm": 1.0478408336639404, |
| "learning_rate": 8.61331062652391e-06, |
| "logits/chosen": -6.978537559509277, |
| "logits/rejected": -6.458289623260498, |
| "logps/chosen": -276.89300537109375, |
| "logps/rejected": -321.28570556640625, |
| "loss": 0.0825, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.9465509653091431, |
| "rewards/margins": 8.534381866455078, |
| "rewards/rejected": -9.48093318939209, |
| "step": 659 |
| }, |
| { |
| "epoch": 0.5299608551641072, |
| "grad_norm": 0.6588581800460815, |
| "learning_rate": 8.568462503591442e-06, |
| "logits/chosen": -7.0561604499816895, |
| "logits/rejected": -6.371212959289551, |
| "logps/chosen": -261.0728759765625, |
| "logps/rejected": -303.8944396972656, |
| "loss": 0.0969, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.9129598140716553, |
| "rewards/margins": 8.645821571350098, |
| "rewards/rejected": -9.558780670166016, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.5307638261567801, |
| "grad_norm": 0.7967017292976379, |
| "learning_rate": 8.523684714922608e-06, |
| "logits/chosen": -6.834590911865234, |
| "logits/rejected": -6.295604228973389, |
| "logps/chosen": -240.04815673828125, |
| "logps/rejected": -309.12799072265625, |
| "loss": 0.0872, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.4322919845581055, |
| "rewards/margins": 8.469236373901367, |
| "rewards/rejected": -9.901527404785156, |
| "step": 661 |
| }, |
| { |
| "epoch": 0.531566797149453, |
| "grad_norm": 0.804474949836731, |
| "learning_rate": 8.478977750200132e-06, |
| "logits/chosen": -6.902838706970215, |
| "logits/rejected": -6.438027858734131, |
| "logps/chosen": -256.29620361328125, |
| "logps/rejected": -308.8296813964844, |
| "loss": 0.082, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.548864483833313, |
| "rewards/margins": 8.697782516479492, |
| "rewards/rejected": -9.246646881103516, |
| "step": 662 |
| }, |
| { |
| "epoch": 0.5323697681421259, |
| "grad_norm": 0.7692179679870605, |
| "learning_rate": 8.434342098332222e-06, |
| "logits/chosen": -6.9271392822265625, |
| "logits/rejected": -6.4990010261535645, |
| "logps/chosen": -287.7195739746094, |
| "logps/rejected": -324.8873596191406, |
| "loss": 0.0866, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.9238299131393433, |
| "rewards/margins": 8.584981918334961, |
| "rewards/rejected": -9.50881290435791, |
| "step": 663 |
| }, |
| { |
| "epoch": 0.5331727391347988, |
| "grad_norm": 1.134378433227539, |
| "learning_rate": 8.389778247447211e-06, |
| "logits/chosen": -7.016768455505371, |
| "logits/rejected": -6.512033462524414, |
| "logps/chosen": -252.52587890625, |
| "logps/rejected": -331.3772277832031, |
| "loss": 0.0823, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.4341472387313843, |
| "rewards/margins": 8.7704439163208, |
| "rewards/rejected": -10.20458984375, |
| "step": 664 |
| }, |
| { |
| "epoch": 0.5339757101274717, |
| "grad_norm": 1.1016666889190674, |
| "learning_rate": 8.345286684888244e-06, |
| "logits/chosen": -7.025629043579102, |
| "logits/rejected": -6.43436861038208, |
| "logps/chosen": -233.0938720703125, |
| "logps/rejected": -296.36004638671875, |
| "loss": 0.0925, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.3858574628829956, |
| "rewards/margins": 8.661656379699707, |
| "rewards/rejected": -10.047513961791992, |
| "step": 665 |
| }, |
| { |
| "epoch": 0.5347786811201445, |
| "grad_norm": 0.83234703540802, |
| "learning_rate": 8.300867897207903e-06, |
| "logits/chosen": -7.058328151702881, |
| "logits/rejected": -6.463898658752441, |
| "logps/chosen": -265.8833923339844, |
| "logps/rejected": -356.85223388671875, |
| "loss": 0.0943, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.1666994094848633, |
| "rewards/margins": 8.357856750488281, |
| "rewards/rejected": -9.524555206298828, |
| "step": 666 |
| }, |
| { |
| "epoch": 0.5355816521128174, |
| "grad_norm": 1.068359613418579, |
| "learning_rate": 8.256522370162949e-06, |
| "logits/chosen": -7.075399875640869, |
| "logits/rejected": -6.466457366943359, |
| "logps/chosen": -246.60830688476562, |
| "logps/rejected": -335.98309326171875, |
| "loss": 0.0882, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.3596584796905518, |
| "rewards/margins": 8.802018165588379, |
| "rewards/rejected": -10.161676406860352, |
| "step": 667 |
| }, |
| { |
| "epoch": 0.5363846231054903, |
| "grad_norm": 0.9530684947967529, |
| "learning_rate": 8.212250588708943e-06, |
| "logits/chosen": -6.768768787384033, |
| "logits/rejected": -6.328291893005371, |
| "logps/chosen": -254.43026733398438, |
| "logps/rejected": -331.96307373046875, |
| "loss": 0.089, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.9084960222244263, |
| "rewards/margins": 8.8890380859375, |
| "rewards/rejected": -9.79753303527832, |
| "step": 668 |
| }, |
| { |
| "epoch": 0.5371875940981632, |
| "grad_norm": 0.8195638060569763, |
| "learning_rate": 8.168053036995011e-06, |
| "logits/chosen": -6.995382308959961, |
| "logits/rejected": -6.383074760437012, |
| "logps/chosen": -248.42758178710938, |
| "logps/rejected": -325.3786926269531, |
| "loss": 0.0963, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.3036795854568481, |
| "rewards/margins": 8.333038330078125, |
| "rewards/rejected": -9.636717796325684, |
| "step": 669 |
| }, |
| { |
| "epoch": 0.5379905650908361, |
| "grad_norm": 0.48641178011894226, |
| "learning_rate": 8.123930198358498e-06, |
| "logits/chosen": -7.103156566619873, |
| "logits/rejected": -6.435365676879883, |
| "logps/chosen": -264.26898193359375, |
| "logps/rejected": -339.1961669921875, |
| "loss": 0.049, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -0.487449586391449, |
| "rewards/margins": 8.90141487121582, |
| "rewards/rejected": -9.388864517211914, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.538793536083509, |
| "grad_norm": 0.9285465478897095, |
| "learning_rate": 8.079882555319685e-06, |
| "logits/chosen": -6.768332004547119, |
| "logits/rejected": -6.268764019012451, |
| "logps/chosen": -271.6624755859375, |
| "logps/rejected": -340.5348205566406, |
| "loss": 0.0891, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.6157209873199463, |
| "rewards/margins": 8.641461372375488, |
| "rewards/rejected": -9.257182121276855, |
| "step": 671 |
| }, |
| { |
| "epoch": 0.5395965070761819, |
| "grad_norm": 0.9320762753486633, |
| "learning_rate": 8.03591058957655e-06, |
| "logits/chosen": -6.868618965148926, |
| "logits/rejected": -6.4160380363464355, |
| "logps/chosen": -234.93861389160156, |
| "logps/rejected": -317.81683349609375, |
| "loss": 0.1001, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.6792980432510376, |
| "rewards/margins": 8.669720649719238, |
| "rewards/rejected": -9.349019050598145, |
| "step": 672 |
| }, |
| { |
| "epoch": 0.5403994780688548, |
| "grad_norm": 0.8494117856025696, |
| "learning_rate": 7.992014781999454e-06, |
| "logits/chosen": -7.124987602233887, |
| "logits/rejected": -6.527643203735352, |
| "logps/chosen": -272.6121520996094, |
| "logps/rejected": -332.18408203125, |
| "loss": 0.0735, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.1065952777862549, |
| "rewards/margins": 8.518379211425781, |
| "rewards/rejected": -9.624975204467773, |
| "step": 673 |
| }, |
| { |
| "epoch": 0.5412024490615277, |
| "grad_norm": 0.807579755783081, |
| "learning_rate": 7.948195612625933e-06, |
| "logits/chosen": -6.997409820556641, |
| "logits/rejected": -6.426229476928711, |
| "logps/chosen": -250.60011291503906, |
| "logps/rejected": -329.26239013671875, |
| "loss": 0.0805, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.8470394611358643, |
| "rewards/margins": 8.789568901062012, |
| "rewards/rejected": -9.636608123779297, |
| "step": 674 |
| }, |
| { |
| "epoch": 0.5420054200542005, |
| "grad_norm": 0.7967342734336853, |
| "learning_rate": 7.904453560655389e-06, |
| "logits/chosen": -6.830033302307129, |
| "logits/rejected": -6.3468122482299805, |
| "logps/chosen": -266.7158203125, |
| "logps/rejected": -310.1424560546875, |
| "loss": 0.0854, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.5239680409431458, |
| "rewards/margins": 8.298447608947754, |
| "rewards/rejected": -8.822416305541992, |
| "step": 675 |
| }, |
| { |
| "epoch": 0.5428083910468734, |
| "grad_norm": 0.8016481399536133, |
| "learning_rate": 7.860789104443897e-06, |
| "logits/chosen": -7.081709861755371, |
| "logits/rejected": -6.474361419677734, |
| "logps/chosen": -247.0025177001953, |
| "logps/rejected": -320.2969665527344, |
| "loss": 0.0774, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7097981572151184, |
| "rewards/margins": 8.952345848083496, |
| "rewards/rejected": -9.66214370727539, |
| "step": 676 |
| }, |
| { |
| "epoch": 0.5436113620395463, |
| "grad_norm": 0.984112024307251, |
| "learning_rate": 7.817202721498955e-06, |
| "logits/chosen": -7.110593318939209, |
| "logits/rejected": -6.52163553237915, |
| "logps/chosen": -224.66893005371094, |
| "logps/rejected": -304.44464111328125, |
| "loss": 0.106, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9855119585990906, |
| "rewards/margins": 8.74248218536377, |
| "rewards/rejected": -9.727994918823242, |
| "step": 677 |
| }, |
| { |
| "epoch": 0.5444143330322192, |
| "grad_norm": 1.0740342140197754, |
| "learning_rate": 7.773694888474268e-06, |
| "logits/chosen": -6.901318073272705, |
| "logits/rejected": -6.246926307678223, |
| "logps/chosen": -255.59103393554688, |
| "logps/rejected": -332.0969543457031, |
| "loss": 0.1322, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.0356650352478027, |
| "rewards/margins": 7.944921493530273, |
| "rewards/rejected": -8.980587005615234, |
| "step": 678 |
| }, |
| { |
| "epoch": 0.5452173040248921, |
| "grad_norm": 0.8492692708969116, |
| "learning_rate": 7.73026608116453e-06, |
| "logits/chosen": -6.956179618835449, |
| "logits/rejected": -6.353421688079834, |
| "logps/chosen": -251.2458953857422, |
| "logps/rejected": -305.417724609375, |
| "loss": 0.1167, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.5975878834724426, |
| "rewards/margins": 8.508864402770996, |
| "rewards/rejected": -9.106452941894531, |
| "step": 679 |
| }, |
| { |
| "epoch": 0.546020275017565, |
| "grad_norm": 0.8984313011169434, |
| "learning_rate": 7.686916774500206e-06, |
| "logits/chosen": -6.821465492248535, |
| "logits/rejected": -6.2749810218811035, |
| "logps/chosen": -248.880615234375, |
| "logps/rejected": -337.4901428222656, |
| "loss": 0.105, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.8504065871238708, |
| "rewards/margins": 8.603238105773926, |
| "rewards/rejected": -9.453644752502441, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.5468232460102379, |
| "grad_norm": 0.910516083240509, |
| "learning_rate": 7.643647442542383e-06, |
| "logits/chosen": -6.7811689376831055, |
| "logits/rejected": -6.1864728927612305, |
| "logps/chosen": -244.49183654785156, |
| "logps/rejected": -323.8446044921875, |
| "loss": 0.119, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.891692042350769, |
| "rewards/margins": 8.119369506835938, |
| "rewards/rejected": -9.011062622070312, |
| "step": 681 |
| }, |
| { |
| "epoch": 0.5476262170029108, |
| "grad_norm": 0.7500264048576355, |
| "learning_rate": 7.600458558477525e-06, |
| "logits/chosen": -6.938991546630859, |
| "logits/rejected": -6.284242153167725, |
| "logps/chosen": -261.8011779785156, |
| "logps/rejected": -342.3115234375, |
| "loss": 0.1107, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.695933997631073, |
| "rewards/margins": 8.7200345993042, |
| "rewards/rejected": -9.415968894958496, |
| "step": 682 |
| }, |
| { |
| "epoch": 0.5484291879955837, |
| "grad_norm": 1.1261390447616577, |
| "learning_rate": 7.557350594612366e-06, |
| "logits/chosen": -6.810909748077393, |
| "logits/rejected": -6.340973854064941, |
| "logps/chosen": -246.64346313476562, |
| "logps/rejected": -322.7760009765625, |
| "loss": 0.1175, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.3812638521194458, |
| "rewards/margins": 8.236270904541016, |
| "rewards/rejected": -9.617534637451172, |
| "step": 683 |
| }, |
| { |
| "epoch": 0.5492321589882565, |
| "grad_norm": 0.7075995802879333, |
| "learning_rate": 7.5143240223686934e-06, |
| "logits/chosen": -6.873748302459717, |
| "logits/rejected": -6.250458717346191, |
| "logps/chosen": -260.7507019042969, |
| "logps/rejected": -327.1451416015625, |
| "loss": 0.0825, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.544756293296814, |
| "rewards/margins": 8.25236701965332, |
| "rewards/rejected": -9.797122955322266, |
| "step": 684 |
| }, |
| { |
| "epoch": 0.5500351299809294, |
| "grad_norm": 0.8708049654960632, |
| "learning_rate": 7.471379312278194e-06, |
| "logits/chosen": -6.913968086242676, |
| "logits/rejected": -6.326502323150635, |
| "logps/chosen": -262.8537902832031, |
| "logps/rejected": -334.64947509765625, |
| "loss": 0.0904, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.4441681206226349, |
| "rewards/margins": 8.81311321258545, |
| "rewards/rejected": -9.257282257080078, |
| "step": 685 |
| }, |
| { |
| "epoch": 0.5508381009736023, |
| "grad_norm": 0.8548024296760559, |
| "learning_rate": 7.4285169339773486e-06, |
| "logits/chosen": -7.090705871582031, |
| "logits/rejected": -6.461982250213623, |
| "logps/chosen": -233.96888732910156, |
| "logps/rejected": -311.4705505371094, |
| "loss": 0.0934, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.2218942642211914, |
| "rewards/margins": 8.347112655639648, |
| "rewards/rejected": -9.569007873535156, |
| "step": 686 |
| }, |
| { |
| "epoch": 0.5516410719662752, |
| "grad_norm": 0.7211710810661316, |
| "learning_rate": 7.385737356202244e-06, |
| "logits/chosen": -7.0706586837768555, |
| "logits/rejected": -6.5209574699401855, |
| "logps/chosen": -246.84091186523438, |
| "logps/rejected": -325.4413146972656, |
| "loss": 0.0862, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.7071864604949951, |
| "rewards/margins": 8.790579795837402, |
| "rewards/rejected": -9.497766494750977, |
| "step": 687 |
| }, |
| { |
| "epoch": 0.5524440429589481, |
| "grad_norm": 0.6711488366127014, |
| "learning_rate": 7.343041046783511e-06, |
| "logits/chosen": -6.973175048828125, |
| "logits/rejected": -6.419558048248291, |
| "logps/chosen": -276.7164306640625, |
| "logps/rejected": -332.6566467285156, |
| "loss": 0.0825, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8948070406913757, |
| "rewards/margins": 8.237979888916016, |
| "rewards/rejected": -9.132787704467773, |
| "step": 688 |
| }, |
| { |
| "epoch": 0.553247013951621, |
| "grad_norm": 0.8788807988166809, |
| "learning_rate": 7.3004284726411315e-06, |
| "logits/chosen": -7.002472400665283, |
| "logits/rejected": -6.322384357452393, |
| "logps/chosen": -267.6522216796875, |
| "logps/rejected": -336.27825927734375, |
| "loss": 0.1276, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.41251686215400696, |
| "rewards/margins": 8.379398345947266, |
| "rewards/rejected": -8.791914939880371, |
| "step": 689 |
| }, |
| { |
| "epoch": 0.5540499849442939, |
| "grad_norm": 0.6286065578460693, |
| "learning_rate": 7.257900099779394e-06, |
| "logits/chosen": -6.955545425415039, |
| "logits/rejected": -6.351616382598877, |
| "logps/chosen": -271.61883544921875, |
| "logps/rejected": -314.3957214355469, |
| "loss": 0.0936, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.4373924136161804, |
| "rewards/margins": 8.69843578338623, |
| "rewards/rejected": -9.135828971862793, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.5548529559369668, |
| "grad_norm": 0.7772742509841919, |
| "learning_rate": 7.215456393281777e-06, |
| "logits/chosen": -7.021120071411133, |
| "logits/rejected": -6.335753917694092, |
| "logps/chosen": -255.6962890625, |
| "logps/rejected": -312.04150390625, |
| "loss": 0.1314, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.1518213748931885, |
| "rewards/margins": 8.247367858886719, |
| "rewards/rejected": -9.399189949035645, |
| "step": 691 |
| }, |
| { |
| "epoch": 0.5556559269296397, |
| "grad_norm": 0.7671638131141663, |
| "learning_rate": 7.173097817305838e-06, |
| "logits/chosen": -6.907026290893555, |
| "logits/rejected": -6.359927177429199, |
| "logps/chosen": -244.78799438476562, |
| "logps/rejected": -308.215087890625, |
| "loss": 0.0871, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.6651936769485474, |
| "rewards/margins": 8.40267276763916, |
| "rewards/rejected": -9.067866325378418, |
| "step": 692 |
| }, |
| { |
| "epoch": 0.5564588979223125, |
| "grad_norm": 0.47320738434791565, |
| "learning_rate": 7.1308248350782e-06, |
| "logits/chosen": -7.127586364746094, |
| "logits/rejected": -6.585206031799316, |
| "logps/chosen": -246.95077514648438, |
| "logps/rejected": -315.8607177734375, |
| "loss": 0.0474, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -0.43731629848480225, |
| "rewards/margins": 8.819184303283691, |
| "rewards/rejected": -9.256500244140625, |
| "step": 693 |
| }, |
| { |
| "epoch": 0.5572618689149854, |
| "grad_norm": 0.7602994441986084, |
| "learning_rate": 7.088637908889406e-06, |
| "logits/chosen": -6.977385520935059, |
| "logits/rejected": -6.469259262084961, |
| "logps/chosen": -251.9960174560547, |
| "logps/rejected": -335.9107666015625, |
| "loss": 0.0944, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5327873229980469, |
| "rewards/margins": 7.9340949058532715, |
| "rewards/rejected": -8.46688175201416, |
| "step": 694 |
| }, |
| { |
| "epoch": 0.5580648399076583, |
| "grad_norm": 0.9828537106513977, |
| "learning_rate": 7.046537500088933e-06, |
| "logits/chosen": -7.099808692932129, |
| "logits/rejected": -6.608901500701904, |
| "logps/chosen": -258.7101745605469, |
| "logps/rejected": -318.8514404296875, |
| "loss": 0.123, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.5670415759086609, |
| "rewards/margins": 8.231430053710938, |
| "rewards/rejected": -8.798471450805664, |
| "step": 695 |
| }, |
| { |
| "epoch": 0.5588678109003312, |
| "grad_norm": 0.9267803430557251, |
| "learning_rate": 7.0045240690800975e-06, |
| "logits/chosen": -6.922746658325195, |
| "logits/rejected": -6.488621234893799, |
| "logps/chosen": -273.5257568359375, |
| "logps/rejected": -334.2964172363281, |
| "loss": 0.0925, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.41908085346221924, |
| "rewards/margins": 8.453338623046875, |
| "rewards/rejected": -8.872418403625488, |
| "step": 696 |
| }, |
| { |
| "epoch": 0.5596707818930041, |
| "grad_norm": 0.7147541046142578, |
| "learning_rate": 6.962598075315047e-06, |
| "logits/chosen": -7.058023452758789, |
| "logits/rejected": -6.4842705726623535, |
| "logps/chosen": -264.17327880859375, |
| "logps/rejected": -312.55218505859375, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.42684197425842285, |
| "rewards/margins": 8.767313957214355, |
| "rewards/rejected": -9.1941556930542, |
| "step": 697 |
| }, |
| { |
| "epoch": 0.560473752885677, |
| "grad_norm": 0.9226776957511902, |
| "learning_rate": 6.920759977289752e-06, |
| "logits/chosen": -7.0287580490112305, |
| "logits/rejected": -6.581677436828613, |
| "logps/chosen": -258.0796813964844, |
| "logps/rejected": -304.3193054199219, |
| "loss": 0.1044, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.7242788672447205, |
| "rewards/margins": 8.268095016479492, |
| "rewards/rejected": -8.9923734664917, |
| "step": 698 |
| }, |
| { |
| "epoch": 0.5612767238783499, |
| "grad_norm": 0.8323525786399841, |
| "learning_rate": 6.87901023253893e-06, |
| "logits/chosen": -7.069052696228027, |
| "logits/rejected": -6.55649471282959, |
| "logps/chosen": -254.30624389648438, |
| "logps/rejected": -314.7563781738281, |
| "loss": 0.1133, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.6567355990409851, |
| "rewards/margins": 8.05506706237793, |
| "rewards/rejected": -8.711803436279297, |
| "step": 699 |
| }, |
| { |
| "epoch": 0.5620796948710228, |
| "grad_norm": 0.5417929887771606, |
| "learning_rate": 6.837349297631114e-06, |
| "logits/chosen": -7.148456573486328, |
| "logits/rejected": -6.532755374908447, |
| "logps/chosen": -240.1038055419922, |
| "logps/rejected": -297.0650329589844, |
| "loss": 0.0601, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -0.1602044403553009, |
| "rewards/margins": 8.838846206665039, |
| "rewards/rejected": -8.999051094055176, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.5628826658636956, |
| "grad_norm": 0.630967378616333, |
| "learning_rate": 6.795777628163599e-06, |
| "logits/chosen": -7.06022834777832, |
| "logits/rejected": -6.577154159545898, |
| "logps/chosen": -240.705810546875, |
| "logps/rejected": -294.53778076171875, |
| "loss": 0.0615, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.5089325904846191, |
| "rewards/margins": 8.647315979003906, |
| "rewards/rejected": -9.156249046325684, |
| "step": 701 |
| }, |
| { |
| "epoch": 0.5636856368563685, |
| "grad_norm": 0.8286001682281494, |
| "learning_rate": 6.754295678757512e-06, |
| "logits/chosen": -6.9650139808654785, |
| "logits/rejected": -6.437186241149902, |
| "logps/chosen": -259.8277587890625, |
| "logps/rejected": -299.0513610839844, |
| "loss": 0.0993, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.8658498525619507, |
| "rewards/margins": 8.4150390625, |
| "rewards/rejected": -9.280889511108398, |
| "step": 702 |
| }, |
| { |
| "epoch": 0.5644886078490414, |
| "grad_norm": 0.6778579354286194, |
| "learning_rate": 6.712903903052802e-06, |
| "logits/chosen": -7.014791011810303, |
| "logits/rejected": -6.4914326667785645, |
| "logps/chosen": -227.64613342285156, |
| "logps/rejected": -312.5353698730469, |
| "loss": 0.0642, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -1.0823395252227783, |
| "rewards/margins": 8.583388328552246, |
| "rewards/rejected": -9.665727615356445, |
| "step": 703 |
| }, |
| { |
| "epoch": 0.5652915788417143, |
| "grad_norm": 0.9949169158935547, |
| "learning_rate": 6.671602753703299e-06, |
| "logits/chosen": -7.091136932373047, |
| "logits/rejected": -6.53409481048584, |
| "logps/chosen": -234.6637420654297, |
| "logps/rejected": -300.4314880371094, |
| "loss": 0.0931, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.8573681116104126, |
| "rewards/margins": 8.236927032470703, |
| "rewards/rejected": -9.094294548034668, |
| "step": 704 |
| }, |
| { |
| "epoch": 0.5660945498343872, |
| "grad_norm": 0.5775994062423706, |
| "learning_rate": 6.630392682371761e-06, |
| "logits/chosen": -7.116576671600342, |
| "logits/rejected": -6.633449554443359, |
| "logps/chosen": -217.08152770996094, |
| "logps/rejected": -299.8333740234375, |
| "loss": 0.0701, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.9338067770004272, |
| "rewards/margins": 8.30478572845459, |
| "rewards/rejected": -9.238592147827148, |
| "step": 705 |
| }, |
| { |
| "epoch": 0.5668975208270601, |
| "grad_norm": 0.8919236660003662, |
| "learning_rate": 6.589274139724911e-06, |
| "logits/chosen": -6.933688640594482, |
| "logits/rejected": -6.49881649017334, |
| "logps/chosen": -253.52719116210938, |
| "logps/rejected": -327.0579528808594, |
| "loss": 0.0896, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.9493825435638428, |
| "rewards/margins": 8.325532913208008, |
| "rewards/rejected": -9.27491569519043, |
| "step": 706 |
| }, |
| { |
| "epoch": 0.567700491819733, |
| "grad_norm": 0.9643837809562683, |
| "learning_rate": 6.5482475754285535e-06, |
| "logits/chosen": -6.929364204406738, |
| "logits/rejected": -6.476649761199951, |
| "logps/chosen": -255.3946075439453, |
| "logps/rejected": -323.0415954589844, |
| "loss": 0.1218, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -1.2197555303573608, |
| "rewards/margins": 8.458791732788086, |
| "rewards/rejected": -9.678546905517578, |
| "step": 707 |
| }, |
| { |
| "epoch": 0.5685034628124059, |
| "grad_norm": 0.7672327756881714, |
| "learning_rate": 6.50731343814262e-06, |
| "logits/chosen": -6.914663791656494, |
| "logits/rejected": -6.260865211486816, |
| "logps/chosen": -225.12811279296875, |
| "logps/rejected": -306.9178161621094, |
| "loss": 0.0923, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.9198747873306274, |
| "rewards/margins": 8.62147045135498, |
| "rewards/rejected": -9.54134464263916, |
| "step": 708 |
| }, |
| { |
| "epoch": 0.5693064338050788, |
| "grad_norm": 0.7010583281517029, |
| "learning_rate": 6.466472175516284e-06, |
| "logits/chosen": -7.069661617279053, |
| "logits/rejected": -6.574976444244385, |
| "logps/chosen": -262.6434020996094, |
| "logps/rejected": -329.60894775390625, |
| "loss": 0.0755, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.49782341718673706, |
| "rewards/margins": 8.650425910949707, |
| "rewards/rejected": -9.148248672485352, |
| "step": 709 |
| }, |
| { |
| "epoch": 0.5701094047977516, |
| "grad_norm": 0.7210307717323303, |
| "learning_rate": 6.425724234183037e-06, |
| "logits/chosen": -7.015939712524414, |
| "logits/rejected": -6.467601776123047, |
| "logps/chosen": -237.75466918945312, |
| "logps/rejected": -299.20135498046875, |
| "loss": 0.0746, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6929614543914795, |
| "rewards/margins": 8.768132209777832, |
| "rewards/rejected": -9.461092948913574, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.5709123757904245, |
| "grad_norm": 0.7920691967010498, |
| "learning_rate": 6.3850700597558465e-06, |
| "logits/chosen": -6.888654708862305, |
| "logits/rejected": -6.366112232208252, |
| "logps/chosen": -289.72802734375, |
| "logps/rejected": -327.2459411621094, |
| "loss": 0.0871, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.4190760850906372, |
| "rewards/margins": 8.769124031066895, |
| "rewards/rejected": -9.188199996948242, |
| "step": 711 |
| }, |
| { |
| "epoch": 0.5717153467830974, |
| "grad_norm": 0.7520655393600464, |
| "learning_rate": 6.344510096822249e-06, |
| "logits/chosen": -6.844126224517822, |
| "logits/rejected": -6.330772399902344, |
| "logps/chosen": -234.70042419433594, |
| "logps/rejected": -316.845458984375, |
| "loss": 0.093, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.1951028108596802, |
| "rewards/margins": 8.21938705444336, |
| "rewards/rejected": -9.41448974609375, |
| "step": 712 |
| }, |
| { |
| "epoch": 0.5725183177757703, |
| "grad_norm": 0.6001507639884949, |
| "learning_rate": 6.304044788939499e-06, |
| "logits/chosen": -7.049295425415039, |
| "logits/rejected": -6.507409572601318, |
| "logps/chosen": -236.15628051757812, |
| "logps/rejected": -295.01934814453125, |
| "loss": 0.0721, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.18693700432777405, |
| "rewards/margins": 8.930510520935059, |
| "rewards/rejected": -9.117447853088379, |
| "step": 713 |
| }, |
| { |
| "epoch": 0.5733212887684432, |
| "grad_norm": 0.66024249792099, |
| "learning_rate": 6.263674578629729e-06, |
| "logits/chosen": -6.934654235839844, |
| "logits/rejected": -6.480746746063232, |
| "logps/chosen": -256.5248718261719, |
| "logps/rejected": -338.6959228515625, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -1.1841962337493896, |
| "rewards/margins": 8.385579109191895, |
| "rewards/rejected": -9.56977653503418, |
| "step": 714 |
| }, |
| { |
| "epoch": 0.5741242597611161, |
| "grad_norm": 0.9683951735496521, |
| "learning_rate": 6.223399907375077e-06, |
| "logits/chosen": -7.026305198669434, |
| "logits/rejected": -6.466622829437256, |
| "logps/chosen": -233.5004425048828, |
| "logps/rejected": -311.3004150390625, |
| "loss": 0.1345, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -1.0816665887832642, |
| "rewards/margins": 8.388959884643555, |
| "rewards/rejected": -9.470626831054688, |
| "step": 715 |
| }, |
| { |
| "epoch": 0.574927230753789, |
| "grad_norm": 0.9349070191383362, |
| "learning_rate": 6.1832212156129045e-06, |
| "logits/chosen": -6.866799831390381, |
| "logits/rejected": -6.3943328857421875, |
| "logps/chosen": -263.0761413574219, |
| "logps/rejected": -327.60162353515625, |
| "loss": 0.1059, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.0279327630996704, |
| "rewards/margins": 8.246272087097168, |
| "rewards/rejected": -9.27420425415039, |
| "step": 716 |
| }, |
| { |
| "epoch": 0.575730201746462, |
| "grad_norm": 0.9828877449035645, |
| "learning_rate": 6.143138942730943e-06, |
| "logits/chosen": -6.871181964874268, |
| "logits/rejected": -6.317739009857178, |
| "logps/chosen": -252.30177307128906, |
| "logps/rejected": -305.6430969238281, |
| "loss": 0.1165, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.827965497970581, |
| "rewards/margins": 8.286823272705078, |
| "rewards/rejected": -9.114788055419922, |
| "step": 717 |
| }, |
| { |
| "epoch": 0.5765331727391348, |
| "grad_norm": 0.7823782563209534, |
| "learning_rate": 6.10315352706251e-06, |
| "logits/chosen": -7.140516757965088, |
| "logits/rejected": -6.615158557891846, |
| "logps/chosen": -247.19140625, |
| "logps/rejected": -313.5113220214844, |
| "loss": 0.0954, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.48452693223953247, |
| "rewards/margins": 8.65560245513916, |
| "rewards/rejected": -9.140130043029785, |
| "step": 718 |
| }, |
| { |
| "epoch": 0.5773361437318076, |
| "grad_norm": 0.8714503645896912, |
| "learning_rate": 6.06326540588171e-06, |
| "logits/chosen": -6.858239650726318, |
| "logits/rejected": -6.3370041847229, |
| "logps/chosen": -226.41912841796875, |
| "logps/rejected": -318.80511474609375, |
| "loss": 0.1216, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -1.2360395193099976, |
| "rewards/margins": 8.385926246643066, |
| "rewards/rejected": -9.621966361999512, |
| "step": 719 |
| }, |
| { |
| "epoch": 0.5781391147244805, |
| "grad_norm": 0.9256161451339722, |
| "learning_rate": 6.023475015398635e-06, |
| "logits/chosen": -6.751380920410156, |
| "logits/rejected": -6.12971305847168, |
| "logps/chosen": -217.2934112548828, |
| "logps/rejected": -295.3868713378906, |
| "loss": 0.1015, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.1291813850402832, |
| "rewards/margins": 8.537799835205078, |
| "rewards/rejected": -9.66698169708252, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.5789420857171534, |
| "grad_norm": 0.8908589482307434, |
| "learning_rate": 5.983782790754624e-06, |
| "logits/chosen": -7.070487976074219, |
| "logits/rejected": -6.509310245513916, |
| "logps/chosen": -263.4241943359375, |
| "logps/rejected": -313.9613952636719, |
| "loss": 0.0982, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.3784397840499878, |
| "rewards/margins": 8.854307174682617, |
| "rewards/rejected": -9.232747077941895, |
| "step": 721 |
| }, |
| { |
| "epoch": 0.5797450567098263, |
| "grad_norm": 0.9128760099411011, |
| "learning_rate": 5.944189166017488e-06, |
| "logits/chosen": -6.992383003234863, |
| "logits/rejected": -6.453734397888184, |
| "logps/chosen": -279.55682373046875, |
| "logps/rejected": -326.1780090332031, |
| "loss": 0.087, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.39648380875587463, |
| "rewards/margins": 8.528565406799316, |
| "rewards/rejected": -8.925049781799316, |
| "step": 722 |
| }, |
| { |
| "epoch": 0.5805480277024992, |
| "grad_norm": 0.5307289361953735, |
| "learning_rate": 5.9046945741767686e-06, |
| "logits/chosen": -7.086591720581055, |
| "logits/rejected": -6.619198799133301, |
| "logps/chosen": -266.15008544921875, |
| "logps/rejected": -325.127197265625, |
| "loss": 0.073, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.7091962099075317, |
| "rewards/margins": 8.710220336914062, |
| "rewards/rejected": -9.419417381286621, |
| "step": 723 |
| }, |
| { |
| "epoch": 0.5813509986951721, |
| "grad_norm": 0.6765527725219727, |
| "learning_rate": 5.8652994471389835e-06, |
| "logits/chosen": -6.995295524597168, |
| "logits/rejected": -6.4750800132751465, |
| "logps/chosen": -226.17315673828125, |
| "logps/rejected": -286.8349609375, |
| "loss": 0.0807, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.7679774761199951, |
| "rewards/margins": 8.605167388916016, |
| "rewards/rejected": -9.373146057128906, |
| "step": 724 |
| }, |
| { |
| "epoch": 0.582153969687845, |
| "grad_norm": 0.8599225282669067, |
| "learning_rate": 5.826004215722937e-06, |
| "logits/chosen": -6.766485214233398, |
| "logits/rejected": -6.269761085510254, |
| "logps/chosen": -250.52175903320312, |
| "logps/rejected": -311.4064636230469, |
| "loss": 0.0942, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.9075266718864441, |
| "rewards/margins": 8.210982322692871, |
| "rewards/rejected": -9.118508338928223, |
| "step": 725 |
| }, |
| { |
| "epoch": 0.582956940680518, |
| "grad_norm": 0.579514741897583, |
| "learning_rate": 5.786809309654983e-06, |
| "logits/chosen": -7.152117729187012, |
| "logits/rejected": -6.64058256149292, |
| "logps/chosen": -221.21621704101562, |
| "logps/rejected": -286.08154296875, |
| "loss": 0.0638, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8867863416671753, |
| "rewards/margins": 8.713176727294922, |
| "rewards/rejected": -9.59996223449707, |
| "step": 726 |
| }, |
| { |
| "epoch": 0.5837599116731909, |
| "grad_norm": 0.7446565628051758, |
| "learning_rate": 5.747715157564335e-06, |
| "logits/chosen": -7.141786575317383, |
| "logits/rejected": -6.517609119415283, |
| "logps/chosen": -266.5047302246094, |
| "logps/rejected": -353.22564697265625, |
| "loss": 0.0577, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.8473352193832397, |
| "rewards/margins": 9.06091594696045, |
| "rewards/rejected": -9.908251762390137, |
| "step": 727 |
| }, |
| { |
| "epoch": 0.5845628826658636, |
| "grad_norm": 1.2561867237091064, |
| "learning_rate": 5.708722186978381e-06, |
| "logits/chosen": -7.106506824493408, |
| "logits/rejected": -6.46436071395874, |
| "logps/chosen": -248.5030975341797, |
| "logps/rejected": -302.48236083984375, |
| "loss": 0.1031, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.5923212766647339, |
| "rewards/margins": 8.972190856933594, |
| "rewards/rejected": -9.564512252807617, |
| "step": 728 |
| }, |
| { |
| "epoch": 0.5853658536585366, |
| "grad_norm": 0.6258772611618042, |
| "learning_rate": 5.669830824317992e-06, |
| "logits/chosen": -6.97711181640625, |
| "logits/rejected": -6.346778392791748, |
| "logps/chosen": -264.7965087890625, |
| "logps/rejected": -319.3843994140625, |
| "loss": 0.0797, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.8063178062438965, |
| "rewards/margins": 8.579811096191406, |
| "rewards/rejected": -9.386128425598145, |
| "step": 729 |
| }, |
| { |
| "epoch": 0.5861688246512095, |
| "grad_norm": 0.6666697263717651, |
| "learning_rate": 5.631041494892883e-06, |
| "logits/chosen": -6.953834056854248, |
| "logits/rejected": -6.477360725402832, |
| "logps/chosen": -260.06195068359375, |
| "logps/rejected": -315.3516845703125, |
| "loss": 0.0691, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.7335094809532166, |
| "rewards/margins": 8.95973014831543, |
| "rewards/rejected": -9.693239212036133, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.5869717956438824, |
| "grad_norm": 0.7699172496795654, |
| "learning_rate": 5.592354622896944e-06, |
| "logits/chosen": -7.010458946228027, |
| "logits/rejected": -6.4375762939453125, |
| "logps/chosen": -253.2987518310547, |
| "logps/rejected": -310.93609619140625, |
| "loss": 0.0803, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.841305136680603, |
| "rewards/margins": 8.616012573242188, |
| "rewards/rejected": -9.457318305969238, |
| "step": 731 |
| }, |
| { |
| "epoch": 0.5877747666365553, |
| "grad_norm": 0.5597600936889648, |
| "learning_rate": 5.553770631403614e-06, |
| "logits/chosen": -6.903814315795898, |
| "logits/rejected": -6.419100761413574, |
| "logps/chosen": -263.9942932128906, |
| "logps/rejected": -330.889404296875, |
| "loss": 0.0548, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -1.0543298721313477, |
| "rewards/margins": 8.587569236755371, |
| "rewards/rejected": -9.641898155212402, |
| "step": 732 |
| }, |
| { |
| "epoch": 0.5885777376292282, |
| "grad_norm": 0.9262829422950745, |
| "learning_rate": 5.515289942361242e-06, |
| "logits/chosen": -6.962847709655762, |
| "logits/rejected": -6.377350807189941, |
| "logps/chosen": -252.26040649414062, |
| "logps/rejected": -303.2207336425781, |
| "loss": 0.1098, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.9544874429702759, |
| "rewards/margins": 8.315235137939453, |
| "rewards/rejected": -9.269721984863281, |
| "step": 733 |
| }, |
| { |
| "epoch": 0.5893807086219011, |
| "grad_norm": 0.6422354578971863, |
| "learning_rate": 5.4769129765884716e-06, |
| "logits/chosen": -7.012918472290039, |
| "logits/rejected": -6.444794654846191, |
| "logps/chosen": -238.55714416503906, |
| "logps/rejected": -304.666015625, |
| "loss": 0.0813, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.0546015501022339, |
| "rewards/margins": 8.887173652648926, |
| "rewards/rejected": -9.94177532196045, |
| "step": 734 |
| }, |
| { |
| "epoch": 0.590183679614574, |
| "grad_norm": 0.9501427412033081, |
| "learning_rate": 5.438640153769654e-06, |
| "logits/chosen": -7.054049491882324, |
| "logits/rejected": -6.516881465911865, |
| "logps/chosen": -239.09054565429688, |
| "logps/rejected": -282.84759521484375, |
| "loss": 0.0872, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.6902347207069397, |
| "rewards/margins": 8.691915512084961, |
| "rewards/rejected": -9.382148742675781, |
| "step": 735 |
| }, |
| { |
| "epoch": 0.5909866506072469, |
| "grad_norm": 0.9050356149673462, |
| "learning_rate": 5.400471892450251e-06, |
| "logits/chosen": -7.233838081359863, |
| "logits/rejected": -6.644248962402344, |
| "logps/chosen": -233.98272705078125, |
| "logps/rejected": -308.3858947753906, |
| "loss": 0.0861, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.8918412923812866, |
| "rewards/margins": 8.69151496887207, |
| "rewards/rejected": -9.583356857299805, |
| "step": 736 |
| }, |
| { |
| "epoch": 0.5917896215999197, |
| "grad_norm": 1.0749232769012451, |
| "learning_rate": 5.362408610032257e-06, |
| "logits/chosen": -6.925085067749023, |
| "logits/rejected": -6.380291938781738, |
| "logps/chosen": -252.7101593017578, |
| "logps/rejected": -312.3319396972656, |
| "loss": 0.1007, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.8772615194320679, |
| "rewards/margins": 8.502469062805176, |
| "rewards/rejected": -9.379731178283691, |
| "step": 737 |
| }, |
| { |
| "epoch": 0.5925925925925926, |
| "grad_norm": 1.0904643535614014, |
| "learning_rate": 5.3244507227696186e-06, |
| "logits/chosen": -6.852221488952637, |
| "logits/rejected": -6.339931488037109, |
| "logps/chosen": -258.4163513183594, |
| "logps/rejected": -321.82171630859375, |
| "loss": 0.1258, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6000339388847351, |
| "rewards/margins": 8.408210754394531, |
| "rewards/rejected": -9.008245468139648, |
| "step": 738 |
| }, |
| { |
| "epoch": 0.5933955635852655, |
| "grad_norm": 0.9109723567962646, |
| "learning_rate": 5.286598645763718e-06, |
| "logits/chosen": -6.96547794342041, |
| "logits/rejected": -6.516641139984131, |
| "logps/chosen": -219.1004180908203, |
| "logps/rejected": -279.8680419921875, |
| "loss": 0.1019, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.8206350207328796, |
| "rewards/margins": 8.246661186218262, |
| "rewards/rejected": -9.067296028137207, |
| "step": 739 |
| }, |
| { |
| "epoch": 0.5941985345779384, |
| "grad_norm": 0.8094525933265686, |
| "learning_rate": 5.248852792958801e-06, |
| "logits/chosen": -6.992488861083984, |
| "logits/rejected": -6.4605889320373535, |
| "logps/chosen": -228.50038146972656, |
| "logps/rejected": -298.7779846191406, |
| "loss": 0.0688, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.3296911716461182, |
| "rewards/margins": 8.739316940307617, |
| "rewards/rejected": -10.069008827209473, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.5950015055706113, |
| "grad_norm": 0.6207690834999084, |
| "learning_rate": 5.21121357713747e-06, |
| "logits/chosen": -6.958118915557861, |
| "logits/rejected": -6.515707492828369, |
| "logps/chosen": -248.33628845214844, |
| "logps/rejected": -314.1471862792969, |
| "loss": 0.0711, |
| "rewards/accuracies": 0.984375, |
| "rewards/chosen": -0.8055965304374695, |
| "rewards/margins": 8.648548126220703, |
| "rewards/rejected": -9.454145431518555, |
| "step": 741 |
| }, |
| { |
| "epoch": 0.5958044765632842, |
| "grad_norm": 0.8237274289131165, |
| "learning_rate": 5.173681409916161e-06, |
| "logits/chosen": -6.915843963623047, |
| "logits/rejected": -6.380044937133789, |
| "logps/chosen": -265.1927795410156, |
| "logps/rejected": -317.678955078125, |
| "loss": 0.097, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.6660754680633545, |
| "rewards/margins": 8.505477905273438, |
| "rewards/rejected": -9.171553611755371, |
| "step": 742 |
| }, |
| { |
| "epoch": 0.5966074475559571, |
| "grad_norm": 1.039599895477295, |
| "learning_rate": 5.136256701740633e-06, |
| "logits/chosen": -7.055418014526367, |
| "logits/rejected": -6.453978061676025, |
| "logps/chosen": -255.86126708984375, |
| "logps/rejected": -329.904296875, |
| "loss": 0.0759, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8573939800262451, |
| "rewards/margins": 8.885194778442383, |
| "rewards/rejected": -9.742588996887207, |
| "step": 743 |
| }, |
| { |
| "epoch": 0.59741041854863, |
| "grad_norm": 0.7146782875061035, |
| "learning_rate": 5.098939861881503e-06, |
| "logits/chosen": -7.066512107849121, |
| "logits/rejected": -6.430700302124023, |
| "logps/chosen": -230.1925811767578, |
| "logps/rejected": -332.1337585449219, |
| "loss": 0.0797, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.643654465675354, |
| "rewards/margins": 8.873724937438965, |
| "rewards/rejected": -9.517379760742188, |
| "step": 744 |
| }, |
| { |
| "epoch": 0.5982133895413029, |
| "grad_norm": 0.6173692941665649, |
| "learning_rate": 5.061731298429755e-06, |
| "logits/chosen": -6.942100524902344, |
| "logits/rejected": -6.427311420440674, |
| "logps/chosen": -239.83836364746094, |
| "logps/rejected": -304.0270080566406, |
| "loss": 0.066, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -0.43390607833862305, |
| "rewards/margins": 9.049518585205078, |
| "rewards/rejected": -9.483424186706543, |
| "step": 745 |
| }, |
| { |
| "epoch": 0.5990163605339757, |
| "grad_norm": 0.5168651342391968, |
| "learning_rate": 5.024631418292275e-06, |
| "logits/chosen": -7.234683036804199, |
| "logits/rejected": -6.731881618499756, |
| "logps/chosen": -256.30377197265625, |
| "logps/rejected": -321.7982177734375, |
| "loss": 0.0553, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.33423835039138794, |
| "rewards/margins": 9.130096435546875, |
| "rewards/rejected": -9.464334487915039, |
| "step": 746 |
| }, |
| { |
| "epoch": 0.5998193315266486, |
| "grad_norm": 0.8141311407089233, |
| "learning_rate": 4.987640627187413e-06, |
| "logits/chosen": -7.075352668762207, |
| "logits/rejected": -6.5317816734313965, |
| "logps/chosen": -212.39854431152344, |
| "logps/rejected": -283.99005126953125, |
| "loss": 0.0657, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.0310208797454834, |
| "rewards/margins": 8.945591926574707, |
| "rewards/rejected": -9.97661304473877, |
| "step": 747 |
| }, |
| { |
| "epoch": 0.6006223025193215, |
| "grad_norm": 0.49748098850250244, |
| "learning_rate": 4.950759329640527e-06, |
| "logits/chosen": -7.0277557373046875, |
| "logits/rejected": -6.488724231719971, |
| "logps/chosen": -224.50326538085938, |
| "logps/rejected": -288.2071533203125, |
| "loss": 0.0557, |
| "rewards/accuracies": 0.984375, |
| "rewards/chosen": -0.956196665763855, |
| "rewards/margins": 9.213329315185547, |
| "rewards/rejected": -10.169526100158691, |
| "step": 748 |
| }, |
| { |
| "epoch": 0.6014252735119944, |
| "grad_norm": 0.7434368133544922, |
| "learning_rate": 4.91398792897958e-06, |
| "logits/chosen": -6.963872909545898, |
| "logits/rejected": -6.431257247924805, |
| "logps/chosen": -249.15399169921875, |
| "logps/rejected": -296.6255187988281, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8942330479621887, |
| "rewards/margins": 8.179784774780273, |
| "rewards/rejected": -9.074018478393555, |
| "step": 749 |
| }, |
| { |
| "epoch": 0.6022282445046673, |
| "grad_norm": 0.7794461846351624, |
| "learning_rate": 4.87732682733072e-06, |
| "logits/chosen": -6.934577465057373, |
| "logits/rejected": -6.422457218170166, |
| "logps/chosen": -248.00518798828125, |
| "logps/rejected": -311.87322998046875, |
| "loss": 0.0888, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.7450020909309387, |
| "rewards/margins": 8.401934623718262, |
| "rewards/rejected": -9.146937370300293, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.6022282445046673, |
| "eval_logits/chosen": -6.943487644195557, |
| "eval_logits/rejected": -6.41221284866333, |
| "eval_logps/chosen": -250.30482482910156, |
| "eval_logps/rejected": -315.5303955078125, |
| "eval_loss": 0.09030003100633621, |
| "eval_rewards/accuracies": 0.9566960334777832, |
| "eval_rewards/chosen": -0.850986897945404, |
| "eval_rewards/margins": 8.525099754333496, |
| "eval_rewards/rejected": -9.376087188720703, |
| "eval_runtime": 714.1665, |
| "eval_samples_per_second": 49.602, |
| "eval_steps_per_second": 1.55, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.6030312154973402, |
| "grad_norm": 0.7635409832000732, |
| "learning_rate": 4.840776425613887e-06, |
| "logits/chosen": -6.97970724105835, |
| "logits/rejected": -6.508722305297852, |
| "logps/chosen": -249.37457275390625, |
| "logps/rejected": -319.44061279296875, |
| "loss": 0.0793, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.6878411173820496, |
| "rewards/margins": 8.84587287902832, |
| "rewards/rejected": -9.533714294433594, |
| "step": 751 |
| }, |
| { |
| "epoch": 0.6038341864900131, |
| "grad_norm": 0.627616822719574, |
| "learning_rate": 4.804337123538406e-06, |
| "logits/chosen": -6.9830474853515625, |
| "logits/rejected": -6.488758087158203, |
| "logps/chosen": -250.8097686767578, |
| "logps/rejected": -316.352783203125, |
| "loss": 0.0627, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.8926767110824585, |
| "rewards/margins": 8.90864086151123, |
| "rewards/rejected": -9.80131721496582, |
| "step": 752 |
| }, |
| { |
| "epoch": 0.604637157482686, |
| "grad_norm": 0.880284309387207, |
| "learning_rate": 4.768009319598653e-06, |
| "logits/chosen": -6.74306583404541, |
| "logits/rejected": -6.347841262817383, |
| "logps/chosen": -279.8773498535156, |
| "logps/rejected": -347.3682861328125, |
| "loss": 0.1001, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.1027377843856812, |
| "rewards/margins": 8.181811332702637, |
| "rewards/rejected": -9.284549713134766, |
| "step": 753 |
| }, |
| { |
| "epoch": 0.6054401284753588, |
| "grad_norm": 1.3451687097549438, |
| "learning_rate": 4.731793411069669e-06, |
| "logits/chosen": -6.809340476989746, |
| "logits/rejected": -6.29241418838501, |
| "logps/chosen": -247.9064483642578, |
| "logps/rejected": -303.164306640625, |
| "loss": 0.1306, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.9983826279640198, |
| "rewards/margins": 8.68941593170166, |
| "rewards/rejected": -9.687797546386719, |
| "step": 754 |
| }, |
| { |
| "epoch": 0.6062430994680317, |
| "grad_norm": 0.9645203351974487, |
| "learning_rate": 4.6956897940028314e-06, |
| "logits/chosen": -6.952563762664795, |
| "logits/rejected": -6.308616638183594, |
| "logps/chosen": -251.18670654296875, |
| "logps/rejected": -328.74041748046875, |
| "loss": 0.1155, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.9498035907745361, |
| "rewards/margins": 8.50449275970459, |
| "rewards/rejected": -9.454297065734863, |
| "step": 755 |
| }, |
| { |
| "epoch": 0.6070460704607046, |
| "grad_norm": 0.8382568955421448, |
| "learning_rate": 4.659698863221513e-06, |
| "logits/chosen": -6.909200668334961, |
| "logits/rejected": -6.348932266235352, |
| "logps/chosen": -240.10342407226562, |
| "logps/rejected": -335.6614685058594, |
| "loss": 0.0995, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6785811185836792, |
| "rewards/margins": 8.998520851135254, |
| "rewards/rejected": -9.677101135253906, |
| "step": 756 |
| }, |
| { |
| "epoch": 0.6078490414533775, |
| "grad_norm": 0.7444134950637817, |
| "learning_rate": 4.623821012316761e-06, |
| "logits/chosen": -7.053999423980713, |
| "logits/rejected": -6.491118431091309, |
| "logps/chosen": -271.21917724609375, |
| "logps/rejected": -333.46136474609375, |
| "loss": 0.0836, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.7820271849632263, |
| "rewards/margins": 8.767594337463379, |
| "rewards/rejected": -9.549620628356934, |
| "step": 757 |
| }, |
| { |
| "epoch": 0.6086520124460504, |
| "grad_norm": 0.7404611110687256, |
| "learning_rate": 4.588056633643011e-06, |
| "logits/chosen": -6.9152302742004395, |
| "logits/rejected": -6.509066581726074, |
| "logps/chosen": -239.07421875, |
| "logps/rejected": -293.39300537109375, |
| "loss": 0.0708, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.9519606828689575, |
| "rewards/margins": 8.604280471801758, |
| "rewards/rejected": -9.556241035461426, |
| "step": 758 |
| }, |
| { |
| "epoch": 0.6094549834387233, |
| "grad_norm": 0.6715630292892456, |
| "learning_rate": 4.552406118313767e-06, |
| "logits/chosen": -7.097541809082031, |
| "logits/rejected": -6.534597396850586, |
| "logps/chosen": -246.92636108398438, |
| "logps/rejected": -314.8747863769531, |
| "loss": 0.0713, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.41669467091560364, |
| "rewards/margins": 8.944201469421387, |
| "rewards/rejected": -9.360896110534668, |
| "step": 759 |
| }, |
| { |
| "epoch": 0.6102579544313962, |
| "grad_norm": 0.8634263873100281, |
| "learning_rate": 4.516869856197363e-06, |
| "logits/chosen": -7.098026752471924, |
| "logits/rejected": -6.463377952575684, |
| "logps/chosen": -262.1308898925781, |
| "logps/rejected": -299.37982177734375, |
| "loss": 0.0821, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8098486065864563, |
| "rewards/margins": 8.458329200744629, |
| "rewards/rejected": -9.268177032470703, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.6110609254240691, |
| "grad_norm": 0.8660572171211243, |
| "learning_rate": 4.481448235912671e-06, |
| "logits/chosen": -6.988103866577148, |
| "logits/rejected": -6.492700099945068, |
| "logps/chosen": -243.95379638671875, |
| "logps/rejected": -303.11199951171875, |
| "loss": 0.0815, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.5675137042999268, |
| "rewards/margins": 8.431657791137695, |
| "rewards/rejected": -8.99917221069336, |
| "step": 761 |
| }, |
| { |
| "epoch": 0.611863896416742, |
| "grad_norm": 0.7459592223167419, |
| "learning_rate": 4.446141644824846e-06, |
| "logits/chosen": -7.135545253753662, |
| "logits/rejected": -6.6236419677734375, |
| "logps/chosen": -255.498291015625, |
| "logps/rejected": -319.11376953125, |
| "loss": 0.0887, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.781557559967041, |
| "rewards/margins": 9.277936935424805, |
| "rewards/rejected": -10.059494018554688, |
| "step": 762 |
| }, |
| { |
| "epoch": 0.6126668674094148, |
| "grad_norm": 0.7405502200126648, |
| "learning_rate": 4.410950469041121e-06, |
| "logits/chosen": -7.053709506988525, |
| "logits/rejected": -6.523200035095215, |
| "logps/chosen": -251.5032958984375, |
| "logps/rejected": -315.4729919433594, |
| "loss": 0.067, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -1.045676350593567, |
| "rewards/margins": 8.44194507598877, |
| "rewards/rejected": -9.487621307373047, |
| "step": 763 |
| }, |
| { |
| "epoch": 0.6134698384020877, |
| "grad_norm": 0.6015735268592834, |
| "learning_rate": 4.3758750934065395e-06, |
| "logits/chosen": -6.955145359039307, |
| "logits/rejected": -6.436923027038574, |
| "logps/chosen": -272.4416809082031, |
| "logps/rejected": -355.66607666015625, |
| "loss": 0.0802, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.007872223854065, |
| "rewards/margins": 8.181082725524902, |
| "rewards/rejected": -9.18895435333252, |
| "step": 764 |
| }, |
| { |
| "epoch": 0.6142728093947606, |
| "grad_norm": 0.6597633361816406, |
| "learning_rate": 4.340915901499813e-06, |
| "logits/chosen": -6.897956848144531, |
| "logits/rejected": -6.440827369689941, |
| "logps/chosen": -243.31478881835938, |
| "logps/rejected": -297.08758544921875, |
| "loss": 0.0801, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.6680831909179688, |
| "rewards/margins": 8.24179744720459, |
| "rewards/rejected": -8.909880638122559, |
| "step": 765 |
| }, |
| { |
| "epoch": 0.6150757803874335, |
| "grad_norm": 0.7382259368896484, |
| "learning_rate": 4.306073275629045e-06, |
| "logits/chosen": -6.988953113555908, |
| "logits/rejected": -6.456672668457031, |
| "logps/chosen": -233.7950439453125, |
| "logps/rejected": -297.27337646484375, |
| "loss": 0.0797, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.8200283646583557, |
| "rewards/margins": 8.407998085021973, |
| "rewards/rejected": -9.228026390075684, |
| "step": 766 |
| }, |
| { |
| "epoch": 0.6158787513801064, |
| "grad_norm": 0.6988097429275513, |
| "learning_rate": 4.27134759682762e-06, |
| "logits/chosen": -7.0872979164123535, |
| "logits/rejected": -6.567286491394043, |
| "logps/chosen": -256.7961120605469, |
| "logps/rejected": -333.62469482421875, |
| "loss": 0.0777, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5543537139892578, |
| "rewards/margins": 8.481685638427734, |
| "rewards/rejected": -9.036039352416992, |
| "step": 767 |
| }, |
| { |
| "epoch": 0.6166817223727793, |
| "grad_norm": 0.7654405236244202, |
| "learning_rate": 4.236739244849991e-06, |
| "logits/chosen": -6.977169990539551, |
| "logits/rejected": -6.5438151359558105, |
| "logps/chosen": -238.3735809326172, |
| "logps/rejected": -304.20556640625, |
| "loss": 0.0937, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.0596084594726562, |
| "rewards/margins": 8.469924926757812, |
| "rewards/rejected": -9.529533386230469, |
| "step": 768 |
| }, |
| { |
| "epoch": 0.6174846933654522, |
| "grad_norm": 0.8412666320800781, |
| "learning_rate": 4.202248598167549e-06, |
| "logits/chosen": -7.121852874755859, |
| "logits/rejected": -6.506812572479248, |
| "logps/chosen": -241.46253967285156, |
| "logps/rejected": -322.79754638671875, |
| "loss": 0.1043, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.9038738012313843, |
| "rewards/margins": 8.622501373291016, |
| "rewards/rejected": -9.526375770568848, |
| "step": 769 |
| }, |
| { |
| "epoch": 0.6182876643581251, |
| "grad_norm": 1.0475529432296753, |
| "learning_rate": 4.167876033964494e-06, |
| "logits/chosen": -6.779577732086182, |
| "logits/rejected": -6.188941478729248, |
| "logps/chosen": -260.16778564453125, |
| "logps/rejected": -321.5066223144531, |
| "loss": 0.1118, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -0.9238437414169312, |
| "rewards/margins": 7.993536949157715, |
| "rewards/rejected": -8.917380332946777, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.619090635350798, |
| "grad_norm": 0.8236129283905029, |
| "learning_rate": 4.133621928133666e-06, |
| "logits/chosen": -6.825661659240723, |
| "logits/rejected": -6.335054874420166, |
| "logps/chosen": -256.1632385253906, |
| "logps/rejected": -327.1282653808594, |
| "loss": 0.1048, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.8888863325119019, |
| "rewards/margins": 8.352470397949219, |
| "rewards/rejected": -9.24135684967041, |
| "step": 771 |
| }, |
| { |
| "epoch": 0.6198936063434708, |
| "grad_norm": 0.5819370150566101, |
| "learning_rate": 4.09948665527249e-06, |
| "logits/chosen": -6.949581623077393, |
| "logits/rejected": -6.473232269287109, |
| "logps/chosen": -239.08428955078125, |
| "logps/rejected": -297.1288757324219, |
| "loss": 0.0657, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -0.985008955001831, |
| "rewards/margins": 8.5289945602417, |
| "rewards/rejected": -9.51400375366211, |
| "step": 772 |
| }, |
| { |
| "epoch": 0.6206965773361437, |
| "grad_norm": 1.2287933826446533, |
| "learning_rate": 4.06547058867883e-06, |
| "logits/chosen": -6.936915397644043, |
| "logits/rejected": -6.527620315551758, |
| "logps/chosen": -248.46786499023438, |
| "logps/rejected": -308.7354431152344, |
| "loss": 0.1492, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -1.2233452796936035, |
| "rewards/margins": 8.312910079956055, |
| "rewards/rejected": -9.536255836486816, |
| "step": 773 |
| }, |
| { |
| "epoch": 0.6214995483288166, |
| "grad_norm": 0.9096470475196838, |
| "learning_rate": 4.031574100346946e-06, |
| "logits/chosen": -7.026955604553223, |
| "logits/rejected": -6.434424877166748, |
| "logps/chosen": -231.12586975097656, |
| "logps/rejected": -309.25433349609375, |
| "loss": 0.097, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.7207227945327759, |
| "rewards/margins": 8.55255126953125, |
| "rewards/rejected": -9.273275375366211, |
| "step": 774 |
| }, |
| { |
| "epoch": 0.6223025193214895, |
| "grad_norm": 1.2915889024734497, |
| "learning_rate": 3.997797560963404e-06, |
| "logits/chosen": -7.092460632324219, |
| "logits/rejected": -6.489498615264893, |
| "logps/chosen": -259.093994140625, |
| "logps/rejected": -323.6791687011719, |
| "loss": 0.0974, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.43507522344589233, |
| "rewards/margins": 8.86640453338623, |
| "rewards/rejected": -9.301480293273926, |
| "step": 775 |
| }, |
| { |
| "epoch": 0.6231054903141624, |
| "grad_norm": 0.7012649774551392, |
| "learning_rate": 3.964141339903026e-06, |
| "logits/chosen": -7.07417631149292, |
| "logits/rejected": -6.490232467651367, |
| "logps/chosen": -280.50164794921875, |
| "logps/rejected": -315.0455627441406, |
| "loss": 0.069, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.6904686093330383, |
| "rewards/margins": 8.775313377380371, |
| "rewards/rejected": -9.465782165527344, |
| "step": 776 |
| }, |
| { |
| "epoch": 0.6239084613068353, |
| "grad_norm": 0.5985205173492432, |
| "learning_rate": 3.930605805224858e-06, |
| "logits/chosen": -6.813873291015625, |
| "logits/rejected": -6.393153190612793, |
| "logps/chosen": -276.38360595703125, |
| "logps/rejected": -312.89239501953125, |
| "loss": 0.0905, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.732748806476593, |
| "rewards/margins": 8.403030395507812, |
| "rewards/rejected": -9.13577938079834, |
| "step": 777 |
| }, |
| { |
| "epoch": 0.6247114322995082, |
| "grad_norm": 0.8872177600860596, |
| "learning_rate": 3.897191323668126e-06, |
| "logits/chosen": -6.856238842010498, |
| "logits/rejected": -6.264463424682617, |
| "logps/chosen": -247.3445587158203, |
| "logps/rejected": -326.99884033203125, |
| "loss": 0.0874, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.6348282098770142, |
| "rewards/margins": 8.921683311462402, |
| "rewards/rejected": -9.556510925292969, |
| "step": 778 |
| }, |
| { |
| "epoch": 0.6255144032921811, |
| "grad_norm": 0.7021169662475586, |
| "learning_rate": 3.8638982606482525e-06, |
| "logits/chosen": -7.118107795715332, |
| "logits/rejected": -6.484702110290527, |
| "logps/chosen": -238.2637939453125, |
| "logps/rejected": -327.7728271484375, |
| "loss": 0.063, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6267252564430237, |
| "rewards/margins": 9.239341735839844, |
| "rewards/rejected": -9.866066932678223, |
| "step": 779 |
| }, |
| { |
| "epoch": 0.626317374284854, |
| "grad_norm": 0.9930330514907837, |
| "learning_rate": 3.830726980252838e-06, |
| "logits/chosen": -6.824446678161621, |
| "logits/rejected": -6.306407451629639, |
| "logps/chosen": -251.5108642578125, |
| "logps/rejected": -313.3147888183594, |
| "loss": 0.0904, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.1878043413162231, |
| "rewards/margins": 8.295165061950684, |
| "rewards/rejected": -9.482969284057617, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.6271203452775268, |
| "grad_norm": 0.8375623822212219, |
| "learning_rate": 3.7976778452376966e-06, |
| "logits/chosen": -6.837080478668213, |
| "logits/rejected": -6.424347400665283, |
| "logps/chosen": -255.79412841796875, |
| "logps/rejected": -316.51202392578125, |
| "loss": 0.1083, |
| "rewards/accuracies": 0.93359375, |
| "rewards/chosen": -0.9681498408317566, |
| "rewards/margins": 8.349898338317871, |
| "rewards/rejected": -9.318046569824219, |
| "step": 781 |
| }, |
| { |
| "epoch": 0.6279233162701997, |
| "grad_norm": 0.5061840415000916, |
| "learning_rate": 3.7647512170228643e-06, |
| "logits/chosen": -6.891932010650635, |
| "logits/rejected": -6.284075736999512, |
| "logps/chosen": -248.99063110351562, |
| "logps/rejected": -314.2254943847656, |
| "loss": 0.0695, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.7286368012428284, |
| "rewards/margins": 8.916427612304688, |
| "rewards/rejected": -9.645063400268555, |
| "step": 782 |
| }, |
| { |
| "epoch": 0.6287262872628726, |
| "grad_norm": 1.0160025358200073, |
| "learning_rate": 3.731947455688677e-06, |
| "logits/chosen": -6.856513977050781, |
| "logits/rejected": -6.301755428314209, |
| "logps/chosen": -266.02508544921875, |
| "logps/rejected": -362.5861511230469, |
| "loss": 0.0976, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.157212257385254, |
| "rewards/margins": 8.525556564331055, |
| "rewards/rejected": -9.682769775390625, |
| "step": 783 |
| }, |
| { |
| "epoch": 0.6295292582555455, |
| "grad_norm": 0.6567729711532593, |
| "learning_rate": 3.699266919971811e-06, |
| "logits/chosen": -6.929780960083008, |
| "logits/rejected": -6.464938163757324, |
| "logps/chosen": -239.8303985595703, |
| "logps/rejected": -305.5951232910156, |
| "loss": 0.0817, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.0645124912261963, |
| "rewards/margins": 8.732171058654785, |
| "rewards/rejected": -9.796684265136719, |
| "step": 784 |
| }, |
| { |
| "epoch": 0.6303322292482184, |
| "grad_norm": 0.7252476215362549, |
| "learning_rate": 3.6667099672613734e-06, |
| "logits/chosen": -6.845086574554443, |
| "logits/rejected": -6.412286758422852, |
| "logps/chosen": -255.58392333984375, |
| "logps/rejected": -313.48345947265625, |
| "loss": 0.0955, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.0981053113937378, |
| "rewards/margins": 8.352322578430176, |
| "rewards/rejected": -9.45042896270752, |
| "step": 785 |
| }, |
| { |
| "epoch": 0.6311352002408913, |
| "grad_norm": 1.1652352809906006, |
| "learning_rate": 3.634276953594982e-06, |
| "logits/chosen": -6.921412467956543, |
| "logits/rejected": -6.341633319854736, |
| "logps/chosen": -267.0845947265625, |
| "logps/rejected": -307.6617126464844, |
| "loss": 0.0996, |
| "rewards/accuracies": 0.97265625, |
| "rewards/chosen": -0.9424811005592346, |
| "rewards/margins": 8.79172134399414, |
| "rewards/rejected": -9.73420238494873, |
| "step": 786 |
| }, |
| { |
| "epoch": 0.6319381712335642, |
| "grad_norm": 0.6850370764732361, |
| "learning_rate": 3.6019682336548736e-06, |
| "logits/chosen": -6.758754253387451, |
| "logits/rejected": -6.279141426086426, |
| "logps/chosen": -257.7626647949219, |
| "logps/rejected": -295.8297424316406, |
| "loss": 0.0723, |
| "rewards/accuracies": 0.98046875, |
| "rewards/chosen": -1.0491838455200195, |
| "rewards/margins": 8.353296279907227, |
| "rewards/rejected": -9.402480125427246, |
| "step": 787 |
| }, |
| { |
| "epoch": 0.6327411422262371, |
| "grad_norm": 0.7349598407745361, |
| "learning_rate": 3.569784160764036e-06, |
| "logits/chosen": -7.088435173034668, |
| "logits/rejected": -6.466038703918457, |
| "logps/chosen": -274.57568359375, |
| "logps/rejected": -338.9457092285156, |
| "loss": 0.0724, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.7119229435920715, |
| "rewards/margins": 8.840612411499023, |
| "rewards/rejected": -9.552535057067871, |
| "step": 788 |
| }, |
| { |
| "epoch": 0.63354411321891, |
| "grad_norm": 0.6719980239868164, |
| "learning_rate": 3.537725086882333e-06, |
| "logits/chosen": -6.920090675354004, |
| "logits/rejected": -6.361482620239258, |
| "logps/chosen": -274.8949279785156, |
| "logps/rejected": -322.67327880859375, |
| "loss": 0.0866, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -1.1468017101287842, |
| "rewards/margins": 8.384204864501953, |
| "rewards/rejected": -9.531005859375, |
| "step": 789 |
| }, |
| { |
| "epoch": 0.6343470842115828, |
| "grad_norm": 0.6633608341217041, |
| "learning_rate": 3.5057913626026616e-06, |
| "logits/chosen": -7.011590003967285, |
| "logits/rejected": -6.4117889404296875, |
| "logps/chosen": -252.76693725585938, |
| "logps/rejected": -339.00067138671875, |
| "loss": 0.0689, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.1228599548339844, |
| "rewards/margins": 8.920897483825684, |
| "rewards/rejected": -10.043756484985352, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.6351500552042557, |
| "grad_norm": 0.732852041721344, |
| "learning_rate": 3.473983337147118e-06, |
| "logits/chosen": -6.926459312438965, |
| "logits/rejected": -6.364645957946777, |
| "logps/chosen": -226.8125457763672, |
| "logps/rejected": -300.0146179199219, |
| "loss": 0.0863, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.9864679574966431, |
| "rewards/margins": 8.449716567993164, |
| "rewards/rejected": -9.43618392944336, |
| "step": 791 |
| }, |
| { |
| "epoch": 0.6359530261969286, |
| "grad_norm": 0.6956843137741089, |
| "learning_rate": 3.442301358363163e-06, |
| "logits/chosen": -6.959061622619629, |
| "logits/rejected": -6.499250411987305, |
| "logps/chosen": -228.44732666015625, |
| "logps/rejected": -291.5260314941406, |
| "loss": 0.0785, |
| "rewards/accuracies": 0.95703125, |
| "rewards/chosen": -0.8826844692230225, |
| "rewards/margins": 8.805318832397461, |
| "rewards/rejected": -9.688003540039062, |
| "step": 792 |
| }, |
| { |
| "epoch": 0.6367559971896015, |
| "grad_norm": 0.6329805254936218, |
| "learning_rate": 3.4107457727198465e-06, |
| "logits/chosen": -6.963720321655273, |
| "logits/rejected": -6.323712348937988, |
| "logps/chosen": -252.61546325683594, |
| "logps/rejected": -328.0885009765625, |
| "loss": 0.0705, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.290144443511963, |
| "rewards/margins": 9.06776237487793, |
| "rewards/rejected": -10.35790729522705, |
| "step": 793 |
| }, |
| { |
| "epoch": 0.6375589681822744, |
| "grad_norm": 0.9175943732261658, |
| "learning_rate": 3.379316925303998e-06, |
| "logits/chosen": -6.835019111633301, |
| "logits/rejected": -6.274209976196289, |
| "logps/chosen": -242.7401580810547, |
| "logps/rejected": -316.0100402832031, |
| "loss": 0.132, |
| "rewards/accuracies": 0.94140625, |
| "rewards/chosen": -0.8641645312309265, |
| "rewards/margins": 8.640796661376953, |
| "rewards/rejected": -9.504961967468262, |
| "step": 794 |
| }, |
| { |
| "epoch": 0.6383619391749473, |
| "grad_norm": 1.0480033159255981, |
| "learning_rate": 3.348015159816458e-06, |
| "logits/chosen": -6.976797580718994, |
| "logits/rejected": -6.433487415313721, |
| "logps/chosen": -281.10308837890625, |
| "logps/rejected": -341.7545166015625, |
| "loss": 0.1052, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -1.5568076372146606, |
| "rewards/margins": 8.337911605834961, |
| "rewards/rejected": -9.894719123840332, |
| "step": 795 |
| }, |
| { |
| "epoch": 0.6391649101676202, |
| "grad_norm": 0.6464856863021851, |
| "learning_rate": 3.3168408185683153e-06, |
| "logits/chosen": -6.778012275695801, |
| "logits/rejected": -6.376547813415527, |
| "logps/chosen": -242.21365356445312, |
| "logps/rejected": -304.88995361328125, |
| "loss": 0.0836, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -1.1265236139297485, |
| "rewards/margins": 8.579636573791504, |
| "rewards/rejected": -9.706159591674805, |
| "step": 796 |
| }, |
| { |
| "epoch": 0.6399678811602931, |
| "grad_norm": 0.5502794981002808, |
| "learning_rate": 3.285794242477173e-06, |
| "logits/chosen": -7.055201530456543, |
| "logits/rejected": -6.475158214569092, |
| "logps/chosen": -249.39747619628906, |
| "logps/rejected": -325.4633483886719, |
| "loss": 0.0495, |
| "rewards/accuracies": 0.99609375, |
| "rewards/chosen": -1.3749713897705078, |
| "rewards/margins": 9.009495735168457, |
| "rewards/rejected": -10.384468078613281, |
| "step": 797 |
| }, |
| { |
| "epoch": 0.6407708521529659, |
| "grad_norm": 0.6813830733299255, |
| "learning_rate": 3.254875771063412e-06, |
| "logits/chosen": -6.882882595062256, |
| "logits/rejected": -6.356379508972168, |
| "logps/chosen": -268.24383544921875, |
| "logps/rejected": -345.9039611816406, |
| "loss": 0.0929, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.9009088277816772, |
| "rewards/margins": 8.757538795471191, |
| "rewards/rejected": -9.658447265625, |
| "step": 798 |
| }, |
| { |
| "epoch": 0.6415738231456388, |
| "grad_norm": 1.1329574584960938, |
| "learning_rate": 3.224085742446484e-06, |
| "logits/chosen": -6.823460102081299, |
| "logits/rejected": -6.317877769470215, |
| "logps/chosen": -242.8865966796875, |
| "logps/rejected": -304.04315185546875, |
| "loss": 0.0842, |
| "rewards/accuracies": 0.96484375, |
| "rewards/chosen": -1.0773792266845703, |
| "rewards/margins": 8.531217575073242, |
| "rewards/rejected": -9.608596801757812, |
| "step": 799 |
| }, |
| { |
| "epoch": 0.6423767941383117, |
| "grad_norm": 1.3927313089370728, |
| "learning_rate": 3.193424493341213e-06, |
| "logits/chosen": -7.01397180557251, |
| "logits/rejected": -6.351844787597656, |
| "logps/chosen": -219.67747497558594, |
| "logps/rejected": -305.4691162109375, |
| "loss": 0.1175, |
| "rewards/accuracies": 0.94921875, |
| "rewards/chosen": -1.3909140825271606, |
| "rewards/margins": 8.750292778015137, |
| "rewards/rejected": -10.141205787658691, |
| "step": 800 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 1000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
