diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,13389 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.999244142101285, + "eval_steps": 100, + "global_step": 661, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "beta_dpo/beta": 0.1015101969242096, + "beta_dpo/beta_margin_grad_mean": -0.5000330209732056, + "beta_dpo/beta_margin_grad_std": 0.006504404824227095, + "beta_dpo/beta_margin_mean": -0.0001328021171502769, + "beta_dpo/beta_margin_std": 0.02602243982255459, + "beta_dpo/beta_used": 0.1015101969242096, + "beta_dpo/beta_used_raw": 0.1015101969242096, + "beta_dpo/gap_mean": 0.00012409687042236328, + "beta_dpo/gap_std": 0.03724822774529457, + "beta_dpo/loss_margin_mean": -0.0013527870178222656, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.0015117157974300832, + "grad_norm": 31.01732063293457, + "learning_rate": 0.0, + "logits/chosen": 0.13337239623069763, + "logits/rejected": 0.12492948770523071, + "loss": 1.3838, + "step": 1 + }, + { + "beta_dpo/beta": 0.1012001484632492, + "beta_dpo/beta_margin_grad_mean": -0.49902671575546265, + "beta_dpo/beta_margin_grad_std": 0.007323236670345068, + "beta_dpo/beta_margin_mean": 0.0038939444348216057, + "beta_dpo/beta_margin_std": 0.029300615191459656, + "beta_dpo/beta_used": 0.1012001484632492, + "beta_dpo/beta_used_raw": 0.1012001484632492, + "beta_dpo/gap_mean": 0.0026843957602977753, + "beta_dpo/gap_std": 0.08112463355064392, + "beta_dpo/loss_margin_mean": 0.03744968771934509, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.0030234315948601664, + "grad_norm": 31.891300201416016, + "learning_rate": 7.462686567164179e-09, + "logits/chosen": 0.09414851665496826, + "logits/rejected": 0.07363267242908478, + "loss": 1.384, + "step": 2 + }, + { + "beta_dpo/beta": 0.10067637264728546, + "beta_dpo/beta_margin_grad_mean": -0.500410258769989, + "beta_dpo/beta_margin_grad_std": 0.009178511798381805, + "beta_dpo/beta_margin_mean": -0.0016434545395895839, + "beta_dpo/beta_margin_std": 0.03673430159687996, + "beta_dpo/beta_used": 0.10067637264728546, + "beta_dpo/beta_used_raw": 0.10067637264728546, + "beta_dpo/gap_mean": 0.00046056427527219057, + "beta_dpo/gap_std": 0.1343999207019806, + "beta_dpo/loss_margin_mean": -0.016742348670959473, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.0045351473922902496, + "grad_norm": 32.93205261230469, + "learning_rate": 1.4925373134328357e-08, + "logits/chosen": 0.09402679651975632, + "logits/rejected": 0.056407660245895386, + "loss": 1.3852, + "step": 3 + }, + { + "beta_dpo/beta": 0.10021987557411194, + "beta_dpo/beta_margin_grad_mean": -0.5003681182861328, + "beta_dpo/beta_margin_grad_std": 0.006939771119505167, + "beta_dpo/beta_margin_mean": -0.0014724871143698692, + "beta_dpo/beta_margin_std": 0.027768485248088837, + "beta_dpo/beta_used": 0.10021987557411194, + "beta_dpo/beta_used_raw": 0.10021987557411194, + "beta_dpo/gap_mean": -0.0026244998443871737, + "beta_dpo/gap_std": 0.1629766970872879, + "beta_dpo/loss_margin_mean": -0.01646369695663452, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.006046863189720333, + "grad_norm": 38.14013671875, + "learning_rate": 2.2388059701492534e-08, + "logits/chosen": 0.10362987220287323, + "logits/rejected": 0.0876229926943779, + "loss": 1.3862, + "step": 4 + }, + { + "beta_dpo/beta": 0.0995863676071167, + "beta_dpo/beta_margin_grad_mean": -0.5007702112197876, + "beta_dpo/beta_margin_grad_std": 0.007658351678401232, + "beta_dpo/beta_margin_mean": -0.003081433940678835, + "beta_dpo/beta_margin_std": 0.030642056837677956, + "beta_dpo/beta_used": 0.0995863676071167, + "beta_dpo/beta_used_raw": 0.0995863676071167, + "beta_dpo/gap_mean": -0.004291285760700703, + "beta_dpo/gap_std": 0.18804097175598145, + "beta_dpo/loss_margin_mean": -0.03093475103378296, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.007558578987150416, + "grad_norm": 31.054513931274414, + "learning_rate": 2.9850746268656714e-08, + "logits/chosen": 0.04905558377504349, + "logits/rejected": 0.011664441786706448, + "loss": 1.3875, + "step": 5 + }, + { + "beta_dpo/beta": 0.1000620573759079, + "beta_dpo/beta_margin_grad_mean": -0.5015894174575806, + "beta_dpo/beta_margin_grad_std": 0.007443729788064957, + "beta_dpo/beta_margin_mean": -0.006359330844134092, + "beta_dpo/beta_margin_std": 0.029781756922602654, + "beta_dpo/beta_used": 0.1000620573759079, + "beta_dpo/beta_used_raw": 0.1000620573759079, + "beta_dpo/gap_mean": -0.012156388722360134, + "beta_dpo/gap_std": 0.21093741059303284, + "beta_dpo/loss_margin_mean": -0.06457433104515076, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.009070294784580499, + "grad_norm": 32.63312911987305, + "learning_rate": 3.731343283582089e-08, + "logits/chosen": 0.12783631682395935, + "logits/rejected": 0.08931756764650345, + "loss": 1.3874, + "step": 6 + }, + { + "beta_dpo/beta": 0.10030151158571243, + "beta_dpo/beta_margin_grad_mean": -0.500143826007843, + "beta_dpo/beta_margin_grad_std": 0.005725794937461615, + "beta_dpo/beta_margin_mean": -0.0005755546153523028, + "beta_dpo/beta_margin_std": 0.02290569618344307, + "beta_dpo/beta_used": 0.10030151158571243, + "beta_dpo/beta_used_raw": 0.10030151158571243, + "beta_dpo/gap_mean": -0.013967369683086872, + "beta_dpo/gap_std": 0.21781358122825623, + "beta_dpo/loss_margin_mean": -0.005991309881210327, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.010582010582010581, + "grad_norm": 29.61240005493164, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": 0.007831894792616367, + "logits/rejected": -0.035432279109954834, + "loss": 1.3873, + "step": 7 + }, + { + "beta_dpo/beta": 0.09883703291416168, + "beta_dpo/beta_margin_grad_mean": -0.4992273151874542, + "beta_dpo/beta_margin_grad_std": 0.00927159283310175, + "beta_dpo/beta_margin_mean": 0.003093534614890814, + "beta_dpo/beta_margin_std": 0.03710191324353218, + "beta_dpo/beta_used": 0.09883703291416168, + "beta_dpo/beta_used_raw": 0.09883703291416168, + "beta_dpo/gap_mean": -0.009043242782354355, + "beta_dpo/gap_std": 0.24341917037963867, + "beta_dpo/loss_margin_mean": 0.031346142292022705, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.012093726379440665, + "grad_norm": 30.12750244140625, + "learning_rate": 5.223880597014925e-08, + "logits/chosen": 0.08428698778152466, + "logits/rejected": 0.061857692897319794, + "loss": 1.3892, + "step": 8 + }, + { + "beta_dpo/beta": 0.10056120157241821, + "beta_dpo/beta_margin_grad_mean": -0.49964460730552673, + "beta_dpo/beta_margin_grad_std": 0.00834520161151886, + "beta_dpo/beta_margin_mean": 0.0014211406232789159, + "beta_dpo/beta_margin_std": 0.033395376056432724, + "beta_dpo/beta_used": 0.10056120157241821, + "beta_dpo/beta_used_raw": 0.10056120157241821, + "beta_dpo/gap_mean": -0.0020596692338585854, + "beta_dpo/gap_std": 0.26306188106536865, + "beta_dpo/loss_margin_mean": 0.01439550518989563, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.013605442176870748, + "grad_norm": 34.624351501464844, + "learning_rate": 5.970149253731343e-08, + "logits/chosen": 0.15184298157691956, + "logits/rejected": 0.09390458464622498, + "loss": 1.3857, + "step": 9 + }, + { + "beta_dpo/beta": 0.10067851096391678, + "beta_dpo/beta_margin_grad_mean": -0.5004361867904663, + "beta_dpo/beta_margin_grad_std": 0.009121380746364594, + "beta_dpo/beta_margin_mean": -0.0017426289850845933, + "beta_dpo/beta_margin_std": 0.036507148295640945, + "beta_dpo/beta_used": 0.10067851096391678, + "beta_dpo/beta_used_raw": 0.10067851096391678, + "beta_dpo/gap_mean": -0.00195028493180871, + "beta_dpo/gap_std": 0.2801288366317749, + "beta_dpo/loss_margin_mean": -0.01787543296813965, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.015117157974300832, + "grad_norm": 30.904678344726562, + "learning_rate": 6.71641791044776e-08, + "logits/chosen": 0.1457955241203308, + "logits/rejected": 0.1140444278717041, + "loss": 1.3855, + "step": 10 + }, + { + "beta_dpo/beta": 0.09911488741636276, + "beta_dpo/beta_margin_grad_mean": -0.4999541640281677, + "beta_dpo/beta_margin_grad_std": 0.008431542664766312, + "beta_dpo/beta_margin_mean": 0.00018064059258904308, + "beta_dpo/beta_margin_std": 0.03374219313263893, + "beta_dpo/beta_used": 0.09911488741636276, + "beta_dpo/beta_used_raw": 0.09911488741636276, + "beta_dpo/gap_mean": -0.0015360511606559157, + "beta_dpo/gap_std": 0.28815028071403503, + "beta_dpo/loss_margin_mean": 0.0008325278759002686, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.016628873771730914, + "grad_norm": 31.81167221069336, + "learning_rate": 7.462686567164178e-08, + "logits/chosen": 0.09544117748737335, + "logits/rejected": 0.0887732282280922, + "loss": 1.388, + "step": 11 + }, + { + "beta_dpo/beta": 0.10047206282615662, + "beta_dpo/beta_margin_grad_mean": -0.5000085830688477, + "beta_dpo/beta_margin_grad_std": 0.006006320007145405, + "beta_dpo/beta_margin_mean": -3.400599962333217e-05, + "beta_dpo/beta_margin_std": 0.02402997761964798, + "beta_dpo/beta_used": 0.10047206282615662, + "beta_dpo/beta_used_raw": 0.10047206282615662, + "beta_dpo/gap_mean": -0.005030278116464615, + "beta_dpo/gap_std": 0.2830773591995239, + "beta_dpo/loss_margin_mean": -0.00034230947494506836, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.018140589569160998, + "grad_norm": 33.425296783447266, + "learning_rate": 8.208955223880596e-08, + "logits/chosen": 0.03307221084833145, + "logits/rejected": 0.016807714477181435, + "loss": 1.3861, + "step": 12 + }, + { + "beta_dpo/beta": 0.09951455891132355, + "beta_dpo/beta_margin_grad_mean": -0.5016134977340698, + "beta_dpo/beta_margin_grad_std": 0.007461505476385355, + "beta_dpo/beta_margin_mean": -0.006455389317125082, + "beta_dpo/beta_margin_std": 0.02985336445271969, + "beta_dpo/beta_used": 0.09951455891132355, + "beta_dpo/beta_used_raw": 0.09951455891132355, + "beta_dpo/gap_mean": -0.013505849055945873, + "beta_dpo/gap_std": 0.28792649507522583, + "beta_dpo/loss_margin_mean": -0.06482848525047302, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.019652305366591082, + "grad_norm": 30.061500549316406, + "learning_rate": 8.955223880597014e-08, + "logits/chosen": 0.10790680348873138, + "logits/rejected": 0.045588694512844086, + "loss": 1.3886, + "step": 13 + }, + { + "beta_dpo/beta": 0.1006593257188797, + "beta_dpo/beta_margin_grad_mean": -0.5000956654548645, + "beta_dpo/beta_margin_grad_std": 0.009061355143785477, + "beta_dpo/beta_margin_mean": -0.0003827259934041649, + "beta_dpo/beta_margin_std": 0.03626179322600365, + "beta_dpo/beta_used": 0.1006593257188797, + "beta_dpo/beta_used_raw": 0.1006593257188797, + "beta_dpo/gap_mean": -0.013502835296094418, + "beta_dpo/gap_std": 0.29340142011642456, + "beta_dpo/loss_margin_mean": -0.004083991050720215, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.021164021164021163, + "grad_norm": 31.050495147705078, + "learning_rate": 9.701492537313432e-08, + "logits/chosen": 0.09658454358577728, + "logits/rejected": 0.07864398509263992, + "loss": 1.3867, + "step": 14 + }, + { + "beta_dpo/beta": 0.10151919722557068, + "beta_dpo/beta_margin_grad_mean": -0.500207245349884, + "beta_dpo/beta_margin_grad_std": 0.00833460595458746, + "beta_dpo/beta_margin_mean": -0.0008312489953823388, + "beta_dpo/beta_margin_std": 0.03335753455758095, + "beta_dpo/beta_used": 0.10151919722557068, + "beta_dpo/beta_used_raw": 0.10151919722557068, + "beta_dpo/gap_mean": -0.013929645530879498, + "beta_dpo/gap_std": 0.30176180601119995, + "beta_dpo/loss_margin_mean": -0.008538007736206055, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.022675736961451247, + "grad_norm": 38.498233795166016, + "learning_rate": 1.044776119402985e-07, + "logits/chosen": 0.05931752920150757, + "logits/rejected": 0.01677715964615345, + "loss": 1.3852, + "step": 15 + }, + { + "beta_dpo/beta": 0.1004377156496048, + "beta_dpo/beta_margin_grad_mean": -0.4998069405555725, + "beta_dpo/beta_margin_grad_std": 0.006501946598291397, + "beta_dpo/beta_margin_mean": 0.0007726156036369503, + "beta_dpo/beta_margin_std": 0.02601255662739277, + "beta_dpo/beta_used": 0.1004377156496048, + "beta_dpo/beta_used_raw": 0.1004377156496048, + "beta_dpo/gap_mean": -0.011293605901300907, + "beta_dpo/gap_std": 0.2983771562576294, + "beta_dpo/loss_margin_mean": 0.007234424352645874, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.02418745275888133, + "grad_norm": 28.71015739440918, + "learning_rate": 1.1194029850746268e-07, + "logits/chosen": 0.13399405777454376, + "logits/rejected": 0.09560250490903854, + "loss": 1.3868, + "step": 16 + }, + { + "beta_dpo/beta": 0.10274288803339005, + "beta_dpo/beta_margin_grad_mean": -0.5008566379547119, + "beta_dpo/beta_margin_grad_std": 0.008705493062734604, + "beta_dpo/beta_margin_mean": -0.0034312044735997915, + "beta_dpo/beta_margin_std": 0.034844666719436646, + "beta_dpo/beta_used": 0.10274288803339005, + "beta_dpo/beta_used_raw": 0.10274288803339005, + "beta_dpo/gap_mean": -0.011791063472628593, + "beta_dpo/gap_std": 0.30600911378860474, + "beta_dpo/loss_margin_mean": -0.03330141305923462, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.025699168556311415, + "grad_norm": 36.072509765625, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": 0.06134633719921112, + "logits/rejected": 0.04228682443499565, + "loss": 1.3829, + "step": 17 + }, + { + "beta_dpo/beta": 0.10039804875850677, + "beta_dpo/beta_margin_grad_mean": -0.49987974762916565, + "beta_dpo/beta_margin_grad_std": 0.006708572618663311, + "beta_dpo/beta_margin_mean": 0.0004805707139894366, + "beta_dpo/beta_margin_std": 0.026840215548872948, + "beta_dpo/beta_used": 0.10039804875850677, + "beta_dpo/beta_used_raw": 0.10039804875850677, + "beta_dpo/gap_mean": -0.013461226597428322, + "beta_dpo/gap_std": 0.29960107803344727, + "beta_dpo/loss_margin_mean": 0.003337264060974121, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.027210884353741496, + "grad_norm": 30.88655662536621, + "learning_rate": 1.2686567164179106e-07, + "logits/chosen": 0.10294324904680252, + "logits/rejected": 0.05656662583351135, + "loss": 1.3869, + "step": 18 + }, + { + "beta_dpo/beta": 0.10079077631235123, + "beta_dpo/beta_margin_grad_mean": -0.49877893924713135, + "beta_dpo/beta_margin_grad_std": 0.008953831158578396, + "beta_dpo/beta_margin_mean": 0.00488430866971612, + "beta_dpo/beta_margin_std": 0.03583861514925957, + "beta_dpo/beta_used": 0.10079077631235123, + "beta_dpo/beta_used_raw": 0.10079077631235123, + "beta_dpo/gap_mean": -0.003621111623942852, + "beta_dpo/gap_std": 0.3067810535430908, + "beta_dpo/loss_margin_mean": 0.049518659710884094, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.02872260015117158, + "grad_norm": 30.9554386138916, + "learning_rate": 1.343283582089552e-07, + "logits/chosen": 0.08978766202926636, + "logits/rejected": 0.07586466521024704, + "loss": 1.3854, + "step": 19 + }, + { + "beta_dpo/beta": 0.09676133096218109, + "beta_dpo/beta_margin_grad_mean": -0.5004621148109436, + "beta_dpo/beta_margin_grad_std": 0.008276589214801788, + "beta_dpo/beta_margin_mean": -0.0018479095306247473, + "beta_dpo/beta_margin_std": 0.0331173837184906, + "beta_dpo/beta_used": 0.09676133096218109, + "beta_dpo/beta_used_raw": 0.09676133096218109, + "beta_dpo/gap_mean": -0.002426680875942111, + "beta_dpo/gap_std": 0.31554800271987915, + "beta_dpo/loss_margin_mean": -0.019725129008293152, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.030234315948601664, + "grad_norm": 30.733718872070312, + "learning_rate": 1.4179104477611938e-07, + "logits/chosen": 0.13740913569927216, + "logits/rejected": 0.11428765952587128, + "loss": 1.3918, + "step": 20 + }, + { + "beta_dpo/beta": 0.1000765711069107, + "beta_dpo/beta_margin_grad_mean": -0.5001194477081299, + "beta_dpo/beta_margin_grad_std": 0.00721387006342411, + "beta_dpo/beta_margin_mean": -0.0004777438298333436, + "beta_dpo/beta_margin_std": 0.028863035142421722, + "beta_dpo/beta_used": 0.1000765711069107, + "beta_dpo/beta_used_raw": 0.1000765711069107, + "beta_dpo/gap_mean": -0.0026407158002257347, + "beta_dpo/gap_std": 0.3122379183769226, + "beta_dpo/loss_margin_mean": -0.0048004984855651855, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.031746031746031744, + "grad_norm": 30.7608699798584, + "learning_rate": 1.4925373134328355e-07, + "logits/chosen": 0.11362561583518982, + "logits/rejected": 0.08779008686542511, + "loss": 1.3866, + "step": 21 + }, + { + "beta_dpo/beta": 0.10177205502986908, + "beta_dpo/beta_margin_grad_mean": -0.49977150559425354, + "beta_dpo/beta_margin_grad_std": 0.006581050809472799, + "beta_dpo/beta_margin_mean": 0.0009137458400800824, + "beta_dpo/beta_margin_std": 0.026328111067414284, + "beta_dpo/beta_used": 0.10177205502986908, + "beta_dpo/beta_used_raw": 0.10177205502986908, + "beta_dpo/gap_mean": -0.0007555157062597573, + "beta_dpo/gap_std": 0.30382484197616577, + "beta_dpo/loss_margin_mean": 0.008973121643066406, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.03325774754346183, + "grad_norm": 31.633691787719727, + "learning_rate": 1.5671641791044775e-07, + "logits/chosen": 0.02553943172097206, + "logits/rejected": 0.0035544242709875107, + "loss": 1.3835, + "step": 22 + }, + { + "beta_dpo/beta": 0.10188088566064835, + "beta_dpo/beta_margin_grad_mean": -0.4981038272380829, + "beta_dpo/beta_margin_grad_std": 0.008175314404070377, + "beta_dpo/beta_margin_mean": 0.007585472427308559, + "beta_dpo/beta_margin_std": 0.032713212072849274, + "beta_dpo/beta_used": 0.10188088566064835, + "beta_dpo/beta_used_raw": 0.10188088566064835, + "beta_dpo/gap_mean": 0.01067442912608385, + "beta_dpo/gap_std": 0.3037213087081909, + "beta_dpo/loss_margin_mean": 0.07446223497390747, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.03476946334089191, + "grad_norm": 35.18233108520508, + "learning_rate": 1.6417910447761193e-07, + "logits/chosen": 0.14777152240276337, + "logits/rejected": 0.12199492752552032, + "loss": 1.3821, + "step": 23 + }, + { + "beta_dpo/beta": 0.10047954320907593, + "beta_dpo/beta_margin_grad_mean": -0.5006471276283264, + "beta_dpo/beta_margin_grad_std": 0.006264388095587492, + "beta_dpo/beta_margin_mean": -0.0025889482349157333, + "beta_dpo/beta_margin_std": 0.025060316547751427, + "beta_dpo/beta_used": 0.10047954320907593, + "beta_dpo/beta_used_raw": 0.10047954320907593, + "beta_dpo/gap_mean": 0.0053078667260706425, + "beta_dpo/gap_std": 0.2973956763744354, + "beta_dpo/loss_margin_mean": -0.028083205223083496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.036281179138321996, + "grad_norm": 29.64365005493164, + "learning_rate": 1.716417910447761e-07, + "logits/chosen": 0.14769330620765686, + "logits/rejected": 0.11441967636346817, + "loss": 1.3848, + "step": 24 + }, + { + "beta_dpo/beta": 0.09820385277271271, + "beta_dpo/beta_margin_grad_mean": -0.49843770265579224, + "beta_dpo/beta_margin_grad_std": 0.008783336728811264, + "beta_dpo/beta_margin_mean": 0.006256488151848316, + "beta_dpo/beta_margin_std": 0.03516627103090286, + "beta_dpo/beta_used": 0.09820385277271271, + "beta_dpo/beta_used_raw": 0.09820385277271271, + "beta_dpo/gap_mean": 0.012951113283634186, + "beta_dpo/gap_std": 0.2982524633407593, + "beta_dpo/loss_margin_mean": 0.06369102001190186, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.03779289493575208, + "grad_norm": 32.16172790527344, + "learning_rate": 1.7910447761194027e-07, + "logits/chosen": 0.10487603396177292, + "logits/rejected": 0.05346622318029404, + "loss": 1.388, + "step": 25 + }, + { + "beta_dpo/beta": 0.0991971343755722, + "beta_dpo/beta_margin_grad_mean": -0.5002507567405701, + "beta_dpo/beta_margin_grad_std": 0.008068394847214222, + "beta_dpo/beta_margin_mean": -0.0010022378992289305, + "beta_dpo/beta_margin_std": 0.03228890150785446, + "beta_dpo/beta_used": 0.0991971343755722, + "beta_dpo/beta_used_raw": 0.0991971343755722, + "beta_dpo/gap_mean": 0.014728277921676636, + "beta_dpo/gap_std": 0.3074049949645996, + "beta_dpo/loss_margin_mean": -0.010433167219161987, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.039304610733182165, + "grad_norm": 30.883846282958984, + "learning_rate": 1.8656716417910447e-07, + "logits/chosen": 0.12802426517009735, + "logits/rejected": 0.1090678796172142, + "loss": 1.3862, + "step": 26 + }, + { + "beta_dpo/beta": 0.10119029879570007, + "beta_dpo/beta_margin_grad_mean": -0.4989292025566101, + "beta_dpo/beta_margin_grad_std": 0.007677622605115175, + "beta_dpo/beta_margin_mean": 0.004283021669834852, + "beta_dpo/beta_margin_std": 0.030718592926859856, + "beta_dpo/beta_used": 0.10119029879570007, + "beta_dpo/beta_used_raw": 0.10119029879570007, + "beta_dpo/gap_mean": 0.018781719729304314, + "beta_dpo/gap_std": 0.30886706709861755, + "beta_dpo/loss_margin_mean": 0.041504472494125366, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.04081632653061224, + "grad_norm": 30.726280212402344, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": 0.1042436882853508, + "logits/rejected": 0.07938185334205627, + "loss": 1.3825, + "step": 27 + }, + { + "beta_dpo/beta": 0.10044533759355545, + "beta_dpo/beta_margin_grad_mean": -0.5000773072242737, + "beta_dpo/beta_margin_grad_std": 0.008852328173816204, + "beta_dpo/beta_margin_mean": -0.00030975634581409395, + "beta_dpo/beta_margin_std": 0.03541991114616394, + "beta_dpo/beta_used": 0.10044533759355545, + "beta_dpo/beta_used_raw": 0.10044533759355545, + "beta_dpo/gap_mean": 0.018769798800349236, + "beta_dpo/gap_std": 0.31470662355422974, + "beta_dpo/loss_margin_mean": -0.0038602352142333984, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.042328042328042326, + "grad_norm": 30.597108840942383, + "learning_rate": 2.0149253731343282e-07, + "logits/chosen": 0.06483221799135208, + "logits/rejected": 0.055861227214336395, + "loss": 1.3838, + "step": 28 + }, + { + "beta_dpo/beta": 0.09717325866222382, + "beta_dpo/beta_margin_grad_mean": -0.5017023086547852, + "beta_dpo/beta_margin_grad_std": 0.007220883388072252, + "beta_dpo/beta_margin_mean": -0.0068104080855846405, + "beta_dpo/beta_margin_std": 0.028891343623399734, + "beta_dpo/beta_used": 0.09717325866222382, + "beta_dpo/beta_used_raw": 0.09717325866222382, + "beta_dpo/gap_mean": -0.00046668609138578176, + "beta_dpo/gap_std": 0.31308144330978394, + "beta_dpo/loss_margin_mean": -0.07001826167106628, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.04383975812547241, + "grad_norm": 34.134544372558594, + "learning_rate": 2.08955223880597e-07, + "logits/chosen": 0.1364022195339203, + "logits/rejected": 0.11881529539823532, + "loss": 1.391, + "step": 29 + }, + { + "beta_dpo/beta": 0.09732764959335327, + "beta_dpo/beta_margin_grad_mean": -0.5010040998458862, + "beta_dpo/beta_margin_grad_std": 0.008385173976421356, + "beta_dpo/beta_margin_mean": -0.00401653815060854, + "beta_dpo/beta_margin_std": 0.03355298191308975, + "beta_dpo/beta_used": 0.09732764959335327, + "beta_dpo/beta_used_raw": 0.09732764959335327, + "beta_dpo/gap_mean": -0.007357730530202389, + "beta_dpo/gap_std": 0.31982719898223877, + "beta_dpo/loss_margin_mean": -0.04151433706283569, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.045351473922902494, + "grad_norm": 31.0594539642334, + "learning_rate": 2.1641791044776117e-07, + "logits/chosen": 0.13864251971244812, + "logits/rejected": 0.08308613300323486, + "loss": 1.3915, + "step": 30 + }, + { + "beta_dpo/beta": 0.1001332476735115, + "beta_dpo/beta_margin_grad_mean": -0.4992826581001282, + "beta_dpo/beta_margin_grad_std": 0.006594392936676741, + "beta_dpo/beta_margin_mean": 0.0028706123121082783, + "beta_dpo/beta_margin_std": 0.02638271264731884, + "beta_dpo/beta_used": 0.1001332476735115, + "beta_dpo/beta_used_raw": 0.1001332476735115, + "beta_dpo/gap_mean": -0.00842762179672718, + "beta_dpo/gap_std": 0.3111611604690552, + "beta_dpo/loss_margin_mean": 0.025867611169815063, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.04686318972033258, + "grad_norm": 37.38663101196289, + "learning_rate": 2.2388059701492537e-07, + "logits/chosen": 0.12416146695613861, + "logits/rejected": 0.07688345015048981, + "loss": 1.3867, + "step": 31 + }, + { + "beta_dpo/beta": 0.10006989538669586, + "beta_dpo/beta_margin_grad_mean": -0.49960097670555115, + "beta_dpo/beta_margin_grad_std": 0.009591592475771904, + "beta_dpo/beta_margin_mean": 0.0015974619891494513, + "beta_dpo/beta_margin_std": 0.038378216326236725, + "beta_dpo/beta_used": 0.10006989538669586, + "beta_dpo/beta_used_raw": 0.10006989538669586, + "beta_dpo/gap_mean": -0.007976244203746319, + "beta_dpo/gap_std": 0.3170028030872345, + "beta_dpo/loss_margin_mean": 0.007406115531921387, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.04837490551776266, + "grad_norm": 33.643096923828125, + "learning_rate": 2.3134328358208954e-07, + "logits/chosen": 0.05975925922393799, + "logits/rejected": 0.05218929052352905, + "loss": 1.3866, + "step": 32 + }, + { + "beta_dpo/beta": 0.10326246917247772, + "beta_dpo/beta_margin_grad_mean": -0.4977298080921173, + "beta_dpo/beta_margin_grad_std": 0.008520841598510742, + "beta_dpo/beta_margin_mean": 0.009082864038646221, + "beta_dpo/beta_margin_std": 0.034096550196409225, + "beta_dpo/beta_used": 0.10326246917247772, + "beta_dpo/beta_used_raw": 0.10326246917247772, + "beta_dpo/gap_mean": 0.010345546528697014, + "beta_dpo/gap_std": 0.3211923837661743, + "beta_dpo/loss_margin_mean": 0.08736389875411987, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.049886621315192746, + "grad_norm": 33.94365310668945, + "learning_rate": 2.388059701492537e-07, + "logits/chosen": 0.03907015174627304, + "logits/rejected": 0.012794176116585732, + "loss": 1.3797, + "step": 33 + }, + { + "beta_dpo/beta": 0.1025274470448494, + "beta_dpo/beta_margin_grad_mean": -0.4979722797870636, + "beta_dpo/beta_margin_grad_std": 0.008004739880561829, + "beta_dpo/beta_margin_mean": 0.008112696930766106, + "beta_dpo/beta_margin_std": 0.032029684633016586, + "beta_dpo/beta_used": 0.1025274470448494, + "beta_dpo/beta_used_raw": 0.1025274470448494, + "beta_dpo/gap_mean": 0.02443467453122139, + "beta_dpo/gap_std": 0.3166555166244507, + "beta_dpo/loss_margin_mean": 0.07865563035011292, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.05139833711262283, + "grad_norm": 29.62582015991211, + "learning_rate": 2.4626865671641786e-07, + "logits/chosen": 0.08650554716587067, + "logits/rejected": 0.05725545063614845, + "loss": 1.3796, + "step": 34 + }, + { + "beta_dpo/beta": 0.09984398633241653, + "beta_dpo/beta_margin_grad_mean": -0.4997754693031311, + "beta_dpo/beta_margin_grad_std": 0.006669824477285147, + "beta_dpo/beta_margin_mean": 0.000898701255209744, + "beta_dpo/beta_margin_std": 0.026683101430535316, + "beta_dpo/beta_used": 0.09984398633241653, + "beta_dpo/beta_used_raw": 0.09984398633241653, + "beta_dpo/gap_mean": 0.024442963302135468, + "beta_dpo/gap_std": 0.31088265776634216, + "beta_dpo/loss_margin_mean": 0.0072622597217559814, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.05291005291005291, + "grad_norm": 31.378681182861328, + "learning_rate": 2.537313432835821e-07, + "logits/chosen": 0.07245808839797974, + "logits/rejected": 0.04945196956396103, + "loss": 1.384, + "step": 35 + }, + { + "beta_dpo/beta": 0.09638850390911102, + "beta_dpo/beta_margin_grad_mean": -0.49970918893814087, + "beta_dpo/beta_margin_grad_std": 0.008332891389727592, + "beta_dpo/beta_margin_mean": 0.0011656360002234578, + "beta_dpo/beta_margin_std": 0.033343665301799774, + "beta_dpo/beta_used": 0.09638850390911102, + "beta_dpo/beta_used_raw": 0.09638850390911102, + "beta_dpo/gap_mean": 0.02393309585750103, + "beta_dpo/gap_std": 0.3164275884628296, + "beta_dpo/loss_margin_mean": 0.012125849723815918, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.05442176870748299, + "grad_norm": 31.100717544555664, + "learning_rate": 2.611940298507462e-07, + "logits/chosen": 0.10495918989181519, + "logits/rejected": 0.05185706540942192, + "loss": 1.3899, + "step": 36 + }, + { + "beta_dpo/beta": 0.10284903645515442, + "beta_dpo/beta_margin_grad_mean": -0.497913122177124, + "beta_dpo/beta_margin_grad_std": 0.008537176996469498, + "beta_dpo/beta_margin_mean": 0.00834999606013298, + "beta_dpo/beta_margin_std": 0.03416427597403526, + "beta_dpo/beta_used": 0.10284903645515442, + "beta_dpo/beta_used_raw": 0.10284903645515442, + "beta_dpo/gap_mean": 0.02981198951601982, + "beta_dpo/gap_std": 0.31800198554992676, + "beta_dpo/loss_margin_mean": 0.0813802182674408, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.055933484504913075, + "grad_norm": 37.75326919555664, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": 0.10214823484420776, + "logits/rejected": 0.022578764706850052, + "loss": 1.3785, + "step": 37 + }, + { + "beta_dpo/beta": 0.09838944673538208, + "beta_dpo/beta_margin_grad_mean": -0.4993027150630951, + "beta_dpo/beta_margin_grad_std": 0.00942269992083311, + "beta_dpo/beta_margin_mean": 0.0027929130010306835, + "beta_dpo/beta_margin_std": 0.03771368786692619, + "beta_dpo/beta_used": 0.09838944673538208, + "beta_dpo/beta_used_raw": 0.09838944673538208, + "beta_dpo/gap_mean": 0.037786953151226044, + "beta_dpo/gap_std": 0.32929813861846924, + "beta_dpo/loss_margin_mean": 0.024533838033676147, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.05744520030234316, + "grad_norm": 31.314617156982422, + "learning_rate": 2.761194029850746e-07, + "logits/chosen": 0.06979146599769592, + "logits/rejected": 0.05586238577961922, + "loss": 1.3852, + "step": 38 + }, + { + "beta_dpo/beta": 0.09809637069702148, + "beta_dpo/beta_margin_grad_mean": -0.4992535710334778, + "beta_dpo/beta_margin_grad_std": 0.010051256977021694, + "beta_dpo/beta_margin_mean": 0.0029852152802050114, + "beta_dpo/beta_margin_std": 0.040224362164735794, + "beta_dpo/beta_used": 0.09809637069702148, + "beta_dpo/beta_used_raw": 0.09809637069702148, + "beta_dpo/gap_mean": 0.029398782178759575, + "beta_dpo/gap_std": 0.33756715059280396, + "beta_dpo/loss_margin_mean": 0.02950763702392578, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.05895691609977324, + "grad_norm": 29.13448715209961, + "learning_rate": 2.8358208955223876e-07, + "logits/chosen": 0.16034747660160065, + "logits/rejected": 0.1335568130016327, + "loss": 1.3866, + "step": 39 + }, + { + "beta_dpo/beta": 0.100420743227005, + "beta_dpo/beta_margin_grad_mean": -0.499179869890213, + "beta_dpo/beta_margin_grad_std": 0.007784340064972639, + "beta_dpo/beta_margin_mean": 0.0032807684037834406, + "beta_dpo/beta_margin_std": 0.031147774308919907, + "beta_dpo/beta_used": 0.100420743227005, + "beta_dpo/beta_used_raw": 0.100420743227005, + "beta_dpo/gap_mean": 0.03405720740556717, + "beta_dpo/gap_std": 0.334450364112854, + "beta_dpo/loss_margin_mean": 0.031633391976356506, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.06046863189720333, + "grad_norm": 35.88178253173828, + "learning_rate": 2.9104477611940296e-07, + "logits/chosen": 0.12306384742259979, + "logits/rejected": 0.07282181829214096, + "loss": 1.3822, + "step": 40 + }, + { + "beta_dpo/beta": 0.10087525099515915, + "beta_dpo/beta_margin_grad_mean": -0.497925728559494, + "beta_dpo/beta_margin_grad_std": 0.008461805991828442, + "beta_dpo/beta_margin_mean": 0.008300484158098698, + "beta_dpo/beta_margin_std": 0.03385802358388901, + "beta_dpo/beta_used": 0.10087525099515915, + "beta_dpo/beta_used_raw": 0.10087525099515915, + "beta_dpo/gap_mean": 0.036321260035037994, + "beta_dpo/gap_std": 0.3385712802410126, + "beta_dpo/loss_margin_mean": 0.080975741147995, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.06198034769463341, + "grad_norm": 35.11070251464844, + "learning_rate": 2.985074626865671e-07, + "logits/chosen": 0.05649389326572418, + "logits/rejected": 0.03625918924808502, + "loss": 1.3812, + "step": 41 + }, + { + "beta_dpo/beta": 0.09852074086666107, + "beta_dpo/beta_margin_grad_mean": -0.4987524747848511, + "beta_dpo/beta_margin_grad_std": 0.009575996547937393, + "beta_dpo/beta_margin_mean": 0.00499336002394557, + "beta_dpo/beta_margin_std": 0.0383220799267292, + "beta_dpo/beta_used": 0.09852074086666107, + "beta_dpo/beta_used_raw": 0.09852074086666107, + "beta_dpo/gap_mean": 0.04520569369196892, + "beta_dpo/gap_std": 0.3435010612010956, + "beta_dpo/loss_margin_mean": 0.047529637813568115, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.06349206349206349, + "grad_norm": 31.087583541870117, + "learning_rate": 3.059701492537313e-07, + "logits/chosen": 0.05465045943856239, + "logits/rejected": 0.03304405137896538, + "loss": 1.3841, + "step": 42 + }, + { + "beta_dpo/beta": 0.10309203714132309, + "beta_dpo/beta_margin_grad_mean": -0.49921998381614685, + "beta_dpo/beta_margin_grad_std": 0.010388917289674282, + "beta_dpo/beta_margin_mean": 0.0031178132630884647, + "beta_dpo/beta_margin_std": 0.041573185473680496, + "beta_dpo/beta_used": 0.10309203714132309, + "beta_dpo/beta_used_raw": 0.10309203714132309, + "beta_dpo/gap_mean": 0.04160957783460617, + "beta_dpo/gap_std": 0.35253000259399414, + "beta_dpo/loss_margin_mean": 0.0297316312789917, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.06500377928949358, + "grad_norm": 39.25471115112305, + "learning_rate": 3.134328358208955e-07, + "logits/chosen": 0.06021273136138916, + "logits/rejected": 0.0498313382267952, + "loss": 1.3769, + "step": 43 + }, + { + "beta_dpo/beta": 0.0943903774023056, + "beta_dpo/beta_margin_grad_mean": -0.5001733899116516, + "beta_dpo/beta_margin_grad_std": 0.009070969186723232, + "beta_dpo/beta_margin_mean": -0.0006922496249899268, + "beta_dpo/beta_margin_std": 0.036296818405389786, + "beta_dpo/beta_used": 0.0943903774023056, + "beta_dpo/beta_used_raw": 0.0943903774023056, + "beta_dpo/gap_mean": 0.0337846614420414, + "beta_dpo/gap_std": 0.36235010623931885, + "beta_dpo/loss_margin_mean": -0.0086250901222229, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.06651549508692366, + "grad_norm": 33.845481872558594, + "learning_rate": 3.2089552238805965e-07, + "logits/chosen": 0.04900500550866127, + "logits/rejected": 0.018543703481554985, + "loss": 1.392, + "step": 44 + }, + { + "beta_dpo/beta": 0.10028617084026337, + "beta_dpo/beta_margin_grad_mean": -0.4976131319999695, + "beta_dpo/beta_margin_grad_std": 0.009892971254885197, + "beta_dpo/beta_margin_mean": 0.009551994502544403, + "beta_dpo/beta_margin_std": 0.03958762809634209, + "beta_dpo/beta_used": 0.10028617084026337, + "beta_dpo/beta_used_raw": 0.10028617084026337, + "beta_dpo/gap_mean": 0.03637174516916275, + "beta_dpo/gap_std": 0.36881065368652344, + "beta_dpo/loss_margin_mean": 0.09402036666870117, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.06802721088435375, + "grad_norm": 36.0006217956543, + "learning_rate": 3.2835820895522385e-07, + "logits/chosen": 0.1208408921957016, + "logits/rejected": 0.06802251189947128, + "loss": 1.3823, + "step": 45 + }, + { + "beta_dpo/beta": 0.09875007718801498, + "beta_dpo/beta_margin_grad_mean": -0.4974738359451294, + "beta_dpo/beta_margin_grad_std": 0.010846122168004513, + "beta_dpo/beta_margin_mean": 0.010113219730556011, + "beta_dpo/beta_margin_std": 0.04341111332178116, + "beta_dpo/beta_used": 0.09875007718801498, + "beta_dpo/beta_used_raw": 0.09875007718801498, + "beta_dpo/gap_mean": 0.050506845116615295, + "beta_dpo/gap_std": 0.38154229521751404, + "beta_dpo/loss_margin_mean": 0.10229560732841492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.06953892668178382, + "grad_norm": 33.66736602783203, + "learning_rate": 3.3582089552238805e-07, + "logits/chosen": 0.12831750512123108, + "logits/rejected": 0.08227770030498505, + "loss": 1.3836, + "step": 46 + }, + { + "beta_dpo/beta": 0.10235325992107391, + "beta_dpo/beta_margin_grad_mean": -0.49693551659584045, + "beta_dpo/beta_margin_grad_std": 0.009976202622056007, + "beta_dpo/beta_margin_mean": 0.01226241048425436, + "beta_dpo/beta_margin_std": 0.039923615753650665, + "beta_dpo/beta_used": 0.10235325992107391, + "beta_dpo/beta_used_raw": 0.10235325992107391, + "beta_dpo/gap_mean": 0.06335177272558212, + "beta_dpo/gap_std": 0.3873853087425232, + "beta_dpo/loss_margin_mean": 0.11981740593910217, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.0710506424792139, + "grad_norm": 29.64479637145996, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": 0.0135899493470788, + "logits/rejected": -0.028094399720430374, + "loss": 1.376, + "step": 47 + }, + { + "beta_dpo/beta": 0.10037179291248322, + "beta_dpo/beta_margin_grad_mean": -0.49941501021385193, + "beta_dpo/beta_margin_grad_std": 0.011483574286103249, + "beta_dpo/beta_margin_mean": 0.0023423591628670692, + "beta_dpo/beta_margin_std": 0.04596313461661339, + "beta_dpo/beta_used": 0.10037179291248322, + "beta_dpo/beta_used_raw": 0.10037179291248322, + "beta_dpo/gap_mean": 0.06056096404790878, + "beta_dpo/gap_std": 0.39652693271636963, + "beta_dpo/loss_margin_mean": 0.023324549198150635, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.07256235827664399, + "grad_norm": 28.561792373657227, + "learning_rate": 3.507462686567164e-07, + "logits/chosen": 0.1474730372428894, + "logits/rejected": 0.11770699918270111, + "loss": 1.3799, + "step": 48 + }, + { + "beta_dpo/beta": 0.10378497838973999, + "beta_dpo/beta_margin_grad_mean": -0.497110515832901, + "beta_dpo/beta_margin_grad_std": 0.012511258944869041, + "beta_dpo/beta_margin_mean": 0.011552227661013603, + "beta_dpo/beta_margin_std": 0.05013538524508476, + "beta_dpo/beta_used": 0.10378497838973999, + "beta_dpo/beta_used_raw": 0.10378497838973999, + "beta_dpo/gap_mean": 0.06303433328866959, + "beta_dpo/gap_std": 0.4152846336364746, + "beta_dpo/loss_margin_mean": 0.11007669568061829, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.07407407407407407, + "grad_norm": 30.38995933532715, + "learning_rate": 3.5820895522388055e-07, + "logits/chosen": 0.0344427265226841, + "logits/rejected": 0.01285216398537159, + "loss": 1.3734, + "step": 49 + }, + { + "beta_dpo/beta": 0.1018582433462143, + "beta_dpo/beta_margin_grad_mean": -0.4980669319629669, + "beta_dpo/beta_margin_grad_std": 0.010723302140831947, + "beta_dpo/beta_margin_mean": 0.00773262232542038, + "beta_dpo/beta_margin_std": 0.042918216437101364, + "beta_dpo/beta_used": 0.1018582433462143, + "beta_dpo/beta_used_raw": 0.1018582433462143, + "beta_dpo/gap_mean": 0.07021433860063553, + "beta_dpo/gap_std": 0.4182177782058716, + "beta_dpo/loss_margin_mean": 0.07544875144958496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.07558578987150416, + "grad_norm": 30.36774253845215, + "learning_rate": 3.6567164179104475e-07, + "logits/chosen": 0.11492250859737396, + "logits/rejected": 0.08501888066530228, + "loss": 1.3761, + "step": 50 + }, + { + "beta_dpo/beta": 0.10719744116067886, + "beta_dpo/beta_margin_grad_mean": -0.4953405559062958, + "beta_dpo/beta_margin_grad_std": 0.012494519352912903, + "beta_dpo/beta_margin_mean": 0.018651319667696953, + "beta_dpo/beta_margin_std": 0.05001518130302429, + "beta_dpo/beta_used": 0.10719744116067886, + "beta_dpo/beta_used_raw": 0.10719744116067886, + "beta_dpo/gap_mean": 0.08684976398944855, + "beta_dpo/gap_std": 0.4211902320384979, + "beta_dpo/loss_margin_mean": 0.17378860712051392, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.07709750566893424, + "grad_norm": 37.90382385253906, + "learning_rate": 3.7313432835820895e-07, + "logits/chosen": 0.08796243369579315, + "logits/rejected": 0.060197874903678894, + "loss": 1.3646, + "step": 51 + }, + { + "beta_dpo/beta": 0.10072717070579529, + "beta_dpo/beta_margin_grad_mean": -0.49583399295806885, + "beta_dpo/beta_margin_grad_std": 0.012241682037711143, + "beta_dpo/beta_margin_mean": 0.016676129773259163, + "beta_dpo/beta_margin_std": 0.049004100263118744, + "beta_dpo/beta_used": 0.10072717070579529, + "beta_dpo/beta_used_raw": 0.10072717070579529, + "beta_dpo/gap_mean": 0.09611259400844574, + "beta_dpo/gap_std": 0.43683481216430664, + "beta_dpo/loss_margin_mean": 0.1634836196899414, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.07860922146636433, + "grad_norm": 35.590213775634766, + "learning_rate": 3.805970149253731e-07, + "logits/chosen": 0.07440444082021713, + "logits/rejected": 0.025928881019353867, + "loss": 1.3756, + "step": 52 + }, + { + "beta_dpo/beta": 0.10719523578882217, + "beta_dpo/beta_margin_grad_mean": -0.49494311213493347, + "beta_dpo/beta_margin_grad_std": 0.012945474125444889, + "beta_dpo/beta_margin_mean": 0.020237509161233902, + "beta_dpo/beta_margin_std": 0.051830440759658813, + "beta_dpo/beta_used": 0.10719523578882217, + "beta_dpo/beta_used_raw": 0.10719523578882217, + "beta_dpo/gap_mean": 0.11741121858358383, + "beta_dpo/gap_std": 0.4450018107891083, + "beta_dpo/loss_margin_mean": 0.1888570785522461, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.0801209372637944, + "grad_norm": 30.624479293823242, + "learning_rate": 3.880597014925373e-07, + "logits/chosen": 0.08777962625026703, + "logits/rejected": 0.07702084630727768, + "loss": 1.3615, + "step": 53 + }, + { + "beta_dpo/beta": 0.10270829498767853, + "beta_dpo/beta_margin_grad_mean": -0.49615994095802307, + "beta_dpo/beta_margin_grad_std": 0.010877852328121662, + "beta_dpo/beta_margin_mean": 0.015367364510893822, + "beta_dpo/beta_margin_std": 0.04353713244199753, + "beta_dpo/beta_used": 0.10270829498767853, + "beta_dpo/beta_used_raw": 0.10270829498767853, + "beta_dpo/gap_mean": 0.1302683800458908, + "beta_dpo/gap_std": 0.44163817167282104, + "beta_dpo/loss_margin_mean": 0.14893117547035217, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.08163265306122448, + "grad_norm": 29.564790725708008, + "learning_rate": 3.9552238805970144e-07, + "logits/chosen": 0.05520813167095184, + "logits/rejected": 0.039937350898981094, + "loss": 1.3686, + "step": 54 + }, + { + "beta_dpo/beta": 0.10008847713470459, + "beta_dpo/beta_margin_grad_mean": -0.49438750743865967, + "beta_dpo/beta_margin_grad_std": 0.01513749547302723, + "beta_dpo/beta_margin_mean": 0.022482367232441902, + "beta_dpo/beta_margin_std": 0.060637783259153366, + "beta_dpo/beta_used": 0.10008847713470459, + "beta_dpo/beta_used_raw": 0.10008847713470459, + "beta_dpo/gap_mean": 0.13691473007202148, + "beta_dpo/gap_std": 0.46075886487960815, + "beta_dpo/loss_margin_mean": 0.2233232706785202, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.08314436885865457, + "grad_norm": 33.72454833984375, + "learning_rate": 4.0298507462686564e-07, + "logits/chosen": 0.02177221141755581, + "logits/rejected": -0.05276140943169594, + "loss": 1.3727, + "step": 55 + }, + { + "beta_dpo/beta": 0.10054030269384384, + "beta_dpo/beta_margin_grad_mean": -0.495496928691864, + "beta_dpo/beta_margin_grad_std": 0.013434402644634247, + "beta_dpo/beta_margin_mean": 0.018017152324318886, + "beta_dpo/beta_margin_std": 0.05379686877131462, + "beta_dpo/beta_used": 0.10054030269384384, + "beta_dpo/beta_used_raw": 0.10054030269384384, + "beta_dpo/gap_mean": 0.1483607143163681, + "beta_dpo/gap_std": 0.48542577028274536, + "beta_dpo/loss_margin_mean": 0.17924529314041138, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.08465608465608465, + "grad_norm": 29.508190155029297, + "learning_rate": 4.1044776119402984e-07, + "logits/chosen": 0.12270273268222809, + "logits/rejected": 0.1003156453371048, + "loss": 1.3708, + "step": 56 + }, + { + "beta_dpo/beta": 0.10854996740818024, + "beta_dpo/beta_margin_grad_mean": -0.49215006828308105, + "beta_dpo/beta_margin_grad_std": 0.017352448776364326, + "beta_dpo/beta_margin_mean": 0.031447600573301315, + "beta_dpo/beta_margin_std": 0.06956035643815994, + "beta_dpo/beta_used": 0.10854996740818024, + "beta_dpo/beta_used_raw": 0.10854996740818024, + "beta_dpo/gap_mean": 0.17608734965324402, + "beta_dpo/gap_std": 0.5068370699882507, + "beta_dpo/loss_margin_mean": 0.29122018814086914, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.08616780045351474, + "grad_norm": 35.71118927001953, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": 0.11377115547657013, + "logits/rejected": 0.04856497421860695, + "loss": 1.3526, + "step": 57 + }, + { + "beta_dpo/beta": 0.09856951981782913, + "beta_dpo/beta_margin_grad_mean": -0.4949773848056793, + "beta_dpo/beta_margin_grad_std": 0.020041456446051598, + "beta_dpo/beta_margin_mean": 0.020195821300148964, + "beta_dpo/beta_margin_std": 0.08059463649988174, + "beta_dpo/beta_used": 0.09856951981782913, + "beta_dpo/beta_used_raw": 0.09856951981782913, + "beta_dpo/gap_mean": 0.18698576092720032, + "beta_dpo/gap_std": 0.5550523996353149, + "beta_dpo/loss_margin_mean": 0.1905546486377716, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.08767951625094482, + "grad_norm": 34.71607971191406, + "learning_rate": 4.253731343283582e-07, + "logits/chosen": 0.06032850965857506, + "logits/rejected": 0.044845663011074066, + "loss": 1.3701, + "step": 58 + }, + { + "beta_dpo/beta": 0.09670780599117279, + "beta_dpo/beta_margin_grad_mean": -0.4935970604419708, + "beta_dpo/beta_margin_grad_std": 0.017941787838935852, + "beta_dpo/beta_margin_mean": 0.02568780817091465, + "beta_dpo/beta_margin_std": 0.07200151681900024, + "beta_dpo/beta_used": 0.09670780599117279, + "beta_dpo/beta_used_raw": 0.09670780599117279, + "beta_dpo/gap_mean": 0.17569701373577118, + "beta_dpo/gap_std": 0.5746784210205078, + "beta_dpo/loss_margin_mean": 0.25069764256477356, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.08919123204837491, + "grad_norm": 27.714921951293945, + "learning_rate": 4.3283582089552234e-07, + "logits/chosen": 0.12836335599422455, + "logits/rejected": 0.06272809952497482, + "loss": 1.3745, + "step": 59 + }, + { + "beta_dpo/beta": 0.09911298751831055, + "beta_dpo/beta_margin_grad_mean": -0.497974693775177, + "beta_dpo/beta_margin_grad_std": 0.01722402684390545, + "beta_dpo/beta_margin_mean": 0.008100156672298908, + "beta_dpo/beta_margin_std": 0.06897980719804764, + "beta_dpo/beta_used": 0.09911298751831055, + "beta_dpo/beta_used_raw": 0.09911298751831055, + "beta_dpo/gap_mean": 0.18243734538555145, + "beta_dpo/gap_std": 0.6014333963394165, + "beta_dpo/loss_margin_mean": 0.07506400346755981, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.09070294784580499, + "grad_norm": 29.543716430664062, + "learning_rate": 4.4029850746268654e-07, + "logits/chosen": 0.11510735005140305, + "logits/rejected": 0.11371426284313202, + "loss": 1.3699, + "step": 60 + }, + { + "beta_dpo/beta": 0.09410177916288376, + "beta_dpo/beta_margin_grad_mean": -0.4979269206523895, + "beta_dpo/beta_margin_grad_std": 0.019729407504200935, + "beta_dpo/beta_margin_mean": 0.008308450691401958, + "beta_dpo/beta_margin_std": 0.07905634492635727, + "beta_dpo/beta_used": 0.09410177916288376, + "beta_dpo/beta_used_raw": 0.09410177916288376, + "beta_dpo/gap_mean": 0.16265742480754852, + "beta_dpo/gap_std": 0.6380031108856201, + "beta_dpo/loss_margin_mean": 0.08492028713226318, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.09221466364323508, + "grad_norm": 26.6069278717041, + "learning_rate": 4.4776119402985074e-07, + "logits/chosen": 0.11286836862564087, + "logits/rejected": 0.08159026503562927, + "loss": 1.3808, + "step": 61 + }, + { + "beta_dpo/beta": 0.10248786211013794, + "beta_dpo/beta_margin_grad_mean": -0.493076890707016, + "beta_dpo/beta_margin_grad_std": 0.019803814589977264, + "beta_dpo/beta_margin_mean": 0.027745075523853302, + "beta_dpo/beta_margin_std": 0.07940506935119629, + "beta_dpo/beta_used": 0.10248786211013794, + "beta_dpo/beta_used_raw": 0.10248786211013794, + "beta_dpo/gap_mean": 0.165227472782135, + "beta_dpo/gap_std": 0.6631143689155579, + "beta_dpo/loss_margin_mean": 0.26825329661369324, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.09372637944066516, + "grad_norm": 33.22285461425781, + "learning_rate": 4.552238805970149e-07, + "logits/chosen": 0.0910121276974678, + "logits/rejected": 0.052763670682907104, + "loss": 1.3659, + "step": 62 + }, + { + "beta_dpo/beta": 0.11158844828605652, + "beta_dpo/beta_margin_grad_mean": -0.4859258234500885, + "beta_dpo/beta_margin_grad_std": 0.021454254165291786, + "beta_dpo/beta_margin_mean": 0.05640276148915291, + "beta_dpo/beta_margin_std": 0.08617250621318817, + "beta_dpo/beta_used": 0.11158844828605652, + "beta_dpo/beta_used_raw": 0.11158844828605652, + "beta_dpo/gap_mean": 0.22149600088596344, + "beta_dpo/gap_std": 0.6870256066322327, + "beta_dpo/loss_margin_mean": 0.5042006373405457, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.09523809523809523, + "grad_norm": 35.34672927856445, + "learning_rate": 4.626865671641791e-07, + "logits/chosen": 0.1868915557861328, + "logits/rejected": 0.16194406151771545, + "loss": 1.3407, + "step": 63 + }, + { + "beta_dpo/beta": 0.09616056084632874, + "beta_dpo/beta_margin_grad_mean": -0.4972890019416809, + "beta_dpo/beta_margin_grad_std": 0.017049264162778854, + "beta_dpo/beta_margin_mean": 0.01085972972214222, + "beta_dpo/beta_margin_std": 0.06834981590509415, + "beta_dpo/beta_used": 0.09616056084632874, + "beta_dpo/beta_used_raw": 0.09616056084632874, + "beta_dpo/gap_mean": 0.223545640707016, + "beta_dpo/gap_std": 0.7015777826309204, + "beta_dpo/loss_margin_mean": 0.11485102772712708, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.09674981103552532, + "grad_norm": 26.68852424621582, + "learning_rate": 4.701492537313433e-07, + "logits/chosen": 0.08099683374166489, + "logits/rejected": 0.056533243507146835, + "loss": 1.3714, + "step": 64 + }, + { + "beta_dpo/beta": 0.11661025881767273, + "beta_dpo/beta_margin_grad_mean": -0.4880481958389282, + "beta_dpo/beta_margin_grad_std": 0.02547621540725231, + "beta_dpo/beta_margin_mean": 0.04795736074447632, + "beta_dpo/beta_margin_std": 0.10229441523551941, + "beta_dpo/beta_used": 0.11661025881767273, + "beta_dpo/beta_used_raw": 0.11661025881767273, + "beta_dpo/gap_mean": 0.24007107317447662, + "beta_dpo/gap_std": 0.7217456698417664, + "beta_dpo/loss_margin_mean": 0.4126676321029663, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.0982615268329554, + "grad_norm": 36.96430587768555, + "learning_rate": 4.776119402985074e-07, + "logits/chosen": 0.20523807406425476, + "logits/rejected": 0.17545229196548462, + "loss": 1.3282, + "step": 65 + }, + { + "beta_dpo/beta": 0.10083754360675812, + "beta_dpo/beta_margin_grad_mean": -0.49288806319236755, + "beta_dpo/beta_margin_grad_std": 0.024221470579504967, + "beta_dpo/beta_margin_mean": 0.02851312793791294, + "beta_dpo/beta_margin_std": 0.09726641327142715, + "beta_dpo/beta_used": 0.10083754360675812, + "beta_dpo/beta_used_raw": 0.10083754360675812, + "beta_dpo/gap_mean": 0.26807379722595215, + "beta_dpo/gap_std": 0.7676073908805847, + "beta_dpo/loss_margin_mean": 0.28801459074020386, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.09977324263038549, + "grad_norm": 32.096954345703125, + "learning_rate": 4.850746268656717e-07, + "logits/chosen": 0.08074181526899338, + "logits/rejected": 0.057395968586206436, + "loss": 1.3589, + "step": 66 + }, + { + "beta_dpo/beta": 0.10163235664367676, + "beta_dpo/beta_margin_grad_mean": -0.49344709515571594, + "beta_dpo/beta_margin_grad_std": 0.01996057853102684, + "beta_dpo/beta_margin_mean": 0.026278013363480568, + "beta_dpo/beta_margin_std": 0.08008788526058197, + "beta_dpo/beta_used": 0.10163235664367676, + "beta_dpo/beta_used_raw": 0.10163235664367676, + "beta_dpo/gap_mean": 0.2551182508468628, + "beta_dpo/gap_std": 0.782842755317688, + "beta_dpo/loss_margin_mean": 0.25911685824394226, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.10128495842781557, + "grad_norm": 32.75022888183594, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": 0.08444076031446457, + "logits/rejected": 0.02903038263320923, + "loss": 1.3585, + "step": 67 + }, + { + "beta_dpo/beta": 0.10411373525857925, + "beta_dpo/beta_margin_grad_mean": -0.49119579792022705, + "beta_dpo/beta_margin_grad_std": 0.027670543640851974, + "beta_dpo/beta_margin_mean": 0.03529214486479759, + "beta_dpo/beta_margin_std": 0.11126258224248886, + "beta_dpo/beta_used": 0.10411373525857925, + "beta_dpo/beta_used_raw": 0.10411373525857925, + "beta_dpo/gap_mean": 0.2772870659828186, + "beta_dpo/gap_std": 0.8147046566009521, + "beta_dpo/loss_margin_mean": 0.34844857454299927, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.10279667422524566, + "grad_norm": 35.180267333984375, + "learning_rate": 5e-07, + "logits/chosen": 0.08185356110334396, + "logits/rejected": 0.05136846378445625, + "loss": 1.351, + "step": 68 + }, + { + "beta_dpo/beta": 0.10379628837108612, + "beta_dpo/beta_margin_grad_mean": -0.4831562936306, + "beta_dpo/beta_margin_grad_std": 0.027017628774046898, + "beta_dpo/beta_margin_mean": 0.06770786643028259, + "beta_dpo/beta_margin_std": 0.10907536745071411, + "beta_dpo/beta_used": 0.10379628837108612, + "beta_dpo/beta_used_raw": 0.10379628837108612, + "beta_dpo/gap_mean": 0.31204187870025635, + "beta_dpo/gap_std": 0.8627355098724365, + "beta_dpo/loss_margin_mean": 0.626896858215332, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.10430839002267574, + "grad_norm": 33.93740463256836, + "learning_rate": 4.999965034812934e-07, + "logits/chosen": 0.10463707894086838, + "logits/rejected": 0.061709824949502945, + "loss": 1.3471, + "step": 69 + }, + { + "beta_dpo/beta": 0.09281350672245026, + "beta_dpo/beta_margin_grad_mean": -0.49325621128082275, + "beta_dpo/beta_margin_grad_std": 0.023107120767235756, + "beta_dpo/beta_margin_mean": 0.027028771117329597, + "beta_dpo/beta_margin_std": 0.09262385219335556, + "beta_dpo/beta_used": 0.09281350672245026, + "beta_dpo/beta_used_raw": 0.09281350672245026, + "beta_dpo/gap_mean": 0.33571797609329224, + "beta_dpo/gap_std": 0.8863239288330078, + "beta_dpo/loss_margin_mean": 0.2911398708820343, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.10582010582010581, + "grad_norm": 29.002174377441406, + "learning_rate": 4.999860140229787e-07, + "logits/chosen": 0.15091601014137268, + "logits/rejected": 0.1275625079870224, + "loss": 1.3677, + "step": 70 + }, + { + "beta_dpo/beta": 0.09932013601064682, + "beta_dpo/beta_margin_grad_mean": -0.4969637989997864, + "beta_dpo/beta_margin_grad_std": 0.030911121517419815, + "beta_dpo/beta_margin_mean": 0.012066809460520744, + "beta_dpo/beta_margin_std": 0.12431972473859787, + "beta_dpo/beta_used": 0.09932013601064682, + "beta_dpo/beta_used_raw": 0.09932013601064682, + "beta_dpo/gap_mean": 0.3090514540672302, + "beta_dpo/gap_std": 0.9437848329544067, + "beta_dpo/loss_margin_mean": 0.1241014152765274, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1073318216175359, + "grad_norm": 30.64402198791504, + "learning_rate": 4.999685319184688e-07, + "logits/chosen": 0.0933336392045021, + "logits/rejected": 0.07769975811243057, + "loss": 1.3585, + "step": 71 + }, + { + "beta_dpo/beta": 0.10620521754026413, + "beta_dpo/beta_margin_grad_mean": -0.4829525053501129, + "beta_dpo/beta_margin_grad_std": 0.029280930757522583, + "beta_dpo/beta_margin_mean": 0.06848964095115662, + "beta_dpo/beta_margin_std": 0.11784832924604416, + "beta_dpo/beta_used": 0.10620521754026413, + "beta_dpo/beta_used_raw": 0.10620521754026413, + "beta_dpo/gap_mean": 0.34875622391700745, + "beta_dpo/gap_std": 0.9951860904693604, + "beta_dpo/loss_margin_mean": 0.6475339531898499, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.10884353741496598, + "grad_norm": 34.11119842529297, + "learning_rate": 4.999440576567755e-07, + "logits/chosen": 0.10905547440052032, + "logits/rejected": 0.04688937962055206, + "loss": 1.3403, + "step": 72 + }, + { + "beta_dpo/beta": 0.08631128817796707, + "beta_dpo/beta_margin_grad_mean": -0.49926990270614624, + "beta_dpo/beta_margin_grad_std": 0.02832244336605072, + "beta_dpo/beta_margin_mean": 0.0030645669903606176, + "beta_dpo/beta_margin_std": 0.11412809789180756, + "beta_dpo/beta_used": 0.08631128817796707, + "beta_dpo/beta_used_raw": 0.08631128817796707, + "beta_dpo/gap_mean": 0.2991209328174591, + "beta_dpo/gap_std": 1.0169306993484497, + "beta_dpo/loss_margin_mean": 0.025026828050613403, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.11035525321239607, + "grad_norm": 28.28575897216797, + "learning_rate": 4.999125919224965e-07, + "logits/chosen": 0.1120387464761734, + "logits/rejected": 0.09779127687215805, + "loss": 1.3813, + "step": 73 + }, + { + "beta_dpo/beta": 0.10641483962535858, + "beta_dpo/beta_margin_grad_mean": -0.4837491810321808, + "beta_dpo/beta_margin_grad_std": 0.030070235952734947, + "beta_dpo/beta_margin_mean": 0.0653383806347847, + "beta_dpo/beta_margin_std": 0.12112236768007278, + "beta_dpo/beta_used": 0.10641483962535858, + "beta_dpo/beta_used_raw": 0.10641483962535858, + "beta_dpo/gap_mean": 0.33124732971191406, + "beta_dpo/gap_std": 1.0620481967926025, + "beta_dpo/loss_margin_mean": 0.6014441251754761, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.11186696900982615, + "grad_norm": 32.007293701171875, + "learning_rate": 4.998741355957963e-07, + "logits/chosen": 0.15119096636772156, + "logits/rejected": 0.09727491438388824, + "loss": 1.3415, + "step": 74 + }, + { + "beta_dpo/beta": 0.10607866197824478, + "beta_dpo/beta_margin_grad_mean": -0.48331546783447266, + "beta_dpo/beta_margin_grad_std": 0.03126617893576622, + "beta_dpo/beta_margin_mean": 0.06704951822757721, + "beta_dpo/beta_margin_std": 0.1258445382118225, + "beta_dpo/beta_used": 0.10607866197824478, + "beta_dpo/beta_used_raw": 0.10607866197824478, + "beta_dpo/gap_mean": 0.40168869495391846, + "beta_dpo/gap_std": 1.076457142829895, + "beta_dpo/loss_margin_mean": 0.6295863389968872, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.11337868480725624, + "grad_norm": 27.578632354736328, + "learning_rate": 4.998286897523808e-07, + "logits/chosen": 0.1281864494085312, + "logits/rejected": 0.0956953763961792, + "loss": 1.3359, + "step": 75 + }, + { + "beta_dpo/beta": 0.09647442400455475, + "beta_dpo/beta_margin_grad_mean": -0.4936937391757965, + "beta_dpo/beta_margin_grad_std": 0.03319334238767624, + "beta_dpo/beta_margin_mean": 0.025323159992694855, + "beta_dpo/beta_margin_std": 0.1335090696811676, + "beta_dpo/beta_used": 0.09647442400455475, + "beta_dpo/beta_used_raw": 0.09647442400455475, + "beta_dpo/gap_mean": 0.38828492164611816, + "beta_dpo/gap_std": 1.133347988128662, + "beta_dpo/loss_margin_mean": 0.2612752318382263, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.11489040060468632, + "grad_norm": 29.8311767578125, + "learning_rate": 4.997762556634679e-07, + "logits/chosen": 0.13822756707668304, + "logits/rejected": 0.09391121566295624, + "loss": 1.3565, + "step": 76 + }, + { + "beta_dpo/beta": 0.1073828637599945, + "beta_dpo/beta_margin_grad_mean": -0.47993001341819763, + "beta_dpo/beta_margin_grad_std": 0.03186849504709244, + "beta_dpo/beta_margin_mean": 0.08071349561214447, + "beta_dpo/beta_margin_std": 0.12836872041225433, + "beta_dpo/beta_used": 0.1073828637599945, + "beta_dpo/beta_used_raw": 0.1073828637599945, + "beta_dpo/gap_mean": 0.4407605528831482, + "beta_dpo/gap_std": 1.1482113599777222, + "beta_dpo/loss_margin_mean": 0.722518801689148, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1164021164021164, + "grad_norm": 33.737342834472656, + "learning_rate": 4.99716834795752e-07, + "logits/chosen": 0.13610659539699554, + "logits/rejected": 0.09622259438037872, + "loss": 1.3264, + "step": 77 + }, + { + "beta_dpo/beta": 0.09310435503721237, + "beta_dpo/beta_margin_grad_mean": -0.49019360542297363, + "beta_dpo/beta_margin_grad_std": 0.027901530265808105, + "beta_dpo/beta_margin_mean": 0.03937191888689995, + "beta_dpo/beta_margin_std": 0.11205900460481644, + "beta_dpo/beta_used": 0.09310435503721237, + "beta_dpo/beta_used_raw": 0.09310435503721237, + "beta_dpo/gap_mean": 0.4376751780509949, + "beta_dpo/gap_std": 1.1631581783294678, + "beta_dpo/loss_margin_mean": 0.4201837480068207, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.11791383219954649, + "grad_norm": 30.198745727539062, + "learning_rate": 4.996504288113623e-07, + "logits/chosen": 0.12471885979175568, + "logits/rejected": 0.1042494922876358, + "loss": 1.3574, + "step": 78 + }, + { + "beta_dpo/beta": 0.10304860770702362, + "beta_dpo/beta_margin_grad_mean": -0.4760061204433441, + "beta_dpo/beta_margin_grad_std": 0.04387785494327545, + "beta_dpo/beta_margin_mean": 0.09730573743581772, + "beta_dpo/beta_margin_std": 0.17926862835884094, + "beta_dpo/beta_used": 0.10304860770702362, + "beta_dpo/beta_used_raw": 0.10304860770702362, + "beta_dpo/gap_mean": 0.5142155289649963, + "beta_dpo/gap_std": 1.2540637254714966, + "beta_dpo/loss_margin_mean": 0.9398516416549683, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.11942554799697656, + "grad_norm": 31.872982025146484, + "learning_rate": 4.995770395678171e-07, + "logits/chosen": 0.15904495120048523, + "logits/rejected": 0.10070707648992538, + "loss": 1.3309, + "step": 79 + }, + { + "beta_dpo/beta": 0.08738920837640762, + "beta_dpo/beta_margin_grad_mean": -0.48728257417678833, + "beta_dpo/beta_margin_grad_std": 0.03742121160030365, + "beta_dpo/beta_margin_mean": 0.051270920783281326, + "beta_dpo/beta_margin_std": 0.1506371945142746, + "beta_dpo/beta_used": 0.08738920837640762, + "beta_dpo/beta_used_raw": 0.08738920837640762, + "beta_dpo/gap_mean": 0.5226389765739441, + "beta_dpo/gap_std": 1.3193085193634033, + "beta_dpo/loss_margin_mean": 0.5483418703079224, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.12093726379440665, + "grad_norm": 27.30435562133789, + "learning_rate": 4.994966691179711e-07, + "logits/chosen": 0.1488003134727478, + "logits/rejected": 0.08947046101093292, + "loss": 1.3597, + "step": 80 + }, + { + "beta_dpo/beta": 0.09669499099254608, + "beta_dpo/beta_margin_grad_mean": -0.48155760765075684, + "beta_dpo/beta_margin_grad_std": 0.03757631033658981, + "beta_dpo/beta_margin_mean": 0.07457643747329712, + "beta_dpo/beta_margin_std": 0.15281039476394653, + "beta_dpo/beta_used": 0.09669499099254608, + "beta_dpo/beta_used_raw": 0.09669499099254608, + "beta_dpo/gap_mean": 0.5826159119606018, + "beta_dpo/gap_std": 1.3762335777282715, + "beta_dpo/loss_margin_mean": 0.7375200390815735, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.12244897959183673, + "grad_norm": 30.987957000732422, + "learning_rate": 4.994093197099587e-07, + "logits/chosen": 0.11368920654058456, + "logits/rejected": 0.08077847957611084, + "loss": 1.3366, + "step": 81 + }, + { + "beta_dpo/beta": 0.11626774817705154, + "beta_dpo/beta_margin_grad_mean": -0.472137451171875, + "beta_dpo/beta_margin_grad_std": 0.03789715841412544, + "beta_dpo/beta_margin_mean": 0.11208187788724899, + "beta_dpo/beta_margin_std": 0.15328720211982727, + "beta_dpo/beta_used": 0.11626774817705154, + "beta_dpo/beta_used_raw": 0.11626774817705154, + "beta_dpo/gap_mean": 0.6363944411277771, + "beta_dpo/gap_std": 1.3633242845535278, + "beta_dpo/loss_margin_mean": 0.9711171388626099, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.12396069538926682, + "grad_norm": 31.890888214111328, + "learning_rate": 4.993149937871306e-07, + "logits/chosen": 0.12913721799850464, + "logits/rejected": 0.06530788540840149, + "loss": 1.2855, + "step": 82 + }, + { + "beta_dpo/beta": 0.11282624304294586, + "beta_dpo/beta_margin_grad_mean": -0.4715110659599304, + "beta_dpo/beta_margin_grad_std": 0.041592177003622055, + "beta_dpo/beta_margin_mean": 0.11523565649986267, + "beta_dpo/beta_margin_std": 0.1703273057937622, + "beta_dpo/beta_used": 0.11282624304294586, + "beta_dpo/beta_used_raw": 0.11282624304294586, + "beta_dpo/gap_mean": 0.7063510417938232, + "beta_dpo/gap_std": 1.3889222145080566, + "beta_dpo/loss_margin_mean": 1.012617826461792, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1254724111866969, + "grad_norm": 33.445735931396484, + "learning_rate": 4.992136939879856e-07, + "logits/chosen": 0.1792718917131424, + "logits/rejected": 0.13057425618171692, + "loss": 1.287, + "step": 83 + }, + { + "beta_dpo/beta": 0.10456011444330215, + "beta_dpo/beta_margin_grad_mean": -0.4815867245197296, + "beta_dpo/beta_margin_grad_std": 0.04160284623503685, + "beta_dpo/beta_margin_mean": 0.07425189018249512, + "beta_dpo/beta_margin_std": 0.16766425967216492, + "beta_dpo/beta_used": 0.10456011444330215, + "beta_dpo/beta_used_raw": 0.10456011444330215, + "beta_dpo/gap_mean": 0.7209852933883667, + "beta_dpo/gap_std": 1.4176113605499268, + "beta_dpo/loss_margin_mean": 0.7021726369857788, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.12698412698412698, + "grad_norm": 34.64680862426758, + "learning_rate": 4.991054231460969e-07, + "logits/chosen": 0.11454911530017853, + "logits/rejected": 0.07707634568214417, + "loss": 1.31, + "step": 84 + }, + { + "beta_dpo/beta": 0.12070846557617188, + "beta_dpo/beta_margin_grad_mean": -0.46796914935112, + "beta_dpo/beta_margin_grad_std": 0.05074300616979599, + "beta_dpo/beta_margin_mean": 0.12967704236507416, + "beta_dpo/beta_margin_std": 0.20634624361991882, + "beta_dpo/beta_used": 0.12070846557617188, + "beta_dpo/beta_used_raw": 0.12070846557617188, + "beta_dpo/gap_mean": 0.757364809513092, + "beta_dpo/gap_std": 1.4538509845733643, + "beta_dpo/loss_margin_mean": 1.0756046772003174, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.12849584278155707, + "grad_norm": 35.5852165222168, + "learning_rate": 4.989901842900325e-07, + "logits/chosen": 0.1759287714958191, + "logits/rejected": 0.13176394999027252, + "loss": 1.2621, + "step": 85 + }, + { + "beta_dpo/beta": 0.08451604843139648, + "beta_dpo/beta_margin_grad_mean": -0.48312827944755554, + "beta_dpo/beta_margin_grad_std": 0.03507286682724953, + "beta_dpo/beta_margin_mean": 0.06791017949581146, + "beta_dpo/beta_margin_std": 0.14122274518013, + "beta_dpo/beta_used": 0.08451604843139648, + "beta_dpo/beta_used_raw": 0.08451604843139648, + "beta_dpo/gap_mean": 0.7781229615211487, + "beta_dpo/gap_std": 1.5088303089141846, + "beta_dpo/loss_margin_mean": 0.7961375713348389, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.13000755857898716, + "grad_norm": 24.137842178344727, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": 0.18794915080070496, + "logits/rejected": 0.16945751011371613, + "loss": 1.3456, + "step": 86 + }, + { + "beta_dpo/beta": 0.10423195362091064, + "beta_dpo/beta_margin_grad_mean": -0.47645512223243713, + "beta_dpo/beta_margin_grad_std": 0.049108896404504776, + "beta_dpo/beta_margin_mean": 0.09508252143859863, + "beta_dpo/beta_margin_std": 0.19925162196159363, + "beta_dpo/beta_used": 0.10423195362091064, + "beta_dpo/beta_used_raw": 0.10423195362091064, + "beta_dpo/gap_mean": 0.8129284381866455, + "beta_dpo/gap_std": 1.571422815322876, + "beta_dpo/loss_margin_mean": 0.8943271636962891, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.13151927437641722, + "grad_norm": 30.79345703125, + "learning_rate": 4.987388156241114e-07, + "logits/chosen": 0.1435273289680481, + "logits/rejected": 0.0876278430223465, + "loss": 1.3003, + "step": 87 + }, + { + "beta_dpo/beta": 0.08814011514186859, + "beta_dpo/beta_margin_grad_mean": -0.4850381910800934, + "beta_dpo/beta_margin_grad_std": 0.05164018273353577, + "beta_dpo/beta_margin_mean": 0.06059926748275757, + "beta_dpo/beta_margin_std": 0.20973937213420868, + "beta_dpo/beta_used": 0.08814011514186859, + "beta_dpo/beta_used_raw": 0.08814011514186859, + "beta_dpo/gap_mean": 0.8027680516242981, + "beta_dpo/gap_std": 1.6989638805389404, + "beta_dpo/loss_margin_mean": 0.6227868795394897, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1330309901738473, + "grad_norm": 25.85154914855957, + "learning_rate": 4.986026928455767e-07, + "logits/chosen": 0.24119189381599426, + "logits/rejected": 0.21431989967823029, + "loss": 1.3356, + "step": 88 + }, + { + "beta_dpo/beta": 0.10085612535476685, + "beta_dpo/beta_margin_grad_mean": -0.4691811800003052, + "beta_dpo/beta_margin_grad_std": 0.0591701865196228, + "beta_dpo/beta_margin_mean": 0.12679243087768555, + "beta_dpo/beta_margin_std": 0.2483137995004654, + "beta_dpo/beta_used": 0.10085612535476685, + "beta_dpo/beta_used_raw": 0.10085612535476685, + "beta_dpo/gap_mean": 0.796160101890564, + "beta_dpo/gap_std": 1.7879228591918945, + "beta_dpo/loss_margin_mean": 1.1158039569854736, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1345427059712774, + "grad_norm": 32.81768798828125, + "learning_rate": 4.984596161153135e-07, + "logits/chosen": 0.23640497028827667, + "logits/rejected": 0.15615656971931458, + "loss": 1.2946, + "step": 89 + }, + { + "beta_dpo/beta": 0.09922297298908234, + "beta_dpo/beta_margin_grad_mean": -0.474229097366333, + "beta_dpo/beta_margin_grad_std": 0.05109386146068573, + "beta_dpo/beta_margin_mean": 0.10420099645853043, + "beta_dpo/beta_margin_std": 0.20693768560886383, + "beta_dpo/beta_used": 0.09922297298908234, + "beta_dpo/beta_used_raw": 0.09922297298908234, + "beta_dpo/gap_mean": 0.8837804794311523, + "beta_dpo/gap_std": 1.8559458255767822, + "beta_dpo/loss_margin_mean": 1.0424596071243286, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1360544217687075, + "grad_norm": 32.41039276123047, + "learning_rate": 4.983095894354857e-07, + "logits/chosen": 0.16189423203468323, + "logits/rejected": 0.10690204054117203, + "loss": 1.3085, + "step": 90 + }, + { + "beta_dpo/beta": 0.09870962798595428, + "beta_dpo/beta_margin_grad_mean": -0.47690755128860474, + "beta_dpo/beta_margin_grad_std": 0.05698744207620621, + "beta_dpo/beta_margin_mean": 0.09375400096178055, + "beta_dpo/beta_margin_std": 0.23264114558696747, + "beta_dpo/beta_used": 0.09870962798595428, + "beta_dpo/beta_used_raw": 0.09870962798595428, + "beta_dpo/gap_mean": 0.8797661662101746, + "beta_dpo/gap_std": 1.9356528520584106, + "beta_dpo/loss_margin_mean": 0.9830507636070251, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.13756613756613756, + "grad_norm": 27.203453063964844, + "learning_rate": 4.98152617002662e-07, + "logits/chosen": 0.14668001234531403, + "logits/rejected": 0.10438361018896103, + "loss": 1.3035, + "step": 91 + }, + { + "beta_dpo/beta": 0.09463343024253845, + "beta_dpo/beta_margin_grad_mean": -0.4683462679386139, + "beta_dpo/beta_margin_grad_std": 0.06076580286026001, + "beta_dpo/beta_margin_mean": 0.12961620092391968, + "beta_dpo/beta_margin_std": 0.25017645955085754, + "beta_dpo/beta_used": 0.09463343024253845, + "beta_dpo/beta_used_raw": 0.09463343024253845, + "beta_dpo/gap_mean": 0.937788188457489, + "beta_dpo/gap_std": 2.049354314804077, + "beta_dpo/loss_margin_mean": 1.3172084093093872, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.13907785336356765, + "grad_norm": 27.962493896484375, + "learning_rate": 4.979887032076988e-07, + "logits/chosen": 0.19484901428222656, + "logits/rejected": 0.15675948560237885, + "loss": 1.3129, + "step": 92 + }, + { + "beta_dpo/beta": 0.08639341592788696, + "beta_dpo/beta_margin_grad_mean": -0.48290809988975525, + "beta_dpo/beta_margin_grad_std": 0.053895775228738785, + "beta_dpo/beta_margin_mean": 0.07019705325365067, + "beta_dpo/beta_margin_std": 0.2225038707256317, + "beta_dpo/beta_used": 0.08639341592788696, + "beta_dpo/beta_used_raw": 0.08639341592788696, + "beta_dpo/gap_mean": 0.9476668834686279, + "beta_dpo/gap_std": 2.1445741653442383, + "beta_dpo/loss_margin_mean": 0.8015139102935791, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.14058956916099774, + "grad_norm": 24.30367088317871, + "learning_rate": 4.978178526356172e-07, + "logits/chosen": 0.19220688939094543, + "logits/rejected": 0.1637325882911682, + "loss": 1.3297, + "step": 93 + }, + { + "beta_dpo/beta": 0.12751072645187378, + "beta_dpo/beta_margin_grad_mean": -0.4522378444671631, + "beta_dpo/beta_margin_grad_std": 0.09412048012018204, + "beta_dpo/beta_margin_mean": 0.19668596982955933, + "beta_dpo/beta_margin_std": 0.4031871557235718, + "beta_dpo/beta_used": 0.12751072645187378, + "beta_dpo/beta_used_raw": 0.12751072645187378, + "beta_dpo/gap_mean": 1.0329408645629883, + "beta_dpo/gap_std": 2.292163372039795, + "beta_dpo/loss_margin_mean": 1.5427203178405762, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1421012849584278, + "grad_norm": 37.51604461669922, + "learning_rate": 4.976400700654751e-07, + "logits/chosen": 0.25582846999168396, + "logits/rejected": 0.21623843908309937, + "loss": 1.217, + "step": 94 + }, + { + "beta_dpo/beta": 0.11038395762443542, + "beta_dpo/beta_margin_grad_mean": -0.46619194746017456, + "beta_dpo/beta_margin_grad_std": 0.07264120876789093, + "beta_dpo/beta_margin_mean": 0.13830190896987915, + "beta_dpo/beta_margin_std": 0.297305703163147, + "beta_dpo/beta_used": 0.11038395762443542, + "beta_dpo/beta_used_raw": 0.11038395762443542, + "beta_dpo/gap_mean": 1.1002323627471924, + "beta_dpo/gap_std": 2.4173107147216797, + "beta_dpo/loss_margin_mean": 1.278725028038025, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1436130007558579, + "grad_norm": 36.0577278137207, + "learning_rate": 4.974553604702332e-07, + "logits/chosen": 0.1208517849445343, + "logits/rejected": 0.06007538363337517, + "loss": 1.2627, + "step": 95 + }, + { + "beta_dpo/beta": 0.07217492908239365, + "beta_dpo/beta_margin_grad_mean": -0.4727436304092407, + "beta_dpo/beta_margin_grad_std": 0.05908944830298424, + "beta_dpo/beta_margin_mean": 0.11143878847360611, + "beta_dpo/beta_margin_std": 0.24230976402759552, + "beta_dpo/beta_used": 0.07217492908239365, + "beta_dpo/beta_used_raw": 0.07217492908239365, + "beta_dpo/gap_mean": 1.1477243900299072, + "beta_dpo/gap_std": 2.6134121417999268, + "beta_dpo/loss_margin_mean": 1.502234935760498, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.14512471655328799, + "grad_norm": 21.8409366607666, + "learning_rate": 4.972637290166157e-07, + "logits/chosen": 0.21506036818027496, + "logits/rejected": 0.17100423574447632, + "loss": 1.3403, + "step": 96 + }, + { + "beta_dpo/beta": 0.07082679867744446, + "beta_dpo/beta_margin_grad_mean": -0.4855877757072449, + "beta_dpo/beta_margin_grad_std": 0.06629322469234467, + "beta_dpo/beta_margin_mean": 0.059007786214351654, + "beta_dpo/beta_margin_std": 0.2750501334667206, + "beta_dpo/beta_used": 0.07082679867744446, + "beta_dpo/beta_used_raw": 0.07082679867744446, + "beta_dpo/gap_mean": 1.14254891872406, + "beta_dpo/gap_std": 2.7921524047851562, + "beta_dpo/loss_margin_mean": 0.8266361951828003, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.14663643235071808, + "grad_norm": 24.70173454284668, + "learning_rate": 4.970651810649666e-07, + "logits/chosen": 0.12239768356084824, + "logits/rejected": 0.07986140251159668, + "loss": 1.3453, + "step": 97 + }, + { + "beta_dpo/beta": 0.09226500988006592, + "beta_dpo/beta_margin_grad_mean": -0.48356226086616516, + "beta_dpo/beta_margin_grad_std": 0.05798405781388283, + "beta_dpo/beta_margin_mean": 0.06667763739824295, + "beta_dpo/beta_margin_std": 0.23548774421215057, + "beta_dpo/beta_used": 0.09226500988006592, + "beta_dpo/beta_used_raw": 0.09226500988006592, + "beta_dpo/gap_mean": 1.052299976348877, + "beta_dpo/gap_std": 2.8152146339416504, + "beta_dpo/loss_margin_mean": 0.6153003573417664, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.14814814814814814, + "grad_norm": 27.817951202392578, + "learning_rate": 4.968597221690985e-07, + "logits/chosen": 0.21289318799972534, + "logits/rejected": 0.1862494945526123, + "loss": 1.3007, + "step": 98 + }, + { + "beta_dpo/beta": 0.07890302687883377, + "beta_dpo/beta_margin_grad_mean": -0.48339250683784485, + "beta_dpo/beta_margin_grad_std": 0.07327800989151001, + "beta_dpo/beta_margin_mean": 0.06827875226736069, + "beta_dpo/beta_margin_std": 0.3037020266056061, + "beta_dpo/beta_used": 0.07890302687883377, + "beta_dpo/beta_used_raw": 0.07890302687883377, + "beta_dpo/gap_mean": 0.9998865723609924, + "beta_dpo/gap_std": 2.9074761867523193, + "beta_dpo/loss_margin_mean": 0.8886429071426392, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.14965986394557823, + "grad_norm": 25.86090660095215, + "learning_rate": 4.966473580761389e-07, + "logits/chosen": 0.247923344373703, + "logits/rejected": 0.20911335945129395, + "loss": 1.3426, + "step": 99 + }, + { + "beta_dpo/beta": 0.10054519027471542, + "beta_dpo/beta_margin_grad_mean": -0.4732176661491394, + "beta_dpo/beta_margin_grad_std": 0.11176663637161255, + "beta_dpo/beta_margin_mean": 0.11648314446210861, + "beta_dpo/beta_margin_std": 0.5131522417068481, + "beta_dpo/beta_used": 0.10054519027471542, + "beta_dpo/beta_used_raw": 0.10054519027471542, + "beta_dpo/gap_mean": 0.979209303855896, + "beta_dpo/gap_std": 3.17752742767334, + "beta_dpo/loss_margin_mean": 1.1264538764953613, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.15117157974300832, + "grad_norm": 35.79426956176758, + "learning_rate": 4.964280947263676e-07, + "logits/chosen": 0.2418309450149536, + "logits/rejected": 0.2337619811296463, + "loss": 1.3014, + "step": 100 + }, + { + "epoch": 0.15117157974300832, + "eval_beta_dpo/beta": 0.11826334148645401, + "eval_beta_dpo/beta_margin_grad_mean": -0.4594725966453552, + "eval_beta_dpo/beta_margin_grad_std": 0.1056964099407196, + "eval_beta_dpo/beta_margin_mean": 0.1789473295211792, + "eval_beta_dpo/beta_margin_std": 0.47487327456474304, + "eval_beta_dpo/beta_used": 0.11826334148645401, + "eval_beta_dpo/beta_used_raw": 0.11826334148645401, + "eval_beta_dpo/gap_mean": 1.0180015563964844, + "eval_beta_dpo/gap_std": 3.335969924926758, + "eval_beta_dpo/loss_margin_mean": 1.3223904371261597, + "eval_beta_dpo/mask_keep_frac": 1.0, + "eval_logits/chosen": 0.25719377398490906, + "eval_logits/rejected": 0.22068104147911072, + "eval_loss": 0.6391391754150391, + "eval_runtime": 39.093, + "eval_samples_per_second": 58.911, + "eval_steps_per_second": 1.842, + "step": 100 + }, + { + "beta_dpo/beta": 0.12229928374290466, + "beta_dpo/beta_margin_grad_mean": -0.4488358199596405, + "beta_dpo/beta_margin_grad_std": 0.0993746891617775, + "beta_dpo/beta_margin_mean": 0.21102577447891235, + "beta_dpo/beta_margin_std": 0.4214361906051636, + "beta_dpo/beta_used": 0.12229928374290466, + "beta_dpo/beta_used_raw": 0.12229928374290466, + "beta_dpo/gap_mean": 1.1088917255401611, + "beta_dpo/gap_std": 3.3312177658081055, + "beta_dpo/loss_margin_mean": 1.7355849742889404, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.15268329554043839, + "grad_norm": 31.71246910095215, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": 0.24451228976249695, + "logits/rejected": 0.20558559894561768, + "loss": 1.2362, + "step": 101 + }, + { + "beta_dpo/beta": 0.05900692939758301, + "beta_dpo/beta_margin_grad_mean": -0.48880329728126526, + "beta_dpo/beta_margin_grad_std": 0.07061181217432022, + "beta_dpo/beta_margin_mean": 0.04781756550073624, + "beta_dpo/beta_margin_std": 0.3020632565021515, + "beta_dpo/beta_used": 0.05900692939758301, + "beta_dpo/beta_used_raw": 0.05900692939758301, + "beta_dpo/gap_mean": 1.0780134201049805, + "beta_dpo/gap_std": 3.5245532989501953, + "beta_dpo/loss_margin_mean": 0.6960800886154175, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.15419501133786848, + "grad_norm": 19.128576278686523, + "learning_rate": 4.959688949822748e-07, + "logits/chosen": 0.1611076295375824, + "logits/rejected": 0.12269513309001923, + "loss": 1.3618, + "step": 102 + }, + { + "beta_dpo/beta": 0.10784563422203064, + "beta_dpo/beta_margin_grad_mean": -0.46517613530158997, + "beta_dpo/beta_margin_grad_std": 0.0859994888305664, + "beta_dpo/beta_margin_mean": 0.14561301469802856, + "beta_dpo/beta_margin_std": 0.3582909107208252, + "beta_dpo/beta_used": 0.10784563422203064, + "beta_dpo/beta_used_raw": 0.10784563422203064, + "beta_dpo/gap_mean": 1.0823962688446045, + "beta_dpo/gap_std": 3.568239450454712, + "beta_dpo/loss_margin_mean": 1.2876253128051758, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.15570672713529857, + "grad_norm": 35.80467987060547, + "learning_rate": 4.957289714327572e-07, + "logits/chosen": 0.2758740782737732, + "logits/rejected": 0.2419765442609787, + "loss": 1.2792, + "step": 103 + }, + { + "beta_dpo/beta": 0.07326889038085938, + "beta_dpo/beta_margin_grad_mean": -0.4682692885398865, + "beta_dpo/beta_margin_grad_std": 0.08652123063802719, + "beta_dpo/beta_margin_mean": 0.13173505663871765, + "beta_dpo/beta_margin_std": 0.3622894287109375, + "beta_dpo/beta_used": 0.07326889038085938, + "beta_dpo/beta_used_raw": 0.07326889038085938, + "beta_dpo/gap_mean": 1.222848892211914, + "beta_dpo/gap_std": 3.759814500808716, + "beta_dpo/loss_margin_mean": 1.9680804014205933, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.15721844293272866, + "grad_norm": 24.454832077026367, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": 0.2790372967720032, + "logits/rejected": 0.19069775938987732, + "loss": 1.3294, + "step": 104 + }, + { + "beta_dpo/beta": 0.09441901743412018, + "beta_dpo/beta_margin_grad_mean": -0.4641573131084442, + "beta_dpo/beta_margin_grad_std": 0.09842989593744278, + "beta_dpo/beta_margin_mean": 0.15091301500797272, + "beta_dpo/beta_margin_std": 0.415006160736084, + "beta_dpo/beta_used": 0.09441901743412018, + "beta_dpo/beta_used_raw": 0.09441901743412018, + "beta_dpo/gap_mean": 1.335235834121704, + "beta_dpo/gap_std": 4.034293174743652, + "beta_dpo/loss_margin_mean": 1.6171789169311523, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.15873015873015872, + "grad_norm": 34.57042694091797, + "learning_rate": 4.952285105344791e-07, + "logits/chosen": 0.18213523924350739, + "logits/rejected": 0.12961822748184204, + "loss": 1.2843, + "step": 105 + }, + { + "beta_dpo/beta": 0.1123632863163948, + "beta_dpo/beta_margin_grad_mean": -0.46128907799720764, + "beta_dpo/beta_margin_grad_std": 0.12345249205827713, + "beta_dpo/beta_margin_mean": 0.16074322164058685, + "beta_dpo/beta_margin_std": 0.5476335287094116, + "beta_dpo/beta_used": 0.1123632863163948, + "beta_dpo/beta_used_raw": 0.1123632863163948, + "beta_dpo/gap_mean": 1.3533254861831665, + "beta_dpo/gap_std": 4.168464660644531, + "beta_dpo/loss_margin_mean": 1.444665551185608, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1602418745275888, + "grad_norm": 35.42899703979492, + "learning_rate": 4.949679871846857e-07, + "logits/chosen": 0.25271597504615784, + "logits/rejected": 0.2385634183883667, + "loss": 1.2505, + "step": 106 + }, + { + "beta_dpo/beta": 0.10497646033763885, + "beta_dpo/beta_margin_grad_mean": -0.475864440202713, + "beta_dpo/beta_margin_grad_std": 0.13249976933002472, + "beta_dpo/beta_margin_mean": 0.09398960322141647, + "beta_dpo/beta_margin_std": 0.5902509093284607, + "beta_dpo/beta_used": 0.10497646033763885, + "beta_dpo/beta_used_raw": 0.10497646033763885, + "beta_dpo/gap_mean": 1.3109784126281738, + "beta_dpo/gap_std": 4.361145973205566, + "beta_dpo/loss_margin_mean": 0.7373759150505066, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1617535903250189, + "grad_norm": 33.42430114746094, + "learning_rate": 4.947006115536947e-07, + "logits/chosen": 0.17871206998825073, + "logits/rejected": 0.1560344398021698, + "loss": 1.2682, + "step": 107 + }, + { + "beta_dpo/beta": 0.13177213072776794, + "beta_dpo/beta_margin_grad_mean": -0.45206108689308167, + "beta_dpo/beta_margin_grad_std": 0.14440619945526123, + "beta_dpo/beta_margin_mean": 0.21699368953704834, + "beta_dpo/beta_margin_std": 0.6584862470626831, + "beta_dpo/beta_used": 0.13177213072776794, + "beta_dpo/beta_used_raw": 0.13177213072776794, + "beta_dpo/gap_mean": 1.2987751960754395, + "beta_dpo/gap_std": 4.507687568664551, + "beta_dpo/loss_margin_mean": 1.6458370685577393, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.16326530612244897, + "grad_norm": 37.21890640258789, + "learning_rate": 4.944263911205772e-07, + "logits/chosen": 0.19411234557628632, + "logits/rejected": 0.16377675533294678, + "loss": 1.2105, + "step": 108 + }, + { + "beta_dpo/beta": 0.08962151408195496, + "beta_dpo/beta_margin_grad_mean": -0.44173574447631836, + "beta_dpo/beta_margin_grad_std": 0.10800231993198395, + "beta_dpo/beta_margin_mean": 0.2577720284461975, + "beta_dpo/beta_margin_std": 0.49331575632095337, + "beta_dpo/beta_used": 0.08962151408195496, + "beta_dpo/beta_used_raw": 0.08962151408195496, + "beta_dpo/gap_mean": 1.5422182083129883, + "beta_dpo/gap_std": 4.703213691711426, + "beta_dpo/loss_margin_mean": 2.883152961730957, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.16477702191987906, + "grad_norm": 26.614168167114258, + "learning_rate": 4.941453335558681e-07, + "logits/chosen": 0.18414027988910675, + "logits/rejected": 0.13302767276763916, + "loss": 1.2804, + "step": 109 + }, + { + "beta_dpo/beta": 0.07986485213041306, + "beta_dpo/beta_margin_grad_mean": -0.49511831998825073, + "beta_dpo/beta_margin_grad_std": 0.10436120629310608, + "beta_dpo/beta_margin_mean": 0.008938713930547237, + "beta_dpo/beta_margin_std": 0.477845698595047, + "beta_dpo/beta_used": 0.07986485213041306, + "beta_dpo/beta_used_raw": 0.07986485213041306, + "beta_dpo/gap_mean": 1.3971412181854248, + "beta_dpo/gap_std": 4.875141143798828, + "beta_dpo/loss_margin_mean": 0.1147775948047638, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.16628873771730915, + "grad_norm": 26.075336456298828, + "learning_rate": 4.938574467213517e-07, + "logits/chosen": 0.19698774814605713, + "logits/rejected": 0.2059447467327118, + "loss": 1.3093, + "step": 110 + }, + { + "beta_dpo/beta": 0.0918872281908989, + "beta_dpo/beta_margin_grad_mean": -0.46057236194610596, + "beta_dpo/beta_margin_grad_std": 0.11816360801458359, + "beta_dpo/beta_margin_mean": 0.17831836640834808, + "beta_dpo/beta_margin_std": 0.5313184857368469, + "beta_dpo/beta_used": 0.0918872281908989, + "beta_dpo/beta_used_raw": 0.0918872281908989, + "beta_dpo/gap_mean": 1.4028397798538208, + "beta_dpo/gap_std": 5.010308742523193, + "beta_dpo/loss_margin_mean": 1.9166357517242432, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.16780045351473924, + "grad_norm": 26.0497989654541, + "learning_rate": 4.935627386698418e-07, + "logits/chosen": 0.28147315979003906, + "logits/rejected": 0.24575236439704895, + "loss": 1.3027, + "step": 111 + }, + { + "beta_dpo/beta": 0.1317475140094757, + "beta_dpo/beta_margin_grad_mean": -0.43314915895462036, + "beta_dpo/beta_margin_grad_std": 0.14976409077644348, + "beta_dpo/beta_margin_mean": 0.2932818531990051, + "beta_dpo/beta_margin_std": 0.6779165863990784, + "beta_dpo/beta_used": 0.1317475140094757, + "beta_dpo/beta_used_raw": 0.1317475140094757, + "beta_dpo/gap_mean": 1.5489156246185303, + "beta_dpo/gap_std": 5.076373100280762, + "beta_dpo/loss_margin_mean": 2.225752353668213, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1693121693121693, + "grad_norm": 42.69919967651367, + "learning_rate": 4.932612176449559e-07, + "logits/chosen": 0.22592046856880188, + "logits/rejected": 0.16073720157146454, + "loss": 1.1894, + "step": 112 + }, + { + "beta_dpo/beta": 0.09787484258413315, + "beta_dpo/beta_margin_grad_mean": -0.47497114539146423, + "beta_dpo/beta_margin_grad_std": 0.12046003341674805, + "beta_dpo/beta_margin_mean": 0.10006435960531235, + "beta_dpo/beta_margin_std": 0.5390375256538391, + "beta_dpo/beta_used": 0.09787484258413315, + "beta_dpo/beta_used_raw": 0.09787484258413315, + "beta_dpo/gap_mean": 1.505769968032837, + "beta_dpo/gap_std": 5.138182640075684, + "beta_dpo/loss_margin_mean": 1.1710374355316162, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1708238851095994, + "grad_norm": 42.932861328125, + "learning_rate": 4.929528920808854e-07, + "logits/chosen": 0.23544499278068542, + "logits/rejected": 0.19660460948944092, + "loss": 1.2689, + "step": 113 + }, + { + "beta_dpo/beta": 0.08954670280218124, + "beta_dpo/beta_margin_grad_mean": -0.4552899897098541, + "beta_dpo/beta_margin_grad_std": 0.11374201625585556, + "beta_dpo/beta_margin_mean": 0.19355130195617676, + "beta_dpo/beta_margin_std": 0.4905747175216675, + "beta_dpo/beta_used": 0.08954670280218124, + "beta_dpo/beta_used_raw": 0.08954670280218124, + "beta_dpo/gap_mean": 1.615609884262085, + "beta_dpo/gap_std": 5.165139198303223, + "beta_dpo/loss_margin_mean": 2.1351757049560547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.17233560090702948, + "grad_norm": 31.22429656982422, + "learning_rate": 4.92637770602159e-07, + "logits/chosen": 0.2743056118488312, + "logits/rejected": 0.21552340686321259, + "loss": 1.2937, + "step": 114 + }, + { + "beta_dpo/beta": 0.09392043948173523, + "beta_dpo/beta_margin_grad_mean": -0.45995837450027466, + "beta_dpo/beta_margin_grad_std": 0.1096334159374237, + "beta_dpo/beta_margin_mean": 0.17287258803844452, + "beta_dpo/beta_margin_std": 0.4752873182296753, + "beta_dpo/beta_used": 0.09392043948173523, + "beta_dpo/beta_used_raw": 0.09392043948173523, + "beta_dpo/gap_mean": 1.685424566268921, + "beta_dpo/gap_std": 5.101430892944336, + "beta_dpo/loss_margin_mean": 1.8901252746582031, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.17384731670445955, + "grad_norm": 28.878787994384766, + "learning_rate": 4.923158620234019e-07, + "logits/chosen": 0.2532087564468384, + "logits/rejected": 0.1925436407327652, + "loss": 1.2787, + "step": 115 + }, + { + "beta_dpo/beta": 0.12234266102313995, + "beta_dpo/beta_margin_grad_mean": -0.4119870066642761, + "beta_dpo/beta_margin_grad_std": 0.1418372392654419, + "beta_dpo/beta_margin_mean": 0.3979705572128296, + "beta_dpo/beta_margin_std": 0.6543411016464233, + "beta_dpo/beta_used": 0.12234266102313995, + "beta_dpo/beta_used_raw": 0.12234266102313995, + "beta_dpo/gap_mean": 1.851987600326538, + "beta_dpo/gap_std": 5.066287994384766, + "beta_dpo/loss_margin_mean": 3.1081788539886475, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.17535903250188964, + "grad_norm": 32.4793701171875, + "learning_rate": 4.91987175349089e-07, + "logits/chosen": 0.2658846974372864, + "logits/rejected": 0.1990557312965393, + "loss": 1.2118, + "step": 116 + }, + { + "beta_dpo/beta": 0.084126777946949, + "beta_dpo/beta_margin_grad_mean": -0.44095876812934875, + "beta_dpo/beta_margin_grad_std": 0.12159502506256104, + "beta_dpo/beta_margin_mean": 0.27267566323280334, + "beta_dpo/beta_margin_std": 0.5938247442245483, + "beta_dpo/beta_used": 0.084126777946949, + "beta_dpo/beta_used_raw": 0.084126777946949, + "beta_dpo/gap_mean": 2.017143726348877, + "beta_dpo/gap_std": 5.090367317199707, + "beta_dpo/loss_margin_mean": 1.8112168312072754, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.17687074829931973, + "grad_norm": 25.763986587524414, + "learning_rate": 4.916517197732933e-07, + "logits/chosen": 0.2703200876712799, + "logits/rejected": 0.23447707295417786, + "loss": 1.2039, + "step": 117 + }, + { + "beta_dpo/beta": 0.12284408509731293, + "beta_dpo/beta_margin_grad_mean": -0.4303743243217468, + "beta_dpo/beta_margin_grad_std": 0.14086388051509857, + "beta_dpo/beta_margin_mean": 0.31298136711120605, + "beta_dpo/beta_margin_std": 0.6518138647079468, + "beta_dpo/beta_used": 0.12284408509731293, + "beta_dpo/beta_used_raw": 0.12284408509731293, + "beta_dpo/gap_mean": 2.001436710357666, + "beta_dpo/gap_std": 5.041218280792236, + "beta_dpo/loss_margin_mean": 2.388618230819702, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.17838246409674982, + "grad_norm": 36.57698059082031, + "learning_rate": 4.913095046794281e-07, + "logits/chosen": 0.3213784098625183, + "logits/rejected": 0.2811124324798584, + "loss": 1.1707, + "step": 118 + }, + { + "beta_dpo/beta": 0.09360114485025406, + "beta_dpo/beta_margin_grad_mean": -0.4581930935382843, + "beta_dpo/beta_margin_grad_std": 0.14400675892829895, + "beta_dpo/beta_margin_mean": 0.19105002284049988, + "beta_dpo/beta_margin_std": 0.678154706954956, + "beta_dpo/beta_used": 0.09360114485025406, + "beta_dpo/beta_used_raw": 0.09360114485025406, + "beta_dpo/gap_mean": 1.9388705492019653, + "beta_dpo/gap_std": 5.184415817260742, + "beta_dpo/loss_margin_mean": 1.7741563320159912, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.17989417989417988, + "grad_norm": 29.395437240600586, + "learning_rate": 4.909605396399855e-07, + "logits/chosen": 0.24336868524551392, + "logits/rejected": 0.2054998278617859, + "loss": 1.2536, + "step": 119 + }, + { + "beta_dpo/beta": 0.11056067794561386, + "beta_dpo/beta_margin_grad_mean": -0.42801910638809204, + "beta_dpo/beta_margin_grad_std": 0.10624414682388306, + "beta_dpo/beta_margin_mean": 0.30804508924484253, + "beta_dpo/beta_margin_std": 0.4607044756412506, + "beta_dpo/beta_used": 0.11056067794561386, + "beta_dpo/beta_used_raw": 0.11056067794561386, + "beta_dpo/gap_mean": 2.0835390090942383, + "beta_dpo/gap_std": 5.089218616485596, + "beta_dpo/loss_margin_mean": 2.8222334384918213, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.18140589569160998, + "grad_norm": 33.18598175048828, + "learning_rate": 4.906048344162676e-07, + "logits/chosen": 0.24057039618492126, + "logits/rejected": 0.1823032796382904, + "loss": 1.1735, + "step": 120 + }, + { + "beta_dpo/beta": 0.06963507831096649, + "beta_dpo/beta_margin_grad_mean": -0.44736799597740173, + "beta_dpo/beta_margin_grad_std": 0.0942336916923523, + "beta_dpo/beta_margin_mean": 0.22871233522891998, + "beta_dpo/beta_margin_std": 0.4231445789337158, + "beta_dpo/beta_used": 0.06963507831096649, + "beta_dpo/beta_used_raw": 0.06963507831096649, + "beta_dpo/gap_mean": 2.3149490356445312, + "beta_dpo/gap_std": 5.239107131958008, + "beta_dpo/loss_margin_mean": 3.3256545066833496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.18291761148904007, + "grad_norm": 21.643383026123047, + "learning_rate": 4.902423989581143e-07, + "logits/chosen": 0.3411806523799896, + "logits/rejected": 0.2576831877231598, + "loss": 1.2754, + "step": 121 + }, + { + "beta_dpo/beta": 0.04715808108448982, + "beta_dpo/beta_margin_grad_mean": -0.46957728266716003, + "beta_dpo/beta_margin_grad_std": 0.09799901396036148, + "beta_dpo/beta_margin_mean": 0.14232568442821503, + "beta_dpo/beta_margin_std": 0.4737836420536041, + "beta_dpo/beta_used": 0.04715808108448982, + "beta_dpo/beta_used_raw": 0.04715808108448982, + "beta_dpo/gap_mean": 2.383568048477173, + "beta_dpo/gap_std": 5.513972282409668, + "beta_dpo/loss_margin_mean": 2.6904125213623047, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.18442932728647016, + "grad_norm": 18.69718360900879, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": 0.27230706810951233, + "logits/rejected": 0.23791071772575378, + "loss": 1.3057, + "step": 122 + }, + { + "beta_dpo/beta": 0.04343951866030693, + "beta_dpo/beta_margin_grad_mean": -0.4723133146762848, + "beta_dpo/beta_margin_grad_std": 0.05662338435649872, + "beta_dpo/beta_margin_mean": 0.11453551054000854, + "beta_dpo/beta_margin_std": 0.2412138134241104, + "beta_dpo/beta_used": 0.04343951866030693, + "beta_dpo/beta_used_raw": 0.04343951866030693, + "beta_dpo/gap_mean": 2.4602584838867188, + "beta_dpo/gap_std": 5.645776748657227, + "beta_dpo/loss_margin_mean": 2.6306912899017334, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.18594104308390022, + "grad_norm": 13.800962448120117, + "learning_rate": 4.894973780788722e-07, + "logits/chosen": 0.26904135942459106, + "logits/rejected": 0.2292679101228714, + "loss": 1.3275, + "step": 123 + }, + { + "beta_dpo/beta": 0.14997775852680206, + "beta_dpo/beta_margin_grad_mean": -0.39104214310646057, + "beta_dpo/beta_margin_grad_std": 0.1786145716905594, + "beta_dpo/beta_margin_mean": 0.5140237808227539, + "beta_dpo/beta_margin_std": 0.888462483882904, + "beta_dpo/beta_used": 0.14997775852680206, + "beta_dpo/beta_used_raw": 0.14997775852680206, + "beta_dpo/gap_mean": 2.6044697761535645, + "beta_dpo/gap_std": 5.672214508056641, + "beta_dpo/loss_margin_mean": 3.4369006156921387, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1874527588813303, + "grad_norm": 37.8648796081543, + "learning_rate": 4.89114813497619e-07, + "logits/chosen": 0.278278112411499, + "logits/rejected": 0.22016727924346924, + "loss": 1.0223, + "step": 124 + }, + { + "beta_dpo/beta": 0.052618607878685, + "beta_dpo/beta_margin_grad_mean": -0.4515383243560791, + "beta_dpo/beta_margin_grad_std": 0.09755320847034454, + "beta_dpo/beta_margin_mean": 0.21529045701026917, + "beta_dpo/beta_margin_std": 0.4597736895084381, + "beta_dpo/beta_used": 0.052618607878685, + "beta_dpo/beta_used_raw": 0.052618607878685, + "beta_dpo/gap_mean": 2.7607717514038086, + "beta_dpo/gap_std": 5.762345314025879, + "beta_dpo/loss_margin_mean": 3.107497453689575, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1889644746787604, + "grad_norm": 24.20044708251953, + "learning_rate": 4.887255603610184e-07, + "logits/chosen": 0.31541913747787476, + "logits/rejected": 0.25735703110694885, + "loss": 1.2743, + "step": 125 + }, + { + "beta_dpo/beta": 0.0405975878238678, + "beta_dpo/beta_margin_grad_mean": -0.48580294847488403, + "beta_dpo/beta_margin_grad_std": 0.0543222539126873, + "beta_dpo/beta_margin_mean": 0.05759422481060028, + "beta_dpo/beta_margin_std": 0.22094084322452545, + "beta_dpo/beta_used": 0.0405975878238678, + "beta_dpo/beta_used_raw": 0.0405975878238678, + "beta_dpo/gap_mean": 2.5174307823181152, + "beta_dpo/gap_std": 5.691348552703857, + "beta_dpo/loss_margin_mean": 1.3669215440750122, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.19047619047619047, + "grad_norm": 14.997529029846191, + "learning_rate": 4.883296295573176e-07, + "logits/chosen": 0.1563548743724823, + "logits/rejected": 0.15073931217193604, + "loss": 1.33, + "step": 126 + }, + { + "beta_dpo/beta": 0.09659086167812347, + "beta_dpo/beta_margin_grad_mean": -0.42623478174209595, + "beta_dpo/beta_margin_grad_std": 0.11462006717920303, + "beta_dpo/beta_margin_mean": 0.32010120153427124, + "beta_dpo/beta_margin_std": 0.5070313811302185, + "beta_dpo/beta_used": 0.09659086167812347, + "beta_dpo/beta_used_raw": 0.09659086167812347, + "beta_dpo/gap_mean": 2.602691650390625, + "beta_dpo/gap_std": 5.580142021179199, + "beta_dpo/loss_margin_mean": 3.193540573120117, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.19198790627362056, + "grad_norm": 29.69304084777832, + "learning_rate": 4.87927032161552e-07, + "logits/chosen": 0.2605654001235962, + "logits/rejected": 0.2306014895439148, + "loss": 1.1924, + "step": 127 + }, + { + "beta_dpo/beta": 0.08403972536325455, + "beta_dpo/beta_margin_grad_mean": -0.4287105202674866, + "beta_dpo/beta_margin_grad_std": 0.14072571694850922, + "beta_dpo/beta_margin_mean": 0.3516583740711212, + "beta_dpo/beta_margin_std": 0.7548319101333618, + "beta_dpo/beta_used": 0.08403972536325455, + "beta_dpo/beta_used_raw": 0.08403972536325455, + "beta_dpo/gap_mean": 2.808605670928955, + "beta_dpo/gap_std": 5.833803176879883, + "beta_dpo/loss_margin_mean": 3.8815970420837402, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.19349962207105065, + "grad_norm": 28.656200408935547, + "learning_rate": 4.875177794352363e-07, + "logits/chosen": 0.26921409368515015, + "logits/rejected": 0.21398761868476868, + "loss": 1.2019, + "step": 128 + }, + { + "beta_dpo/beta": 0.033706970512866974, + "beta_dpo/beta_margin_grad_mean": -0.4868035614490509, + "beta_dpo/beta_margin_grad_std": 0.060135770589113235, + "beta_dpo/beta_margin_mean": 0.05301497131586075, + "beta_dpo/beta_margin_std": 0.24528354406356812, + "beta_dpo/beta_used": 0.033706970512866974, + "beta_dpo/beta_used_raw": 0.033706970512866974, + "beta_dpo/gap_mean": 2.6513404846191406, + "beta_dpo/gap_std": 6.111024856567383, + "beta_dpo/loss_margin_mean": 1.6255993843078613, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.19501133786848074, + "grad_norm": 12.075440406799316, + "learning_rate": 4.871018828260491e-07, + "logits/chosen": 0.2676212191581726, + "logits/rejected": 0.2616123557090759, + "loss": 1.3404, + "step": 129 + }, + { + "beta_dpo/beta": 0.10842250287532806, + "beta_dpo/beta_margin_grad_mean": -0.4172552824020386, + "beta_dpo/beta_margin_grad_std": 0.1448642611503601, + "beta_dpo/beta_margin_mean": 0.3701942563056946, + "beta_dpo/beta_margin_std": 0.6463952660560608, + "beta_dpo/beta_used": 0.10842250287532806, + "beta_dpo/beta_used_raw": 0.10842250287532806, + "beta_dpo/gap_mean": 2.757774829864502, + "beta_dpo/gap_std": 6.210426330566406, + "beta_dpo/loss_margin_mean": 3.5002007484436035, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1965230536659108, + "grad_norm": 35.02021408081055, + "learning_rate": 4.866793539675126e-07, + "logits/chosen": 0.23043951392173767, + "logits/rejected": 0.1808605194091797, + "loss": 1.1603, + "step": 130 + }, + { + "beta_dpo/beta": 0.09228923916816711, + "beta_dpo/beta_margin_grad_mean": -0.41555356979370117, + "beta_dpo/beta_margin_grad_std": 0.1676713079214096, + "beta_dpo/beta_margin_mean": 0.4410023093223572, + "beta_dpo/beta_margin_std": 0.9455744028091431, + "beta_dpo/beta_used": 0.09228923916816711, + "beta_dpo/beta_used_raw": 0.09228923916816711, + "beta_dpo/gap_mean": 3.0267586708068848, + "beta_dpo/gap_std": 6.380265235900879, + "beta_dpo/loss_margin_mean": 4.280169486999512, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.1980347694633409, + "grad_norm": 31.57815170288086, + "learning_rate": 4.86250204678667e-07, + "logits/chosen": 0.2520222067832947, + "logits/rejected": 0.1882476657629013, + "loss": 1.2054, + "step": 131 + }, + { + "beta_dpo/beta": 0.15973003208637238, + "beta_dpo/beta_margin_grad_mean": -0.37915053963661194, + "beta_dpo/beta_margin_grad_std": 0.18757346272468567, + "beta_dpo/beta_margin_mean": 0.572222888469696, + "beta_dpo/beta_margin_std": 0.9784995317459106, + "beta_dpo/beta_used": 0.15973003208637238, + "beta_dpo/beta_used_raw": 0.15973003208637238, + "beta_dpo/gap_mean": 3.1300902366638184, + "beta_dpo/gap_std": 6.296309471130371, + "beta_dpo/loss_margin_mean": 3.5487518310546875, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.19954648526077098, + "grad_norm": 39.60453414916992, + "learning_rate": 4.858144469637408e-07, + "logits/chosen": 0.32432320713996887, + "logits/rejected": 0.2931896150112152, + "loss": 0.9575, + "step": 132 + }, + { + "beta_dpo/beta": 0.07123492658138275, + "beta_dpo/beta_margin_grad_mean": -0.4693993330001831, + "beta_dpo/beta_margin_grad_std": 0.13667266070842743, + "beta_dpo/beta_margin_mean": 0.13288940489292145, + "beta_dpo/beta_margin_std": 0.6505784392356873, + "beta_dpo/beta_used": 0.07123492658138275, + "beta_dpo/beta_used_raw": 0.07123492658138275, + "beta_dpo/gap_mean": 2.9774646759033203, + "beta_dpo/gap_std": 6.303676605224609, + "beta_dpo/loss_margin_mean": 2.090972423553467, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.20105820105820105, + "grad_norm": 26.4464168548584, + "learning_rate": 4.853720930118138e-07, + "logits/chosen": 0.24306905269622803, + "logits/rejected": 0.23324322700500488, + "loss": 1.2258, + "step": 133 + }, + { + "beta_dpo/beta": 0.11694416403770447, + "beta_dpo/beta_margin_grad_mean": -0.3908437192440033, + "beta_dpo/beta_margin_grad_std": 0.16718235611915588, + "beta_dpo/beta_margin_mean": 0.5406786799430847, + "beta_dpo/beta_margin_std": 0.9178445935249329, + "beta_dpo/beta_used": 0.11694416403770447, + "beta_dpo/beta_used_raw": 0.11694416403770447, + "beta_dpo/gap_mean": 3.1626126766204834, + "beta_dpo/gap_std": 6.352079391479492, + "beta_dpo/loss_margin_mean": 4.546454429626465, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.20256991685563114, + "grad_norm": 36.3588981628418, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": 0.3457499146461487, + "logits/rejected": 0.29014575481414795, + "loss": 1.094, + "step": 134 + }, + { + "beta_dpo/beta": 0.07570680230855942, + "beta_dpo/beta_margin_grad_mean": -0.44854554533958435, + "beta_dpo/beta_margin_grad_std": 0.1104312390089035, + "beta_dpo/beta_margin_mean": 0.2286955565214157, + "beta_dpo/beta_margin_std": 0.5155693888664246, + "beta_dpo/beta_used": 0.07570680230855942, + "beta_dpo/beta_used_raw": 0.07570680230855942, + "beta_dpo/gap_mean": 3.166677951812744, + "beta_dpo/gap_std": 6.442228317260742, + "beta_dpo/loss_margin_mean": 2.8143091201782227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.20408163265306123, + "grad_norm": 22.227096557617188, + "learning_rate": 4.844676460754862e-07, + "logits/chosen": 0.293022096157074, + "logits/rejected": 0.2606502175331116, + "loss": 1.2059, + "step": 135 + }, + { + "beta_dpo/beta": 0.14603909850120544, + "beta_dpo/beta_margin_grad_mean": -0.39259859919548035, + "beta_dpo/beta_margin_grad_std": 0.23135711252689362, + "beta_dpo/beta_margin_mean": 0.6742245554924011, + "beta_dpo/beta_margin_std": 1.4196631908416748, + "beta_dpo/beta_used": 0.14603909850120544, + "beta_dpo/beta_used_raw": 0.14603909850120544, + "beta_dpo/gap_mean": 3.3520989418029785, + "beta_dpo/gap_std": 6.7299604415893555, + "beta_dpo/loss_margin_mean": 3.902292251586914, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.20559334845049132, + "grad_norm": 55.59946823120117, + "learning_rate": 4.840055783904106e-07, + "logits/chosen": 0.26997774839401245, + "logits/rejected": 0.1992480307817459, + "loss": 1.0577, + "step": 136 + }, + { + "beta_dpo/beta": 0.06228652596473694, + "beta_dpo/beta_margin_grad_mean": -0.4408150017261505, + "beta_dpo/beta_margin_grad_std": 0.10646083205938339, + "beta_dpo/beta_margin_mean": 0.2699366807937622, + "beta_dpo/beta_margin_std": 0.5479843616485596, + "beta_dpo/beta_used": 0.06228652596473694, + "beta_dpo/beta_used_raw": 0.06228652596473694, + "beta_dpo/gap_mean": 3.378662586212158, + "beta_dpo/gap_std": 6.867051124572754, + "beta_dpo/loss_margin_mean": 4.064416408538818, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.20710506424792138, + "grad_norm": 19.85221290588379, + "learning_rate": 4.835369650662767e-07, + "logits/chosen": 0.30249646306037903, + "logits/rejected": 0.2738552689552307, + "loss": 1.2387, + "step": 137 + }, + { + "beta_dpo/beta": 0.1027345061302185, + "beta_dpo/beta_margin_grad_mean": -0.4208414554595947, + "beta_dpo/beta_margin_grad_std": 0.17435140907764435, + "beta_dpo/beta_margin_mean": 0.3642386198043823, + "beta_dpo/beta_margin_std": 0.8839937448501587, + "beta_dpo/beta_used": 0.1027345061302185, + "beta_dpo/beta_used_raw": 0.1027345061302185, + "beta_dpo/gap_mean": 3.448456287384033, + "beta_dpo/gap_std": 7.06696891784668, + "beta_dpo/loss_margin_mean": 3.50492525100708, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.20861678004535147, + "grad_norm": 28.683637619018555, + "learning_rate": 4.830618192112065e-07, + "logits/chosen": 0.3061959147453308, + "logits/rejected": 0.2705356180667877, + "loss": 1.1284, + "step": 138 + }, + { + "beta_dpo/beta": 0.1404586136341095, + "beta_dpo/beta_margin_grad_mean": -0.38947877287864685, + "beta_dpo/beta_margin_grad_std": 0.19302061200141907, + "beta_dpo/beta_margin_mean": 0.5290707945823669, + "beta_dpo/beta_margin_std": 1.0226430892944336, + "beta_dpo/beta_used": 0.1404586136341095, + "beta_dpo/beta_used_raw": 0.1404586136341095, + "beta_dpo/gap_mean": 3.507274866104126, + "beta_dpo/gap_std": 7.193138599395752, + "beta_dpo/loss_margin_mean": 3.7624733448028564, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.21012849584278157, + "grad_norm": 43.497894287109375, + "learning_rate": 4.825801541160509e-07, + "logits/chosen": 0.2549942433834076, + "logits/rejected": 0.2260269820690155, + "loss": 0.9876, + "step": 139 + }, + { + "beta_dpo/beta": 0.14053229987621307, + "beta_dpo/beta_margin_grad_mean": -0.3765046298503876, + "beta_dpo/beta_margin_grad_std": 0.24486418068408966, + "beta_dpo/beta_margin_mean": 0.8039000630378723, + "beta_dpo/beta_margin_std": 1.6034449338912964, + "beta_dpo/beta_used": 0.14053229987621307, + "beta_dpo/beta_used_raw": 0.14053229987621307, + "beta_dpo/gap_mean": 3.81668758392334, + "beta_dpo/gap_std": 7.505254745483398, + "beta_dpo/loss_margin_mean": 5.339648723602295, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.21164021164021163, + "grad_norm": 47.931549072265625, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": 0.26516544818878174, + "logits/rejected": 0.2187662273645401, + "loss": 1.0943, + "step": 140 + }, + { + "beta_dpo/beta": 0.10588505864143372, + "beta_dpo/beta_margin_grad_mean": -0.3871758282184601, + "beta_dpo/beta_margin_grad_std": 0.2020079493522644, + "beta_dpo/beta_margin_mean": 0.6185064911842346, + "beta_dpo/beta_margin_std": 1.2682307958602905, + "beta_dpo/beta_used": 0.10588505864143372, + "beta_dpo/beta_used_raw": 0.10588505864143372, + "beta_dpo/gap_mean": 4.12210750579834, + "beta_dpo/gap_std": 7.9551897048950195, + "beta_dpo/loss_margin_mean": 5.243870735168457, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.21315192743764172, + "grad_norm": 29.578947067260742, + "learning_rate": 4.815973202802966e-07, + "logits/chosen": 0.3229524493217468, + "logits/rejected": 0.27724385261535645, + "loss": 1.0874, + "step": 141 + }, + { + "beta_dpo/beta": 0.027559388428926468, + "beta_dpo/beta_margin_grad_mean": -0.4702756702899933, + "beta_dpo/beta_margin_grad_std": 0.07220163941383362, + "beta_dpo/beta_margin_mean": 0.1247277557849884, + "beta_dpo/beta_margin_std": 0.30607911944389343, + "beta_dpo/beta_used": 0.027559388428926468, + "beta_dpo/beta_used_raw": 0.02024449221789837, + "beta_dpo/gap_mean": 4.11570930480957, + "beta_dpo/gap_std": 7.797653675079346, + "beta_dpo/loss_margin_mean": 4.000240802764893, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2146636432350718, + "grad_norm": 11.837271690368652, + "learning_rate": 4.810961790316729e-07, + "logits/chosen": 0.31370168924331665, + "logits/rejected": 0.2887398898601532, + "loss": 1.3053, + "step": 142 + }, + { + "beta_dpo/beta": 0.12018455564975739, + "beta_dpo/beta_margin_grad_mean": -0.4178207814693451, + "beta_dpo/beta_margin_grad_std": 0.2122364342212677, + "beta_dpo/beta_margin_mean": 0.3589272201061249, + "beta_dpo/beta_margin_std": 1.4373321533203125, + "beta_dpo/beta_used": 0.12018455564975739, + "beta_dpo/beta_used_raw": 0.12018455564975739, + "beta_dpo/gap_mean": 3.8963608741760254, + "beta_dpo/gap_std": 8.00349235534668, + "beta_dpo/loss_margin_mean": 2.8756353855133057, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2161753590325019, + "grad_norm": 34.828216552734375, + "learning_rate": 4.805885735261454e-07, + "logits/chosen": 0.3244457244873047, + "logits/rejected": 0.30710387229919434, + "loss": 1.0544, + "step": 143 + }, + { + "beta_dpo/beta": 0.03807094693183899, + "beta_dpo/beta_margin_grad_mean": -0.4691088795661926, + "beta_dpo/beta_margin_grad_std": 0.10766191780567169, + "beta_dpo/beta_margin_mean": 0.13813741505146027, + "beta_dpo/beta_margin_std": 0.4963976740837097, + "beta_dpo/beta_used": 0.03807094693183899, + "beta_dpo/beta_used_raw": 0.03807094693183899, + "beta_dpo/gap_mean": 3.9589571952819824, + "beta_dpo/gap_std": 8.451248168945312, + "beta_dpo/loss_margin_mean": 4.306517124176025, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.21768707482993196, + "grad_norm": 15.574111938476562, + "learning_rate": 4.800745179625307e-07, + "logits/chosen": 0.2954272925853729, + "logits/rejected": 0.26464635133743286, + "loss": 1.2931, + "step": 144 + }, + { + "beta_dpo/beta": 0.1702955812215805, + "beta_dpo/beta_margin_grad_mean": -0.3686128854751587, + "beta_dpo/beta_margin_grad_std": 0.2736431062221527, + "beta_dpo/beta_margin_mean": 0.7588291168212891, + "beta_dpo/beta_margin_std": 1.8663489818572998, + "beta_dpo/beta_used": 0.1702955812215805, + "beta_dpo/beta_used_raw": 0.1702955812215805, + "beta_dpo/gap_mean": 4.025421142578125, + "beta_dpo/gap_std": 8.940174102783203, + "beta_dpo/loss_margin_mean": 4.432011604309082, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.21919879062736206, + "grad_norm": 57.19875717163086, + "learning_rate": 4.795540267200686e-07, + "logits/chosen": 0.26614999771118164, + "logits/rejected": 0.2815524935722351, + "loss": 0.9421, + "step": 145 + }, + { + "beta_dpo/beta": 0.08469145745038986, + "beta_dpo/beta_margin_grad_mean": -0.4177340567111969, + "beta_dpo/beta_margin_grad_std": 0.16123804450035095, + "beta_dpo/beta_margin_mean": 0.3749273121356964, + "beta_dpo/beta_margin_std": 0.7374047636985779, + "beta_dpo/beta_used": 0.08469145745038986, + "beta_dpo/beta_used_raw": 0.08469145745038986, + "beta_dpo/gap_mean": 4.05902099609375, + "beta_dpo/gap_std": 8.936864852905273, + "beta_dpo/loss_margin_mean": 4.456746578216553, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.22071050642479215, + "grad_norm": 32.773834228515625, + "learning_rate": 4.790271143580173e-07, + "logits/chosen": 0.26889604330062866, + "logits/rejected": 0.254508912563324, + "loss": 1.187, + "step": 146 + }, + { + "beta_dpo/beta": 0.1501540094614029, + "beta_dpo/beta_margin_grad_mean": -0.4042907953262329, + "beta_dpo/beta_margin_grad_std": 0.24906255304813385, + "beta_dpo/beta_margin_mean": 0.5283809900283813, + "beta_dpo/beta_margin_std": 1.4151684045791626, + "beta_dpo/beta_used": 0.1501540094614029, + "beta_dpo/beta_used_raw": 0.1501540094614029, + "beta_dpo/gap_mean": 4.006381988525391, + "beta_dpo/gap_std": 9.127963066101074, + "beta_dpo/loss_margin_mean": 3.4659228324890137, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2222222222222222, + "grad_norm": 46.82968521118164, + "learning_rate": 4.784937956152489e-07, + "logits/chosen": 0.2905007600784302, + "logits/rejected": 0.24713094532489777, + "loss": 1.0305, + "step": 147 + }, + { + "beta_dpo/beta": 0.19147343933582306, + "beta_dpo/beta_margin_grad_mean": -0.32735762000083923, + "beta_dpo/beta_margin_grad_std": 0.26872846484184265, + "beta_dpo/beta_margin_mean": 1.068808913230896, + "beta_dpo/beta_margin_std": 1.695504069328308, + "beta_dpo/beta_used": 0.19147343933582306, + "beta_dpo/beta_used_raw": 0.19147343933582306, + "beta_dpo/gap_mean": 4.158587455749512, + "beta_dpo/gap_std": 9.099699974060059, + "beta_dpo/loss_margin_mean": 5.4793381690979, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2237339380196523, + "grad_norm": 47.708614349365234, + "learning_rate": 4.779540854098347e-07, + "logits/chosen": 0.41925525665283203, + "logits/rejected": 0.3467964828014374, + "loss": 0.8812, + "step": 148 + }, + { + "beta_dpo/beta": 0.10122452676296234, + "beta_dpo/beta_margin_grad_mean": -0.3874892592430115, + "beta_dpo/beta_margin_grad_std": 0.16896942257881165, + "beta_dpo/beta_margin_mean": 0.563135027885437, + "beta_dpo/beta_margin_std": 0.9112203121185303, + "beta_dpo/beta_used": 0.10122452676296234, + "beta_dpo/beta_used_raw": 0.10122452676296234, + "beta_dpo/gap_mean": 4.419680595397949, + "beta_dpo/gap_std": 8.921280860900879, + "beta_dpo/loss_margin_mean": 5.370478630065918, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2252456538170824, + "grad_norm": 31.664949417114258, + "learning_rate": 4.774079988386296e-07, + "logits/chosen": 0.2955988049507141, + "logits/rejected": 0.2491205632686615, + "loss": 1.0825, + "step": 149 + }, + { + "beta_dpo/beta": 0.12354602664709091, + "beta_dpo/beta_margin_grad_mean": -0.3436400294303894, + "beta_dpo/beta_margin_grad_std": 0.21282432973384857, + "beta_dpo/beta_margin_mean": 0.9323195815086365, + "beta_dpo/beta_margin_std": 1.350142478942871, + "beta_dpo/beta_used": 0.12354602664709091, + "beta_dpo/beta_used_raw": 0.12354602664709091, + "beta_dpo/gap_mean": 4.944735527038574, + "beta_dpo/gap_std": 9.273387908935547, + "beta_dpo/loss_margin_mean": 7.580916881561279, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.22675736961451248, + "grad_norm": 46.71488952636719, + "learning_rate": 4.768555511768486e-07, + "logits/chosen": 0.3046860694885254, + "logits/rejected": 0.2608673572540283, + "loss": 1.0507, + "step": 150 + }, + { + "beta_dpo/beta": 0.21111351251602173, + "beta_dpo/beta_margin_grad_mean": -0.2585567235946655, + "beta_dpo/beta_margin_grad_std": 0.25030454993247986, + "beta_dpo/beta_margin_mean": 1.8264737129211426, + "beta_dpo/beta_margin_std": 2.1160757541656494, + "beta_dpo/beta_used": 0.21111351251602173, + "beta_dpo/beta_used_raw": 0.21111351251602173, + "beta_dpo/gap_mean": 5.5986528396606445, + "beta_dpo/gap_std": 9.438240051269531, + "beta_dpo/loss_margin_mean": 8.652307510375977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.22826908541194255, + "grad_norm": 55.5644645690918, + "learning_rate": 4.762967578776406e-07, + "logits/chosen": 0.313462495803833, + "logits/rejected": 0.25910061597824097, + "loss": 0.7352, + "step": 151 + }, + { + "beta_dpo/beta": 0.09550273418426514, + "beta_dpo/beta_margin_grad_mean": -0.38633495569229126, + "beta_dpo/beta_margin_grad_std": 0.20238004624843597, + "beta_dpo/beta_margin_mean": 0.5673766136169434, + "beta_dpo/beta_margin_std": 1.0258516073226929, + "beta_dpo/beta_used": 0.09550273418426514, + "beta_dpo/beta_used_raw": 0.09550273418426514, + "beta_dpo/gap_mean": 5.716914176940918, + "beta_dpo/gap_std": 9.608884811401367, + "beta_dpo/loss_margin_mean": 5.932858943939209, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.22978080120937264, + "grad_norm": 29.89628791809082, + "learning_rate": 4.757316345716553e-07, + "logits/chosen": 0.3986983299255371, + "logits/rejected": 0.3447926342487335, + "loss": 1.0198, + "step": 152 + }, + { + "beta_dpo/beta": 0.11710208654403687, + "beta_dpo/beta_margin_grad_mean": -0.35504305362701416, + "beta_dpo/beta_margin_grad_std": 0.19991737604141235, + "beta_dpo/beta_margin_mean": 0.740251898765564, + "beta_dpo/beta_margin_std": 1.0371873378753662, + "beta_dpo/beta_used": 0.11710208654403687, + "beta_dpo/beta_used_raw": 0.11710208654403687, + "beta_dpo/gap_mean": 5.847277641296387, + "beta_dpo/gap_std": 9.527088165283203, + "beta_dpo/loss_margin_mean": 6.285402297973633, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.23129251700680273, + "grad_norm": 38.12724304199219, + "learning_rate": 4.751601970666064e-07, + "logits/chosen": 0.2800205647945404, + "logits/rejected": 0.24503669142723083, + "loss": 0.9413, + "step": 153 + }, + { + "beta_dpo/beta": 0.12367129325866699, + "beta_dpo/beta_margin_grad_mean": -0.38842007517814636, + "beta_dpo/beta_margin_grad_std": 0.24438461661338806, + "beta_dpo/beta_margin_mean": 0.5933310985565186, + "beta_dpo/beta_margin_std": 1.3632546663284302, + "beta_dpo/beta_used": 0.12367129325866699, + "beta_dpo/beta_used_raw": 0.12367129325866699, + "beta_dpo/gap_mean": 5.747869968414307, + "beta_dpo/gap_std": 9.679136276245117, + "beta_dpo/loss_margin_mean": 4.650402545928955, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2328042328042328, + "grad_norm": 34.96685028076172, + "learning_rate": 4.745824613468292e-07, + "logits/chosen": 0.3880733847618103, + "logits/rejected": 0.38508933782577515, + "loss": 0.9928, + "step": 154 + }, + { + "beta_dpo/beta": 0.1392279863357544, + "beta_dpo/beta_margin_grad_mean": -0.3571988344192505, + "beta_dpo/beta_margin_grad_std": 0.2581491768360138, + "beta_dpo/beta_margin_mean": 0.9176416397094727, + "beta_dpo/beta_margin_std": 1.843299150466919, + "beta_dpo/beta_used": 0.1392279863357544, + "beta_dpo/beta_used_raw": 0.1392279863357544, + "beta_dpo/gap_mean": 5.793259620666504, + "beta_dpo/gap_std": 10.092991828918457, + "beta_dpo/loss_margin_mean": 6.254579544067383, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.23431594860166288, + "grad_norm": 54.003902435302734, + "learning_rate": 4.7399844357283393e-07, + "logits/chosen": 0.3745535910129547, + "logits/rejected": 0.3542165756225586, + "loss": 0.9385, + "step": 155 + }, + { + "beta_dpo/beta": 0.1403558999300003, + "beta_dpo/beta_margin_grad_mean": -0.3198482096195221, + "beta_dpo/beta_margin_grad_std": 0.23029401898384094, + "beta_dpo/beta_margin_mean": 1.1250354051589966, + "beta_dpo/beta_margin_std": 1.623459815979004, + "beta_dpo/beta_used": 0.1403558999300003, + "beta_dpo/beta_used_raw": 0.1403558999300003, + "beta_dpo/gap_mean": 6.089911460876465, + "beta_dpo/gap_std": 10.31209659576416, + "beta_dpo/loss_margin_mean": 8.206597328186035, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.23582766439909297, + "grad_norm": 41.47368621826172, + "learning_rate": 4.7340816008085305e-07, + "logits/chosen": 0.3209989070892334, + "logits/rejected": 0.27650001645088196, + "loss": 0.8772, + "step": 156 + }, + { + "beta_dpo/beta": 0.10112761706113815, + "beta_dpo/beta_margin_grad_mean": -0.37136074900627136, + "beta_dpo/beta_margin_grad_std": 0.22732508182525635, + "beta_dpo/beta_margin_mean": 0.9316520690917969, + "beta_dpo/beta_margin_std": 1.8109554052352905, + "beta_dpo/beta_used": 0.10112761706113815, + "beta_dpo/beta_used_raw": 0.06065015494823456, + "beta_dpo/gap_mean": 6.362764358520508, + "beta_dpo/gap_std": 10.283683776855469, + "beta_dpo/loss_margin_mean": 6.171655654907227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.23733938019652306, + "grad_norm": 34.023895263671875, + "learning_rate": 4.728116273823847e-07, + "logits/chosen": 0.31635981798171997, + "logits/rejected": 0.29805511236190796, + "loss": 1.0313, + "step": 157 + }, + { + "beta_dpo/beta": 0.09633895754814148, + "beta_dpo/beta_margin_grad_mean": -0.3940832018852234, + "beta_dpo/beta_margin_grad_std": 0.21066348254680634, + "beta_dpo/beta_margin_mean": 0.6347554326057434, + "beta_dpo/beta_margin_std": 1.3230136632919312, + "beta_dpo/beta_used": 0.09633895754814148, + "beta_dpo/beta_used_raw": 0.09633895754814148, + "beta_dpo/gap_mean": 6.245730400085449, + "beta_dpo/gap_std": 10.449724197387695, + "beta_dpo/loss_margin_mean": 6.507798194885254, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.23885109599395313, + "grad_norm": 35.92675018310547, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": 0.34792205691337585, + "logits/rejected": 0.31133797764778137, + "loss": 1.0874, + "step": 158 + }, + { + "beta_dpo/beta": 0.1303880512714386, + "beta_dpo/beta_margin_grad_mean": -0.33156701922416687, + "beta_dpo/beta_margin_grad_std": 0.23368042707443237, + "beta_dpo/beta_margin_mean": 1.0686310529708862, + "beta_dpo/beta_margin_std": 1.632267713546753, + "beta_dpo/beta_used": 0.1303880512714386, + "beta_dpo/beta_used_raw": 0.1303880512714386, + "beta_dpo/gap_mean": 6.408536911010742, + "beta_dpo/gap_std": 10.668962478637695, + "beta_dpo/loss_margin_mean": 7.6960601806640625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.24036281179138322, + "grad_norm": 44.9423828125, + "learning_rate": 4.715998812855304e-07, + "logits/chosen": 0.3547423779964447, + "logits/rejected": 0.3168538212776184, + "loss": 0.9662, + "step": 159 + }, + { + "beta_dpo/beta": 0.043398354202508926, + "beta_dpo/beta_margin_grad_mean": -0.44123727083206177, + "beta_dpo/beta_margin_grad_std": 0.15287929773330688, + "beta_dpo/beta_margin_mean": 0.29121875762939453, + "beta_dpo/beta_margin_std": 0.7604160904884338, + "beta_dpo/beta_used": 0.043398354202508926, + "beta_dpo/beta_used_raw": 0.04327729716897011, + "beta_dpo/gap_mean": 6.495538711547852, + "beta_dpo/gap_std": 10.80352783203125, + "beta_dpo/loss_margin_mean": 6.3689141273498535, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2418745275888133, + "grad_norm": 17.05034828186035, + "learning_rate": 4.7098470178228755e-07, + "logits/chosen": 0.24205464124679565, + "logits/rejected": 0.1971985101699829, + "loss": 1.2066, + "step": 160 + }, + { + "beta_dpo/beta": 0.19278591871261597, + "beta_dpo/beta_margin_grad_mean": -0.30224671959877014, + "beta_dpo/beta_margin_grad_std": 0.27746814489364624, + "beta_dpo/beta_margin_mean": 1.3429075479507446, + "beta_dpo/beta_margin_std": 2.287931442260742, + "beta_dpo/beta_used": 0.19278591871261597, + "beta_dpo/beta_used_raw": 0.19278591871261597, + "beta_dpo/gap_mean": 6.569012641906738, + "beta_dpo/gap_std": 10.928247451782227, + "beta_dpo/loss_margin_mean": 7.049097537994385, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.24338624338624337, + "grad_norm": 47.56407928466797, + "learning_rate": 4.703633408618955e-07, + "logits/chosen": 0.360584020614624, + "logits/rejected": 0.3227540850639343, + "loss": 0.7116, + "step": 161 + }, + { + "beta_dpo/beta": 0.2703293561935425, + "beta_dpo/beta_margin_grad_mean": -0.23558209836483002, + "beta_dpo/beta_margin_grad_std": 0.2916191816329956, + "beta_dpo/beta_margin_mean": 2.5003085136413574, + "beta_dpo/beta_margin_std": 3.345001220703125, + "beta_dpo/beta_used": 0.2703293561935425, + "beta_dpo/beta_used_raw": 0.2703293561935425, + "beta_dpo/gap_mean": 6.996166229248047, + "beta_dpo/gap_std": 10.952880859375, + "beta_dpo/loss_margin_mean": 9.228057861328125, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.24489795918367346, + "grad_norm": 70.73652648925781, + "learning_rate": 4.697358159051549e-07, + "logits/chosen": 0.4289015531539917, + "logits/rejected": 0.3779623508453369, + "loss": 0.7172, + "step": 162 + }, + { + "beta_dpo/beta": 0.12281934171915054, + "beta_dpo/beta_margin_grad_mean": -0.3348690867424011, + "beta_dpo/beta_margin_grad_std": 0.23672305047512054, + "beta_dpo/beta_margin_mean": 1.490291953086853, + "beta_dpo/beta_margin_std": 2.317763328552246, + "beta_dpo/beta_used": 0.12281934171915054, + "beta_dpo/beta_used_raw": 0.08148720860481262, + "beta_dpo/gap_mean": 7.235516548156738, + "beta_dpo/gap_std": 11.109076499938965, + "beta_dpo/loss_margin_mean": 8.966986656188965, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.24640967498110355, + "grad_norm": 42.12664794921875, + "learning_rate": 4.691021444652876e-07, + "logits/chosen": 0.34994274377822876, + "logits/rejected": 0.30174481868743896, + "loss": 0.9681, + "step": 163 + }, + { + "beta_dpo/beta": 0.15474146604537964, + "beta_dpo/beta_margin_grad_mean": -0.28919804096221924, + "beta_dpo/beta_margin_grad_std": 0.27007272839546204, + "beta_dpo/beta_margin_mean": 1.4409639835357666, + "beta_dpo/beta_margin_std": 1.9009385108947754, + "beta_dpo/beta_used": 0.15474146604537964, + "beta_dpo/beta_used_raw": 0.15474146604537964, + "beta_dpo/gap_mean": 7.648365020751953, + "beta_dpo/gap_std": 11.386979103088379, + "beta_dpo/loss_margin_mean": 9.083514213562012, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.24792139077853365, + "grad_norm": 40.19808578491211, + "learning_rate": 4.6846234426744624e-07, + "logits/chosen": 0.33891743421554565, + "logits/rejected": 0.2750347852706909, + "loss": 0.7839, + "step": 164 + }, + { + "beta_dpo/beta": 0.07602068781852722, + "beta_dpo/beta_margin_grad_mean": -0.37450677156448364, + "beta_dpo/beta_margin_grad_std": 0.19640876352787018, + "beta_dpo/beta_margin_mean": 0.8033673167228699, + "beta_dpo/beta_margin_std": 1.3503183126449585, + "beta_dpo/beta_used": 0.07602068781852722, + "beta_dpo/beta_used_raw": 0.07242593914270401, + "beta_dpo/gap_mean": 7.7765655517578125, + "beta_dpo/gap_std": 11.585585594177246, + "beta_dpo/loss_margin_mean": 8.278116226196289, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2494331065759637, + "grad_norm": 25.78687858581543, + "learning_rate": 4.678164332082175e-07, + "logits/chosen": 0.3837462067604065, + "logits/rejected": 0.32918989658355713, + "loss": 1.0428, + "step": 165 + }, + { + "beta_dpo/beta": 0.01979365386068821, + "beta_dpo/beta_margin_grad_mean": -0.46458086371421814, + "beta_dpo/beta_margin_grad_std": 0.07628511637449265, + "beta_dpo/beta_margin_mean": 0.14900416135787964, + "beta_dpo/beta_margin_std": 0.3222452998161316, + "beta_dpo/beta_used": 0.01979365386068821, + "beta_dpo/beta_used_raw": 0.015947217121720314, + "beta_dpo/gap_mean": 7.783942222595215, + "beta_dpo/gap_std": 11.442436218261719, + "beta_dpo/loss_margin_mean": 6.6719818115234375, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2509448223733938, + "grad_norm": 10.664765357971191, + "learning_rate": 4.6716442935512214e-07, + "logits/chosen": 0.3633756637573242, + "logits/rejected": 0.27223867177963257, + "loss": 1.2742, + "step": 166 + }, + { + "beta_dpo/beta": 0.13996389508247375, + "beta_dpo/beta_margin_grad_mean": -0.3225991129875183, + "beta_dpo/beta_margin_grad_std": 0.2330418974161148, + "beta_dpo/beta_margin_mean": 1.5369998216629028, + "beta_dpo/beta_margin_std": 2.3832523822784424, + "beta_dpo/beta_used": 0.13996389508247375, + "beta_dpo/beta_used_raw": 0.13996389508247375, + "beta_dpo/gap_mean": 8.021318435668945, + "beta_dpo/gap_std": 11.419715881347656, + "beta_dpo/loss_margin_mean": 9.416756629943848, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.25245653817082386, + "grad_norm": 44.863494873046875, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": 0.29010146856307983, + "logits/rejected": 0.2558974623680115, + "loss": 0.8469, + "step": 167 + }, + { + "beta_dpo/beta": 0.12838992476463318, + "beta_dpo/beta_margin_grad_mean": -0.35723546147346497, + "beta_dpo/beta_margin_grad_std": 0.21953362226486206, + "beta_dpo/beta_margin_mean": 1.0821810960769653, + "beta_dpo/beta_margin_std": 1.8164141178131104, + "beta_dpo/beta_used": 0.12838992476463318, + "beta_dpo/beta_used_raw": 0.12838992476463318, + "beta_dpo/gap_mean": 8.001766204833984, + "beta_dpo/gap_std": 11.373407363891602, + "beta_dpo/loss_margin_mean": 7.806889533996582, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.25396825396825395, + "grad_norm": 37.25100326538086, + "learning_rate": 4.6584221638904767e-07, + "logits/chosen": 0.316717267036438, + "logits/rejected": 0.2821142077445984, + "loss": 0.9152, + "step": 168 + }, + { + "beta_dpo/beta": 0.08584662526845932, + "beta_dpo/beta_margin_grad_mean": -0.3747517466545105, + "beta_dpo/beta_margin_grad_std": 0.22064180672168732, + "beta_dpo/beta_margin_mean": 0.7022613883018494, + "beta_dpo/beta_margin_std": 1.4712177515029907, + "beta_dpo/beta_used": 0.08584662526845932, + "beta_dpo/beta_used_raw": 0.08584662526845932, + "beta_dpo/gap_mean": 7.975764274597168, + "beta_dpo/gap_std": 11.88857364654541, + "beta_dpo/loss_margin_mean": 8.126168251037598, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.25547996976568405, + "grad_norm": 37.95653533935547, + "learning_rate": 4.651720442612075e-07, + "logits/chosen": 0.38414955139160156, + "logits/rejected": 0.3503633141517639, + "loss": 1.0402, + "step": 169 + }, + { + "beta_dpo/beta": 0.1892240047454834, + "beta_dpo/beta_margin_grad_mean": -0.35289227962493896, + "beta_dpo/beta_margin_grad_std": 0.2945278286933899, + "beta_dpo/beta_margin_mean": 1.7924374341964722, + "beta_dpo/beta_margin_std": 4.277266979217529, + "beta_dpo/beta_used": 0.1892240047454834, + "beta_dpo/beta_used_raw": 0.1892240047454834, + "beta_dpo/gap_mean": 8.009968757629395, + "beta_dpo/gap_std": 12.184398651123047, + "beta_dpo/loss_margin_mean": 8.559287071228027, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.25699168556311414, + "grad_norm": 62.05488967895508, + "learning_rate": 4.6449585330874425e-07, + "logits/chosen": 0.33764785528182983, + "logits/rejected": 0.33556270599365234, + "loss": 1.0906, + "step": 170 + }, + { + "beta_dpo/beta": 0.060626909136772156, + "beta_dpo/beta_margin_grad_mean": -0.38109874725341797, + "beta_dpo/beta_margin_grad_std": 0.21113894879817963, + "beta_dpo/beta_margin_mean": 0.7707568407058716, + "beta_dpo/beta_margin_std": 1.3697214126586914, + "beta_dpo/beta_used": 0.060626909136772156, + "beta_dpo/beta_used_raw": 0.033127665519714355, + "beta_dpo/gap_mean": 8.439079284667969, + "beta_dpo/gap_std": 12.717405319213867, + "beta_dpo/loss_margin_mean": 9.387589454650879, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2585034013605442, + "grad_norm": 30.833776473999023, + "learning_rate": 4.6381366244617224e-07, + "logits/chosen": 0.3928905725479126, + "logits/rejected": 0.3435228765010834, + "loss": 1.1744, + "step": 171 + }, + { + "beta_dpo/beta": 0.05887819081544876, + "beta_dpo/beta_margin_grad_mean": -0.3980772793292999, + "beta_dpo/beta_margin_grad_std": 0.1916777342557907, + "beta_dpo/beta_margin_mean": 0.6465792059898376, + "beta_dpo/beta_margin_std": 1.2853487730026245, + "beta_dpo/beta_used": 0.05887819081544876, + "beta_dpo/beta_used_raw": 0.015539560467004776, + "beta_dpo/gap_mean": 8.218719482421875, + "beta_dpo/gap_std": 12.945394515991211, + "beta_dpo/loss_margin_mean": 8.467949867248535, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2600151171579743, + "grad_norm": 28.302942276000977, + "learning_rate": 4.631254907558365e-07, + "logits/chosen": 0.39802086353302, + "logits/rejected": 0.34046101570129395, + "loss": 1.1582, + "step": 172 + }, + { + "beta_dpo/beta": 0.14484825730323792, + "beta_dpo/beta_margin_grad_mean": -0.38113656640052795, + "beta_dpo/beta_margin_grad_std": 0.2808246612548828, + "beta_dpo/beta_margin_mean": 1.866429328918457, + "beta_dpo/beta_margin_std": 3.806476593017578, + "beta_dpo/beta_used": 0.14484825730323792, + "beta_dpo/beta_used_raw": 0.11751250922679901, + "beta_dpo/gap_mean": 8.664399147033691, + "beta_dpo/gap_std": 13.51144790649414, + "beta_dpo/loss_margin_mean": 9.301438331604004, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2615268329554044, + "grad_norm": 59.29345703125, + "learning_rate": 4.624313574873786e-07, + "logits/chosen": 0.3995574712753296, + "logits/rejected": 0.3124532103538513, + "loss": 1.2498, + "step": 173 + }, + { + "beta_dpo/beta": 0.06351954489946365, + "beta_dpo/beta_margin_grad_mean": -0.3894718587398529, + "beta_dpo/beta_margin_grad_std": 0.20763230323791504, + "beta_dpo/beta_margin_mean": 0.6753450632095337, + "beta_dpo/beta_margin_std": 1.3690553903579712, + "beta_dpo/beta_used": 0.06351954489946365, + "beta_dpo/beta_used_raw": 0.017194651067256927, + "beta_dpo/gap_mean": 8.727994918823242, + "beta_dpo/gap_std": 13.735832214355469, + "beta_dpo/loss_margin_mean": 9.745848655700684, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.26303854875283444, + "grad_norm": 27.401628494262695, + "learning_rate": 4.61731282057198e-07, + "logits/chosen": 0.3719063401222229, + "logits/rejected": 0.3064553737640381, + "loss": 1.116, + "step": 174 + }, + { + "beta_dpo/beta": 0.17716515064239502, + "beta_dpo/beta_margin_grad_mean": -0.30274447798728943, + "beta_dpo/beta_margin_grad_std": 0.307978093624115, + "beta_dpo/beta_margin_mean": 1.780694842338562, + "beta_dpo/beta_margin_std": 2.6141295433044434, + "beta_dpo/beta_used": 0.17716515064239502, + "beta_dpo/beta_used_raw": 0.17716515064239502, + "beta_dpo/gap_mean": 8.922683715820312, + "beta_dpo/gap_std": 13.874872207641602, + "beta_dpo/loss_margin_mean": 10.052102088928223, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.26455026455026454, + "grad_norm": 61.10188674926758, + "learning_rate": 4.6102528404790965e-07, + "logits/chosen": 0.42207950353622437, + "logits/rejected": 0.3914017975330353, + "loss": 0.9454, + "step": 175 + }, + { + "beta_dpo/beta": 0.014193031936883926, + "beta_dpo/beta_margin_grad_mean": -0.46745988726615906, + "beta_dpo/beta_margin_grad_std": 0.06953144818544388, + "beta_dpo/beta_margin_mean": 0.13555732369422913, + "beta_dpo/beta_margin_std": 0.29187914729118347, + "beta_dpo/beta_used": 0.014193031936883926, + "beta_dpo/beta_used_raw": -0.012513306923210621, + "beta_dpo/gap_mean": 8.849132537841797, + "beta_dpo/gap_std": 14.094968795776367, + "beta_dpo/loss_margin_mean": 7.293295860290527, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2660619803476946, + "grad_norm": 8.840620040893555, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": 0.3553032875061035, + "logits/rejected": 0.32939857244491577, + "loss": 1.2904, + "step": 176 + }, + { + "beta_dpo/beta": 0.23863446712493896, + "beta_dpo/beta_margin_grad_mean": -0.2016785889863968, + "beta_dpo/beta_margin_grad_std": 0.2579502761363983, + "beta_dpo/beta_margin_mean": 3.2435951232910156, + "beta_dpo/beta_margin_std": 3.862337589263916, + "beta_dpo/beta_used": 0.23863446712493896, + "beta_dpo/beta_used_raw": 0.23863446712493896, + "beta_dpo/gap_mean": 9.347793579101562, + "beta_dpo/gap_std": 14.189004898071289, + "beta_dpo/loss_margin_mean": 13.310332298278809, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2675736961451247, + "grad_norm": 49.86887741088867, + "learning_rate": 4.5959559945025183e-07, + "logits/chosen": 0.4710957705974579, + "logits/rejected": 0.377672016620636, + "loss": 0.5513, + "step": 177 + }, + { + "beta_dpo/beta": 0.1357753425836563, + "beta_dpo/beta_margin_grad_mean": -0.35512369871139526, + "beta_dpo/beta_margin_grad_std": 0.22710704803466797, + "beta_dpo/beta_margin_mean": 1.5265370607376099, + "beta_dpo/beta_margin_std": 2.7655467987060547, + "beta_dpo/beta_used": 0.1357753425836563, + "beta_dpo/beta_used_raw": 0.10194461047649384, + "beta_dpo/gap_mean": 9.464914321899414, + "beta_dpo/gap_std": 13.956880569458008, + "beta_dpo/loss_margin_mean": 8.183725357055664, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2690854119425548, + "grad_norm": 28.31778335571289, + "learning_rate": 4.588719528532341e-07, + "logits/chosen": 0.3640734553337097, + "logits/rejected": 0.3152313828468323, + "loss": 0.8726, + "step": 178 + }, + { + "beta_dpo/beta": 0.07843155413866043, + "beta_dpo/beta_margin_grad_mean": -0.3692859709262848, + "beta_dpo/beta_margin_grad_std": 0.22836384177207947, + "beta_dpo/beta_margin_mean": 0.7024539709091187, + "beta_dpo/beta_margin_std": 1.2696468830108643, + "beta_dpo/beta_used": 0.07843155413866043, + "beta_dpo/beta_used_raw": 0.07843155413866043, + "beta_dpo/gap_mean": 9.213695526123047, + "beta_dpo/gap_std": 14.17374038696289, + "beta_dpo/loss_margin_mean": 9.025459289550781, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2705971277399849, + "grad_norm": 31.640764236450195, + "learning_rate": 4.581424636586928e-07, + "logits/chosen": 0.388230562210083, + "logits/rejected": 0.3721519112586975, + "loss": 1.0333, + "step": 179 + }, + { + "beta_dpo/beta": 0.053619977086782455, + "beta_dpo/beta_margin_grad_mean": -0.4183306396007538, + "beta_dpo/beta_margin_grad_std": 0.18744249641895294, + "beta_dpo/beta_margin_mean": 0.4771144390106201, + "beta_dpo/beta_margin_std": 1.1917914152145386, + "beta_dpo/beta_used": 0.053619977086782455, + "beta_dpo/beta_used_raw": -0.014215823262929916, + "beta_dpo/gap_mean": 8.941102981567383, + "beta_dpo/gap_std": 14.313918113708496, + "beta_dpo/loss_margin_mean": 7.800354957580566, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.272108843537415, + "grad_norm": 19.132511138916016, + "learning_rate": 4.5740715227200897e-07, + "logits/chosen": 0.21985140442848206, + "logits/rejected": 0.20140361785888672, + "loss": 1.1503, + "step": 180 + }, + { + "beta_dpo/beta": 0.10039281845092773, + "beta_dpo/beta_margin_grad_mean": -0.30794665217399597, + "beta_dpo/beta_margin_grad_std": 0.21378229558467865, + "beta_dpo/beta_margin_mean": 1.109846830368042, + "beta_dpo/beta_margin_std": 1.3198515176773071, + "beta_dpo/beta_used": 0.10039281845092773, + "beta_dpo/beta_used_raw": 0.10039281845092773, + "beta_dpo/gap_mean": 9.251337051391602, + "beta_dpo/gap_std": 14.210281372070312, + "beta_dpo/loss_margin_mean": 11.050002098083496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.273620559334845, + "grad_norm": 36.42921447753906, + "learning_rate": 4.566660392614228e-07, + "logits/chosen": 0.41873055696487427, + "logits/rejected": 0.37909412384033203, + "loss": 0.8726, + "step": 181 + }, + { + "beta_dpo/beta": 0.3119150400161743, + "beta_dpo/beta_margin_grad_mean": -0.242029070854187, + "beta_dpo/beta_margin_grad_std": 0.33485910296440125, + "beta_dpo/beta_margin_mean": 3.7719202041625977, + "beta_dpo/beta_margin_std": 5.102834224700928, + "beta_dpo/beta_used": 0.3119150400161743, + "beta_dpo/beta_used_raw": 0.3119150400161743, + "beta_dpo/gap_mean": 9.723394393920898, + "beta_dpo/gap_std": 14.290912628173828, + "beta_dpo/loss_margin_mean": 12.058207511901855, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2751322751322751, + "grad_norm": 89.14917755126953, + "learning_rate": 4.5591914535745817e-07, + "logits/chosen": 0.39306944608688354, + "logits/rejected": 0.3127225339412689, + "loss": 0.62, + "step": 182 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4987727999687195, + "beta_dpo/beta_margin_grad_std": 0.0033296155743300915, + "beta_dpo/beta_margin_mean": 0.004908976145088673, + "beta_dpo/beta_margin_std": 0.013319132849574089, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.06012484058737755, + "beta_dpo/gap_mean": 9.176264762878418, + "beta_dpo/gap_std": 14.319192886352539, + "beta_dpo/loss_margin_mean": 4.908976078033447, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2766439909297052, + "grad_norm": 0.4539560079574585, + "learning_rate": 4.551664914523433e-07, + "logits/chosen": 0.35062384605407715, + "logits/rejected": 0.3301568031311035, + "loss": 1.3798, + "step": 183 + }, + { + "beta_dpo/beta": 0.061808113008737564, + "beta_dpo/beta_margin_grad_mean": -0.3990829586982727, + "beta_dpo/beta_margin_grad_std": 0.19997593760490417, + "beta_dpo/beta_margin_mean": 0.6703276634216309, + "beta_dpo/beta_margin_std": 1.3407752513885498, + "beta_dpo/beta_used": 0.061808113008737564, + "beta_dpo/beta_used_raw": 0.024503856897354126, + "beta_dpo/gap_mean": 8.988155364990234, + "beta_dpo/gap_std": 14.048759460449219, + "beta_dpo/loss_margin_mean": 8.803841590881348, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2781557067271353, + "grad_norm": 26.691272735595703, + "learning_rate": 4.544080985994258e-07, + "logits/chosen": 0.45184823870658875, + "logits/rejected": 0.3895701766014099, + "loss": 1.1332, + "step": 184 + }, + { + "beta_dpo/beta": 0.12395273894071579, + "beta_dpo/beta_margin_grad_mean": -0.37329500913619995, + "beta_dpo/beta_margin_grad_std": 0.2816545069217682, + "beta_dpo/beta_margin_mean": 1.4491159915924072, + "beta_dpo/beta_margin_std": 3.2485547065734863, + "beta_dpo/beta_used": 0.12395273894071579, + "beta_dpo/beta_used_raw": 0.09359581768512726, + "beta_dpo/gap_mean": 9.074477195739746, + "beta_dpo/gap_std": 14.342059135437012, + "beta_dpo/loss_margin_mean": 9.644372940063477, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2796674225245654, + "grad_norm": 58.39167785644531, + "learning_rate": 4.5364398801258394e-07, + "logits/chosen": 0.41287195682525635, + "logits/rejected": 0.36629611253738403, + "loss": 1.1856, + "step": 185 + }, + { + "beta_dpo/beta": 0.08933384716510773, + "beta_dpo/beta_margin_grad_mean": -0.3376484215259552, + "beta_dpo/beta_margin_grad_std": 0.23757222294807434, + "beta_dpo/beta_margin_mean": 0.938936710357666, + "beta_dpo/beta_margin_std": 1.409091830253601, + "beta_dpo/beta_used": 0.08933384716510773, + "beta_dpo/beta_used_raw": 0.08933384716510773, + "beta_dpo/gap_mean": 9.258342742919922, + "beta_dpo/gap_std": 14.744714736938477, + "beta_dpo/loss_margin_mean": 10.580622673034668, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2811791383219955, + "grad_norm": 32.79323959350586, + "learning_rate": 4.5287418106563354e-07, + "logits/chosen": 0.319606214761734, + "logits/rejected": 0.2793455719947815, + "loss": 0.9711, + "step": 186 + }, + { + "beta_dpo/beta": 0.16774369776248932, + "beta_dpo/beta_margin_grad_mean": -0.3941460847854614, + "beta_dpo/beta_margin_grad_std": 0.2943778336048126, + "beta_dpo/beta_margin_mean": 1.7276290655136108, + "beta_dpo/beta_margin_std": 4.444072723388672, + "beta_dpo/beta_used": 0.16774369776248932, + "beta_dpo/beta_used_raw": 0.15828540921211243, + "beta_dpo/gap_mean": 9.301834106445312, + "beta_dpo/gap_std": 15.03553295135498, + "beta_dpo/loss_margin_mean": 8.952919006347656, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.28269085411942557, + "grad_norm": 77.14328002929688, + "learning_rate": 4.520986992917297e-07, + "logits/chosen": 0.34573984146118164, + "logits/rejected": 0.29307812452316284, + "loss": 1.1231, + "step": 187 + }, + { + "beta_dpo/beta": 0.04338323697447777, + "beta_dpo/beta_margin_grad_mean": -0.4201662838459015, + "beta_dpo/beta_margin_grad_std": 0.1753028780221939, + "beta_dpo/beta_margin_mean": 0.4367694556713104, + "beta_dpo/beta_margin_std": 1.02838933467865, + "beta_dpo/beta_used": 0.04338323697447777, + "beta_dpo/beta_used_raw": 0.007783316075801849, + "beta_dpo/gap_mean": 9.230745315551758, + "beta_dpo/gap_std": 14.944974899291992, + "beta_dpo/loss_margin_mean": 9.515567779541016, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2842025699168556, + "grad_norm": 19.2230281829834, + "learning_rate": 4.5131756438276466e-07, + "logits/chosen": 0.3678765296936035, + "logits/rejected": 0.32579484581947327, + "loss": 1.1893, + "step": 188 + }, + { + "beta_dpo/beta": 0.13889284431934357, + "beta_dpo/beta_margin_grad_mean": -0.3419681489467621, + "beta_dpo/beta_margin_grad_std": 0.23903344571590424, + "beta_dpo/beta_margin_mean": 1.627152442932129, + "beta_dpo/beta_margin_std": 2.859347343444824, + "beta_dpo/beta_used": 0.13889284431934357, + "beta_dpo/beta_used_raw": 0.07579651474952698, + "beta_dpo/gap_mean": 9.295833587646484, + "beta_dpo/gap_std": 14.690962791442871, + "beta_dpo/loss_margin_mean": 8.370240211486816, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2857142857142857, + "grad_norm": 41.23509979248047, + "learning_rate": 4.5053079818876096e-07, + "logits/chosen": 0.3950865566730499, + "logits/rejected": 0.4080202877521515, + "loss": 0.8801, + "step": 189 + }, + { + "beta_dpo/beta": 0.09667577594518661, + "beta_dpo/beta_margin_grad_mean": -0.3044895529747009, + "beta_dpo/beta_margin_grad_std": 0.21711069345474243, + "beta_dpo/beta_margin_mean": 1.1539376974105835, + "beta_dpo/beta_margin_std": 1.3543658256530762, + "beta_dpo/beta_used": 0.09667577594518661, + "beta_dpo/beta_used_raw": 0.09667577594518661, + "beta_dpo/gap_mean": 9.486055374145508, + "beta_dpo/gap_std": 14.589920997619629, + "beta_dpo/loss_margin_mean": 11.935791015625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2872260015117158, + "grad_norm": 41.752498626708984, + "learning_rate": 4.4973842271726024e-07, + "logits/chosen": 0.45457923412323, + "logits/rejected": 0.31400829553604126, + "loss": 0.9074, + "step": 190 + }, + { + "beta_dpo/beta": 0.09922586381435394, + "beta_dpo/beta_margin_grad_mean": -0.34414324164390564, + "beta_dpo/beta_margin_grad_std": 0.24596984684467316, + "beta_dpo/beta_margin_mean": 0.8614938259124756, + "beta_dpo/beta_margin_std": 1.517560601234436, + "beta_dpo/beta_used": 0.09922586381435394, + "beta_dpo/beta_used_raw": 0.09922586381435394, + "beta_dpo/gap_mean": 9.516761779785156, + "beta_dpo/gap_std": 14.810906410217285, + "beta_dpo/loss_margin_mean": 8.773695945739746, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2887377173091459, + "grad_norm": 35.169071197509766, + "learning_rate": 4.48940460132708e-07, + "logits/chosen": 0.4336588382720947, + "logits/rejected": 0.4057428538799286, + "loss": 0.8965, + "step": 191 + }, + { + "beta_dpo/beta": 0.018604885786771774, + "beta_dpo/beta_margin_grad_mean": -0.4708011746406555, + "beta_dpo/beta_margin_grad_std": 0.07631269097328186, + "beta_dpo/beta_margin_mean": 0.12362504750490189, + "beta_dpo/beta_margin_std": 0.3264995813369751, + "beta_dpo/beta_used": 0.018604885786771774, + "beta_dpo/beta_used_raw": -0.016573412343859673, + "beta_dpo/gap_mean": 8.850980758666992, + "beta_dpo/gap_std": 14.339606285095215, + "beta_dpo/loss_margin_mean": 5.650508403778076, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.29024943310657597, + "grad_norm": 9.701313018798828, + "learning_rate": 4.481369327558329e-07, + "logits/chosen": 0.43346714973449707, + "logits/rejected": 0.4073938727378845, + "loss": 1.2685, + "step": 192 + }, + { + "beta_dpo/beta": 0.13145090639591217, + "beta_dpo/beta_margin_grad_mean": -0.35873836278915405, + "beta_dpo/beta_margin_grad_std": 0.231857031583786, + "beta_dpo/beta_margin_mean": 1.6050541400909424, + "beta_dpo/beta_margin_std": 2.9745678901672363, + "beta_dpo/beta_used": 0.13145090639591217, + "beta_dpo/beta_used_raw": 0.12950916588306427, + "beta_dpo/gap_mean": 9.072196960449219, + "beta_dpo/gap_std": 14.12684154510498, + "beta_dpo/loss_margin_mean": 10.512983322143555, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.29176114890400606, + "grad_norm": 35.8497200012207, + "learning_rate": 4.47327863063023e-07, + "logits/chosen": 0.3378722667694092, + "logits/rejected": 0.3161700367927551, + "loss": 0.9555, + "step": 193 + }, + { + "beta_dpo/beta": 0.05448709428310394, + "beta_dpo/beta_margin_grad_mean": -0.4038701355457306, + "beta_dpo/beta_margin_grad_std": 0.22828541696071625, + "beta_dpo/beta_margin_mean": 0.6580454111099243, + "beta_dpo/beta_margin_std": 1.4924321174621582, + "beta_dpo/beta_used": 0.05448709428310394, + "beta_dpo/beta_used_raw": -0.0076342858374118805, + "beta_dpo/gap_mean": 9.176326751708984, + "beta_dpo/gap_std": 14.445403099060059, + "beta_dpo/loss_margin_mean": 8.855490684509277, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.29327286470143615, + "grad_norm": 29.742713928222656, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": 0.39154088497161865, + "logits/rejected": 0.36346444487571716, + "loss": 1.2431, + "step": 194 + }, + { + "beta_dpo/beta": 0.16575054824352264, + "beta_dpo/beta_margin_grad_mean": -0.2906278073787689, + "beta_dpo/beta_margin_grad_std": 0.28971052169799805, + "beta_dpo/beta_margin_mean": 1.8691999912261963, + "beta_dpo/beta_margin_std": 2.65287184715271, + "beta_dpo/beta_used": 0.16575054824352264, + "beta_dpo/beta_used_raw": 0.16575054824352264, + "beta_dpo/gap_mean": 9.309657096862793, + "beta_dpo/gap_std": 14.455615997314453, + "beta_dpo/loss_margin_mean": 10.742607116699219, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2947845804988662, + "grad_norm": 59.959014892578125, + "learning_rate": 4.4569318740967043e-07, + "logits/chosen": 0.30429327487945557, + "logits/rejected": 0.30695855617523193, + "loss": 0.9126, + "step": 195 + }, + { + "beta_dpo/beta": 0.0779055505990982, + "beta_dpo/beta_margin_grad_mean": -0.3585551679134369, + "beta_dpo/beta_margin_grad_std": 0.1866200864315033, + "beta_dpo/beta_margin_mean": 0.8284059166908264, + "beta_dpo/beta_margin_std": 1.2458693981170654, + "beta_dpo/beta_used": 0.0779055505990982, + "beta_dpo/beta_used_raw": 0.0779055505990982, + "beta_dpo/gap_mean": 9.129471778869629, + "beta_dpo/gap_std": 14.351459503173828, + "beta_dpo/loss_margin_mean": 8.850098609924316, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.2962962962962963, + "grad_norm": 30.147045135498047, + "learning_rate": 4.448676271745197e-07, + "logits/chosen": 0.41115450859069824, + "logits/rejected": 0.37019413709640503, + "loss": 0.9658, + "step": 196 + }, + { + "beta_dpo/beta": 0.17337322235107422, + "beta_dpo/beta_margin_grad_mean": -0.3025214970111847, + "beta_dpo/beta_margin_grad_std": 0.3217240571975708, + "beta_dpo/beta_margin_mean": 1.8553613424301147, + "beta_dpo/beta_margin_std": 3.1213772296905518, + "beta_dpo/beta_used": 0.17337322235107422, + "beta_dpo/beta_used_raw": 0.17337322235107422, + "beta_dpo/gap_mean": 9.257464408874512, + "beta_dpo/gap_std": 14.713043212890625, + "beta_dpo/loss_margin_mean": 10.203736305236816, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.29780801209372637, + "grad_norm": 74.21235656738281, + "learning_rate": 4.440366160729392e-07, + "logits/chosen": 0.46007847785949707, + "logits/rejected": 0.41251420974731445, + "loss": 0.9402, + "step": 197 + }, + { + "beta_dpo/beta": 0.14420902729034424, + "beta_dpo/beta_margin_grad_mean": -0.33038392663002014, + "beta_dpo/beta_margin_grad_std": 0.25793397426605225, + "beta_dpo/beta_margin_mean": 1.8629796504974365, + "beta_dpo/beta_margin_std": 3.182447910308838, + "beta_dpo/beta_used": 0.14420902729034424, + "beta_dpo/beta_used_raw": 0.14420902729034424, + "beta_dpo/gap_mean": 9.538639068603516, + "beta_dpo/gap_std": 14.523576736450195, + "beta_dpo/loss_margin_mean": 10.75778579711914, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.29931972789115646, + "grad_norm": 32.6105842590332, + "learning_rate": 4.432001773500957e-07, + "logits/chosen": 0.4632105827331543, + "logits/rejected": 0.423373818397522, + "loss": 0.8424, + "step": 198 + }, + { + "beta_dpo/beta": 0.2410755455493927, + "beta_dpo/beta_margin_grad_mean": -0.3472752869129181, + "beta_dpo/beta_margin_grad_std": 0.28805285692214966, + "beta_dpo/beta_margin_mean": 2.90079665184021, + "beta_dpo/beta_margin_std": 5.994080066680908, + "beta_dpo/beta_used": 0.2410755455493927, + "beta_dpo/beta_used_raw": 0.19233468174934387, + "beta_dpo/gap_mean": 9.373411178588867, + "beta_dpo/gap_std": 14.620790481567383, + "beta_dpo/loss_margin_mean": 8.427955627441406, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.30083144368858655, + "grad_norm": 90.61835479736328, + "learning_rate": 4.4235833440297856e-07, + "logits/chosen": 0.4073561728000641, + "logits/rejected": 0.32114654779434204, + "loss": 0.8905, + "step": 199 + }, + { + "beta_dpo/beta": 0.14784272015094757, + "beta_dpo/beta_margin_grad_mean": -0.3243432939052582, + "beta_dpo/beta_margin_grad_std": 0.24798478186130524, + "beta_dpo/beta_margin_mean": 2.166560173034668, + "beta_dpo/beta_margin_std": 3.9794764518737793, + "beta_dpo/beta_used": 0.14784272015094757, + "beta_dpo/beta_used_raw": 0.14784272015094757, + "beta_dpo/gap_mean": 10.113958358764648, + "beta_dpo/gap_std": 15.15192985534668, + "beta_dpo/loss_margin_mean": 13.069280624389648, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.30234315948601664, + "grad_norm": 64.76429748535156, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": 0.43764257431030273, + "logits/rejected": 0.3673984110355377, + "loss": 0.9318, + "step": 200 + }, + { + "epoch": 0.30234315948601664, + "eval_beta_dpo/beta": 0.0751558393239975, + "eval_beta_dpo/beta_margin_grad_mean": -0.39745259284973145, + "eval_beta_dpo/beta_margin_grad_std": 0.13326896727085114, + "eval_beta_dpo/beta_margin_mean": 0.866983950138092, + "eval_beta_dpo/beta_margin_std": 1.1565665006637573, + "eval_beta_dpo/beta_used": 0.0751558393239975, + "eval_beta_dpo/beta_used_raw": 0.03455571457743645, + "eval_beta_dpo/gap_mean": 10.170903205871582, + "eval_beta_dpo/gap_std": 15.263747215270996, + "eval_beta_dpo/loss_margin_mean": 9.080164909362793, + "eval_beta_dpo/mask_keep_frac": 1.0, + "eval_logits/chosen": 0.42500150203704834, + "eval_logits/rejected": 0.37864118814468384, + "eval_loss": 0.5938660502433777, + "eval_runtime": 38.8927, + "eval_samples_per_second": 59.214, + "eval_steps_per_second": 1.851, + "step": 200 + }, + { + "beta_dpo/beta": 0.10797238349914551, + "beta_dpo/beta_margin_grad_mean": -0.34527766704559326, + "beta_dpo/beta_margin_grad_std": 0.2624824643135071, + "beta_dpo/beta_margin_mean": 0.9581461548805237, + "beta_dpo/beta_margin_std": 1.753610372543335, + "beta_dpo/beta_used": 0.10797238349914551, + "beta_dpo/beta_used_raw": 0.10797238349914551, + "beta_dpo/gap_mean": 10.185378074645996, + "beta_dpo/gap_std": 15.321168899536133, + "beta_dpo/loss_margin_mean": 9.576545715332031, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.30385487528344673, + "grad_norm": 41.83230972290039, + "learning_rate": 4.4065853017905953e-07, + "logits/chosen": 0.4699372947216034, + "logits/rejected": 0.4243282973766327, + "loss": 0.8809, + "step": 201 + }, + { + "beta_dpo/beta": 0.09276492148637772, + "beta_dpo/beta_margin_grad_mean": -0.3580490052700043, + "beta_dpo/beta_margin_grad_std": 0.2220529168844223, + "beta_dpo/beta_margin_mean": 1.1636439561843872, + "beta_dpo/beta_margin_std": 2.07939076423645, + "beta_dpo/beta_used": 0.09276492148637772, + "beta_dpo/beta_used_raw": 0.036593932658433914, + "beta_dpo/gap_mean": 9.951011657714844, + "beta_dpo/gap_std": 15.453784942626953, + "beta_dpo/loss_margin_mean": 10.190545082092285, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.30536659108087677, + "grad_norm": 27.87430191040039, + "learning_rate": 4.3980061644943575e-07, + "logits/chosen": 0.362282931804657, + "logits/rejected": 0.292508989572525, + "loss": 0.9849, + "step": 202 + }, + { + "beta_dpo/beta": 0.009305215440690517, + "beta_dpo/beta_margin_grad_mean": -0.48150211572647095, + "beta_dpo/beta_margin_grad_std": 0.04638690501451492, + "beta_dpo/beta_margin_mean": 0.0751611739397049, + "beta_dpo/beta_margin_std": 0.18903209269046783, + "beta_dpo/beta_used": 0.009305215440690517, + "beta_dpo/beta_used_raw": -0.0015188101679086685, + "beta_dpo/gap_mean": 9.854846000671387, + "beta_dpo/gap_std": 15.199752807617188, + "beta_dpo/loss_margin_mean": 8.687176704406738, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.30687830687830686, + "grad_norm": 7.202058792114258, + "learning_rate": 4.3893739358856455e-07, + "logits/chosen": 0.42288321256637573, + "logits/rejected": 0.3511812090873718, + "loss": 1.3163, + "step": 203 + }, + { + "beta_dpo/beta": 0.12975779175758362, + "beta_dpo/beta_margin_grad_mean": -0.3484925925731659, + "beta_dpo/beta_margin_grad_std": 0.2547266483306885, + "beta_dpo/beta_margin_mean": 1.9150761365890503, + "beta_dpo/beta_margin_std": 3.3663294315338135, + "beta_dpo/beta_used": 0.12975779175758362, + "beta_dpo/beta_used_raw": 0.09229342639446259, + "beta_dpo/gap_mean": 10.125568389892578, + "beta_dpo/gap_std": 15.098621368408203, + "beta_dpo/loss_margin_mean": 10.558574676513672, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.30839002267573695, + "grad_norm": 46.554901123046875, + "learning_rate": 4.380688857426449e-07, + "logits/chosen": 0.36739447712898254, + "logits/rejected": 0.3011772036552429, + "loss": 1.0569, + "step": 204 + }, + { + "beta_dpo/beta": 0.056526899337768555, + "beta_dpo/beta_margin_grad_mean": -0.3798976540565491, + "beta_dpo/beta_margin_grad_std": 0.176364466547966, + "beta_dpo/beta_margin_mean": 0.6287395358085632, + "beta_dpo/beta_margin_std": 1.0148792266845703, + "beta_dpo/beta_used": 0.056526899337768555, + "beta_dpo/beta_used_raw": 0.056526899337768555, + "beta_dpo/gap_mean": 10.036584854125977, + "beta_dpo/gap_std": 15.158401489257812, + "beta_dpo/loss_margin_mean": 10.209553718566895, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.30990173847316704, + "grad_norm": 26.912302017211914, + "learning_rate": 4.3719511720570814e-07, + "logits/chosen": 0.4105345606803894, + "logits/rejected": 0.349745512008667, + "loss": 1.0787, + "step": 205 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4979967772960663, + "beta_dpo/beta_margin_grad_std": 0.004184938967227936, + "beta_dpo/beta_margin_mean": 0.008013593032956123, + "beta_dpo/beta_margin_std": 0.016741132363677025, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.09804756939411163, + "beta_dpo/gap_mean": 9.589317321777344, + "beta_dpo/gap_std": 15.620407104492188, + "beta_dpo/loss_margin_mean": 8.013592720031738, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.31141345427059713, + "grad_norm": 0.5026677846908569, + "learning_rate": 4.363161124189387e-07, + "logits/chosen": 0.43905892968177795, + "logits/rejected": 0.42348912358283997, + "loss": 1.3801, + "step": 206 + }, + { + "beta_dpo/beta": 0.14289897680282593, + "beta_dpo/beta_margin_grad_mean": -0.29581886529922485, + "beta_dpo/beta_margin_grad_std": 0.2654317021369934, + "beta_dpo/beta_margin_mean": 1.6500314474105835, + "beta_dpo/beta_margin_std": 2.359097480773926, + "beta_dpo/beta_used": 0.14289897680282593, + "beta_dpo/beta_used_raw": 0.14289897680282593, + "beta_dpo/gap_mean": 9.915831565856934, + "beta_dpo/gap_std": 15.770639419555664, + "beta_dpo/loss_margin_mean": 11.355326652526855, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3129251700680272, + "grad_norm": 43.67290115356445, + "learning_rate": 4.3543189596998986e-07, + "logits/chosen": 0.3800593316555023, + "logits/rejected": 0.31542912125587463, + "loss": 0.7969, + "step": 207 + }, + { + "beta_dpo/beta": 0.02445879578590393, + "beta_dpo/beta_margin_grad_mean": -0.4628857374191284, + "beta_dpo/beta_margin_grad_std": 0.12461158633232117, + "beta_dpo/beta_margin_mean": 0.1690797209739685, + "beta_dpo/beta_margin_std": 0.5674142241477966, + "beta_dpo/beta_used": 0.02445879578590393, + "beta_dpo/beta_used_raw": -0.005164986476302147, + "beta_dpo/gap_mean": 9.330613136291504, + "beta_dpo/gap_std": 15.561910629272461, + "beta_dpo/loss_margin_mean": 6.052608013153076, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3144368858654573, + "grad_norm": 14.582758903503418, + "learning_rate": 4.3454249259229664e-07, + "logits/chosen": 0.3913588225841522, + "logits/rejected": 0.365945041179657, + "loss": 1.2406, + "step": 208 + }, + { + "beta_dpo/beta": 0.26314157247543335, + "beta_dpo/beta_margin_grad_mean": -0.26778027415275574, + "beta_dpo/beta_margin_grad_std": 0.3371621072292328, + "beta_dpo/beta_margin_mean": 3.1177279949188232, + "beta_dpo/beta_margin_std": 4.769398212432861, + "beta_dpo/beta_used": 0.26314157247543335, + "beta_dpo/beta_used_raw": 0.26314157247543335, + "beta_dpo/gap_mean": 9.51311206817627, + "beta_dpo/gap_std": 15.985190391540527, + "beta_dpo/loss_margin_mean": 11.842841148376465, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.31594860166288735, + "grad_norm": 107.09064483642578, + "learning_rate": 4.336479271643833e-07, + "logits/chosen": 0.3666651248931885, + "logits/rejected": 0.3131159543991089, + "loss": 1.0, + "step": 209 + }, + { + "beta_dpo/beta": 0.1870449036359787, + "beta_dpo/beta_margin_grad_mean": -0.26491236686706543, + "beta_dpo/beta_margin_grad_std": 0.30507174134254456, + "beta_dpo/beta_margin_mean": 2.4757566452026367, + "beta_dpo/beta_margin_std": 3.589545488357544, + "beta_dpo/beta_used": 0.1870449036359787, + "beta_dpo/beta_used_raw": 0.1870449036359787, + "beta_dpo/gap_mean": 10.071860313415527, + "beta_dpo/gap_std": 16.347694396972656, + "beta_dpo/loss_margin_mean": 12.725071907043457, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.31746031746031744, + "grad_norm": 56.52892303466797, + "learning_rate": 4.327482247091679e-07, + "logits/chosen": 0.45084625482559204, + "logits/rejected": 0.3572552800178528, + "loss": 0.8648, + "step": 210 + }, + { + "beta_dpo/beta": 0.0382014662027359, + "beta_dpo/beta_margin_grad_mean": -0.43724095821380615, + "beta_dpo/beta_margin_grad_std": 0.17051011323928833, + "beta_dpo/beta_margin_mean": 0.3267289698123932, + "beta_dpo/beta_margin_std": 0.8928489685058594, + "beta_dpo/beta_used": 0.0382014662027359, + "beta_dpo/beta_used_raw": 0.012627029791474342, + "beta_dpo/gap_mean": 10.269024848937988, + "beta_dpo/gap_std": 16.395240783691406, + "beta_dpo/loss_margin_mean": 10.694591522216797, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.31897203325774753, + "grad_norm": 24.648975372314453, + "learning_rate": 4.3184341039326217e-07, + "logits/chosen": 0.46954113245010376, + "logits/rejected": 0.3781422972679138, + "loss": 1.1807, + "step": 211 + }, + { + "beta_dpo/beta": 0.3232237696647644, + "beta_dpo/beta_margin_grad_mean": -0.2582166790962219, + "beta_dpo/beta_margin_grad_std": 0.3422297537326813, + "beta_dpo/beta_margin_mean": 4.010345935821533, + "beta_dpo/beta_margin_std": 6.0359578132629395, + "beta_dpo/beta_used": 0.3232237696647644, + "beta_dpo/beta_used_raw": 0.3232237696647644, + "beta_dpo/gap_mean": 10.692178726196289, + "beta_dpo/gap_std": 16.397342681884766, + "beta_dpo/loss_margin_mean": 12.392468452453613, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3204837490551776, + "grad_norm": 106.18115234375, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": 0.44062310457229614, + "logits/rejected": 0.37093472480773926, + "loss": 0.8162, + "step": 212 + }, + { + "beta_dpo/beta": 0.11237598955631256, + "beta_dpo/beta_margin_grad_mean": -0.3015235662460327, + "beta_dpo/beta_margin_grad_std": 0.26578059792518616, + "beta_dpo/beta_margin_mean": 1.4179470539093018, + "beta_dpo/beta_margin_std": 2.069363594055176, + "beta_dpo/beta_used": 0.11237598955631256, + "beta_dpo/beta_used_raw": 0.11237598955631256, + "beta_dpo/gap_mean": 11.056885719299316, + "beta_dpo/gap_std": 16.766132354736328, + "beta_dpo/loss_margin_mean": 12.505507469177246, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3219954648526077, + "grad_norm": 40.509246826171875, + "learning_rate": 4.3001854756006724e-07, + "logits/chosen": 0.43783697485923767, + "logits/rejected": 0.41364121437072754, + "loss": 0.9027, + "step": 213 + }, + { + "beta_dpo/beta": 0.0608704648911953, + "beta_dpo/beta_margin_grad_mean": -0.3818773627281189, + "beta_dpo/beta_margin_grad_std": 0.23320011794567108, + "beta_dpo/beta_margin_mean": 0.9389346837997437, + "beta_dpo/beta_margin_std": 1.8776214122772217, + "beta_dpo/beta_used": 0.0608704648911953, + "beta_dpo/beta_used_raw": 0.01937410980463028, + "beta_dpo/gap_mean": 11.113325119018555, + "beta_dpo/gap_std": 17.1014404296875, + "beta_dpo/loss_margin_mean": 12.176145553588867, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3235071806500378, + "grad_norm": 35.15140151977539, + "learning_rate": 4.290985500881143e-07, + "logits/chosen": 0.3065475821495056, + "logits/rejected": 0.2855311930179596, + "loss": 1.1407, + "step": 214 + }, + { + "beta_dpo/beta": 0.21405133605003357, + "beta_dpo/beta_margin_grad_mean": -0.27612632513046265, + "beta_dpo/beta_margin_grad_std": 0.3283703327178955, + "beta_dpo/beta_margin_mean": 2.8124191761016846, + "beta_dpo/beta_margin_std": 3.740720748901367, + "beta_dpo/beta_used": 0.21405133605003357, + "beta_dpo/beta_used_raw": 0.21405133605003357, + "beta_dpo/gap_mean": 11.513839721679688, + "beta_dpo/gap_std": 17.267501831054688, + "beta_dpo/loss_margin_mean": 13.468052864074707, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3250188964474679, + "grad_norm": 83.70057678222656, + "learning_rate": 4.281735428447157e-07, + "logits/chosen": 0.33656060695648193, + "logits/rejected": 0.23366260528564453, + "loss": 0.8325, + "step": 215 + }, + { + "beta_dpo/beta": 0.063032366335392, + "beta_dpo/beta_margin_grad_mean": -0.3830406665802002, + "beta_dpo/beta_margin_grad_std": 0.21951407194137573, + "beta_dpo/beta_margin_mean": 0.8744828701019287, + "beta_dpo/beta_margin_std": 1.6632479429244995, + "beta_dpo/beta_used": 0.063032366335392, + "beta_dpo/beta_used_raw": 0.04040754958987236, + "beta_dpo/gap_mean": 12.008759498596191, + "beta_dpo/gap_std": 17.251224517822266, + "beta_dpo/loss_margin_mean": 13.142531394958496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.32653061224489793, + "grad_norm": 26.42040252685547, + "learning_rate": 4.2724355170431247e-07, + "logits/chosen": 0.45972394943237305, + "logits/rejected": 0.3764275014400482, + "loss": 1.0786, + "step": 216 + }, + { + "beta_dpo/beta": 0.05261437967419624, + "beta_dpo/beta_margin_grad_mean": -0.3919501006603241, + "beta_dpo/beta_margin_grad_std": 0.20264987647533417, + "beta_dpo/beta_margin_mean": 0.7517213821411133, + "beta_dpo/beta_margin_std": 1.5161195993423462, + "beta_dpo/beta_used": 0.05261437967419624, + "beta_dpo/beta_used_raw": 0.01570678874850273, + "beta_dpo/gap_mean": 12.255045890808105, + "beta_dpo/gap_std": 17.351184844970703, + "beta_dpo/loss_margin_mean": 13.261373519897461, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.328042328042328, + "grad_norm": 24.293655395507812, + "learning_rate": 4.26308602680756e-07, + "logits/chosen": 0.4059142768383026, + "logits/rejected": 0.30515235662460327, + "loss": 1.121, + "step": 217 + }, + { + "beta_dpo/beta": 0.14698970317840576, + "beta_dpo/beta_margin_grad_mean": -0.38040444254875183, + "beta_dpo/beta_margin_grad_std": 0.28120723366737366, + "beta_dpo/beta_margin_mean": 1.6690462827682495, + "beta_dpo/beta_margin_std": 4.68047571182251, + "beta_dpo/beta_used": 0.14698970317840576, + "beta_dpo/beta_used_raw": 0.14698970317840576, + "beta_dpo/gap_mean": 11.52371597290039, + "beta_dpo/gap_std": 17.293601989746094, + "beta_dpo/loss_margin_mean": 8.26142406463623, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3295540438397581, + "grad_norm": 56.007667541503906, + "learning_rate": 4.253687219265803e-07, + "logits/chosen": 0.2987746000289917, + "logits/rejected": 0.2939714789390564, + "loss": 1.1388, + "step": 218 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4981619417667389, + "beta_dpo/beta_margin_grad_std": 0.0033710116986185312, + "beta_dpo/beta_margin_mean": 0.007352730259299278, + "beta_dpo/beta_margin_std": 0.013484997674822807, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.06018597632646561, + "beta_dpo/gap_mean": 10.95039176940918, + "beta_dpo/gap_std": 16.972551345825195, + "beta_dpo/loss_margin_mean": 7.3527302742004395, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3310657596371882, + "grad_norm": 0.6384725570678711, + "learning_rate": 4.2442393573227043e-07, + "logits/chosen": 0.3558601140975952, + "logits/rejected": 0.3152506947517395, + "loss": 1.3781, + "step": 219 + }, + { + "beta_dpo/beta": 0.12556403875350952, + "beta_dpo/beta_margin_grad_mean": -0.36618298292160034, + "beta_dpo/beta_margin_grad_std": 0.24885612726211548, + "beta_dpo/beta_margin_mean": 1.5560061931610107, + "beta_dpo/beta_margin_std": 2.9427649974823, + "beta_dpo/beta_used": 0.12556403875350952, + "beta_dpo/beta_used_raw": 0.055669739842414856, + "beta_dpo/gap_mean": 10.764374732971191, + "beta_dpo/gap_std": 16.656213760375977, + "beta_dpo/loss_margin_mean": 10.330924987792969, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3325774754346183, + "grad_norm": 60.86797332763672, + "learning_rate": 4.234742705255272e-07, + "logits/chosen": 0.4525846242904663, + "logits/rejected": 0.39114877581596375, + "loss": 1.0217, + "step": 220 + }, + { + "beta_dpo/beta": 0.029629409313201904, + "beta_dpo/beta_margin_grad_mean": -0.4422619044780731, + "beta_dpo/beta_margin_grad_std": 0.1514252871274948, + "beta_dpo/beta_margin_mean": 0.2931990623474121, + "beta_dpo/beta_margin_std": 0.7789700031280518, + "beta_dpo/beta_used": 0.029629409313201904, + "beta_dpo/beta_used_raw": 0.015723902732133865, + "beta_dpo/gap_mean": 10.685160636901855, + "beta_dpo/gap_std": 16.907085418701172, + "beta_dpo/loss_margin_mean": 11.518767356872559, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3340891912320484, + "grad_norm": 17.142822265625, + "learning_rate": 4.22519752870528e-07, + "logits/chosen": 0.398847758769989, + "logits/rejected": 0.3308085799217224, + "loss": 1.1883, + "step": 221 + }, + { + "beta_dpo/beta": 0.21331076323986053, + "beta_dpo/beta_margin_grad_mean": -0.23689118027687073, + "beta_dpo/beta_margin_grad_std": 0.292352557182312, + "beta_dpo/beta_margin_mean": 3.1503472328186035, + "beta_dpo/beta_margin_std": 3.831242322921753, + "beta_dpo/beta_used": 0.21331076323986053, + "beta_dpo/beta_used_raw": 0.21331076323986053, + "beta_dpo/gap_mean": 11.39797592163086, + "beta_dpo/gap_std": 16.90200424194336, + "beta_dpo/loss_margin_mean": 14.945018768310547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3356009070294785, + "grad_norm": 59.19512939453125, + "learning_rate": 4.2156040946718343e-07, + "logits/chosen": 0.43866443634033203, + "logits/rejected": 0.35765448212623596, + "loss": 0.7302, + "step": 222 + }, + { + "beta_dpo/beta": 0.052638933062553406, + "beta_dpo/beta_margin_grad_mean": -0.40419548749923706, + "beta_dpo/beta_margin_grad_std": 0.18599951267242432, + "beta_dpo/beta_margin_mean": 0.5999060869216919, + "beta_dpo/beta_margin_std": 1.2042607069015503, + "beta_dpo/beta_used": 0.052638933062553406, + "beta_dpo/beta_used_raw": 0.03331661969423294, + "beta_dpo/gap_mean": 11.813493728637695, + "beta_dpo/gap_std": 16.868061065673828, + "beta_dpo/loss_margin_mean": 12.644187927246094, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3371126228269085, + "grad_norm": 23.424663543701172, + "learning_rate": 4.2059626715039065e-07, + "logits/chosen": 0.43733513355255127, + "logits/rejected": 0.3827732801437378, + "loss": 1.0815, + "step": 223 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49811801314353943, + "beta_dpo/beta_margin_grad_std": 0.0036412279587239027, + "beta_dpo/beta_margin_mean": 0.0075285304337739944, + "beta_dpo/beta_margin_std": 0.014566164463758469, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.14365257322788239, + "beta_dpo/gap_mean": 11.2182035446167, + "beta_dpo/gap_std": 16.46889877319336, + "beta_dpo/loss_margin_mean": 7.528530120849609, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3386243386243386, + "grad_norm": 0.497575044631958, + "learning_rate": 4.1962735288928304e-07, + "logits/chosen": 0.45619451999664307, + "logits/rejected": 0.43575888872146606, + "loss": 1.3792, + "step": 224 + }, + { + "beta_dpo/beta": 0.0709293931722641, + "beta_dpo/beta_margin_grad_mean": -0.3631777763366699, + "beta_dpo/beta_margin_grad_std": 0.20055457949638367, + "beta_dpo/beta_margin_mean": 1.091381311416626, + "beta_dpo/beta_margin_std": 1.866531252861023, + "beta_dpo/beta_used": 0.0709293931722641, + "beta_dpo/beta_used_raw": 0.05940817669034004, + "beta_dpo/gap_mean": 11.353107452392578, + "beta_dpo/gap_std": 16.404705047607422, + "beta_dpo/loss_margin_mean": 12.793769836425781, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3401360544217687, + "grad_norm": 28.656009674072266, + "learning_rate": 4.186536937864752e-07, + "logits/chosen": 0.4412558972835541, + "logits/rejected": 0.3315754532814026, + "loss": 1.033, + "step": 225 + }, + { + "beta_dpo/beta": 0.08705855906009674, + "beta_dpo/beta_margin_grad_mean": -0.4210311770439148, + "beta_dpo/beta_margin_grad_std": 0.25707387924194336, + "beta_dpo/beta_margin_mean": 0.7551378011703491, + "beta_dpo/beta_margin_std": 2.270979881286621, + "beta_dpo/beta_used": 0.08705855906009674, + "beta_dpo/beta_used_raw": 0.004039667546749115, + "beta_dpo/gap_mean": 10.987207412719727, + "beta_dpo/gap_std": 16.678972244262695, + "beta_dpo/loss_margin_mean": 9.073394775390625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3416477702191988, + "grad_norm": 35.07719039916992, + "learning_rate": 4.176753170773052e-07, + "logits/chosen": 0.4656580984592438, + "logits/rejected": 0.41961991786956787, + "loss": 1.1192, + "step": 226 + }, + { + "beta_dpo/beta": 0.17244452238082886, + "beta_dpo/beta_margin_grad_mean": -0.3985823690891266, + "beta_dpo/beta_margin_grad_std": 0.2724515199661255, + "beta_dpo/beta_margin_mean": 1.867832064628601, + "beta_dpo/beta_margin_std": 4.624873161315918, + "beta_dpo/beta_used": 0.17244452238082886, + "beta_dpo/beta_used_raw": 0.1496300846338272, + "beta_dpo/gap_mean": 10.793601036071777, + "beta_dpo/gap_std": 16.972103118896484, + "beta_dpo/loss_margin_mean": 10.16253662109375, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3431594860166289, + "grad_norm": 114.27677917480469, + "learning_rate": 4.166922501290729e-07, + "logits/chosen": 0.4805440604686737, + "logits/rejected": 0.4414859414100647, + "loss": 1.1064, + "step": 227 + }, + { + "beta_dpo/beta": 0.09896055608987808, + "beta_dpo/beta_margin_grad_mean": -0.38118550181388855, + "beta_dpo/beta_margin_grad_std": 0.26663917303085327, + "beta_dpo/beta_margin_mean": 1.2820067405700684, + "beta_dpo/beta_margin_std": 2.9501523971557617, + "beta_dpo/beta_used": 0.09896055608987808, + "beta_dpo/beta_used_raw": 0.09224580228328705, + "beta_dpo/gap_mean": 10.714456558227539, + "beta_dpo/gap_std": 17.144290924072266, + "beta_dpo/loss_margin_mean": 11.201956748962402, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.34467120181405897, + "grad_norm": 55.93511962890625, + "learning_rate": 4.1570452044027405e-07, + "logits/chosen": 0.4445973336696625, + "logits/rejected": 0.3688391447067261, + "loss": 1.0708, + "step": 228 + }, + { + "beta_dpo/beta": 0.07893217355012894, + "beta_dpo/beta_margin_grad_mean": -0.4054025709629059, + "beta_dpo/beta_margin_grad_std": 0.23098501563072205, + "beta_dpo/beta_margin_mean": 0.7533602714538574, + "beta_dpo/beta_margin_std": 1.986180067062378, + "beta_dpo/beta_used": 0.07893217355012894, + "beta_dpo/beta_used_raw": 0.07893217355012894, + "beta_dpo/gap_mean": 10.609885215759277, + "beta_dpo/gap_std": 17.362579345703125, + "beta_dpo/loss_margin_mean": 9.359911918640137, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.34618291761148906, + "grad_norm": 29.154024124145508, + "learning_rate": 4.147121556398312e-07, + "logits/chosen": 0.5463353395462036, + "logits/rejected": 0.4801861047744751, + "loss": 1.0969, + "step": 229 + }, + { + "beta_dpo/beta": 0.025358129292726517, + "beta_dpo/beta_margin_grad_mean": -0.42037683725357056, + "beta_dpo/beta_margin_grad_std": 0.1302468180656433, + "beta_dpo/beta_margin_mean": 0.37733587622642517, + "beta_dpo/beta_margin_std": 0.6492464542388916, + "beta_dpo/beta_used": 0.025358129292726517, + "beta_dpo/beta_used_raw": 0.02071252465248108, + "beta_dpo/gap_mean": 10.497218132019043, + "beta_dpo/gap_std": 17.413190841674805, + "beta_dpo/loss_margin_mean": 11.307621002197266, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3476946334089191, + "grad_norm": 12.894368171691895, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": 0.434891939163208, + "logits/rejected": 0.43139857053756714, + "loss": 1.2022, + "step": 230 + }, + { + "beta_dpo/beta": 0.3272258937358856, + "beta_dpo/beta_margin_grad_mean": -0.22254712879657745, + "beta_dpo/beta_margin_grad_std": 0.31797823309898376, + "beta_dpo/beta_margin_mean": 5.628799915313721, + "beta_dpo/beta_margin_std": 6.833213806152344, + "beta_dpo/beta_used": 0.3272258937358856, + "beta_dpo/beta_used_raw": 0.3272258937358856, + "beta_dpo/gap_mean": 11.607155799865723, + "beta_dpo/gap_std": 17.173091888427734, + "beta_dpo/loss_margin_mean": 16.309707641601562, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3492063492063492, + "grad_norm": 165.0520477294922, + "learning_rate": 4.1271363186719835e-07, + "logits/chosen": 0.35244348645210266, + "logits/rejected": 0.3390952944755554, + "loss": 0.8224, + "step": 231 + }, + { + "beta_dpo/beta": 0.06888004392385483, + "beta_dpo/beta_margin_grad_mean": -0.40324854850769043, + "beta_dpo/beta_margin_grad_std": 0.20974577963352203, + "beta_dpo/beta_margin_mean": 0.798814594745636, + "beta_dpo/beta_margin_std": 1.7441960573196411, + "beta_dpo/beta_used": 0.06888004392385483, + "beta_dpo/beta_used_raw": -0.003230072557926178, + "beta_dpo/gap_mean": 11.649392127990723, + "beta_dpo/gap_std": 17.554821014404297, + "beta_dpo/loss_margin_mean": 10.54377269744873, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3507180650037793, + "grad_norm": 36.105350494384766, + "learning_rate": 4.1170752879801436e-07, + "logits/chosen": 0.41157642006874084, + "logits/rejected": 0.38223710656166077, + "loss": 1.1257, + "step": 232 + }, + { + "beta_dpo/beta": 0.09651569277048111, + "beta_dpo/beta_margin_grad_mean": -0.37120458483695984, + "beta_dpo/beta_margin_grad_std": 0.2465994656085968, + "beta_dpo/beta_margin_mean": 1.4080229997634888, + "beta_dpo/beta_margin_std": 2.7828683853149414, + "beta_dpo/beta_used": 0.09651569277048111, + "beta_dpo/beta_used_raw": -0.13416676223278046, + "beta_dpo/gap_mean": 11.465993881225586, + "beta_dpo/gap_std": 17.789779663085938, + "beta_dpo/loss_margin_mean": 9.44713020324707, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.35222978080120937, + "grad_norm": 45.80852508544922, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": 0.45560508966445923, + "logits/rejected": 0.39847636222839355, + "loss": 1.0658, + "step": 233 + }, + { + "beta_dpo/beta": 0.06583171337842941, + "beta_dpo/beta_margin_grad_mean": -0.3836709260940552, + "beta_dpo/beta_margin_grad_std": 0.22087500989437103, + "beta_dpo/beta_margin_mean": 0.8249198198318481, + "beta_dpo/beta_margin_std": 1.6976362466812134, + "beta_dpo/beta_used": 0.06583171337842941, + "beta_dpo/beta_used_raw": -0.054439254105091095, + "beta_dpo/gap_mean": 10.425298690795898, + "beta_dpo/gap_std": 17.674182891845703, + "beta_dpo/loss_margin_mean": 7.9661054611206055, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.35374149659863946, + "grad_norm": 30.462419509887695, + "learning_rate": 4.09681781007452e-07, + "logits/chosen": 0.37714219093322754, + "logits/rejected": 0.3655141592025757, + "loss": 1.1052, + "step": 234 + }, + { + "beta_dpo/beta": 0.21070542931556702, + "beta_dpo/beta_margin_grad_mean": -0.20869186520576477, + "beta_dpo/beta_margin_grad_std": 0.26472654938697815, + "beta_dpo/beta_margin_mean": 3.020737886428833, + "beta_dpo/beta_margin_std": 3.6254777908325195, + "beta_dpo/beta_used": 0.21070542931556702, + "beta_dpo/beta_used_raw": 0.21070542931556702, + "beta_dpo/gap_mean": 11.103967666625977, + "beta_dpo/gap_std": 17.23556137084961, + "beta_dpo/loss_margin_mean": 14.643046379089355, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.35525321239606955, + "grad_norm": 96.36847686767578, + "learning_rate": 4.08662192950594e-07, + "logits/chosen": 0.4545882046222687, + "logits/rejected": 0.43633896112442017, + "loss": 0.8063, + "step": 235 + }, + { + "beta_dpo/beta": 0.2329292595386505, + "beta_dpo/beta_margin_grad_mean": -0.2999168634414673, + "beta_dpo/beta_margin_grad_std": 0.33858177065849304, + "beta_dpo/beta_margin_mean": 2.892549514770508, + "beta_dpo/beta_margin_std": 5.401739597320557, + "beta_dpo/beta_used": 0.2329292595386505, + "beta_dpo/beta_used_raw": 0.2329292595386505, + "beta_dpo/gap_mean": 11.491912841796875, + "beta_dpo/gap_std": 17.67310905456543, + "beta_dpo/loss_margin_mean": 11.814598083496094, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.35676492819349964, + "grad_norm": 98.90902709960938, + "learning_rate": 4.076381667711306e-07, + "logits/chosen": 0.39827001094818115, + "logits/rejected": 0.38383930921554565, + "loss": 0.8167, + "step": 236 + }, + { + "beta_dpo/beta": 0.3174073100090027, + "beta_dpo/beta_margin_grad_mean": -0.21913442015647888, + "beta_dpo/beta_margin_grad_std": 0.3325287401676178, + "beta_dpo/beta_margin_mean": 3.888357639312744, + "beta_dpo/beta_margin_std": 7.1055073738098145, + "beta_dpo/beta_used": 0.3174073100090027, + "beta_dpo/beta_used_raw": 0.3174073100090027, + "beta_dpo/gap_mean": 11.519922256469727, + "beta_dpo/gap_std": 17.956809997558594, + "beta_dpo/loss_margin_mean": 12.32577133178711, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.35827664399092973, + "grad_norm": 120.24292755126953, + "learning_rate": 4.066097311132753e-07, + "logits/chosen": 0.4687383472919464, + "logits/rejected": 0.45551079511642456, + "loss": 0.6911, + "step": 237 + }, + { + "beta_dpo/beta": 0.01583448052406311, + "beta_dpo/beta_margin_grad_mean": -0.4526436924934387, + "beta_dpo/beta_margin_grad_std": 0.07343795150518417, + "beta_dpo/beta_margin_mean": 0.19448736310005188, + "beta_dpo/beta_margin_std": 0.30200886726379395, + "beta_dpo/beta_used": 0.01583448052406311, + "beta_dpo/beta_used_raw": 0.01583448052406311, + "beta_dpo/gap_mean": 11.880534172058105, + "beta_dpo/gap_std": 17.80138397216797, + "beta_dpo/loss_margin_mean": 12.886168479919434, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.35978835978835977, + "grad_norm": 8.267656326293945, + "learning_rate": 4.0557691474458414e-07, + "logits/chosen": 0.3795435428619385, + "logits/rejected": 0.3663020730018616, + "loss": 1.2449, + "step": 238 + }, + { + "beta_dpo/beta": 0.101829394698143, + "beta_dpo/beta_margin_grad_mean": -0.33910471200942993, + "beta_dpo/beta_margin_grad_std": 0.2562665641307831, + "beta_dpo/beta_margin_mean": 1.7569659948349, + "beta_dpo/beta_margin_std": 3.0229148864746094, + "beta_dpo/beta_used": 0.101829394698143, + "beta_dpo/beta_used_raw": 0.101829394698143, + "beta_dpo/gap_mean": 11.849081039428711, + "beta_dpo/gap_std": 18.05896759033203, + "beta_dpo/loss_margin_mean": 13.367530822753906, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.36130007558578986, + "grad_norm": 40.64457321166992, + "learning_rate": 4.045397465551513e-07, + "logits/chosen": 0.5176012516021729, + "logits/rejected": 0.395955353975296, + "loss": 1.0144, + "step": 239 + }, + { + "beta_dpo/beta": 0.4317074418067932, + "beta_dpo/beta_margin_grad_mean": -0.2166636437177658, + "beta_dpo/beta_margin_grad_std": 0.3384714126586914, + "beta_dpo/beta_margin_mean": 7.427967071533203, + "beta_dpo/beta_margin_std": 9.432563781738281, + "beta_dpo/beta_used": 0.4317074418067932, + "beta_dpo/beta_used_raw": 0.4317074418067932, + "beta_dpo/gap_mean": 12.70523452758789, + "beta_dpo/gap_std": 18.133880615234375, + "beta_dpo/loss_margin_mean": 16.595781326293945, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.36281179138321995, + "grad_norm": 113.23979187011719, + "learning_rate": 4.0349825555680045e-07, + "logits/chosen": 0.4384820759296417, + "logits/rejected": 0.3469165563583374, + "loss": 0.5406, + "step": 240 + }, + { + "beta_dpo/beta": 0.045368727296590805, + "beta_dpo/beta_margin_grad_mean": -0.424957811832428, + "beta_dpo/beta_margin_grad_std": 0.19752579927444458, + "beta_dpo/beta_margin_mean": 0.4823284447193146, + "beta_dpo/beta_margin_std": 1.2542048692703247, + "beta_dpo/beta_used": 0.045368727296590805, + "beta_dpo/beta_used_raw": -0.007718522101640701, + "beta_dpo/gap_mean": 12.458725929260254, + "beta_dpo/gap_std": 18.17209243774414, + "beta_dpo/loss_margin_mean": 9.584371566772461, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.36432350718065004, + "grad_norm": 23.7581729888916, + "learning_rate": 4.0245247088227377e-07, + "logits/chosen": 0.41160500049591064, + "logits/rejected": 0.37446707487106323, + "loss": 1.1377, + "step": 241 + }, + { + "beta_dpo/beta": 0.06172889471054077, + "beta_dpo/beta_margin_grad_mean": -0.35529595613479614, + "beta_dpo/beta_margin_grad_std": 0.21127015352249146, + "beta_dpo/beta_margin_mean": 1.0536497831344604, + "beta_dpo/beta_margin_std": 1.8481637239456177, + "beta_dpo/beta_used": 0.06172889471054077, + "beta_dpo/beta_used_raw": 0.06172889471054077, + "beta_dpo/gap_mean": 12.84834098815918, + "beta_dpo/gap_std": 18.49911880493164, + "beta_dpo/loss_margin_mean": 14.712586402893066, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.36583522297808013, + "grad_norm": 20.04178237915039, + "learning_rate": 4.0140242178441665e-07, + "logits/chosen": 0.3890725374221802, + "logits/rejected": 0.3670586049556732, + "loss": 1.0399, + "step": 242 + }, + { + "beta_dpo/beta": 0.1559908539056778, + "beta_dpo/beta_margin_grad_mean": -0.28708603978157043, + "beta_dpo/beta_margin_grad_std": 0.3101195693016052, + "beta_dpo/beta_margin_mean": 1.9018021821975708, + "beta_dpo/beta_margin_std": 2.786595582962036, + "beta_dpo/beta_used": 0.1559908539056778, + "beta_dpo/beta_used_raw": 0.1559908539056778, + "beta_dpo/gap_mean": 12.671987533569336, + "beta_dpo/gap_std": 18.41918182373047, + "beta_dpo/loss_margin_mean": 12.152036666870117, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3673469387755102, + "grad_norm": 55.34590530395508, + "learning_rate": 4.003481376353596e-07, + "logits/chosen": 0.407698392868042, + "logits/rejected": 0.4090770483016968, + "loss": 0.8206, + "step": 243 + }, + { + "beta_dpo/beta": 0.2864580452442169, + "beta_dpo/beta_margin_grad_mean": -0.1638103872537613, + "beta_dpo/beta_margin_grad_std": 0.24098889529705048, + "beta_dpo/beta_margin_mean": 4.863481044769287, + "beta_dpo/beta_margin_std": 5.131604194641113, + "beta_dpo/beta_used": 0.2864580452442169, + "beta_dpo/beta_used_raw": 0.2864580452442169, + "beta_dpo/gap_mean": 13.105916976928711, + "beta_dpo/gap_std": 18.159984588623047, + "beta_dpo/loss_margin_mean": 16.353925704956055, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3688586545729403, + "grad_norm": 60.079708099365234, + "learning_rate": 3.9928964792569654e-07, + "logits/chosen": 0.4592773914337158, + "logits/rejected": 0.37424206733703613, + "loss": 0.4978, + "step": 244 + }, + { + "beta_dpo/beta": 0.22604097425937653, + "beta_dpo/beta_margin_grad_mean": -0.1703149378299713, + "beta_dpo/beta_margin_grad_std": 0.2654411196708679, + "beta_dpo/beta_margin_mean": 4.111567497253418, + "beta_dpo/beta_margin_std": 4.208682060241699, + "beta_dpo/beta_used": 0.22604097425937653, + "beta_dpo/beta_used_raw": 0.22604097425937653, + "beta_dpo/gap_mean": 14.10053539276123, + "beta_dpo/gap_std": 18.00957489013672, + "beta_dpo/loss_margin_mean": 18.244796752929688, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.37037037037037035, + "grad_norm": 65.31201171875, + "learning_rate": 3.982269822636601e-07, + "logits/chosen": 0.47270846366882324, + "logits/rejected": 0.4419899582862854, + "loss": 0.4987, + "step": 245 + }, + { + "beta_dpo/beta": 0.004375386517494917, + "beta_dpo/beta_margin_grad_mean": -0.4861513674259186, + "beta_dpo/beta_margin_grad_std": 0.03189266845583916, + "beta_dpo/beta_margin_mean": 0.05574238672852516, + "beta_dpo/beta_margin_std": 0.12858904898166656, + "beta_dpo/beta_used": 0.004375386517494917, + "beta_dpo/beta_used_raw": -0.028331764042377472, + "beta_dpo/gap_mean": 14.207629203796387, + "beta_dpo/gap_std": 18.656829833984375, + "beta_dpo/loss_margin_mean": 14.23241901397705, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.37188208616780044, + "grad_norm": 3.1414432525634766, + "learning_rate": 3.971601703742932e-07, + "logits/chosen": 0.46958476305007935, + "logits/rejected": 0.4090319275856018, + "loss": 1.3347, + "step": 246 + }, + { + "beta_dpo/beta": 0.03395707905292511, + "beta_dpo/beta_margin_grad_mean": -0.41882583498954773, + "beta_dpo/beta_margin_grad_std": 0.16061575710773468, + "beta_dpo/beta_margin_mean": 0.4101926386356354, + "beta_dpo/beta_margin_std": 0.8106998801231384, + "beta_dpo/beta_used": 0.03395707905292511, + "beta_dpo/beta_used_raw": -0.03948557376861572, + "beta_dpo/gap_mean": 13.277643203735352, + "beta_dpo/gap_std": 18.8796329498291, + "beta_dpo/loss_margin_mean": 8.37158203125, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.37339380196523053, + "grad_norm": 23.345308303833008, + "learning_rate": 3.960892420986177e-07, + "logits/chosen": 0.4679937958717346, + "logits/rejected": 0.45675230026245117, + "loss": 1.1301, + "step": 247 + }, + { + "beta_dpo/beta": 0.06966085731983185, + "beta_dpo/beta_margin_grad_mean": -0.3950624465942383, + "beta_dpo/beta_margin_grad_std": 0.24403807520866394, + "beta_dpo/beta_margin_mean": 1.0330671072006226, + "beta_dpo/beta_margin_std": 2.318099021911621, + "beta_dpo/beta_used": 0.06966085731983185, + "beta_dpo/beta_used_raw": 0.06966085731983185, + "beta_dpo/gap_mean": 13.289308547973633, + "beta_dpo/gap_std": 19.19415283203125, + "beta_dpo/loss_margin_mean": 13.971484184265137, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3749055177626606, + "grad_norm": 45.709678649902344, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": 0.43965810537338257, + "logits/rejected": 0.48121294379234314, + "loss": 1.1869, + "step": 248 + }, + { + "beta_dpo/beta": 0.05159245431423187, + "beta_dpo/beta_margin_grad_mean": -0.4309861660003662, + "beta_dpo/beta_margin_grad_std": 0.24051769077777863, + "beta_dpo/beta_margin_mean": 0.4380229711532593, + "beta_dpo/beta_margin_std": 1.5990570783615112, + "beta_dpo/beta_used": 0.05159245431423187, + "beta_dpo/beta_used_raw": 0.009713437408208847, + "beta_dpo/gap_mean": 12.065717697143555, + "beta_dpo/gap_std": 19.33898162841797, + "beta_dpo/loss_margin_mean": 5.354761123657227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3764172335600907, + "grad_norm": 33.44173812866211, + "learning_rate": 3.9393515632731094e-07, + "logits/chosen": 0.42937660217285156, + "logits/rejected": 0.4625554084777832, + "loss": 1.2139, + "step": 249 + }, + { + "beta_dpo/beta": 0.20074601471424103, + "beta_dpo/beta_margin_grad_mean": -0.2688792049884796, + "beta_dpo/beta_margin_grad_std": 0.29901885986328125, + "beta_dpo/beta_margin_mean": 3.485891819000244, + "beta_dpo/beta_margin_std": 5.231993675231934, + "beta_dpo/beta_used": 0.20074601471424103, + "beta_dpo/beta_used_raw": 0.20074601471424103, + "beta_dpo/gap_mean": 12.264188766479492, + "beta_dpo/gap_std": 19.683074951171875, + "beta_dpo/loss_margin_mean": 15.519104957580566, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3779289493575208, + "grad_norm": 116.9253921508789, + "learning_rate": 3.9285205908608934e-07, + "logits/chosen": 0.5390890836715698, + "logits/rejected": 0.4927697777748108, + "loss": 1.1048, + "step": 250 + }, + { + "beta_dpo/beta": 0.10495083034038544, + "beta_dpo/beta_margin_grad_mean": -0.3397623598575592, + "beta_dpo/beta_margin_grad_std": 0.288044810295105, + "beta_dpo/beta_margin_mean": 1.0604958534240723, + "beta_dpo/beta_margin_std": 1.9399701356887817, + "beta_dpo/beta_used": 0.10495083034038544, + "beta_dpo/beta_used_raw": 0.10495083034038544, + "beta_dpo/gap_mean": 12.307474136352539, + "beta_dpo/gap_std": 19.697185516357422, + "beta_dpo/loss_margin_mean": 10.240516662597656, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3794406651549509, + "grad_norm": 40.252906799316406, + "learning_rate": 3.9176496596569265e-07, + "logits/chosen": 0.5220420360565186, + "logits/rejected": 0.4805965721607208, + "loss": 0.7686, + "step": 251 + }, + { + "beta_dpo/beta": 0.05897517874836922, + "beta_dpo/beta_margin_grad_mean": -0.3872048556804657, + "beta_dpo/beta_margin_grad_std": 0.2150556445121765, + "beta_dpo/beta_margin_mean": 0.8916309475898743, + "beta_dpo/beta_margin_std": 1.7850096225738525, + "beta_dpo/beta_used": 0.05897517874836922, + "beta_dpo/beta_used_raw": 0.009286485612392426, + "beta_dpo/gap_mean": 11.468460083007812, + "beta_dpo/gap_std": 19.796926498413086, + "beta_dpo/loss_margin_mean": 9.313298225402832, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.38095238095238093, + "grad_norm": 26.423254013061523, + "learning_rate": 3.9067390737445254e-07, + "logits/chosen": 0.3940281867980957, + "logits/rejected": 0.3418418765068054, + "loss": 1.0894, + "step": 252 + }, + { + "beta_dpo/beta": 0.10484018921852112, + "beta_dpo/beta_margin_grad_mean": -0.3770482540130615, + "beta_dpo/beta_margin_grad_std": 0.2779637575149536, + "beta_dpo/beta_margin_mean": 1.4820187091827393, + "beta_dpo/beta_margin_std": 3.32963228225708, + "beta_dpo/beta_used": 0.10484018921852112, + "beta_dpo/beta_used_raw": 0.08652335405349731, + "beta_dpo/gap_mean": 11.674590110778809, + "beta_dpo/gap_std": 19.611839294433594, + "beta_dpo/loss_margin_mean": 10.626477241516113, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.382464096749811, + "grad_norm": 59.38056182861328, + "learning_rate": 3.8957891383162304e-07, + "logits/chosen": 0.48830246925354004, + "logits/rejected": 0.4461994767189026, + "loss": 1.3494, + "step": 253 + }, + { + "beta_dpo/beta": 0.058685217052698135, + "beta_dpo/beta_margin_grad_mean": -0.426761269569397, + "beta_dpo/beta_margin_grad_std": 0.21793650090694427, + "beta_dpo/beta_margin_mean": 0.5492849349975586, + "beta_dpo/beta_margin_std": 1.6022003889083862, + "beta_dpo/beta_used": 0.058685217052698135, + "beta_dpo/beta_used_raw": 0.053593721240758896, + "beta_dpo/gap_mean": 11.511938095092773, + "beta_dpo/gap_std": 19.289674758911133, + "beta_dpo/loss_margin_mean": 11.349576950073242, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3839758125472411, + "grad_norm": 36.390403747558594, + "learning_rate": 3.884800159665276e-07, + "logits/chosen": 0.4373748302459717, + "logits/rejected": 0.38516467809677124, + "loss": 1.1464, + "step": 254 + }, + { + "beta_dpo/beta": 0.11067987978458405, + "beta_dpo/beta_margin_grad_mean": -0.3657572865486145, + "beta_dpo/beta_margin_grad_std": 0.27411824464797974, + "beta_dpo/beta_margin_mean": 1.7994710206985474, + "beta_dpo/beta_margin_std": 3.8073818683624268, + "beta_dpo/beta_used": 0.11067987978458405, + "beta_dpo/beta_used_raw": 0.007904693484306335, + "beta_dpo/gap_mean": 11.95395565032959, + "beta_dpo/gap_std": 19.61341667175293, + "beta_dpo/loss_margin_mean": 14.774104118347168, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3854875283446712, + "grad_norm": 89.89997100830078, + "learning_rate": 3.873772445177015e-07, + "logits/chosen": 0.42883598804473877, + "logits/rejected": 0.39614689350128174, + "loss": 1.2533, + "step": 255 + }, + { + "beta_dpo/beta": 0.117228664457798, + "beta_dpo/beta_margin_grad_mean": -0.40933340787887573, + "beta_dpo/beta_margin_grad_std": 0.2813352644443512, + "beta_dpo/beta_margin_mean": 1.6359100341796875, + "beta_dpo/beta_margin_std": 4.08366060256958, + "beta_dpo/beta_used": 0.117228664457798, + "beta_dpo/beta_used_raw": 0.037854887545108795, + "beta_dpo/gap_mean": 12.13801383972168, + "beta_dpo/gap_std": 20.043987274169922, + "beta_dpo/loss_margin_mean": 12.292617797851562, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3869992441421013, + "grad_norm": 99.5350341796875, + "learning_rate": 3.862706303320329e-07, + "logits/chosen": 0.4363447427749634, + "logits/rejected": 0.374740332365036, + "loss": 1.2428, + "step": 256 + }, + { + "beta_dpo/beta": 0.038477640599012375, + "beta_dpo/beta_margin_grad_mean": -0.4051564931869507, + "beta_dpo/beta_margin_grad_std": 0.1941753476858139, + "beta_dpo/beta_margin_mean": 0.611764669418335, + "beta_dpo/beta_margin_std": 1.3536244630813599, + "beta_dpo/beta_used": 0.038477640599012375, + "beta_dpo/beta_used_raw": 0.01731099747121334, + "beta_dpo/gap_mean": 12.789517402648926, + "beta_dpo/gap_std": 20.492897033691406, + "beta_dpo/loss_margin_mean": 16.812488555908203, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3885109599395314, + "grad_norm": 24.06647491455078, + "learning_rate": 3.851602043638994e-07, + "logits/chosen": 0.43763476610183716, + "logits/rejected": 0.374079167842865, + "loss": 1.1426, + "step": 257 + }, + { + "beta_dpo/beta": 0.03051311895251274, + "beta_dpo/beta_margin_grad_mean": -0.40624645352363586, + "beta_dpo/beta_margin_grad_std": 0.14975614845752716, + "beta_dpo/beta_margin_mean": 0.45043420791625977, + "beta_dpo/beta_margin_std": 0.7583449482917786, + "beta_dpo/beta_used": 0.03051311895251274, + "beta_dpo/beta_used_raw": -0.1679498702287674, + "beta_dpo/gap_mean": 12.876970291137695, + "beta_dpo/gap_std": 20.080921173095703, + "beta_dpo/loss_margin_mean": 13.000361442565918, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3900226757369615, + "grad_norm": 21.060880661010742, + "learning_rate": 3.840459976743023e-07, + "logits/chosen": 0.47881019115448, + "logits/rejected": 0.42693912982940674, + "loss": 1.1519, + "step": 258 + }, + { + "beta_dpo/beta": 0.2693525552749634, + "beta_dpo/beta_margin_grad_mean": -0.19658398628234863, + "beta_dpo/beta_margin_grad_std": 0.3391178548336029, + "beta_dpo/beta_margin_mean": 5.206562519073486, + "beta_dpo/beta_margin_std": 5.524745464324951, + "beta_dpo/beta_used": 0.2693525552749634, + "beta_dpo/beta_used_raw": 0.2693525552749634, + "beta_dpo/gap_mean": 14.02800178527832, + "beta_dpo/gap_std": 20.028091430664062, + "beta_dpo/loss_margin_mean": 19.475568771362305, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3915343915343915, + "grad_norm": 133.27005004882812, + "learning_rate": 3.8292804142999796e-07, + "logits/chosen": 0.36390572786331177, + "logits/rejected": 0.2654324173927307, + "loss": 1.1567, + "step": 259 + }, + { + "beta_dpo/beta": 0.04879666119813919, + "beta_dpo/beta_margin_grad_mean": -0.3880171477794647, + "beta_dpo/beta_margin_grad_std": 0.2359832376241684, + "beta_dpo/beta_margin_mean": 0.7953795790672302, + "beta_dpo/beta_margin_std": 1.7049293518066406, + "beta_dpo/beta_used": 0.04879666119813919, + "beta_dpo/beta_used_raw": 0.046260517090559006, + "beta_dpo/gap_mean": 14.264688491821289, + "beta_dpo/gap_std": 20.290300369262695, + "beta_dpo/loss_margin_mean": 13.966394424438477, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3930461073318216, + "grad_norm": 27.301448822021484, + "learning_rate": 3.818063669026256e-07, + "logits/chosen": 0.4362449049949646, + "logits/rejected": 0.349369078874588, + "loss": 1.1159, + "step": 260 + }, + { + "beta_dpo/beta": 0.06346012651920319, + "beta_dpo/beta_margin_grad_mean": -0.3791761100292206, + "beta_dpo/beta_margin_grad_std": 0.21919672191143036, + "beta_dpo/beta_margin_mean": 0.7327264547348022, + "beta_dpo/beta_margin_std": 1.4131560325622559, + "beta_dpo/beta_used": 0.06346012651920319, + "beta_dpo/beta_used_raw": 0.06346012651920319, + "beta_dpo/gap_mean": 13.676494598388672, + "beta_dpo/gap_std": 20.232555389404297, + "beta_dpo/loss_margin_mean": 11.213432312011719, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3945578231292517, + "grad_norm": 32.0900993347168, + "learning_rate": 3.806810054678331e-07, + "logits/chosen": 0.32648757100105286, + "logits/rejected": 0.3490898013114929, + "loss": 0.9706, + "step": 261 + }, + { + "beta_dpo/beta": 0.0835457369685173, + "beta_dpo/beta_margin_grad_mean": -0.38975584506988525, + "beta_dpo/beta_margin_grad_std": 0.26027020812034607, + "beta_dpo/beta_margin_mean": 1.2454676628112793, + "beta_dpo/beta_margin_std": 2.6653995513916016, + "beta_dpo/beta_used": 0.0835457369685173, + "beta_dpo/beta_used_raw": 0.0018124952912330627, + "beta_dpo/gap_mean": 13.374794960021973, + "beta_dpo/gap_std": 19.955730438232422, + "beta_dpo/loss_margin_mean": 12.837187767028809, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3960695389266818, + "grad_norm": 51.011417388916016, + "learning_rate": 3.7955198860439887e-07, + "logits/chosen": 0.46894973516464233, + "logits/rejected": 0.40835195779800415, + "loss": 1.2075, + "step": 262 + }, + { + "beta_dpo/beta": 0.0994168296456337, + "beta_dpo/beta_margin_grad_mean": -0.3875061571598053, + "beta_dpo/beta_margin_grad_std": 0.27189844846725464, + "beta_dpo/beta_margin_mean": 1.2055269479751587, + "beta_dpo/beta_margin_std": 2.890883207321167, + "beta_dpo/beta_used": 0.0994168296456337, + "beta_dpo/beta_used_raw": 0.015077054500579834, + "beta_dpo/gap_mean": 13.045722961425781, + "beta_dpo/gap_std": 19.6978759765625, + "beta_dpo/loss_margin_mean": 10.980850219726562, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.3975812547241119, + "grad_norm": 48.162906646728516, + "learning_rate": 3.784193478933516e-07, + "logits/chosen": 0.42282721400260925, + "logits/rejected": 0.31909075379371643, + "loss": 1.003, + "step": 263 + }, + { + "beta_dpo/beta": 0.038988277316093445, + "beta_dpo/beta_margin_grad_mean": -0.4159775674343109, + "beta_dpo/beta_margin_grad_std": 0.21583546698093414, + "beta_dpo/beta_margin_mean": 0.5544974207878113, + "beta_dpo/beta_margin_std": 1.3794368505477905, + "beta_dpo/beta_used": 0.038988277316093445, + "beta_dpo/beta_used_raw": -0.004001736640930176, + "beta_dpo/gap_mean": 13.082950592041016, + "beta_dpo/gap_std": 19.840484619140625, + "beta_dpo/loss_margin_mean": 13.738378524780273, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.39909297052154197, + "grad_norm": 26.92840576171875, + "learning_rate": 3.7728311501708674e-07, + "logits/chosen": 0.34600526094436646, + "logits/rejected": 0.2996159791946411, + "loss": 1.1644, + "step": 264 + }, + { + "beta_dpo/beta": 0.2847278118133545, + "beta_dpo/beta_margin_grad_mean": -0.2701014578342438, + "beta_dpo/beta_margin_grad_std": 0.35836902260780334, + "beta_dpo/beta_margin_mean": 4.273728370666504, + "beta_dpo/beta_margin_std": 7.470240592956543, + "beta_dpo/beta_used": 0.2847278118133545, + "beta_dpo/beta_used_raw": 0.2847278118133545, + "beta_dpo/gap_mean": 13.191385269165039, + "beta_dpo/gap_std": 20.41083526611328, + "beta_dpo/loss_margin_mean": 14.18088436126709, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.40060468631897206, + "grad_norm": 130.78456115722656, + "learning_rate": 3.7614332175848027e-07, + "logits/chosen": 0.5080592632293701, + "logits/rejected": 0.440662145614624, + "loss": 1.2185, + "step": 265 + }, + { + "beta_dpo/beta": 0.08126115798950195, + "beta_dpo/beta_margin_grad_mean": -0.33244937658309937, + "beta_dpo/beta_margin_grad_std": 0.2310456931591034, + "beta_dpo/beta_margin_mean": 1.032869815826416, + "beta_dpo/beta_margin_std": 1.767521858215332, + "beta_dpo/beta_used": 0.08126115798950195, + "beta_dpo/beta_used_raw": 0.08126115798950195, + "beta_dpo/gap_mean": 13.363494873046875, + "beta_dpo/gap_std": 20.626605987548828, + "beta_dpo/loss_margin_mean": 13.082228660583496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4021164021164021, + "grad_norm": 36.103389739990234, + "learning_rate": 3.75e-07, + "logits/chosen": 0.4579857587814331, + "logits/rejected": 0.38330739736557007, + "loss": 0.8273, + "step": 266 + }, + { + "beta_dpo/beta": 0.072122722864151, + "beta_dpo/beta_margin_grad_mean": -0.37835559248924255, + "beta_dpo/beta_margin_grad_std": 0.2315768450498581, + "beta_dpo/beta_margin_mean": 1.1022335290908813, + "beta_dpo/beta_margin_std": 2.124861717224121, + "beta_dpo/beta_used": 0.072122722864151, + "beta_dpo/beta_used_raw": -0.060242317616939545, + "beta_dpo/gap_mean": 12.627429962158203, + "beta_dpo/gap_std": 20.575902938842773, + "beta_dpo/loss_margin_mean": 10.242201805114746, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4036281179138322, + "grad_norm": 32.35624313354492, + "learning_rate": 3.738531817228131e-07, + "logits/chosen": 0.4831930100917816, + "logits/rejected": 0.46250009536743164, + "loss": 1.0788, + "step": 267 + }, + { + "beta_dpo/beta": 0.013414818793535233, + "beta_dpo/beta_margin_grad_mean": -0.45541560649871826, + "beta_dpo/beta_margin_grad_std": 0.07803189009428024, + "beta_dpo/beta_margin_mean": 0.18836283683776855, + "beta_dpo/beta_margin_std": 0.33558791875839233, + "beta_dpo/beta_used": 0.013414818793535233, + "beta_dpo/beta_used_raw": 0.004978269338607788, + "beta_dpo/gap_mean": 12.683042526245117, + "beta_dpo/gap_std": 19.99517059326172, + "beta_dpo/loss_margin_mean": 12.636152267456055, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4051398337112623, + "grad_norm": 7.843941688537598, + "learning_rate": 3.7270289900589204e-07, + "logits/chosen": 0.32715708017349243, + "logits/rejected": 0.3075103759765625, + "loss": 1.2591, + "step": 268 + }, + { + "beta_dpo/beta": 0.21129584312438965, + "beta_dpo/beta_margin_grad_mean": -0.2817341685295105, + "beta_dpo/beta_margin_grad_std": 0.2654723525047302, + "beta_dpo/beta_margin_mean": 3.7788643836975098, + "beta_dpo/beta_margin_std": 5.652310848236084, + "beta_dpo/beta_used": 0.21129584312438965, + "beta_dpo/beta_used_raw": 0.21129584312438965, + "beta_dpo/gap_mean": 12.821964263916016, + "beta_dpo/gap_std": 19.65093231201172, + "beta_dpo/loss_margin_mean": 14.411201477050781, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.40665154950869237, + "grad_norm": 82.85317993164062, + "learning_rate": 3.7154918402511714e-07, + "logits/chosen": 0.5275648832321167, + "logits/rejected": 0.47768309712409973, + "loss": 0.85, + "step": 269 + }, + { + "beta_dpo/beta": 0.027644401416182518, + "beta_dpo/beta_margin_grad_mean": -0.43374115228652954, + "beta_dpo/beta_margin_grad_std": 0.14928385615348816, + "beta_dpo/beta_margin_mean": 0.3241073489189148, + "beta_dpo/beta_margin_std": 0.7700070142745972, + "beta_dpo/beta_used": 0.027644401416182518, + "beta_dpo/beta_used_raw": -0.021652307361364365, + "beta_dpo/gap_mean": 12.841148376464844, + "beta_dpo/gap_std": 19.287689208984375, + "beta_dpo/loss_margin_mean": 11.001672744750977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.40816326530612246, + "grad_norm": 16.57350730895996, + "learning_rate": 3.7039206905237656e-07, + "logits/chosen": 0.44649437069892883, + "logits/rejected": 0.3714543879032135, + "loss": 1.1677, + "step": 270 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49744880199432373, + "beta_dpo/beta_margin_grad_std": 0.005622472148388624, + "beta_dpo/beta_margin_mean": 0.010206691920757294, + "beta_dpo/beta_margin_std": 0.022493908181786537, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.14274440705776215, + "beta_dpo/gap_mean": 12.42354965209961, + "beta_dpo/gap_std": 19.610389709472656, + "beta_dpo/loss_margin_mean": 10.20669174194336, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.40967498110355255, + "grad_norm": 0.6051694750785828, + "learning_rate": 3.692315864546635e-07, + "logits/chosen": 0.45940130949020386, + "logits/rejected": 0.39874905347824097, + "loss": 1.378, + "step": 271 + }, + { + "beta_dpo/beta": 0.18113896250724792, + "beta_dpo/beta_margin_grad_mean": -0.2012496441602707, + "beta_dpo/beta_margin_grad_std": 0.25230535864830017, + "beta_dpo/beta_margin_mean": 3.179334878921509, + "beta_dpo/beta_margin_std": 3.390836238861084, + "beta_dpo/beta_used": 0.18113896250724792, + "beta_dpo/beta_used_raw": 0.18113896250724792, + "beta_dpo/gap_mean": 12.827465057373047, + "beta_dpo/gap_std": 19.45254898071289, + "beta_dpo/loss_margin_mean": 17.20043182373047, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.41118669690098264, + "grad_norm": 79.047607421875, + "learning_rate": 3.6806776869317067e-07, + "logits/chosen": 0.45520102977752686, + "logits/rejected": 0.4643624424934387, + "loss": 0.7339, + "step": 272 + }, + { + "beta_dpo/beta": 0.11468373984098434, + "beta_dpo/beta_margin_grad_mean": -0.39956751465797424, + "beta_dpo/beta_margin_grad_std": 0.2712042033672333, + "beta_dpo/beta_margin_mean": 1.4363672733306885, + "beta_dpo/beta_margin_std": 3.4752349853515625, + "beta_dpo/beta_used": 0.11468373984098434, + "beta_dpo/beta_used_raw": 0.10186155885457993, + "beta_dpo/gap_mean": 13.144290924072266, + "beta_dpo/gap_std": 19.811664581298828, + "beta_dpo/loss_margin_mean": 13.085746765136719, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4126984126984127, + "grad_norm": 65.30044555664062, + "learning_rate": 3.669006483223828e-07, + "logits/chosen": 0.4667285084724426, + "logits/rejected": 0.4003213047981262, + "loss": 1.1074, + "step": 273 + }, + { + "beta_dpo/beta": 0.0694403201341629, + "beta_dpo/beta_margin_grad_mean": -0.40486380457878113, + "beta_dpo/beta_margin_grad_std": 0.24759718775749207, + "beta_dpo/beta_margin_mean": 0.8668053150177002, + "beta_dpo/beta_margin_std": 2.210718870162964, + "beta_dpo/beta_used": 0.0694403201341629, + "beta_dpo/beta_used_raw": 0.03696062043309212, + "beta_dpo/gap_mean": 13.18702507019043, + "beta_dpo/gap_std": 20.19510269165039, + "beta_dpo/loss_margin_mean": 13.648605346679688, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.41421012849584277, + "grad_norm": 39.4247932434082, + "learning_rate": 3.657302579891656e-07, + "logits/chosen": 0.33252406120300293, + "logits/rejected": 0.30951541662216187, + "loss": 1.1161, + "step": 274 + }, + { + "beta_dpo/beta": 0.08022014796733856, + "beta_dpo/beta_margin_grad_mean": -0.32078537344932556, + "beta_dpo/beta_margin_grad_std": 0.2081281989812851, + "beta_dpo/beta_margin_mean": 1.29566490650177, + "beta_dpo/beta_margin_std": 1.8696632385253906, + "beta_dpo/beta_used": 0.08022014796733856, + "beta_dpo/beta_used_raw": 0.08022014796733856, + "beta_dpo/gap_mean": 13.405573844909668, + "beta_dpo/gap_std": 20.12377166748047, + "beta_dpo/loss_margin_mean": 14.838817596435547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.41572184429327286, + "grad_norm": 28.87332534790039, + "learning_rate": 3.645566304318526e-07, + "logits/chosen": 0.41394883394241333, + "logits/rejected": 0.3336278200149536, + "loss": 0.8851, + "step": 275 + }, + { + "beta_dpo/beta": 0.07089600712060928, + "beta_dpo/beta_margin_grad_mean": -0.37287554144859314, + "beta_dpo/beta_margin_grad_std": 0.22276493906974792, + "beta_dpo/beta_margin_mean": 0.9248519539833069, + "beta_dpo/beta_margin_std": 1.8152260780334473, + "beta_dpo/beta_used": 0.07089600712060928, + "beta_dpo/beta_used_raw": 0.03236527368426323, + "beta_dpo/gap_mean": 13.642976760864258, + "beta_dpo/gap_std": 19.716773986816406, + "beta_dpo/loss_margin_mean": 13.728145599365234, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.41723356009070295, + "grad_norm": 29.26936912536621, + "learning_rate": 3.633797984793294e-07, + "logits/chosen": 0.3696138560771942, + "logits/rejected": 0.33492618799209595, + "loss": 0.9796, + "step": 276 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49800875782966614, + "beta_dpo/beta_margin_grad_std": 0.004511854145675898, + "beta_dpo/beta_margin_mean": 0.007965884171426296, + "beta_dpo/beta_margin_std": 0.01804952323436737, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.1568543016910553, + "beta_dpo/gap_mean": 12.772453308105469, + "beta_dpo/gap_std": 19.320093154907227, + "beta_dpo/loss_margin_mean": 7.965883255004883, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.41874527588813304, + "grad_norm": 0.6365845203399658, + "learning_rate": 3.6219979505011555e-07, + "logits/chosen": 0.46259212493896484, + "logits/rejected": 0.48363691568374634, + "loss": 1.3779, + "step": 277 + }, + { + "beta_dpo/beta": 0.15876400470733643, + "beta_dpo/beta_margin_grad_mean": -0.30526286363601685, + "beta_dpo/beta_margin_grad_std": 0.2431895136833191, + "beta_dpo/beta_margin_mean": 2.8034675121307373, + "beta_dpo/beta_margin_std": 5.201408863067627, + "beta_dpo/beta_used": 0.15876400470733643, + "beta_dpo/beta_used_raw": 0.15876400470733643, + "beta_dpo/gap_mean": 12.495584487915039, + "beta_dpo/gap_std": 19.255725860595703, + "beta_dpo/loss_margin_mean": 13.553324699401855, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.42025699168556313, + "grad_norm": 33.650630950927734, + "learning_rate": 3.6101665315144353e-07, + "logits/chosen": 0.3732357621192932, + "logits/rejected": 0.3219851851463318, + "loss": 0.7781, + "step": 278 + }, + { + "beta_dpo/beta": 0.23024505376815796, + "beta_dpo/beta_margin_grad_mean": -0.19230079650878906, + "beta_dpo/beta_margin_grad_std": 0.28550806641578674, + "beta_dpo/beta_margin_mean": 3.9565861225128174, + "beta_dpo/beta_margin_std": 4.337207317352295, + "beta_dpo/beta_used": 0.23024505376815796, + "beta_dpo/beta_used_raw": 0.23024505376815796, + "beta_dpo/gap_mean": 13.335415840148926, + "beta_dpo/gap_std": 19.329914093017578, + "beta_dpo/loss_margin_mean": 17.311302185058594, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4217687074829932, + "grad_norm": 63.41184616088867, + "learning_rate": 3.5983040587833563e-07, + "logits/chosen": 0.3997264504432678, + "logits/rejected": 0.35758689045906067, + "loss": 0.5583, + "step": 279 + }, + { + "beta_dpo/beta": 0.13080888986587524, + "beta_dpo/beta_margin_grad_mean": -0.22343085706233978, + "beta_dpo/beta_margin_grad_std": 0.21184687316417694, + "beta_dpo/beta_margin_mean": 2.2733867168426514, + "beta_dpo/beta_margin_std": 2.2885751724243164, + "beta_dpo/beta_used": 0.13080888986587524, + "beta_dpo/beta_used_raw": 0.13080888986587524, + "beta_dpo/gap_mean": 14.12271499633789, + "beta_dpo/gap_std": 18.93375015258789, + "beta_dpo/loss_margin_mean": 17.238388061523438, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.42328042328042326, + "grad_norm": 38.145751953125, + "learning_rate": 3.586410864126781e-07, + "logits/chosen": 0.4410172700881958, + "logits/rejected": 0.39698508381843567, + "loss": 0.5901, + "step": 280 + }, + { + "beta_dpo/beta": 0.030089804902672768, + "beta_dpo/beta_margin_grad_mean": -0.4115253686904907, + "beta_dpo/beta_margin_grad_std": 0.16938680410385132, + "beta_dpo/beta_margin_mean": 0.4769037663936615, + "beta_dpo/beta_margin_std": 0.9234161376953125, + "beta_dpo/beta_used": 0.030089804902672768, + "beta_dpo/beta_used_raw": 0.015273205004632473, + "beta_dpo/gap_mean": 14.474631309509277, + "beta_dpo/gap_std": 18.93929672241211, + "beta_dpo/loss_margin_mean": 14.873807907104492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.42479213907785335, + "grad_norm": 15.947169303894043, + "learning_rate": 3.574487280222929e-07, + "logits/chosen": 0.4040282964706421, + "logits/rejected": 0.41597017645835876, + "loss": 1.158, + "step": 281 + }, + { + "beta_dpo/beta": 0.10513893514871597, + "beta_dpo/beta_margin_grad_mean": -0.3416615426540375, + "beta_dpo/beta_margin_grad_std": 0.2618146240711212, + "beta_dpo/beta_margin_mean": 2.1443495750427246, + "beta_dpo/beta_margin_std": 3.675140857696533, + "beta_dpo/beta_used": 0.10513893514871597, + "beta_dpo/beta_used_raw": -0.015448860824108124, + "beta_dpo/gap_mean": 14.205320358276367, + "beta_dpo/gap_std": 19.232637405395508, + "beta_dpo/loss_margin_mean": 14.617086410522461, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.42630385487528344, + "grad_norm": 67.41303253173828, + "learning_rate": 3.562533640600075e-07, + "logits/chosen": 0.3522970676422119, + "logits/rejected": 0.3013116419315338, + "loss": 1.1754, + "step": 282 + }, + { + "beta_dpo/beta": 0.0794285237789154, + "beta_dpo/beta_margin_grad_mean": -0.3860606551170349, + "beta_dpo/beta_margin_grad_std": 0.23796863853931427, + "beta_dpo/beta_margin_mean": 1.2510942220687866, + "beta_dpo/beta_margin_std": 2.778822422027588, + "beta_dpo/beta_used": 0.0794285237789154, + "beta_dpo/beta_used_raw": 0.030920717865228653, + "beta_dpo/gap_mean": 14.144121170043945, + "beta_dpo/gap_std": 19.416378021240234, + "beta_dpo/loss_margin_mean": 12.779112815856934, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.42781557067271353, + "grad_norm": 49.29255294799805, + "learning_rate": 3.550550279627215e-07, + "logits/chosen": 0.40588125586509705, + "logits/rejected": 0.31223565340042114, + "loss": 1.1026, + "step": 283 + }, + { + "beta_dpo/beta": 0.04316014051437378, + "beta_dpo/beta_margin_grad_mean": -0.39386722445487976, + "beta_dpo/beta_margin_grad_std": 0.16511482000350952, + "beta_dpo/beta_margin_mean": 0.6583900451660156, + "beta_dpo/beta_margin_std": 1.2365429401397705, + "beta_dpo/beta_used": 0.04316014051437378, + "beta_dpo/beta_used_raw": 0.04316014051437378, + "beta_dpo/gap_mean": 14.177618026733398, + "beta_dpo/gap_std": 19.436182022094727, + "beta_dpo/loss_margin_mean": 13.194892883300781, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4293272864701436, + "grad_norm": 23.569568634033203, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": 0.4499743580818176, + "logits/rejected": 0.38757190108299255, + "loss": 1.0673, + "step": 284 + }, + { + "beta_dpo/beta": 0.0996825098991394, + "beta_dpo/beta_margin_grad_mean": -0.35815978050231934, + "beta_dpo/beta_margin_grad_std": 0.2535557746887207, + "beta_dpo/beta_margin_mean": 1.3248549699783325, + "beta_dpo/beta_margin_std": 2.5650172233581543, + "beta_dpo/beta_used": 0.0996825098991394, + "beta_dpo/beta_used_raw": 0.0996825098991394, + "beta_dpo/gap_mean": 13.592554092407227, + "beta_dpo/gap_std": 19.435794830322266, + "beta_dpo/loss_margin_mean": 11.820451736450195, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4308390022675737, + "grad_norm": 50.08279037475586, + "learning_rate": 3.5264957352549375e-07, + "logits/chosen": 0.46355849504470825, + "logits/rejected": 0.43509894609451294, + "loss": 0.8775, + "step": 285 + }, + { + "beta_dpo/beta": 0.19798368215560913, + "beta_dpo/beta_margin_grad_mean": -0.3241359293460846, + "beta_dpo/beta_margin_grad_std": 0.265657901763916, + "beta_dpo/beta_margin_mean": 3.4995052814483643, + "beta_dpo/beta_margin_std": 7.216194152832031, + "beta_dpo/beta_used": 0.19798368215560913, + "beta_dpo/beta_used_raw": 0.19475844502449036, + "beta_dpo/gap_mean": 14.27553653717041, + "beta_dpo/gap_std": 19.55514144897461, + "beta_dpo/loss_margin_mean": 18.03702735900879, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4323507180650038, + "grad_norm": 50.606544494628906, + "learning_rate": 3.514425224712835e-07, + "logits/chosen": 0.38016998767852783, + "logits/rejected": 0.2861751317977905, + "loss": 0.7735, + "step": 286 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49570053815841675, + "beta_dpo/beta_margin_grad_std": 0.005138974636793137, + "beta_dpo/beta_margin_mean": 0.017200568690896034, + "beta_dpo/beta_margin_std": 0.020560333505272865, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.12309788912534714, + "beta_dpo/gap_mean": 14.72789192199707, + "beta_dpo/gap_std": 19.78351593017578, + "beta_dpo/loss_margin_mean": 17.20056915283203, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.43386243386243384, + "grad_norm": 0.901676595211029, + "learning_rate": 3.502326338516534e-07, + "logits/chosen": 0.4146936237812042, + "logits/rejected": 0.373552143573761, + "loss": 1.3754, + "step": 287 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4973767399787903, + "beta_dpo/beta_margin_grad_std": 0.00516990851610899, + "beta_dpo/beta_margin_mean": 0.010494444519281387, + "beta_dpo/beta_margin_std": 0.02068295143544674, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.20959964394569397, + "beta_dpo/gap_mean": 14.297950744628906, + "beta_dpo/gap_std": 20.072662353515625, + "beta_dpo/loss_margin_mean": 10.494444847106934, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.43537414965986393, + "grad_norm": 0.7291444540023804, + "learning_rate": 3.490199415097892e-07, + "logits/chosen": 0.3069779574871063, + "logits/rejected": 0.25205767154693604, + "loss": 1.3772, + "step": 288 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49683305621147156, + "beta_dpo/beta_margin_grad_std": 0.00543037336319685, + "beta_dpo/beta_margin_mean": 0.012669811025261879, + "beta_dpo/beta_margin_std": 0.02172556333243847, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.048285968601703644, + "beta_dpo/gap_mean": 13.913497924804688, + "beta_dpo/gap_std": 20.553274154663086, + "beta_dpo/loss_margin_mean": 12.66981029510498, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.436885865457294, + "grad_norm": 0.7524048089981079, + "learning_rate": 3.4780447936730247e-07, + "logits/chosen": 0.513412356376648, + "logits/rejected": 0.47326600551605225, + "loss": 1.375, + "step": 289 + }, + { + "beta_dpo/beta": 0.09844214469194412, + "beta_dpo/beta_margin_grad_mean": -0.38945475220680237, + "beta_dpo/beta_margin_grad_std": 0.2678634226322174, + "beta_dpo/beta_margin_mean": 1.5054290294647217, + "beta_dpo/beta_margin_std": 3.634999990463257, + "beta_dpo/beta_used": 0.09844214469194412, + "beta_dpo/beta_used_raw": 0.09844214469194412, + "beta_dpo/gap_mean": 14.4169282913208, + "beta_dpo/gap_std": 20.567150115966797, + "beta_dpo/loss_margin_mean": 16.994667053222656, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4383975812547241, + "grad_norm": 59.73664474487305, + "learning_rate": 3.465862814232821e-07, + "logits/chosen": 0.49665823578834534, + "logits/rejected": 0.42759448289871216, + "loss": 1.1353, + "step": 290 + }, + { + "beta_dpo/beta": 0.25055256485939026, + "beta_dpo/beta_margin_grad_mean": -0.33395248651504517, + "beta_dpo/beta_margin_grad_std": 0.2863628566265106, + "beta_dpo/beta_margin_mean": 5.20906400680542, + "beta_dpo/beta_margin_std": 10.68582534790039, + "beta_dpo/beta_used": 0.25055256485939026, + "beta_dpo/beta_used_raw": 0.2299947887659073, + "beta_dpo/gap_mean": 14.44134521484375, + "beta_dpo/gap_std": 21.393653869628906, + "beta_dpo/loss_margin_mean": 15.887751579284668, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4399092970521542, + "grad_norm": 58.57472229003906, + "learning_rate": 3.4536538175334343e-07, + "logits/chosen": 0.5883669853210449, + "logits/rejected": 0.5197364091873169, + "loss": 0.8488, + "step": 291 + }, + { + "beta_dpo/beta": 0.007076106034219265, + "beta_dpo/beta_margin_grad_mean": -0.47282010316848755, + "beta_dpo/beta_margin_grad_std": 0.050678376108407974, + "beta_dpo/beta_margin_mean": 0.11135596036911011, + "beta_dpo/beta_margin_std": 0.2094070464372635, + "beta_dpo/beta_used": 0.007076106034219265, + "beta_dpo/beta_used_raw": -0.014458229765295982, + "beta_dpo/gap_mean": 14.619291305541992, + "beta_dpo/gap_std": 21.718494415283203, + "beta_dpo/loss_margin_mean": 14.234935760498047, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4414210128495843, + "grad_norm": 5.891993999481201, + "learning_rate": 3.4414181450867465e-07, + "logits/chosen": 0.45743709802627563, + "logits/rejected": 0.40714502334594727, + "loss": 1.3021, + "step": 292 + }, + { + "beta_dpo/beta": 0.23176653683185577, + "beta_dpo/beta_margin_grad_mean": -0.2546376585960388, + "beta_dpo/beta_margin_grad_std": 0.36235809326171875, + "beta_dpo/beta_margin_mean": 4.366588592529297, + "beta_dpo/beta_margin_std": 5.932827472686768, + "beta_dpo/beta_used": 0.23176653683185577, + "beta_dpo/beta_used_raw": 0.23176653683185577, + "beta_dpo/gap_mean": 15.188974380493164, + "beta_dpo/gap_std": 22.373016357421875, + "beta_dpo/loss_margin_mean": 18.706132888793945, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4429327286470144, + "grad_norm": 113.21065521240234, + "learning_rate": 3.4291561391508185e-07, + "logits/chosen": 0.5480353832244873, + "logits/rejected": 0.45960021018981934, + "loss": 1.0772, + "step": 293 + }, + { + "beta_dpo/beta": 0.1309327632188797, + "beta_dpo/beta_margin_grad_mean": -0.3639461398124695, + "beta_dpo/beta_margin_grad_std": 0.24621707201004028, + "beta_dpo/beta_margin_mean": 2.632067918777466, + "beta_dpo/beta_margin_std": 4.834066867828369, + "beta_dpo/beta_used": 0.1309327632188797, + "beta_dpo/beta_used_raw": -0.024740561842918396, + "beta_dpo/gap_mean": 15.85311222076416, + "beta_dpo/gap_std": 22.474395751953125, + "beta_dpo/loss_margin_mean": 17.183238983154297, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4444444444444444, + "grad_norm": 131.88706970214844, + "learning_rate": 3.4168681427203153e-07, + "logits/chosen": 0.45878100395202637, + "logits/rejected": 0.4147722125053406, + "loss": 1.0422, + "step": 294 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49661287665367126, + "beta_dpo/beta_margin_grad_std": 0.005212538409978151, + "beta_dpo/beta_margin_mean": 0.013550843112170696, + "beta_dpo/beta_margin_std": 0.0208545234054327, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.119144506752491, + "beta_dpo/gap_mean": 15.471101760864258, + "beta_dpo/gap_std": 22.285518646240234, + "beta_dpo/loss_margin_mean": 13.55084228515625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4459561602418745, + "grad_norm": 0.7092771530151367, + "learning_rate": 3.4045544995169125e-07, + "logits/chosen": 0.444843053817749, + "logits/rejected": 0.35121992230415344, + "loss": 1.3746, + "step": 295 + }, + { + "beta_dpo/beta": 0.05136363208293915, + "beta_dpo/beta_margin_grad_mean": -0.3628765046596527, + "beta_dpo/beta_margin_grad_std": 0.21744120121002197, + "beta_dpo/beta_margin_mean": 1.1004339456558228, + "beta_dpo/beta_margin_std": 1.9954913854599, + "beta_dpo/beta_used": 0.05136363208293915, + "beta_dpo/beta_used_raw": 0.04004165530204773, + "beta_dpo/gap_mean": 15.906424522399902, + "beta_dpo/gap_std": 22.500926971435547, + "beta_dpo/loss_margin_mean": 18.224267959594727, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4474678760393046, + "grad_norm": 27.477766036987305, + "learning_rate": 3.392215553979679e-07, + "logits/chosen": 0.4178934097290039, + "logits/rejected": 0.3756554126739502, + "loss": 1.0375, + "step": 296 + }, + { + "beta_dpo/beta": 0.2311479151248932, + "beta_dpo/beta_margin_grad_mean": -0.24369271099567413, + "beta_dpo/beta_margin_grad_std": 0.2907634675502777, + "beta_dpo/beta_margin_mean": 4.739898681640625, + "beta_dpo/beta_margin_std": 6.735095977783203, + "beta_dpo/beta_used": 0.2311479151248932, + "beta_dpo/beta_used_raw": 0.2311479151248932, + "beta_dpo/gap_mean": 16.063180923461914, + "beta_dpo/gap_std": 22.32459259033203, + "beta_dpo/loss_margin_mean": 18.45130729675293, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4489795918367347, + "grad_norm": 151.92831420898438, + "learning_rate": 3.3798516512554485e-07, + "logits/chosen": 0.4511827230453491, + "logits/rejected": 0.39176806807518005, + "loss": 0.8429, + "step": 297 + }, + { + "beta_dpo/beta": 0.0763932392001152, + "beta_dpo/beta_margin_grad_mean": -0.3957996666431427, + "beta_dpo/beta_margin_grad_std": 0.25355854630470276, + "beta_dpo/beta_margin_mean": 1.2725530862808228, + "beta_dpo/beta_margin_std": 2.942795753479004, + "beta_dpo/beta_used": 0.0763932392001152, + "beta_dpo/beta_used_raw": 0.057013507932424545, + "beta_dpo/gap_mean": 16.267505645751953, + "beta_dpo/gap_std": 22.456554412841797, + "beta_dpo/loss_margin_mean": 15.823343276977539, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4504913076341648, + "grad_norm": 51.522674560546875, + "learning_rate": 3.367463137189156e-07, + "logits/chosen": 0.5388308763504028, + "logits/rejected": 0.4800010323524475, + "loss": 1.0912, + "step": 298 + }, + { + "beta_dpo/beta": 0.0421302504837513, + "beta_dpo/beta_margin_grad_mean": -0.39610981941223145, + "beta_dpo/beta_margin_grad_std": 0.21745111048221588, + "beta_dpo/beta_margin_mean": 0.6996102929115295, + "beta_dpo/beta_margin_std": 1.5129352807998657, + "beta_dpo/beta_used": 0.0421302504837513, + "beta_dpo/beta_used_raw": -0.040616486221551895, + "beta_dpo/gap_mean": 16.089981079101562, + "beta_dpo/gap_std": 22.930587768554688, + "beta_dpo/loss_margin_mean": 14.132258415222168, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4520030234315949, + "grad_norm": 25.89242172241211, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": 0.3590264618396759, + "logits/rejected": 0.3310147523880005, + "loss": 1.0826, + "step": 299 + }, + { + "beta_dpo/beta": 0.05767098441720009, + "beta_dpo/beta_margin_grad_mean": -0.38832151889801025, + "beta_dpo/beta_margin_grad_std": 0.2553543150424957, + "beta_dpo/beta_margin_mean": 0.9388990998268127, + "beta_dpo/beta_margin_std": 2.173591375350952, + "beta_dpo/beta_used": 0.05767098441720009, + "beta_dpo/beta_used_raw": 0.04955215007066727, + "beta_dpo/gap_mean": 15.694482803344727, + "beta_dpo/gap_std": 23.181346893310547, + "beta_dpo/loss_margin_mean": 15.122093200683594, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.45351473922902497, + "grad_norm": 46.98778533935547, + "learning_rate": 3.3426136618426043e-07, + "logits/chosen": 0.46122419834136963, + "logits/rejected": 0.39728420972824097, + "loss": 1.1289, + "step": 300 + }, + { + "epoch": 0.45351473922902497, + "eval_beta_dpo/beta": 0.11505560576915741, + "eval_beta_dpo/beta_margin_grad_mean": -0.3684498369693756, + "eval_beta_dpo/beta_margin_grad_std": 0.17674221098423004, + "eval_beta_dpo/beta_margin_mean": 2.143476963043213, + "eval_beta_dpo/beta_margin_std": 2.818143129348755, + "eval_beta_dpo/beta_used": 0.11505560576915741, + "eval_beta_dpo/beta_used_raw": 0.03927391394972801, + "eval_beta_dpo/gap_mean": 15.726388931274414, + "eval_beta_dpo/gap_std": 23.26214599609375, + "eval_beta_dpo/loss_margin_mean": 14.714288711547852, + "eval_beta_dpo/mask_keep_frac": 1.0, + "eval_logits/chosen": 0.5036243200302124, + "eval_logits/rejected": 0.45220693945884705, + "eval_loss": 0.6938130855560303, + "eval_runtime": 38.89, + "eval_samples_per_second": 59.218, + "eval_steps_per_second": 1.851, + "step": 300 + }, + { + "beta_dpo/beta": 0.13246102631092072, + "beta_dpo/beta_margin_grad_mean": -0.3911619782447815, + "beta_dpo/beta_margin_grad_std": 0.27769890427589417, + "beta_dpo/beta_margin_mean": 1.5554008483886719, + "beta_dpo/beta_margin_std": 4.746208667755127, + "beta_dpo/beta_used": 0.13246102631092072, + "beta_dpo/beta_used_raw": 0.07967150211334229, + "beta_dpo/gap_mean": 15.842938423156738, + "beta_dpo/gap_std": 23.74286460876465, + "beta_dpo/loss_margin_mean": 15.3896484375, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.455026455026455, + "grad_norm": 85.52210998535156, + "learning_rate": 3.3301533956555885e-07, + "logits/chosen": 0.5230924487113953, + "logits/rejected": 0.4926201105117798, + "loss": 0.9855, + "step": 301 + }, + { + "beta_dpo/beta": 0.046855803579092026, + "beta_dpo/beta_margin_grad_mean": -0.42220836877822876, + "beta_dpo/beta_margin_grad_std": 0.20177949965000153, + "beta_dpo/beta_margin_mean": 0.46218007802963257, + "beta_dpo/beta_margin_std": 1.4732623100280762, + "beta_dpo/beta_used": 0.046855803579092026, + "beta_dpo/beta_used_raw": -0.1274799406528473, + "beta_dpo/gap_mean": 14.743999481201172, + "beta_dpo/gap_std": 23.574613571166992, + "beta_dpo/loss_margin_mean": 9.621975898742676, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4565381708238851, + "grad_norm": 27.584754943847656, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": 0.34268122911453247, + "logits/rejected": 0.28931474685668945, + "loss": 1.0507, + "step": 302 + }, + { + "beta_dpo/beta": 0.21900054812431335, + "beta_dpo/beta_margin_grad_mean": -0.2753453850746155, + "beta_dpo/beta_margin_grad_std": 0.3416510820388794, + "beta_dpo/beta_margin_mean": 4.497269630432129, + "beta_dpo/beta_margin_std": 7.897620677947998, + "beta_dpo/beta_used": 0.21900054812431335, + "beta_dpo/beta_used_raw": 0.21900054812431335, + "beta_dpo/gap_mean": 15.209549903869629, + "beta_dpo/gap_std": 23.96515464782715, + "beta_dpo/loss_margin_mean": 19.80791473388672, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4580498866213152, + "grad_norm": 149.34068298339844, + "learning_rate": 3.3051635489464793e-07, + "logits/chosen": 0.46690768003463745, + "logits/rejected": 0.40056318044662476, + "loss": 1.5522, + "step": 303 + }, + { + "beta_dpo/beta": 0.1770317256450653, + "beta_dpo/beta_margin_grad_mean": -0.2056574821472168, + "beta_dpo/beta_margin_grad_std": 0.27400633692741394, + "beta_dpo/beta_margin_mean": 3.444209575653076, + "beta_dpo/beta_margin_std": 3.6619412899017334, + "beta_dpo/beta_used": 0.1770317256450653, + "beta_dpo/beta_used_raw": 0.1770317256450653, + "beta_dpo/gap_mean": 16.013572692871094, + "beta_dpo/gap_std": 23.73634147644043, + "beta_dpo/loss_margin_mean": 19.397008895874023, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4595616024187453, + "grad_norm": 76.03279113769531, + "learning_rate": 3.292634667444117e-07, + "logits/chosen": 0.43441325426101685, + "logits/rejected": 0.380365252494812, + "loss": 0.6161, + "step": 304 + }, + { + "beta_dpo/beta": 0.25595974922180176, + "beta_dpo/beta_margin_grad_mean": -0.2995953857898712, + "beta_dpo/beta_margin_grad_std": 0.3861447870731354, + "beta_dpo/beta_margin_mean": 3.4279513359069824, + "beta_dpo/beta_margin_std": 6.85020637512207, + "beta_dpo/beta_used": 0.25595974922180176, + "beta_dpo/beta_used_raw": 0.25595974922180176, + "beta_dpo/gap_mean": 15.928811073303223, + "beta_dpo/gap_std": 23.983856201171875, + "beta_dpo/loss_margin_mean": 13.063897132873535, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.46107331821617537, + "grad_norm": 132.59515380859375, + "learning_rate": 3.280083614246217e-07, + "logits/chosen": 0.4133915603160858, + "logits/rejected": 0.4339390695095062, + "loss": 1.0019, + "step": 305 + }, + { + "beta_dpo/beta": 0.17636467516422272, + "beta_dpo/beta_margin_grad_mean": -0.3082793056964874, + "beta_dpo/beta_margin_grad_std": 0.2324202060699463, + "beta_dpo/beta_margin_mean": 3.2350707054138184, + "beta_dpo/beta_margin_std": 5.149374485015869, + "beta_dpo/beta_used": 0.17636467516422272, + "beta_dpo/beta_used_raw": -0.025320664048194885, + "beta_dpo/gap_mean": 15.603142738342285, + "beta_dpo/gap_std": 23.355175018310547, + "beta_dpo/loss_margin_mean": 14.176777839660645, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.46258503401360546, + "grad_norm": 81.64033508300781, + "learning_rate": 3.267510740432719e-07, + "logits/chosen": 0.4885938763618469, + "logits/rejected": 0.37870633602142334, + "loss": 0.8295, + "step": 306 + }, + { + "beta_dpo/beta": 0.1279929280281067, + "beta_dpo/beta_margin_grad_mean": -0.43924155831336975, + "beta_dpo/beta_margin_grad_std": 0.29262182116508484, + "beta_dpo/beta_margin_mean": 1.2105649709701538, + "beta_dpo/beta_margin_std": 4.71697998046875, + "beta_dpo/beta_used": 0.1279929280281067, + "beta_dpo/beta_used_raw": 0.09390576928853989, + "beta_dpo/gap_mean": 14.28852653503418, + "beta_dpo/gap_std": 23.57619285583496, + "beta_dpo/loss_margin_mean": 8.560576438903809, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.46409674981103555, + "grad_norm": 115.50189208984375, + "learning_rate": 3.2549163976939285e-07, + "logits/chosen": 0.49088621139526367, + "logits/rejected": 0.4447266459465027, + "loss": 1.4104, + "step": 307 + }, + { + "beta_dpo/beta": 0.10693937540054321, + "beta_dpo/beta_margin_grad_mean": -0.3734705448150635, + "beta_dpo/beta_margin_grad_std": 0.2782239615917206, + "beta_dpo/beta_margin_mean": 1.970252275466919, + "beta_dpo/beta_margin_std": 4.069371223449707, + "beta_dpo/beta_used": 0.10693937540054321, + "beta_dpo/beta_used_raw": 0.09306029975414276, + "beta_dpo/gap_mean": 14.391298294067383, + "beta_dpo/gap_std": 23.804813385009766, + "beta_dpo/loss_margin_mean": 15.613865852355957, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4656084656084656, + "grad_norm": 86.94285583496094, + "learning_rate": 3.2423009383206874e-07, + "logits/chosen": 0.4649581015110016, + "logits/rejected": 0.4495544135570526, + "loss": 1.5807, + "step": 308 + }, + { + "beta_dpo/beta": 0.00609197374433279, + "beta_dpo/beta_margin_grad_mean": -0.4813478887081146, + "beta_dpo/beta_margin_grad_std": 0.04934345558285713, + "beta_dpo/beta_margin_mean": 0.07604541629552841, + "beta_dpo/beta_margin_std": 0.20145538449287415, + "beta_dpo/beta_used": 0.00609197374433279, + "beta_dpo/beta_used_raw": 0.006033188197761774, + "beta_dpo/gap_mean": 14.475622177124023, + "beta_dpo/gap_std": 23.469074249267578, + "beta_dpo/loss_margin_mean": 14.663956642150879, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4671201814058957, + "grad_norm": 5.302502155303955, + "learning_rate": 3.229664715194511e-07, + "logits/chosen": 0.5176172256469727, + "logits/rejected": 0.45871448516845703, + "loss": 1.3167, + "step": 309 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4975871741771698, + "beta_dpo/beta_margin_grad_std": 0.005759851075708866, + "beta_dpo/beta_margin_mean": 0.009653035551309586, + "beta_dpo/beta_margin_std": 0.023043153807520866, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.15084339678287506, + "beta_dpo/gap_mean": 13.659213066101074, + "beta_dpo/gap_std": 23.398651123046875, + "beta_dpo/loss_margin_mean": 9.653035163879395, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.46863189720332576, + "grad_norm": 0.7138440012931824, + "learning_rate": 3.2170080817777257e-07, + "logits/chosen": 0.48263025283813477, + "logits/rejected": 0.46910643577575684, + "loss": 1.3769, + "step": 310 + }, + { + "beta_dpo/beta": 0.028796648606657982, + "beta_dpo/beta_margin_grad_mean": -0.4123019874095917, + "beta_dpo/beta_margin_grad_std": 0.16005714237689972, + "beta_dpo/beta_margin_mean": 0.462990939617157, + "beta_dpo/beta_margin_std": 0.9138634204864502, + "beta_dpo/beta_used": 0.028796648606657982, + "beta_dpo/beta_used_raw": -0.09092193841934204, + "beta_dpo/gap_mean": 13.479878425598145, + "beta_dpo/gap_std": 23.28680419921875, + "beta_dpo/loss_margin_mean": 12.439473152160645, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.47014361300075586, + "grad_norm": 16.826684951782227, + "learning_rate": 3.204331392103574e-07, + "logits/chosen": 0.3971518874168396, + "logits/rejected": 0.2650872468948364, + "loss": 1.1307, + "step": 311 + }, + { + "beta_dpo/beta": 0.163363978266716, + "beta_dpo/beta_margin_grad_mean": -0.3334113359451294, + "beta_dpo/beta_margin_grad_std": 0.25571146607398987, + "beta_dpo/beta_margin_mean": 2.983691692352295, + "beta_dpo/beta_margin_std": 5.67026424407959, + "beta_dpo/beta_used": 0.163363978266716, + "beta_dpo/beta_used_raw": 0.06641173362731934, + "beta_dpo/gap_mean": 13.949535369873047, + "beta_dpo/gap_std": 23.26689910888672, + "beta_dpo/loss_margin_mean": 18.187755584716797, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.47165532879818595, + "grad_norm": 41.266841888427734, + "learning_rate": 3.1916350007663176e-07, + "logits/chosen": 0.4562729001045227, + "logits/rejected": 0.3602554500102997, + "loss": 0.8424, + "step": 312 + }, + { + "beta_dpo/beta": 0.04959503188729286, + "beta_dpo/beta_margin_grad_mean": -0.42779460549354553, + "beta_dpo/beta_margin_grad_std": 0.23276080191135406, + "beta_dpo/beta_margin_mean": 0.5780282616615295, + "beta_dpo/beta_margin_std": 1.984850287437439, + "beta_dpo/beta_used": 0.04959503188729286, + "beta_dpo/beta_used_raw": 0.011893421411514282, + "beta_dpo/gap_mean": 13.817769050598145, + "beta_dpo/gap_std": 23.424896240234375, + "beta_dpo/loss_margin_mean": 11.726629257202148, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.47316704459561604, + "grad_norm": 39.96240234375, + "learning_rate": 3.178919262911314e-07, + "logits/chosen": 0.5323947668075562, + "logits/rejected": 0.510050356388092, + "loss": 1.1167, + "step": 313 + }, + { + "beta_dpo/beta": 0.4231081008911133, + "beta_dpo/beta_margin_grad_mean": -0.1942075788974762, + "beta_dpo/beta_margin_grad_std": 0.3291747570037842, + "beta_dpo/beta_margin_mean": 8.420723915100098, + "beta_dpo/beta_margin_std": 9.913713455200195, + "beta_dpo/beta_used": 0.4231081008911133, + "beta_dpo/beta_used_raw": 0.4231081008911133, + "beta_dpo/gap_mean": 14.603485107421875, + "beta_dpo/gap_std": 23.527881622314453, + "beta_dpo/loss_margin_mean": 19.856849670410156, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.47467876039304613, + "grad_norm": 202.71646118164062, + "learning_rate": 3.166184534225087e-07, + "logits/chosen": 0.45087888836860657, + "logits/rejected": 0.47414296865463257, + "loss": 1.258, + "step": 314 + }, + { + "beta_dpo/beta": 0.1378045380115509, + "beta_dpo/beta_margin_grad_mean": -0.3461618721485138, + "beta_dpo/beta_margin_grad_std": 0.282050758600235, + "beta_dpo/beta_margin_mean": 2.926037549972534, + "beta_dpo/beta_margin_std": 5.34313440322876, + "beta_dpo/beta_used": 0.1378045380115509, + "beta_dpo/beta_used_raw": -0.007274851202964783, + "beta_dpo/gap_mean": 14.610097885131836, + "beta_dpo/gap_std": 23.17371940612793, + "beta_dpo/loss_margin_mean": 15.014217376708984, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.47619047619047616, + "grad_norm": 92.25020599365234, + "learning_rate": 3.1534311709253723e-07, + "logits/chosen": 0.4022720456123352, + "logits/rejected": 0.361319899559021, + "loss": 1.3093, + "step": 315 + }, + { + "beta_dpo/beta": 0.2009076625108719, + "beta_dpo/beta_margin_grad_mean": -0.30123385787010193, + "beta_dpo/beta_margin_grad_std": 0.2647503912448883, + "beta_dpo/beta_margin_mean": 5.084220886230469, + "beta_dpo/beta_margin_std": 7.214354038238525, + "beta_dpo/beta_used": 0.2009076625108719, + "beta_dpo/beta_used_raw": 0.18065881729125977, + "beta_dpo/gap_mean": 15.16646957397461, + "beta_dpo/gap_std": 22.370153427124023, + "beta_dpo/loss_margin_mean": 18.50032615661621, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.47770219198790626, + "grad_norm": 95.88639068603516, + "learning_rate": 3.1406595297511564e-07, + "logits/chosen": 0.3557189106941223, + "logits/rejected": 0.23355801403522491, + "loss": 1.0698, + "step": 316 + }, + { + "beta_dpo/beta": 0.1612454503774643, + "beta_dpo/beta_margin_grad_mean": -0.3476071357727051, + "beta_dpo/beta_margin_grad_std": 0.286365807056427, + "beta_dpo/beta_margin_mean": 3.270941734313965, + "beta_dpo/beta_margin_std": 6.323551654815674, + "beta_dpo/beta_used": 0.1612454503774643, + "beta_dpo/beta_used_raw": 0.07801032066345215, + "beta_dpo/gap_mean": 16.22469711303711, + "beta_dpo/gap_std": 22.18575668334961, + "beta_dpo/loss_margin_mean": 18.86298179626465, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.47921390778533635, + "grad_norm": 82.9071273803711, + "learning_rate": 3.1278699679526975e-07, + "logits/chosen": 0.5267002582550049, + "logits/rejected": 0.4774794578552246, + "loss": 1.2068, + "step": 317 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4968032240867615, + "beta_dpo/beta_margin_grad_std": 0.0065883188508450985, + "beta_dpo/beta_margin_mean": 0.01278975885361433, + "beta_dpo/beta_margin_std": 0.02635836973786354, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.18052691221237183, + "beta_dpo/gap_mean": 15.857492446899414, + "beta_dpo/gap_std": 22.781883239746094, + "beta_dpo/loss_margin_mean": 12.789758682250977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.48072562358276644, + "grad_norm": 0.6836572885513306, + "learning_rate": 3.1150628432815336e-07, + "logits/chosen": 0.4887760281562805, + "logits/rejected": 0.42186325788497925, + "loss": 1.3752, + "step": 318 + }, + { + "beta_dpo/beta": 0.1261560469865799, + "beta_dpo/beta_margin_grad_mean": -0.3443688154220581, + "beta_dpo/beta_margin_grad_std": 0.261486679315567, + "beta_dpo/beta_margin_mean": 2.205345869064331, + "beta_dpo/beta_margin_std": 4.338085174560547, + "beta_dpo/beta_used": 0.1261560469865799, + "beta_dpo/beta_used_raw": 0.1261560469865799, + "beta_dpo/gap_mean": 15.758548736572266, + "beta_dpo/gap_std": 22.95236587524414, + "beta_dpo/loss_margin_mean": 16.208959579467773, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.48223733938019653, + "grad_norm": 70.87828826904297, + "learning_rate": 3.1022385139804707e-07, + "logits/chosen": 0.4163881838321686, + "logits/rejected": 0.3962825834751129, + "loss": 1.0191, + "step": 319 + }, + { + "beta_dpo/beta": 0.08117100596427917, + "beta_dpo/beta_margin_grad_mean": -0.3801567256450653, + "beta_dpo/beta_margin_grad_std": 0.25498995184898376, + "beta_dpo/beta_margin_mean": 1.6039687395095825, + "beta_dpo/beta_margin_std": 3.3591623306274414, + "beta_dpo/beta_used": 0.08117100596427917, + "beta_dpo/beta_used_raw": -0.06516657024621964, + "beta_dpo/gap_mean": 14.905145645141602, + "beta_dpo/gap_std": 22.96976089477539, + "beta_dpo/loss_margin_mean": 12.470100402832031, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4837490551776266, + "grad_norm": 35.630069732666016, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": 0.3363953232765198, + "logits/rejected": 0.2939741611480713, + "loss": 1.1093, + "step": 320 + }, + { + "beta_dpo/beta": 0.07048396021127701, + "beta_dpo/beta_margin_grad_mean": -0.3663146197795868, + "beta_dpo/beta_margin_grad_std": 0.22248594462871552, + "beta_dpo/beta_margin_mean": 1.050384521484375, + "beta_dpo/beta_margin_std": 2.262019157409668, + "beta_dpo/beta_used": 0.07048396021127701, + "beta_dpo/beta_used_raw": -0.09373751282691956, + "beta_dpo/gap_mean": 14.79505729675293, + "beta_dpo/gap_std": 22.987442016601562, + "beta_dpo/loss_margin_mean": 11.55805492401123, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4852607709750567, + "grad_norm": 28.83603286743164, + "learning_rate": 3.0765396768561004e-07, + "logits/chosen": 0.42309537529945374, + "logits/rejected": 0.40167927742004395, + "loss": 0.9358, + "step": 321 + }, + { + "beta_dpo/beta": 0.3060827851295471, + "beta_dpo/beta_margin_grad_mean": -0.1633433699607849, + "beta_dpo/beta_margin_grad_std": 0.28638920187950134, + "beta_dpo/beta_margin_mean": 5.767949104309082, + "beta_dpo/beta_margin_std": 6.40022087097168, + "beta_dpo/beta_used": 0.3060827851295471, + "beta_dpo/beta_used_raw": 0.3060827851295471, + "beta_dpo/gap_mean": 14.88734245300293, + "beta_dpo/gap_std": 22.760725021362305, + "beta_dpo/loss_margin_mean": 18.466712951660156, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.48677248677248675, + "grad_norm": 120.95641326904297, + "learning_rate": 3.063665887884511e-07, + "logits/chosen": 0.47183048725128174, + "logits/rejected": 0.39079803228378296, + "loss": 0.8343, + "step": 322 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4965234100818634, + "beta_dpo/beta_margin_grad_std": 0.00671707559376955, + "beta_dpo/beta_margin_mean": 0.01391004491597414, + "beta_dpo/beta_margin_std": 0.026876913383603096, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.13449180126190186, + "beta_dpo/gap_mean": 14.877632141113281, + "beta_dpo/gap_std": 23.045122146606445, + "beta_dpo/loss_margin_mean": 13.910043716430664, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.48828420256991684, + "grad_norm": 0.7591775059700012, + "learning_rate": 3.0507763319663517e-07, + "logits/chosen": 0.3593803644180298, + "logits/rejected": 0.28365999460220337, + "loss": 1.3754, + "step": 323 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4968474507331848, + "beta_dpo/beta_margin_grad_std": 0.004732394125312567, + "beta_dpo/beta_margin_mean": 0.012611806392669678, + "beta_dpo/beta_margin_std": 0.018933270126581192, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.12538644671440125, + "beta_dpo/gap_mean": 14.813908576965332, + "beta_dpo/gap_std": 22.670183181762695, + "beta_dpo/loss_margin_mean": 12.61180591583252, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4897959183673469, + "grad_norm": 0.7829446196556091, + "learning_rate": 3.0378713696502097e-07, + "logits/chosen": 0.47413748502731323, + "logits/rejected": 0.41598939895629883, + "loss": 1.3753, + "step": 324 + }, + { + "beta_dpo/beta": 0.22583173215389252, + "beta_dpo/beta_margin_grad_mean": -0.3542848229408264, + "beta_dpo/beta_margin_grad_std": 0.301741361618042, + "beta_dpo/beta_margin_mean": 4.75588846206665, + "beta_dpo/beta_margin_std": 9.164177894592285, + "beta_dpo/beta_used": 0.22583173215389252, + "beta_dpo/beta_used_raw": 0.10879484564065933, + "beta_dpo/gap_mean": 14.971842765808105, + "beta_dpo/gap_std": 22.737186431884766, + "beta_dpo/loss_margin_mean": 15.938600540161133, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.491307634164777, + "grad_norm": 86.0190658569336, + "learning_rate": 3.0249513619156206e-07, + "logits/chosen": 0.42567065358161926, + "logits/rejected": 0.36200234293937683, + "loss": 1.1197, + "step": 325 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49821144342422485, + "beta_dpo/beta_margin_grad_std": 0.0051818969659507275, + "beta_dpo/beta_margin_mean": 0.00715494342148304, + "beta_dpo/beta_margin_std": 0.02072981186211109, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.12947367131710052, + "beta_dpo/gap_mean": 13.62185001373291, + "beta_dpo/gap_std": 22.390117645263672, + "beta_dpo/loss_margin_mean": 7.154942989349365, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4928193499622071, + "grad_norm": 0.8826065063476562, + "learning_rate": 3.012016670162977e-07, + "logits/chosen": 0.3907411992549896, + "logits/rejected": 0.39254793524742126, + "loss": 1.3766, + "step": 326 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4969516396522522, + "beta_dpo/beta_margin_grad_std": 0.006606688257306814, + "beta_dpo/beta_margin_mean": 0.012196130119264126, + "beta_dpo/beta_margin_std": 0.026434360072016716, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.05859054625034332, + "beta_dpo/gap_mean": 13.037698745727539, + "beta_dpo/gap_std": 22.96277618408203, + "beta_dpo/loss_margin_mean": 12.19612979888916, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4943310657596372, + "grad_norm": 0.7743061184883118, + "learning_rate": 2.99906765620341e-07, + "logits/chosen": 0.35184246301651, + "logits/rejected": 0.3126526474952698, + "loss": 1.376, + "step": 327 + }, + { + "beta_dpo/beta": 0.09753447771072388, + "beta_dpo/beta_margin_grad_mean": -0.38534635305404663, + "beta_dpo/beta_margin_grad_std": 0.25296443700790405, + "beta_dpo/beta_margin_mean": 1.4550480842590332, + "beta_dpo/beta_margin_std": 3.0800600051879883, + "beta_dpo/beta_used": 0.09753447771072388, + "beta_dpo/beta_used_raw": 0.0254751518368721, + "beta_dpo/gap_mean": 13.520404815673828, + "beta_dpo/gap_std": 22.76117515563965, + "beta_dpo/loss_margin_mean": 15.616045951843262, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4958427815570673, + "grad_norm": 72.21566009521484, + "learning_rate": 2.9861046822486766e-07, + "logits/chosen": 0.3937995433807373, + "logits/rejected": 0.36060845851898193, + "loss": 1.1729, + "step": 328 + }, + { + "beta_dpo/beta": 0.002249703276902437, + "beta_dpo/beta_margin_grad_mean": -0.49035826325416565, + "beta_dpo/beta_margin_grad_std": 0.016022533178329468, + "beta_dpo/beta_margin_mean": 0.03864653781056404, + "beta_dpo/beta_margin_std": 0.06428585946559906, + "beta_dpo/beta_used": 0.002249703276902437, + "beta_dpo/beta_used_raw": -0.004807803314179182, + "beta_dpo/gap_mean": 14.01812744140625, + "beta_dpo/gap_std": 22.638988494873047, + "beta_dpo/loss_margin_mean": 15.689536094665527, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4973544973544973, + "grad_norm": 2.2578487396240234, + "learning_rate": 2.9731281109010253e-07, + "logits/chosen": 0.4657662510871887, + "logits/rejected": 0.41007131338119507, + "loss": 1.3591, + "step": 329 + }, + { + "beta_dpo/beta": 0.10219182074069977, + "beta_dpo/beta_margin_grad_mean": -0.3714367747306824, + "beta_dpo/beta_margin_grad_std": 0.2565270662307739, + "beta_dpo/beta_margin_mean": 1.8236949443817139, + "beta_dpo/beta_margin_std": 3.910278081893921, + "beta_dpo/beta_used": 0.10219182074069977, + "beta_dpo/beta_used_raw": 0.04957320913672447, + "beta_dpo/gap_mean": 14.02891731262207, + "beta_dpo/gap_std": 23.100826263427734, + "beta_dpo/loss_margin_mean": 15.226579666137695, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.4988662131519274, + "grad_norm": 45.77909851074219, + "learning_rate": 2.9601383051430505e-07, + "logits/chosen": 0.43188655376434326, + "logits/rejected": 0.3610241115093231, + "loss": 0.9899, + "step": 330 + }, + { + "beta_dpo/beta": 0.10813254117965698, + "beta_dpo/beta_margin_grad_mean": -0.3179372251033783, + "beta_dpo/beta_margin_grad_std": 0.2648589611053467, + "beta_dpo/beta_margin_mean": 2.2522261142730713, + "beta_dpo/beta_margin_std": 4.484652519226074, + "beta_dpo/beta_used": 0.10813254117965698, + "beta_dpo/beta_used_raw": 0.10813254117965698, + "beta_dpo/gap_mean": 15.494121551513672, + "beta_dpo/gap_std": 23.874298095703125, + "beta_dpo/loss_margin_mean": 22.335155487060547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5003779289493575, + "grad_norm": 85.85868835449219, + "learning_rate": 2.947135628327544e-07, + "logits/chosen": 0.5212812423706055, + "logits/rejected": 0.49004465341567993, + "loss": 1.0558, + "step": 331 + }, + { + "beta_dpo/beta": 0.055088166147470474, + "beta_dpo/beta_margin_grad_mean": -0.3655672073364258, + "beta_dpo/beta_margin_grad_std": 0.22148992121219635, + "beta_dpo/beta_margin_mean": 1.311160922050476, + "beta_dpo/beta_margin_std": 2.3807129859924316, + "beta_dpo/beta_used": 0.055088166147470474, + "beta_dpo/beta_used_raw": -0.038188386708498, + "beta_dpo/gap_mean": 15.760552406311035, + "beta_dpo/gap_std": 24.142658233642578, + "beta_dpo/loss_margin_mean": 17.73267364501953, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5018896447467877, + "grad_norm": 47.309410095214844, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": 0.4019997715950012, + "logits/rejected": 0.35436803102493286, + "loss": 1.1193, + "step": 332 + }, + { + "beta_dpo/beta": 0.3347168564796448, + "beta_dpo/beta_margin_grad_mean": -0.2177586406469345, + "beta_dpo/beta_margin_grad_std": 0.3292071223258972, + "beta_dpo/beta_margin_mean": 7.595628261566162, + "beta_dpo/beta_margin_std": 8.555011749267578, + "beta_dpo/beta_used": 0.3347168564796448, + "beta_dpo/beta_used_raw": 0.3347168564796448, + "beta_dpo/gap_mean": 17.009662628173828, + "beta_dpo/gap_std": 24.363880157470703, + "beta_dpo/loss_margin_mean": 22.31917953491211, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5034013605442177, + "grad_norm": 173.7876434326172, + "learning_rate": 2.921093116725076e-07, + "logits/chosen": 0.4317076802253723, + "logits/rejected": 0.3626272678375244, + "loss": 0.8614, + "step": 333 + }, + { + "beta_dpo/beta": 0.0760747492313385, + "beta_dpo/beta_margin_grad_mean": -0.40545621514320374, + "beta_dpo/beta_margin_grad_std": 0.2751104235649109, + "beta_dpo/beta_margin_mean": 1.072293758392334, + "beta_dpo/beta_margin_std": 3.0955679416656494, + "beta_dpo/beta_used": 0.0760747492313385, + "beta_dpo/beta_used_raw": 0.06470038741827011, + "beta_dpo/gap_mean": 16.923879623413086, + "beta_dpo/gap_std": 24.878725051879883, + "beta_dpo/loss_margin_mean": 14.388141632080078, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5049130763416477, + "grad_norm": 81.30701446533203, + "learning_rate": 2.9080540104031484e-07, + "logits/chosen": 0.46789878606796265, + "logits/rejected": 0.4239296317100525, + "loss": 1.3485, + "step": 334 + }, + { + "beta_dpo/beta": 0.14732955396175385, + "beta_dpo/beta_margin_grad_mean": -0.35019081830978394, + "beta_dpo/beta_margin_grad_std": 0.27779415249824524, + "beta_dpo/beta_margin_mean": 3.1584525108337402, + "beta_dpo/beta_margin_std": 5.779693603515625, + "beta_dpo/beta_used": 0.14732955396175385, + "beta_dpo/beta_used_raw": 0.06579115241765976, + "beta_dpo/gap_mean": 17.025259017944336, + "beta_dpo/gap_std": 25.54525375366211, + "beta_dpo/loss_margin_mean": 19.060718536376953, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5064247921390779, + "grad_norm": 115.02545928955078, + "learning_rate": 2.895003489933375e-07, + "logits/chosen": 0.4890259802341461, + "logits/rejected": 0.45069044828414917, + "loss": 1.3095, + "step": 335 + }, + { + "beta_dpo/beta": 0.006434235721826553, + "beta_dpo/beta_margin_grad_mean": -0.4622502028942108, + "beta_dpo/beta_margin_grad_std": 0.07217463850975037, + "beta_dpo/beta_margin_mean": 0.1582019180059433, + "beta_dpo/beta_margin_std": 0.307355672121048, + "beta_dpo/beta_used": 0.006434235721826553, + "beta_dpo/beta_used_raw": -0.16895954310894012, + "beta_dpo/gap_mean": 17.870014190673828, + "beta_dpo/gap_std": 26.2718505859375, + "beta_dpo/loss_margin_mean": 19.669475555419922, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5079365079365079, + "grad_norm": 6.488018989562988, + "learning_rate": 2.8819419203668675e-07, + "logits/chosen": 0.391056627035141, + "logits/rejected": 0.3693464994430542, + "loss": 1.2986, + "step": 336 + }, + { + "beta_dpo/beta": 0.0338905043900013, + "beta_dpo/beta_margin_grad_mean": -0.4215489327907562, + "beta_dpo/beta_margin_grad_std": 0.18948403000831604, + "beta_dpo/beta_margin_mean": 0.4833620488643646, + "beta_dpo/beta_margin_std": 1.2267965078353882, + "beta_dpo/beta_used": 0.0338905043900013, + "beta_dpo/beta_used_raw": -0.11094224452972412, + "beta_dpo/gap_mean": 17.074710845947266, + "beta_dpo/gap_std": 26.033262252807617, + "beta_dpo/loss_margin_mean": 13.338370323181152, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.509448223733938, + "grad_norm": 26.08516502380371, + "learning_rate": 2.8688696670638053e-07, + "logits/chosen": 0.3497691750526428, + "logits/rejected": 0.316182017326355, + "loss": 1.1495, + "step": 337 + }, + { + "beta_dpo/beta": 0.046072110533714294, + "beta_dpo/beta_margin_grad_mean": -0.41082024574279785, + "beta_dpo/beta_margin_grad_std": 0.20698098838329315, + "beta_dpo/beta_margin_mean": 0.5744975805282593, + "beta_dpo/beta_margin_std": 1.5593328475952148, + "beta_dpo/beta_used": 0.046072110533714294, + "beta_dpo/beta_used_raw": -0.08032860606908798, + "beta_dpo/gap_mean": 16.35885238647461, + "beta_dpo/gap_std": 25.77899932861328, + "beta_dpo/loss_margin_mean": 14.114729881286621, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5109599395313681, + "grad_norm": 40.451805114746094, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": 0.4158708155155182, + "logits/rejected": 0.38753098249435425, + "loss": 1.0496, + "step": 338 + }, + { + "beta_dpo/beta": 0.08386623114347458, + "beta_dpo/beta_margin_grad_mean": -0.38753587007522583, + "beta_dpo/beta_margin_grad_std": 0.25693479180336, + "beta_dpo/beta_margin_mean": 1.5702204704284668, + "beta_dpo/beta_margin_std": 3.792813539505005, + "beta_dpo/beta_used": 0.08386623114347458, + "beta_dpo/beta_used_raw": -0.0034884288907051086, + "beta_dpo/gap_mean": 16.51103973388672, + "beta_dpo/gap_std": 26.04473304748535, + "beta_dpo/loss_margin_mean": 17.060956954956055, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5124716553287982, + "grad_norm": 49.29536819458008, + "learning_rate": 2.842694572172736e-07, + "logits/chosen": 0.5705593824386597, + "logits/rejected": 0.48631036281585693, + "loss": 1.158, + "step": 339 + }, + { + "beta_dpo/beta": 0.16217856109142303, + "beta_dpo/beta_margin_grad_mean": -0.3174978196620941, + "beta_dpo/beta_margin_grad_std": 0.37058526277542114, + "beta_dpo/beta_margin_mean": 3.037602186203003, + "beta_dpo/beta_margin_std": 4.942996978759766, + "beta_dpo/beta_used": 0.16217856109142303, + "beta_dpo/beta_used_raw": 0.16217856109142303, + "beta_dpo/gap_mean": 16.645965576171875, + "beta_dpo/gap_std": 26.448713302612305, + "beta_dpo/loss_margin_mean": 19.0429744720459, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5139833711262283, + "grad_norm": 103.46623229980469, + "learning_rate": 2.8295924627584004e-07, + "logits/chosen": 0.46611109375953674, + "logits/rejected": 0.4412732720375061, + "loss": 1.4731, + "step": 340 + }, + { + "beta_dpo/beta": 0.42890098690986633, + "beta_dpo/beta_margin_grad_mean": -0.3295966386795044, + "beta_dpo/beta_margin_grad_std": 0.29606911540031433, + "beta_dpo/beta_margin_mean": 12.761946678161621, + "beta_dpo/beta_margin_std": 20.45010757446289, + "beta_dpo/beta_used": 0.42890098690986633, + "beta_dpo/beta_used_raw": 0.3651895523071289, + "beta_dpo/gap_mean": 17.889223098754883, + "beta_dpo/gap_std": 26.904251098632812, + "beta_dpo/loss_margin_mean": 20.278278350830078, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5154950869236583, + "grad_norm": 190.08685302734375, + "learning_rate": 2.816481133934373e-07, + "logits/chosen": 0.4816865622997284, + "logits/rejected": 0.43140339851379395, + "loss": 1.0551, + "step": 341 + }, + { + "beta_dpo/beta": 0.10002905875444412, + "beta_dpo/beta_margin_grad_mean": -0.39862194657325745, + "beta_dpo/beta_margin_grad_std": 0.2727389633655548, + "beta_dpo/beta_margin_mean": 2.2917656898498535, + "beta_dpo/beta_margin_std": 5.209559917449951, + "beta_dpo/beta_used": 0.10002905875444412, + "beta_dpo/beta_used_raw": 0.04651253670454025, + "beta_dpo/gap_mean": 18.16006851196289, + "beta_dpo/gap_std": 27.659713745117188, + "beta_dpo/loss_margin_mean": 21.531021118164062, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5170068027210885, + "grad_norm": 89.060546875, + "learning_rate": 2.8033609524527046e-07, + "logits/chosen": 0.5078241229057312, + "logits/rejected": 0.46389999985694885, + "loss": 1.4452, + "step": 342 + }, + { + "beta_dpo/beta": 0.020842978730797768, + "beta_dpo/beta_margin_grad_mean": -0.41583141684532166, + "beta_dpo/beta_margin_grad_std": 0.1556914746761322, + "beta_dpo/beta_margin_mean": 0.4192487895488739, + "beta_dpo/beta_margin_std": 0.8365549445152283, + "beta_dpo/beta_used": 0.020842978730797768, + "beta_dpo/beta_used_raw": -0.29891690611839294, + "beta_dpo/gap_mean": 17.80866813659668, + "beta_dpo/gap_std": 27.227333068847656, + "beta_dpo/loss_margin_mean": 12.708889961242676, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5185185185185185, + "grad_norm": 11.837695121765137, + "learning_rate": 2.7902322853130753e-07, + "logits/chosen": 0.3714722990989685, + "logits/rejected": 0.3623930811882019, + "loss": 1.1458, + "step": 343 + }, + { + "beta_dpo/beta": 0.017244327813386917, + "beta_dpo/beta_margin_grad_mean": -0.4226335287094116, + "beta_dpo/beta_margin_grad_std": 0.14330071210861206, + "beta_dpo/beta_margin_mean": 0.3823748230934143, + "beta_dpo/beta_margin_std": 0.7459104657173157, + "beta_dpo/beta_used": 0.017244327813386917, + "beta_dpo/beta_used_raw": 0.0008646678179502487, + "beta_dpo/gap_mean": 17.61762237548828, + "beta_dpo/gap_std": 27.113140106201172, + "beta_dpo/loss_margin_mean": 19.65960121154785, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5200302343159486, + "grad_norm": 13.51938533782959, + "learning_rate": 2.7770954997525274e-07, + "logits/chosen": 0.49157804250717163, + "logits/rejected": 0.42625850439071655, + "loss": 1.1898, + "step": 344 + }, + { + "beta_dpo/beta": 0.2315424680709839, + "beta_dpo/beta_margin_grad_mean": -0.28378885984420776, + "beta_dpo/beta_margin_grad_std": 0.36535513401031494, + "beta_dpo/beta_margin_mean": 3.849392890930176, + "beta_dpo/beta_margin_std": 6.788583278656006, + "beta_dpo/beta_used": 0.2315424680709839, + "beta_dpo/beta_used_raw": 0.2315424680709839, + "beta_dpo/gap_mean": 17.418254852294922, + "beta_dpo/gap_std": 27.31433868408203, + "beta_dpo/loss_margin_mean": 16.526798248291016, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5215419501133787, + "grad_norm": 133.7179412841797, + "learning_rate": 2.7639509632351927e-07, + "logits/chosen": 0.5586827397346497, + "logits/rejected": 0.5119316577911377, + "loss": 1.0315, + "step": 345 + }, + { + "beta_dpo/beta": 0.19482511281967163, + "beta_dpo/beta_margin_grad_mean": -0.2792256474494934, + "beta_dpo/beta_margin_grad_std": 0.3250068128108978, + "beta_dpo/beta_margin_mean": 4.4295735359191895, + "beta_dpo/beta_margin_std": 8.381417274475098, + "beta_dpo/beta_used": 0.19482511281967163, + "beta_dpo/beta_used_raw": 0.19482511281967163, + "beta_dpo/gap_mean": 17.792781829833984, + "beta_dpo/gap_std": 27.38411521911621, + "beta_dpo/loss_margin_mean": 21.0446720123291, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5230536659108088, + "grad_norm": 143.16773986816406, + "learning_rate": 2.7507990434420123e-07, + "logits/chosen": 0.5135695934295654, + "logits/rejected": 0.43501418828964233, + "loss": 1.4406, + "step": 346 + }, + { + "beta_dpo/beta": 0.02533472701907158, + "beta_dpo/beta_margin_grad_mean": -0.4315335750579834, + "beta_dpo/beta_margin_grad_std": 0.17104710638523102, + "beta_dpo/beta_margin_mean": 0.3843132555484772, + "beta_dpo/beta_margin_std": 0.9899463653564453, + "beta_dpo/beta_used": 0.02533472701907158, + "beta_dpo/beta_used_raw": -0.21560555696487427, + "beta_dpo/gap_mean": 17.808048248291016, + "beta_dpo/gap_std": 27.780567169189453, + "beta_dpo/loss_margin_mean": 16.444530487060547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5245653817082389, + "grad_norm": 18.07720375061035, + "learning_rate": 2.737640108260456e-07, + "logits/chosen": 0.5739601850509644, + "logits/rejected": 0.5248545408248901, + "loss": 1.1428, + "step": 347 + }, + { + "beta_dpo/beta": 0.001988435396924615, + "beta_dpo/beta_margin_grad_mean": -0.4891984760761261, + "beta_dpo/beta_margin_grad_std": 0.01943393610417843, + "beta_dpo/beta_margin_mean": 0.04333081468939781, + "beta_dpo/beta_margin_std": 0.07802307605743408, + "beta_dpo/beta_used": 0.001988435396924615, + "beta_dpo/beta_used_raw": -0.11356782168149948, + "beta_dpo/gap_mean": 18.29491424560547, + "beta_dpo/gap_std": 28.60049057006836, + "beta_dpo/loss_margin_mean": 20.155725479125977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5260770975056689, + "grad_norm": 1.6436684131622314, + "learning_rate": 2.724474525774229e-07, + "logits/chosen": 0.6003262400627136, + "logits/rejected": 0.5684754252433777, + "loss": 1.3561, + "step": 348 + }, + { + "beta_dpo/beta": 0.3341373801231384, + "beta_dpo/beta_margin_grad_mean": -0.2516961991786957, + "beta_dpo/beta_margin_grad_std": 0.36008405685424805, + "beta_dpo/beta_margin_mean": 7.000345706939697, + "beta_dpo/beta_margin_std": 9.868422508239746, + "beta_dpo/beta_used": 0.3341373801231384, + "beta_dpo/beta_used_raw": 0.3341373801231384, + "beta_dpo/gap_mean": 18.376079559326172, + "beta_dpo/gap_std": 29.04025650024414, + "beta_dpo/loss_margin_mean": 20.50440216064453, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.527588813303099, + "grad_norm": 210.76205444335938, + "learning_rate": 2.711302664252973e-07, + "logits/chosen": 0.49958592653274536, + "logits/rejected": 0.4125804305076599, + "loss": 1.4889, + "step": 349 + }, + { + "beta_dpo/beta": 0.20781663060188293, + "beta_dpo/beta_margin_grad_mean": -0.23984435200691223, + "beta_dpo/beta_margin_grad_std": 0.3351220190525055, + "beta_dpo/beta_margin_mean": 5.478157043457031, + "beta_dpo/beta_margin_std": 7.281736850738525, + "beta_dpo/beta_used": 0.20781663060188293, + "beta_dpo/beta_used_raw": 0.20781663060188293, + "beta_dpo/gap_mean": 19.763946533203125, + "beta_dpo/gap_std": 29.05038070678711, + "beta_dpo/loss_margin_mean": 25.382022857666016, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5291005291005291, + "grad_norm": 110.43148040771484, + "learning_rate": 2.698124892141971e-07, + "logits/chosen": 0.48420655727386475, + "logits/rejected": 0.40342360734939575, + "loss": 1.1176, + "step": 350 + }, + { + "beta_dpo/beta": 0.3125341534614563, + "beta_dpo/beta_margin_grad_mean": -0.24661174416542053, + "beta_dpo/beta_margin_grad_std": 0.33939671516418457, + "beta_dpo/beta_margin_mean": 5.621974468231201, + "beta_dpo/beta_margin_std": 14.418107032775879, + "beta_dpo/beta_used": 0.3125341534614563, + "beta_dpo/beta_used_raw": 0.3125341534614563, + "beta_dpo/gap_mean": 19.951894760131836, + "beta_dpo/gap_std": 29.225894927978516, + "beta_dpo/loss_margin_mean": 19.42828941345215, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5306122448979592, + "grad_norm": 142.16448974609375, + "learning_rate": 2.6849415780518357e-07, + "logits/chosen": 0.4121101200580597, + "logits/rejected": 0.33884552121162415, + "loss": 0.777, + "step": 351 + }, + { + "beta_dpo/beta": 0.09537062793970108, + "beta_dpo/beta_margin_grad_mean": -0.35886648297309875, + "beta_dpo/beta_margin_grad_std": 0.26381397247314453, + "beta_dpo/beta_margin_mean": 2.0912771224975586, + "beta_dpo/beta_margin_std": 4.213487148284912, + "beta_dpo/beta_used": 0.09537062793970108, + "beta_dpo/beta_used_raw": 0.08282928168773651, + "beta_dpo/gap_mean": 19.89529800415039, + "beta_dpo/gap_std": 29.30970001220703, + "beta_dpo/loss_margin_mean": 19.572755813598633, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5321239606953893, + "grad_norm": 67.9634780883789, + "learning_rate": 2.6717530907482024e-07, + "logits/chosen": 0.5165045261383057, + "logits/rejected": 0.4606800675392151, + "loss": 1.2334, + "step": 352 + }, + { + "beta_dpo/beta": 0.15907520055770874, + "beta_dpo/beta_margin_grad_mean": -0.38286593556404114, + "beta_dpo/beta_margin_grad_std": 0.29630035161972046, + "beta_dpo/beta_margin_mean": 3.0601959228515625, + "beta_dpo/beta_margin_std": 7.680595397949219, + "beta_dpo/beta_used": 0.15907520055770874, + "beta_dpo/beta_used_raw": 0.15692223608493805, + "beta_dpo/gap_mean": 19.871681213378906, + "beta_dpo/gap_std": 29.349777221679688, + "beta_dpo/loss_margin_mean": 21.176862716674805, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5336356764928194, + "grad_norm": 143.2347412109375, + "learning_rate": 2.658559799141411e-07, + "logits/chosen": 0.46999743580818176, + "logits/rejected": 0.4698483347892761, + "loss": 1.4572, + "step": 353 + }, + { + "beta_dpo/beta": 0.20546159148216248, + "beta_dpo/beta_margin_grad_mean": -0.32804247736930847, + "beta_dpo/beta_margin_grad_std": 0.2996892035007477, + "beta_dpo/beta_margin_mean": 5.386460304260254, + "beta_dpo/beta_margin_std": 9.564033508300781, + "beta_dpo/beta_used": 0.20546159148216248, + "beta_dpo/beta_used_raw": 0.20546159148216248, + "beta_dpo/gap_mean": 20.4854793548584, + "beta_dpo/gap_std": 29.17739486694336, + "beta_dpo/loss_margin_mean": 21.667699813842773, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5351473922902494, + "grad_norm": 75.140869140625, + "learning_rate": 2.6453620722761895e-07, + "logits/chosen": 0.5394304394721985, + "logits/rejected": 0.4111761450767517, + "loss": 0.992, + "step": 354 + }, + { + "beta_dpo/beta": 0.321403831243515, + "beta_dpo/beta_margin_grad_mean": -0.2626912295818329, + "beta_dpo/beta_margin_grad_std": 0.38087186217308044, + "beta_dpo/beta_margin_mean": 6.422491073608398, + "beta_dpo/beta_margin_std": 12.237265586853027, + "beta_dpo/beta_used": 0.321403831243515, + "beta_dpo/beta_used_raw": 0.321403831243515, + "beta_dpo/gap_mean": 20.343292236328125, + "beta_dpo/gap_std": 29.79207420349121, + "beta_dpo/loss_margin_mean": 21.592735290527344, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5366591080876795, + "grad_norm": 187.55264282226562, + "learning_rate": 2.632160279321328e-07, + "logits/chosen": 0.5159712433815002, + "logits/rejected": 0.3886120319366455, + "loss": 1.3602, + "step": 355 + }, + { + "beta_dpo/beta": 0.1345498114824295, + "beta_dpo/beta_margin_grad_mean": -0.3875563144683838, + "beta_dpo/beta_margin_grad_std": 0.28731241822242737, + "beta_dpo/beta_margin_mean": 2.935427188873291, + "beta_dpo/beta_margin_std": 7.447274208068848, + "beta_dpo/beta_used": 0.1345498114824295, + "beta_dpo/beta_used_raw": 0.059556834399700165, + "beta_dpo/gap_mean": 20.36905288696289, + "beta_dpo/gap_std": 30.672847747802734, + "beta_dpo/loss_margin_mean": 17.87114715576172, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5381708238851096, + "grad_norm": 79.05123901367188, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": 0.4842956066131592, + "logits/rejected": 0.4091559946537018, + "loss": 1.4938, + "step": 356 + }, + { + "beta_dpo/beta": 0.2215581089258194, + "beta_dpo/beta_margin_grad_mean": -0.3200496733188629, + "beta_dpo/beta_margin_grad_std": 0.29043084383010864, + "beta_dpo/beta_margin_mean": 5.6213555335998535, + "beta_dpo/beta_margin_std": 10.097216606140137, + "beta_dpo/beta_used": 0.2215581089258194, + "beta_dpo/beta_used_raw": -0.05693148076534271, + "beta_dpo/gap_mean": 19.813404083251953, + "beta_dpo/gap_std": 30.41214370727539, + "beta_dpo/loss_margin_mean": 15.453861236572266, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5396825396825397, + "grad_norm": 83.3572998046875, + "learning_rate": 2.6057459723762076e-07, + "logits/chosen": 0.4960511028766632, + "logits/rejected": 0.46816837787628174, + "loss": 1.069, + "step": 357 + }, + { + "beta_dpo/beta": 0.2554855942726135, + "beta_dpo/beta_margin_grad_mean": -0.2960377633571625, + "beta_dpo/beta_margin_grad_std": 0.3241449296474457, + "beta_dpo/beta_margin_mean": 5.539848327636719, + "beta_dpo/beta_margin_std": 10.479439735412598, + "beta_dpo/beta_used": 0.2554855942726135, + "beta_dpo/beta_used_raw": 0.2554855942726135, + "beta_dpo/gap_mean": 19.572742462158203, + "beta_dpo/gap_std": 30.08466148376465, + "beta_dpo/loss_margin_mean": 23.14980125427246, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5411942554799698, + "grad_norm": 239.4638671875, + "learning_rate": 2.5925341972508954e-07, + "logits/chosen": 0.42851513624191284, + "logits/rejected": 0.43600502610206604, + "loss": 1.5906, + "step": 358 + }, + { + "beta_dpo/beta": 0.03553559258580208, + "beta_dpo/beta_margin_grad_mean": -0.416434109210968, + "beta_dpo/beta_margin_grad_std": 0.20974458754062653, + "beta_dpo/beta_margin_mean": 0.5857033729553223, + "beta_dpo/beta_margin_std": 1.500328540802002, + "beta_dpo/beta_used": 0.03553559258580208, + "beta_dpo/beta_used_raw": -0.23965869843959808, + "beta_dpo/gap_mean": 18.273534774780273, + "beta_dpo/gap_std": 29.011173248291016, + "beta_dpo/loss_margin_mean": 10.77413272857666, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5427059712773998, + "grad_norm": 25.886608123779297, + "learning_rate": 2.579319833745169e-07, + "logits/chosen": 0.44077521562576294, + "logits/rejected": 0.4130966365337372, + "loss": 1.0985, + "step": 359 + }, + { + "beta_dpo/beta": 0.033735036849975586, + "beta_dpo/beta_margin_grad_mean": -0.41033679246902466, + "beta_dpo/beta_margin_grad_std": 0.18321087956428528, + "beta_dpo/beta_margin_mean": 0.5736103653907776, + "beta_dpo/beta_margin_std": 1.292583703994751, + "beta_dpo/beta_used": 0.033735036849975586, + "beta_dpo/beta_used_raw": -0.03152921795845032, + "beta_dpo/gap_mean": 18.03290367126465, + "beta_dpo/gap_std": 28.54608917236328, + "beta_dpo/loss_margin_mean": 17.090740203857422, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.54421768707483, + "grad_norm": 20.45010757446289, + "learning_rate": 2.5661032514931834e-07, + "logits/chosen": 0.39448633790016174, + "logits/rejected": 0.30303341150283813, + "loss": 1.095, + "step": 360 + }, + { + "beta_dpo/beta": 0.11376432329416275, + "beta_dpo/beta_margin_grad_mean": -0.268765389919281, + "beta_dpo/beta_margin_grad_std": 0.27982643246650696, + "beta_dpo/beta_margin_mean": 2.057039737701416, + "beta_dpo/beta_margin_std": 3.291944980621338, + "beta_dpo/beta_used": 0.11376432329416275, + "beta_dpo/beta_used_raw": 0.11376432329416275, + "beta_dpo/gap_mean": 18.154455184936523, + "beta_dpo/gap_std": 28.302597045898438, + "beta_dpo/loss_margin_mean": 18.740699768066406, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.54572940287226, + "grad_norm": 60.24184799194336, + "learning_rate": 2.552884820191154e-07, + "logits/chosen": 0.5338229537010193, + "logits/rejected": 0.48165255784988403, + "loss": 0.7756, + "step": 361 + }, + { + "beta_dpo/beta": 0.21435359120368958, + "beta_dpo/beta_margin_grad_mean": -0.3343871533870697, + "beta_dpo/beta_margin_grad_std": 0.2788733243942261, + "beta_dpo/beta_margin_mean": 6.487202167510986, + "beta_dpo/beta_margin_std": 11.84697437286377, + "beta_dpo/beta_used": 0.21435359120368958, + "beta_dpo/beta_used_raw": 0.10896323621273041, + "beta_dpo/gap_mean": 18.30129623413086, + "beta_dpo/gap_std": 27.87055206298828, + "beta_dpo/loss_margin_mean": 22.009746551513672, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.54724111866969, + "grad_norm": 104.80538940429688, + "learning_rate": 2.53966490958702e-07, + "logits/chosen": 0.5371885299682617, + "logits/rejected": 0.4334333539009094, + "loss": 1.3946, + "step": 362 + }, + { + "beta_dpo/beta": 0.06475654244422913, + "beta_dpo/beta_margin_grad_mean": -0.35036513209342957, + "beta_dpo/beta_margin_grad_std": 0.2489209771156311, + "beta_dpo/beta_margin_mean": 1.5037075281143188, + "beta_dpo/beta_margin_std": 2.7491438388824463, + "beta_dpo/beta_used": 0.06475654244422913, + "beta_dpo/beta_used_raw": -0.055414482951164246, + "beta_dpo/gap_mean": 19.423065185546875, + "beta_dpo/gap_std": 27.947891235351562, + "beta_dpo/loss_margin_mean": 21.895166397094727, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5487528344671202, + "grad_norm": 30.40268325805664, + "learning_rate": 2.526443889470099e-07, + "logits/chosen": 0.5154651403427124, + "logits/rejected": 0.3899514079093933, + "loss": 1.0825, + "step": 363 + }, + { + "beta_dpo/beta": 0.17884472012519836, + "beta_dpo/beta_margin_grad_mean": -0.2793542444705963, + "beta_dpo/beta_margin_grad_std": 0.3477945923805237, + "beta_dpo/beta_margin_mean": 3.8306987285614014, + "beta_dpo/beta_margin_std": 5.927258491516113, + "beta_dpo/beta_used": 0.17884472012519836, + "beta_dpo/beta_used_raw": 0.17884472012519836, + "beta_dpo/gap_mean": 19.666500091552734, + "beta_dpo/gap_std": 28.525062561035156, + "beta_dpo/loss_margin_mean": 21.15984535217285, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5502645502645502, + "grad_norm": 92.59161376953125, + "learning_rate": 2.513222129660744e-07, + "logits/chosen": 0.35895150899887085, + "logits/rejected": 0.26696082949638367, + "loss": 1.1274, + "step": 364 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4951821565628052, + "beta_dpo/beta_margin_grad_std": 0.005717262625694275, + "beta_dpo/beta_margin_mean": 0.01927482709288597, + "beta_dpo/beta_margin_std": 0.022875025868415833, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.03625158965587616, + "beta_dpo/gap_mean": 19.75531578063965, + "beta_dpo/gap_std": 27.833091735839844, + "beta_dpo/loss_margin_mean": 19.27482795715332, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5517762660619804, + "grad_norm": 0.978028416633606, + "learning_rate": 2.5e-07, + "logits/chosen": 0.505312442779541, + "logits/rejected": 0.4941166639328003, + "loss": 1.369, + "step": 365 + }, + { + "beta_dpo/beta": 0.034532975405454636, + "beta_dpo/beta_margin_grad_mean": -0.3938365876674652, + "beta_dpo/beta_margin_grad_std": 0.20211590826511383, + "beta_dpo/beta_margin_mean": 0.7358689308166504, + "beta_dpo/beta_margin_std": 1.4430540800094604, + "beta_dpo/beta_used": 0.034532975405454636, + "beta_dpo/beta_used_raw": 0.0041874852031469345, + "beta_dpo/gap_mean": 19.195980072021484, + "beta_dpo/gap_std": 27.799442291259766, + "beta_dpo/loss_margin_mean": 17.422622680664062, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5532879818594104, + "grad_norm": 19.836706161499023, + "learning_rate": 2.486777870339255e-07, + "logits/chosen": 0.4261232316493988, + "logits/rejected": 0.4047813415527344, + "loss": 1.1212, + "step": 366 + }, + { + "beta_dpo/beta": 0.06913218647241592, + "beta_dpo/beta_margin_grad_mean": -0.38716188073158264, + "beta_dpo/beta_margin_grad_std": 0.24359238147735596, + "beta_dpo/beta_margin_mean": 0.9459067583084106, + "beta_dpo/beta_margin_std": 2.751627206802368, + "beta_dpo/beta_used": 0.06913218647241592, + "beta_dpo/beta_used_raw": 0.01255800575017929, + "beta_dpo/gap_mean": 18.8731689453125, + "beta_dpo/gap_std": 27.13546371459961, + "beta_dpo/loss_margin_mean": 15.656370162963867, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5547996976568406, + "grad_norm": 35.99217987060547, + "learning_rate": 2.4735561105299014e-07, + "logits/chosen": 0.46010124683380127, + "logits/rejected": 0.3569197952747345, + "loss": 0.9947, + "step": 367 + }, + { + "beta_dpo/beta": 0.06575921177864075, + "beta_dpo/beta_margin_grad_mean": -0.3926805555820465, + "beta_dpo/beta_margin_grad_std": 0.25895747542381287, + "beta_dpo/beta_margin_mean": 1.0753775835037231, + "beta_dpo/beta_margin_std": 2.560882091522217, + "beta_dpo/beta_used": 0.06575921177864075, + "beta_dpo/beta_used_raw": 0.008796077221632004, + "beta_dpo/gap_mean": 18.227333068847656, + "beta_dpo/gap_std": 26.96753692626953, + "beta_dpo/loss_margin_mean": 15.894394874572754, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5563114134542706, + "grad_norm": 40.85283279418945, + "learning_rate": 2.46033509041298e-07, + "logits/chosen": 0.31978198885917664, + "logits/rejected": 0.31229010224342346, + "loss": 1.1472, + "step": 368 + }, + { + "beta_dpo/beta": 0.04814423620700836, + "beta_dpo/beta_margin_grad_mean": -0.39616596698760986, + "beta_dpo/beta_margin_grad_std": 0.2142905592918396, + "beta_dpo/beta_margin_mean": 0.8517483472824097, + "beta_dpo/beta_margin_std": 1.803131103515625, + "beta_dpo/beta_used": 0.04814423620700836, + "beta_dpo/beta_used_raw": -0.09424681216478348, + "beta_dpo/gap_mean": 17.42707061767578, + "beta_dpo/gap_std": 26.661537170410156, + "beta_dpo/loss_margin_mean": 14.567131996154785, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5578231292517006, + "grad_norm": 40.79011917114258, + "learning_rate": 2.447115179808846e-07, + "logits/chosen": 0.4484362304210663, + "logits/rejected": 0.3975934386253357, + "loss": 1.1383, + "step": 369 + }, + { + "beta_dpo/beta": 0.19350400567054749, + "beta_dpo/beta_margin_grad_mean": -0.2489045411348343, + "beta_dpo/beta_margin_grad_std": 0.32901516556739807, + "beta_dpo/beta_margin_mean": 3.838660478591919, + "beta_dpo/beta_margin_std": 5.751165866851807, + "beta_dpo/beta_used": 0.19350400567054749, + "beta_dpo/beta_used_raw": 0.19350400567054749, + "beta_dpo/gap_mean": 17.788787841796875, + "beta_dpo/gap_std": 27.057907104492188, + "beta_dpo/loss_margin_mean": 19.834497451782227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5593348450491308, + "grad_norm": 87.48202514648438, + "learning_rate": 2.4338967485068164e-07, + "logits/chosen": 0.5482546091079712, + "logits/rejected": 0.47974199056625366, + "loss": 0.9597, + "step": 370 + }, + { + "beta_dpo/beta": 0.07270210981369019, + "beta_dpo/beta_margin_grad_mean": -0.3535962700843811, + "beta_dpo/beta_margin_grad_std": 0.263553649187088, + "beta_dpo/beta_margin_mean": 1.738492727279663, + "beta_dpo/beta_margin_std": 3.044748306274414, + "beta_dpo/beta_used": 0.07270210981369019, + "beta_dpo/beta_used_raw": 0.0545848049223423, + "beta_dpo/gap_mean": 17.900829315185547, + "beta_dpo/gap_std": 27.141389846801758, + "beta_dpo/loss_margin_mean": 19.347623825073242, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5608465608465608, + "grad_norm": 53.165611267089844, + "learning_rate": 2.420680166254831e-07, + "logits/chosen": 0.5655019283294678, + "logits/rejected": 0.5300487279891968, + "loss": 1.2018, + "step": 371 + }, + { + "beta_dpo/beta": 0.053923849016427994, + "beta_dpo/beta_margin_grad_mean": -0.4136897623538971, + "beta_dpo/beta_margin_grad_std": 0.2468746155500412, + "beta_dpo/beta_margin_mean": 0.8222647309303284, + "beta_dpo/beta_margin_std": 2.5193469524383545, + "beta_dpo/beta_used": 0.053923849016427994, + "beta_dpo/beta_used_raw": -0.09299886226654053, + "beta_dpo/gap_mean": 16.81853485107422, + "beta_dpo/gap_std": 27.437904357910156, + "beta_dpo/loss_margin_mean": 10.160256385803223, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.562358276643991, + "grad_norm": 40.79886245727539, + "learning_rate": 2.4074658027491044e-07, + "logits/chosen": 0.4797401428222656, + "logits/rejected": 0.39019930362701416, + "loss": 1.1626, + "step": 372 + }, + { + "beta_dpo/beta": 0.019156690686941147, + "beta_dpo/beta_margin_grad_mean": -0.4477452337741852, + "beta_dpo/beta_margin_grad_std": 0.15734830498695374, + "beta_dpo/beta_margin_mean": 0.2678249180316925, + "beta_dpo/beta_margin_std": 0.8076358437538147, + "beta_dpo/beta_used": 0.019156690686941147, + "beta_dpo/beta_used_raw": -0.09059305489063263, + "beta_dpo/gap_mean": 16.431793212890625, + "beta_dpo/gap_std": 27.95029640197754, + "beta_dpo/loss_margin_mean": 14.914463996887207, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.563869992441421, + "grad_norm": 18.376066207885742, + "learning_rate": 2.394254027623792e-07, + "logits/chosen": 0.47552430629730225, + "logits/rejected": 0.40688779950141907, + "loss": 1.2204, + "step": 373 + }, + { + "beta_dpo/beta": 0.30811965465545654, + "beta_dpo/beta_margin_grad_mean": -0.1607130467891693, + "beta_dpo/beta_margin_grad_std": 0.29871711134910583, + "beta_dpo/beta_margin_mean": 7.594576835632324, + "beta_dpo/beta_margin_std": 9.025672912597656, + "beta_dpo/beta_used": 0.30811965465545654, + "beta_dpo/beta_used_raw": 0.30811965465545654, + "beta_dpo/gap_mean": 17.549823760986328, + "beta_dpo/gap_std": 27.397924423217773, + "beta_dpo/loss_margin_mean": 23.819591522216797, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5653817082388511, + "grad_norm": 150.54312133789062, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": 0.38389527797698975, + "logits/rejected": 0.37589797377586365, + "loss": 0.9835, + "step": 374 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49679815769195557, + "beta_dpo/beta_margin_grad_std": 0.006316898390650749, + "beta_dpo/beta_margin_mean": 0.012809698469936848, + "beta_dpo/beta_margin_std": 0.0252725537866354, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.31367141008377075, + "beta_dpo/gap_mean": 17.115184783935547, + "beta_dpo/gap_std": 26.947261810302734, + "beta_dpo/loss_margin_mean": 12.809698104858398, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5668934240362812, + "grad_norm": 0.853339433670044, + "learning_rate": 2.3678397206786715e-07, + "logits/chosen": 0.495971143245697, + "logits/rejected": 0.43806982040405273, + "loss": 1.3762, + "step": 375 + }, + { + "beta_dpo/beta": 0.1979072391986847, + "beta_dpo/beta_margin_grad_mean": -0.24381671845912933, + "beta_dpo/beta_margin_grad_std": 0.3681492805480957, + "beta_dpo/beta_margin_mean": 4.5390448570251465, + "beta_dpo/beta_margin_std": 6.06004524230957, + "beta_dpo/beta_used": 0.1979072391986847, + "beta_dpo/beta_used_raw": 0.1979072391986847, + "beta_dpo/gap_mean": 17.752330780029297, + "beta_dpo/gap_std": 27.663818359375, + "beta_dpo/loss_margin_mean": 22.940746307373047, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5684051398337112, + "grad_norm": 116.034912109375, + "learning_rate": 2.3546379277238103e-07, + "logits/chosen": 0.531902551651001, + "logits/rejected": 0.46265655755996704, + "loss": 1.6455, + "step": 376 + }, + { + "beta_dpo/beta": 0.12244483828544617, + "beta_dpo/beta_margin_grad_mean": -0.3793109059333801, + "beta_dpo/beta_margin_grad_std": 0.2850496768951416, + "beta_dpo/beta_margin_mean": 2.0889053344726562, + "beta_dpo/beta_margin_std": 4.836654186248779, + "beta_dpo/beta_used": 0.12244483828544617, + "beta_dpo/beta_used_raw": 0.09535021334886551, + "beta_dpo/gap_mean": 17.380741119384766, + "beta_dpo/gap_std": 27.402048110961914, + "beta_dpo/loss_margin_mean": 14.553292274475098, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5699168556311414, + "grad_norm": 68.49671936035156, + "learning_rate": 2.3414402008585886e-07, + "logits/chosen": 0.44863414764404297, + "logits/rejected": 0.42397451400756836, + "loss": 1.0434, + "step": 377 + }, + { + "beta_dpo/beta": 0.10824133455753326, + "beta_dpo/beta_margin_grad_mean": -0.3810761570930481, + "beta_dpo/beta_margin_grad_std": 0.2576172351837158, + "beta_dpo/beta_margin_mean": 1.9034552574157715, + "beta_dpo/beta_margin_std": 4.89661979675293, + "beta_dpo/beta_used": 0.10824133455753326, + "beta_dpo/beta_used_raw": 0.014959946274757385, + "beta_dpo/gap_mean": 16.712974548339844, + "beta_dpo/gap_std": 26.61426544189453, + "beta_dpo/loss_margin_mean": 13.993646621704102, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5714285714285714, + "grad_norm": 92.63347625732422, + "learning_rate": 2.3282469092517977e-07, + "logits/chosen": 0.519462525844574, + "logits/rejected": 0.46947789192199707, + "loss": 1.3392, + "step": 378 + }, + { + "beta_dpo/beta": 0.3307998776435852, + "beta_dpo/beta_margin_grad_mean": -0.21893391013145447, + "beta_dpo/beta_margin_grad_std": 0.35074496269226074, + "beta_dpo/beta_margin_mean": 6.119668006896973, + "beta_dpo/beta_margin_std": 8.400911331176758, + "beta_dpo/beta_used": 0.3307998776435852, + "beta_dpo/beta_used_raw": 0.3307998776435852, + "beta_dpo/gap_mean": 17.011268615722656, + "beta_dpo/gap_std": 26.57973861694336, + "beta_dpo/loss_margin_mean": 18.500185012817383, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5729402872260015, + "grad_norm": 151.25961303710938, + "learning_rate": 2.3150584219481643e-07, + "logits/chosen": 0.4980725049972534, + "logits/rejected": 0.4293912947177887, + "loss": 0.7841, + "step": 379 + }, + { + "beta_dpo/beta": 0.29439330101013184, + "beta_dpo/beta_margin_grad_mean": -0.22058521211147308, + "beta_dpo/beta_margin_grad_std": 0.3584253489971161, + "beta_dpo/beta_margin_mean": 6.252655982971191, + "beta_dpo/beta_margin_std": 8.074928283691406, + "beta_dpo/beta_used": 0.29439330101013184, + "beta_dpo/beta_used_raw": 0.29439330101013184, + "beta_dpo/gap_mean": 17.592838287353516, + "beta_dpo/gap_std": 26.798145294189453, + "beta_dpo/loss_margin_mean": 20.89601707458496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5744520030234316, + "grad_norm": 136.94004821777344, + "learning_rate": 2.3018751078580283e-07, + "logits/chosen": 0.4958919882774353, + "logits/rejected": 0.45453202724456787, + "loss": 1.1291, + "step": 380 + }, + { + "beta_dpo/beta": 0.10175088793039322, + "beta_dpo/beta_margin_grad_mean": -0.42357251048088074, + "beta_dpo/beta_margin_grad_std": 0.2839849889278412, + "beta_dpo/beta_margin_mean": 1.3963819742202759, + "beta_dpo/beta_margin_std": 4.324319839477539, + "beta_dpo/beta_used": 0.10175088793039322, + "beta_dpo/beta_used_raw": -0.13142962753772736, + "beta_dpo/gap_mean": 16.102523803710938, + "beta_dpo/gap_std": 26.190248489379883, + "beta_dpo/loss_margin_mean": 7.799356460571289, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5759637188208617, + "grad_norm": 76.48500061035156, + "learning_rate": 2.288697335747027e-07, + "logits/chosen": 0.4466308355331421, + "logits/rejected": 0.4188675880432129, + "loss": 1.1505, + "step": 381 + }, + { + "beta_dpo/beta": 0.11290884017944336, + "beta_dpo/beta_margin_grad_mean": -0.33594417572021484, + "beta_dpo/beta_margin_grad_std": 0.2575390040874481, + "beta_dpo/beta_margin_mean": 2.236764907836914, + "beta_dpo/beta_margin_std": 4.038408279418945, + "beta_dpo/beta_used": 0.11290884017944336, + "beta_dpo/beta_used_raw": -0.006284750998020172, + "beta_dpo/gap_mean": 15.629575729370117, + "beta_dpo/gap_std": 25.694416046142578, + "beta_dpo/loss_margin_mean": 15.05109691619873, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5774754346182918, + "grad_norm": 31.48938751220703, + "learning_rate": 2.2755254742257706e-07, + "logits/chosen": 0.4852759838104248, + "logits/rejected": 0.43432578444480896, + "loss": 0.8542, + "step": 382 + }, + { + "beta_dpo/beta": 0.0896899476647377, + "beta_dpo/beta_margin_grad_mean": -0.38577207922935486, + "beta_dpo/beta_margin_grad_std": 0.25467216968536377, + "beta_dpo/beta_margin_mean": 1.8165364265441895, + "beta_dpo/beta_margin_std": 3.8192477226257324, + "beta_dpo/beta_used": 0.0896899476647377, + "beta_dpo/beta_used_raw": 0.011013053357601166, + "beta_dpo/gap_mean": 16.150833129882812, + "beta_dpo/gap_std": 25.653362274169922, + "beta_dpo/loss_margin_mean": 16.843584060668945, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5789871504157218, + "grad_norm": 63.09295654296875, + "learning_rate": 2.2623598917395436e-07, + "logits/chosen": 0.34624814987182617, + "logits/rejected": 0.3639281094074249, + "loss": 1.1991, + "step": 383 + }, + { + "beta_dpo/beta": 0.07606925070285797, + "beta_dpo/beta_margin_grad_mean": -0.37067925930023193, + "beta_dpo/beta_margin_grad_std": 0.26238465309143066, + "beta_dpo/beta_margin_mean": 1.464687466621399, + "beta_dpo/beta_margin_std": 2.8831002712249756, + "beta_dpo/beta_used": 0.07606925070285797, + "beta_dpo/beta_used_raw": 0.03083164617419243, + "beta_dpo/gap_mean": 16.12920379638672, + "beta_dpo/gap_std": 25.395118713378906, + "beta_dpo/loss_margin_mean": 17.43052101135254, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5804988662131519, + "grad_norm": 55.40347671508789, + "learning_rate": 2.2492009565579875e-07, + "logits/chosen": 0.48950695991516113, + "logits/rejected": 0.44139331579208374, + "loss": 1.2499, + "step": 384 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49544164538383484, + "beta_dpo/beta_margin_grad_std": 0.005564127117395401, + "beta_dpo/beta_margin_mean": 0.018236415460705757, + "beta_dpo/beta_margin_std": 0.022261304780840874, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.07840212434530258, + "beta_dpo/gap_mean": 16.50790786743164, + "beta_dpo/gap_std": 24.757612228393555, + "beta_dpo/loss_margin_mean": 18.23641586303711, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.582010582010582, + "grad_norm": 0.7727059721946716, + "learning_rate": 2.2360490367648084e-07, + "logits/chosen": 0.40697604417800903, + "logits/rejected": 0.36528295278549194, + "loss": 1.3729, + "step": 385 + }, + { + "beta_dpo/beta": 0.10942035913467407, + "beta_dpo/beta_margin_grad_mean": -0.36618587374687195, + "beta_dpo/beta_margin_grad_std": 0.2598530054092407, + "beta_dpo/beta_margin_mean": 1.4770482778549194, + "beta_dpo/beta_margin_std": 3.5450892448425293, + "beta_dpo/beta_used": 0.10942035913467407, + "beta_dpo/beta_used_raw": 0.09165161103010178, + "beta_dpo/gap_mean": 16.45754623413086, + "beta_dpo/gap_std": 24.44674301147461, + "beta_dpo/loss_margin_mean": 14.757094383239746, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5835222978080121, + "grad_norm": 58.915008544921875, + "learning_rate": 2.2229045002474724e-07, + "logits/chosen": 0.3973620533943176, + "logits/rejected": 0.3377264738082886, + "loss": 0.9296, + "step": 386 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49545782804489136, + "beta_dpo/beta_margin_grad_std": 0.005244361702352762, + "beta_dpo/beta_margin_mean": 0.01817174255847931, + "beta_dpo/beta_margin_std": 0.020982708781957626, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.047689471393823624, + "beta_dpo/gap_mean": 16.546337127685547, + "beta_dpo/gap_std": 23.793880462646484, + "beta_dpo/loss_margin_mean": 18.171741485595703, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5850340136054422, + "grad_norm": 0.7535995841026306, + "learning_rate": 2.209767714686924e-07, + "logits/chosen": 0.4398246109485626, + "logits/rejected": 0.344159871339798, + "loss": 1.3723, + "step": 387 + }, + { + "beta_dpo/beta": 0.07745764404535294, + "beta_dpo/beta_margin_grad_mean": -0.4048055410385132, + "beta_dpo/beta_margin_grad_std": 0.2543579638004303, + "beta_dpo/beta_margin_mean": 1.2141345739364624, + "beta_dpo/beta_margin_std": 2.8941874504089355, + "beta_dpo/beta_used": 0.07745764404535294, + "beta_dpo/beta_used_raw": -0.088335320353508, + "beta_dpo/gap_mean": 16.209470748901367, + "beta_dpo/gap_std": 23.847801208496094, + "beta_dpo/loss_margin_mean": 12.559057235717773, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5865457294028723, + "grad_norm": 61.19523620605469, + "learning_rate": 2.1966390475472954e-07, + "logits/chosen": 0.45595115423202515, + "logits/rejected": 0.4419313073158264, + "loss": 1.1601, + "step": 388 + }, + { + "beta_dpo/beta": 0.12304967641830444, + "beta_dpo/beta_margin_grad_mean": -0.38289767503738403, + "beta_dpo/beta_margin_grad_std": 0.2779510021209717, + "beta_dpo/beta_margin_mean": 2.046694755554199, + "beta_dpo/beta_margin_std": 4.120178699493408, + "beta_dpo/beta_used": 0.12304967641830444, + "beta_dpo/beta_used_raw": 0.048085957765579224, + "beta_dpo/gap_mean": 15.835220336914062, + "beta_dpo/gap_std": 24.056732177734375, + "beta_dpo/loss_margin_mean": 15.953550338745117, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5880574452003023, + "grad_norm": 83.82422637939453, + "learning_rate": 2.1835188660656265e-07, + "logits/chosen": 0.468747079372406, + "logits/rejected": 0.43019360303878784, + "loss": 1.186, + "step": 389 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49623188376426697, + "beta_dpo/beta_margin_grad_std": 0.0054491744376719, + "beta_dpo/beta_margin_mean": 0.015074868686497211, + "beta_dpo/beta_margin_std": 0.021800970658659935, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.18968456983566284, + "beta_dpo/gap_mean": 15.827681541442871, + "beta_dpo/gap_std": 23.630695343017578, + "beta_dpo/loss_margin_mean": 15.074868202209473, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5895691609977324, + "grad_norm": 0.6730204224586487, + "learning_rate": 2.170407537241599e-07, + "logits/chosen": 0.5367964506149292, + "logits/rejected": 0.4671306014060974, + "loss": 1.3754, + "step": 390 + }, + { + "beta_dpo/beta": 0.3387356102466583, + "beta_dpo/beta_margin_grad_mean": -0.3062863051891327, + "beta_dpo/beta_margin_grad_std": 0.3581731617450714, + "beta_dpo/beta_margin_mean": 5.966565132141113, + "beta_dpo/beta_margin_std": 12.230363845825195, + "beta_dpo/beta_used": 0.3387356102466583, + "beta_dpo/beta_used_raw": 0.3387356102466583, + "beta_dpo/gap_mean": 15.984790802001953, + "beta_dpo/gap_std": 24.179485321044922, + "beta_dpo/loss_margin_mean": 17.596973419189453, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5910808767951625, + "grad_norm": 227.28407287597656, + "learning_rate": 2.1573054278272636e-07, + "logits/chosen": 0.47585949301719666, + "logits/rejected": 0.4066176414489746, + "loss": 1.151, + "step": 391 + }, + { + "beta_dpo/beta": 0.07082900404930115, + "beta_dpo/beta_margin_grad_mean": -0.36467647552490234, + "beta_dpo/beta_margin_grad_std": 0.2492174655199051, + "beta_dpo/beta_margin_mean": 1.7737425565719604, + "beta_dpo/beta_margin_std": 3.359530448913574, + "beta_dpo/beta_used": 0.07082900404930115, + "beta_dpo/beta_used_raw": 0.06858093291521072, + "beta_dpo/gap_mean": 16.43636131286621, + "beta_dpo/gap_std": 25.151107788085938, + "beta_dpo/loss_margin_mean": 20.14677619934082, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5925925925925926, + "grad_norm": 48.511722564697266, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": 0.5203168392181396, + "logits/rejected": 0.46064090728759766, + "loss": 1.152, + "step": 392 + }, + { + "beta_dpo/beta": 0.11964704096317291, + "beta_dpo/beta_margin_grad_mean": -0.34089747071266174, + "beta_dpo/beta_margin_grad_std": 0.254536509513855, + "beta_dpo/beta_margin_mean": 2.8861536979675293, + "beta_dpo/beta_margin_std": 5.0302629470825195, + "beta_dpo/beta_used": 0.11964704096317291, + "beta_dpo/beta_used_raw": 0.06222714111208916, + "beta_dpo/gap_mean": 17.573963165283203, + "beta_dpo/gap_std": 25.1307373046875, + "beta_dpo/loss_margin_mean": 20.435420989990234, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5941043083900227, + "grad_norm": 52.03828430175781, + "learning_rate": 2.131130332936195e-07, + "logits/chosen": 0.4791126251220703, + "logits/rejected": 0.4386283755302429, + "loss": 1.1132, + "step": 393 + }, + { + "beta_dpo/beta": 0.06266598403453827, + "beta_dpo/beta_margin_grad_mean": -0.37125974893569946, + "beta_dpo/beta_margin_grad_std": 0.2566405236721039, + "beta_dpo/beta_margin_mean": 1.087797999382019, + "beta_dpo/beta_margin_std": 2.204742908477783, + "beta_dpo/beta_used": 0.06266598403453827, + "beta_dpo/beta_used_raw": -0.004001658409833908, + "beta_dpo/gap_mean": 17.02603530883789, + "beta_dpo/gap_std": 24.27169418334961, + "beta_dpo/loss_margin_mean": 14.85161018371582, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5956160241874527, + "grad_norm": 37.43368148803711, + "learning_rate": 2.1180580796331323e-07, + "logits/chosen": 0.5228984355926514, + "logits/rejected": 0.48896557092666626, + "loss": 1.1447, + "step": 394 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49653303623199463, + "beta_dpo/beta_margin_grad_std": 0.0054858047515153885, + "beta_dpo/beta_margin_mean": 0.013870153576135635, + "beta_dpo/beta_margin_std": 0.021947424858808517, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.23153652250766754, + "beta_dpo/gap_mean": 16.453784942626953, + "beta_dpo/gap_std": 23.770017623901367, + "beta_dpo/loss_margin_mean": 13.870152473449707, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5971277399848829, + "grad_norm": 0.7639675736427307, + "learning_rate": 2.104996510066625e-07, + "logits/chosen": 0.4781748056411743, + "logits/rejected": 0.38682687282562256, + "loss": 1.3755, + "step": 395 + }, + { + "beta_dpo/beta": 0.0550997294485569, + "beta_dpo/beta_margin_grad_mean": -0.3593597114086151, + "beta_dpo/beta_margin_grad_std": 0.2185526043176651, + "beta_dpo/beta_margin_mean": 1.2726523876190186, + "beta_dpo/beta_margin_std": 2.2179882526397705, + "beta_dpo/beta_used": 0.0550997294485569, + "beta_dpo/beta_used_raw": -0.1203291267156601, + "beta_dpo/gap_mean": 16.714210510253906, + "beta_dpo/gap_std": 23.554920196533203, + "beta_dpo/loss_margin_mean": 15.856443405151367, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.5986394557823129, + "grad_norm": 24.56127166748047, + "learning_rate": 2.0919459895968517e-07, + "logits/chosen": 0.4733230471611023, + "logits/rejected": 0.3763098418712616, + "loss": 1.0211, + "step": 396 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49783027172088623, + "beta_dpo/beta_margin_grad_std": 0.005984327290207148, + "beta_dpo/beta_margin_mean": 0.008679854683578014, + "beta_dpo/beta_margin_std": 0.023941440507769585, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.13786454498767853, + "beta_dpo/gap_mean": 15.07978630065918, + "beta_dpo/gap_std": 23.18680191040039, + "beta_dpo/loss_margin_mean": 8.679854393005371, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.600151171579743, + "grad_norm": 0.7217801809310913, + "learning_rate": 2.078906883274924e-07, + "logits/chosen": 0.4001700282096863, + "logits/rejected": 0.35347041487693787, + "loss": 1.3753, + "step": 397 + }, + { + "beta_dpo/beta": 0.1426246017217636, + "beta_dpo/beta_margin_grad_mean": -0.25918036699295044, + "beta_dpo/beta_margin_grad_std": 0.27966228127479553, + "beta_dpo/beta_margin_mean": 2.813566207885742, + "beta_dpo/beta_margin_std": 3.6704766750335693, + "beta_dpo/beta_used": 0.1426246017217636, + "beta_dpo/beta_used_raw": 0.1426246017217636, + "beta_dpo/gap_mean": 15.561747550964355, + "beta_dpo/gap_std": 23.492074966430664, + "beta_dpo/loss_margin_mean": 19.25560760498047, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6016628873771731, + "grad_norm": 48.140342712402344, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 0.4392428994178772, + "logits/rejected": 0.3709562420845032, + "loss": 0.7647, + "step": 398 + }, + { + "beta_dpo/beta": 0.058677032589912415, + "beta_dpo/beta_margin_grad_mean": -0.36431384086608887, + "beta_dpo/beta_margin_grad_std": 0.22912275791168213, + "beta_dpo/beta_margin_mean": 1.1983758211135864, + "beta_dpo/beta_margin_std": 2.138871908187866, + "beta_dpo/beta_used": 0.058677032589912415, + "beta_dpo/beta_used_raw": -0.1429281234741211, + "beta_dpo/gap_mean": 16.37626838684082, + "beta_dpo/gap_std": 23.693418502807617, + "beta_dpo/loss_margin_mean": 20.51258659362793, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6031746031746031, + "grad_norm": 35.35142517089844, + "learning_rate": 2.052864371672457e-07, + "logits/chosen": 0.37037575244903564, + "logits/rejected": 0.23645275831222534, + "loss": 1.0667, + "step": 399 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4972189664840698, + "beta_dpo/beta_margin_grad_std": 0.005795391276478767, + "beta_dpo/beta_margin_mean": 0.011126162484288216, + "beta_dpo/beta_margin_std": 0.023186147212982178, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.33826786279678345, + "beta_dpo/gap_mean": 16.00902557373047, + "beta_dpo/gap_std": 23.945209503173828, + "beta_dpo/loss_margin_mean": 11.1261625289917, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6046863189720333, + "grad_norm": 0.7984305024147034, + "learning_rate": 2.0398616948569493e-07, + "logits/chosen": 0.47489333152770996, + "logits/rejected": 0.4180806279182434, + "loss": 1.3777, + "step": 400 + }, + { + "epoch": 0.6046863189720333, + "eval_beta_dpo/beta": 0.06983860582113266, + "eval_beta_dpo/beta_margin_grad_mean": -0.40659934282302856, + "eval_beta_dpo/beta_margin_grad_std": 0.12279289215803146, + "eval_beta_dpo/beta_margin_mean": 1.232630729675293, + "eval_beta_dpo/beta_margin_std": 1.670194387435913, + "eval_beta_dpo/beta_used": 0.06983860582113266, + "eval_beta_dpo/beta_used_raw": -0.03454193100333214, + "eval_beta_dpo/gap_mean": 15.513669967651367, + "eval_beta_dpo/gap_std": 23.837387084960938, + "eval_beta_dpo/loss_margin_mean": 13.271306037902832, + "eval_beta_dpo/mask_keep_frac": 1.0, + "eval_logits/chosen": 0.43915027379989624, + "eval_logits/rejected": 0.387622207403183, + "eval_loss": 0.6486362814903259, + "eval_runtime": 38.9939, + "eval_samples_per_second": 59.061, + "eval_steps_per_second": 1.846, + "step": 400 + }, + { + "beta_dpo/beta": 0.10424748808145523, + "beta_dpo/beta_margin_grad_mean": -0.3355239927768707, + "beta_dpo/beta_margin_grad_std": 0.2324715554714203, + "beta_dpo/beta_margin_mean": 2.10429310798645, + "beta_dpo/beta_margin_std": 3.528271436691284, + "beta_dpo/beta_used": 0.10424748808145523, + "beta_dpo/beta_used_raw": 0.08866387605667114, + "beta_dpo/gap_mean": 16.252836227416992, + "beta_dpo/gap_std": 23.609046936035156, + "beta_dpo/loss_margin_mean": 20.805694580078125, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6061980347694633, + "grad_norm": 37.258792877197266, + "learning_rate": 2.0268718890989752e-07, + "logits/chosen": 0.45804572105407715, + "logits/rejected": 0.3601095676422119, + "loss": 0.9848, + "step": 401 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49628403782844543, + "beta_dpo/beta_margin_grad_std": 0.006008894182741642, + "beta_dpo/beta_margin_mean": 0.014866580255329609, + "beta_dpo/beta_margin_std": 0.024040691554546356, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.08317705988883972, + "beta_dpo/gap_mean": 16.39299201965332, + "beta_dpo/gap_std": 23.771438598632812, + "beta_dpo/loss_margin_mean": 14.86658000946045, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6077097505668935, + "grad_norm": 0.9669122099876404, + "learning_rate": 2.013895317751323e-07, + "logits/chosen": 0.47899359464645386, + "logits/rejected": 0.4380500316619873, + "loss": 1.3731, + "step": 402 + }, + { + "beta_dpo/beta": 0.03571583703160286, + "beta_dpo/beta_margin_grad_mean": -0.3757665455341339, + "beta_dpo/beta_margin_grad_std": 0.1899324208498001, + "beta_dpo/beta_margin_mean": 0.8030893802642822, + "beta_dpo/beta_margin_std": 1.3427276611328125, + "beta_dpo/beta_used": 0.03571583703160286, + "beta_dpo/beta_used_raw": -0.032127734273672104, + "beta_dpo/gap_mean": 16.863525390625, + "beta_dpo/gap_std": 24.415699005126953, + "beta_dpo/loss_margin_mean": 21.25139808654785, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6092214663643235, + "grad_norm": 18.598398208618164, + "learning_rate": 2.0009323437965898e-07, + "logits/chosen": 0.5733494758605957, + "logits/rejected": 0.4917227625846863, + "loss": 1.1002, + "step": 403 + }, + { + "beta_dpo/beta": 0.25357943773269653, + "beta_dpo/beta_margin_grad_mean": -0.25755763053894043, + "beta_dpo/beta_margin_grad_std": 0.35744139552116394, + "beta_dpo/beta_margin_mean": 4.395455837249756, + "beta_dpo/beta_margin_std": 6.034870624542236, + "beta_dpo/beta_used": 0.25357943773269653, + "beta_dpo/beta_used_raw": 0.25357943773269653, + "beta_dpo/gap_mean": 17.377187728881836, + "beta_dpo/gap_std": 24.109294891357422, + "beta_dpo/loss_margin_mean": 16.861284255981445, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6107331821617535, + "grad_norm": 109.06430053710938, + "learning_rate": 1.9879833298370237e-07, + "logits/chosen": 0.4435925781726837, + "logits/rejected": 0.3511613607406616, + "loss": 0.7406, + "step": 404 + }, + { + "beta_dpo/beta": 0.1809747815132141, + "beta_dpo/beta_margin_grad_mean": -0.3554416596889496, + "beta_dpo/beta_margin_grad_std": 0.29358693957328796, + "beta_dpo/beta_margin_mean": 4.457952499389648, + "beta_dpo/beta_margin_std": 7.733155250549316, + "beta_dpo/beta_used": 0.1809747815132141, + "beta_dpo/beta_used_raw": -0.05717755854129791, + "beta_dpo/gap_mean": 17.331336975097656, + "beta_dpo/gap_std": 23.831462860107422, + "beta_dpo/loss_margin_mean": 16.45929527282715, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6122448979591837, + "grad_norm": 89.80956268310547, + "learning_rate": 1.975048638084379e-07, + "logits/chosen": 0.49917006492614746, + "logits/rejected": 0.45142000913619995, + "loss": 1.1742, + "step": 405 + }, + { + "beta_dpo/beta": 0.21984750032424927, + "beta_dpo/beta_margin_grad_mean": -0.37012386322021484, + "beta_dpo/beta_margin_grad_std": 0.30180418491363525, + "beta_dpo/beta_margin_mean": 4.523665904998779, + "beta_dpo/beta_margin_std": 10.00094223022461, + "beta_dpo/beta_used": 0.21984750032424927, + "beta_dpo/beta_used_raw": 0.13835027813911438, + "beta_dpo/gap_mean": 17.194957733154297, + "beta_dpo/gap_std": 23.892921447753906, + "beta_dpo/loss_margin_mean": 18.594226837158203, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6137566137566137, + "grad_norm": 122.70568084716797, + "learning_rate": 1.9621286303497914e-07, + "logits/chosen": 0.4927158057689667, + "logits/rejected": 0.339374303817749, + "loss": 1.1442, + "step": 406 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49648579955101013, + "beta_dpo/beta_margin_grad_std": 0.005789801478385925, + "beta_dpo/beta_margin_mean": 0.014059151522815228, + "beta_dpo/beta_margin_std": 0.02316402457654476, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.06442619860172272, + "beta_dpo/gap_mean": 16.722686767578125, + "beta_dpo/gap_std": 23.689287185668945, + "beta_dpo/loss_margin_mean": 14.059150695800781, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6152683295540439, + "grad_norm": 0.9176772832870483, + "learning_rate": 1.9492236680336483e-07, + "logits/chosen": 0.37719011306762695, + "logits/rejected": 0.29988420009613037, + "loss": 1.3724, + "step": 407 + }, + { + "beta_dpo/beta": 0.059294041246175766, + "beta_dpo/beta_margin_grad_mean": -0.3586428463459015, + "beta_dpo/beta_margin_grad_std": 0.2213546484708786, + "beta_dpo/beta_margin_mean": 1.363776683807373, + "beta_dpo/beta_margin_std": 2.3988959789276123, + "beta_dpo/beta_used": 0.059294041246175766, + "beta_dpo/beta_used_raw": 0.0582197904586792, + "beta_dpo/gap_mean": 16.845844268798828, + "beta_dpo/gap_std": 23.545011520385742, + "beta_dpo/loss_margin_mean": 19.550216674804688, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6167800453514739, + "grad_norm": 34.37969207763672, + "learning_rate": 1.9363341121154895e-07, + "logits/chosen": 0.46227675676345825, + "logits/rejected": 0.37937504053115845, + "loss": 1.0517, + "step": 408 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.496993750333786, + "beta_dpo/beta_margin_grad_std": 0.005834223236888647, + "beta_dpo/beta_margin_mean": 0.01202741451561451, + "beta_dpo/beta_margin_std": 0.023341767489910126, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.300281822681427, + "beta_dpo/gap_mean": 16.437042236328125, + "beta_dpo/gap_std": 23.63758659362793, + "beta_dpo/loss_margin_mean": 12.027414321899414, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.618291761148904, + "grad_norm": 0.6476607918739319, + "learning_rate": 1.9234603231438994e-07, + "logits/chosen": 0.4304888844490051, + "logits/rejected": 0.4324306547641754, + "loss": 1.3767, + "step": 409 + }, + { + "beta_dpo/beta": 0.07325400412082672, + "beta_dpo/beta_margin_grad_mean": -0.3431945741176605, + "beta_dpo/beta_margin_grad_std": 0.22117945551872253, + "beta_dpo/beta_margin_mean": 1.494828701019287, + "beta_dpo/beta_margin_std": 2.5302703380584717, + "beta_dpo/beta_used": 0.07325400412082672, + "beta_dpo/beta_used_raw": -0.053586445748806, + "beta_dpo/gap_mean": 16.29637336730957, + "beta_dpo/gap_std": 22.9282283782959, + "beta_dpo/loss_margin_mean": 15.299408912658691, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6198034769463341, + "grad_norm": 41.123111724853516, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": 0.47349417209625244, + "logits/rejected": 0.4450224041938782, + "loss": 0.971, + "step": 410 + }, + { + "beta_dpo/beta": 0.12284256517887115, + "beta_dpo/beta_margin_grad_mean": -0.3937680721282959, + "beta_dpo/beta_margin_grad_std": 0.28876185417175293, + "beta_dpo/beta_margin_mean": 2.466118812561035, + "beta_dpo/beta_margin_std": 5.714894771575928, + "beta_dpo/beta_used": 0.12284256517887115, + "beta_dpo/beta_used_raw": 0.04404173046350479, + "beta_dpo/gap_mean": 16.310779571533203, + "beta_dpo/gap_std": 23.467342376708984, + "beta_dpo/loss_margin_mean": 17.508848190307617, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6213151927437641, + "grad_norm": 110.05622100830078, + "learning_rate": 1.8977614860195296e-07, + "logits/chosen": 0.43857282400131226, + "logits/rejected": 0.37754371762275696, + "loss": 1.4855, + "step": 411 + }, + { + "beta_dpo/beta": 0.06893683224916458, + "beta_dpo/beta_margin_grad_mean": -0.4188035726547241, + "beta_dpo/beta_margin_grad_std": 0.247576504945755, + "beta_dpo/beta_margin_mean": 0.9057201743125916, + "beta_dpo/beta_margin_std": 2.4244892597198486, + "beta_dpo/beta_used": 0.06893683224916458, + "beta_dpo/beta_used_raw": -0.06521739065647125, + "beta_dpo/gap_mean": 16.284269332885742, + "beta_dpo/gap_std": 23.78243637084961, + "beta_dpo/loss_margin_mean": 15.739941596984863, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6228269085411943, + "grad_norm": 51.01114273071289, + "learning_rate": 1.8849371567184662e-07, + "logits/chosen": 0.4643993377685547, + "logits/rejected": 0.39351195096969604, + "loss": 1.1684, + "step": 412 + }, + { + "beta_dpo/beta": 0.12662924826145172, + "beta_dpo/beta_margin_grad_mean": -0.3909075856208801, + "beta_dpo/beta_margin_grad_std": 0.3073519468307495, + "beta_dpo/beta_margin_mean": 2.064873456954956, + "beta_dpo/beta_margin_std": 5.546405792236328, + "beta_dpo/beta_used": 0.12662924826145172, + "beta_dpo/beta_used_raw": -0.011943697929382324, + "beta_dpo/gap_mean": 16.021413803100586, + "beta_dpo/gap_std": 24.061824798583984, + "beta_dpo/loss_margin_mean": 15.6614351272583, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6243386243386243, + "grad_norm": 105.40924072265625, + "learning_rate": 1.872130032047302e-07, + "logits/chosen": 0.3003043532371521, + "logits/rejected": 0.26333707571029663, + "loss": 1.159, + "step": 413 + }, + { + "beta_dpo/beta": 0.1279238611459732, + "beta_dpo/beta_margin_grad_mean": -0.36717891693115234, + "beta_dpo/beta_margin_grad_std": 0.2865369915962219, + "beta_dpo/beta_margin_mean": 2.5722670555114746, + "beta_dpo/beta_margin_std": 5.616209983825684, + "beta_dpo/beta_used": 0.1279238611459732, + "beta_dpo/beta_used_raw": 0.05033845454454422, + "beta_dpo/gap_mean": 16.612018585205078, + "beta_dpo/gap_std": 24.592979431152344, + "beta_dpo/loss_margin_mean": 19.95748519897461, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6258503401360545, + "grad_norm": 103.1255111694336, + "learning_rate": 1.8593404702488436e-07, + "logits/chosen": 0.44454312324523926, + "logits/rejected": 0.38254380226135254, + "loss": 1.0349, + "step": 414 + }, + { + "beta_dpo/beta": 0.17535589635372162, + "beta_dpo/beta_margin_grad_mean": -0.40026840567588806, + "beta_dpo/beta_margin_grad_std": 0.3133196234703064, + "beta_dpo/beta_margin_mean": 2.7742631435394287, + "beta_dpo/beta_margin_std": 7.570990562438965, + "beta_dpo/beta_used": 0.17535589635372162, + "beta_dpo/beta_used_raw": 0.1638440191745758, + "beta_dpo/gap_mean": 17.03055191040039, + "beta_dpo/gap_std": 24.755905151367188, + "beta_dpo/loss_margin_mean": 17.868186950683594, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6273620559334845, + "grad_norm": 211.03265380859375, + "learning_rate": 1.846568829074628e-07, + "logits/chosen": 0.46807390451431274, + "logits/rejected": 0.4464624226093292, + "loss": 1.6837, + "step": 415 + }, + { + "beta_dpo/beta": 0.1974441409111023, + "beta_dpo/beta_margin_grad_mean": -0.3462623953819275, + "beta_dpo/beta_margin_grad_std": 0.2804645299911499, + "beta_dpo/beta_margin_mean": 4.031702518463135, + "beta_dpo/beta_margin_std": 8.079400062561035, + "beta_dpo/beta_used": 0.1974441409111023, + "beta_dpo/beta_used_raw": 0.06509672105312347, + "beta_dpo/gap_mean": 15.797903060913086, + "beta_dpo/gap_std": 24.798934936523438, + "beta_dpo/loss_margin_mean": 11.64545726776123, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6288737717309146, + "grad_norm": 102.91788482666016, + "learning_rate": 1.8338154657749128e-07, + "logits/chosen": 0.4295591711997986, + "logits/rejected": 0.3772777318954468, + "loss": 0.9866, + "step": 416 + }, + { + "beta_dpo/beta": 0.1781310886144638, + "beta_dpo/beta_margin_grad_mean": -0.25753018260002136, + "beta_dpo/beta_margin_grad_std": 0.32876917719841003, + "beta_dpo/beta_margin_mean": 3.546206474304199, + "beta_dpo/beta_margin_std": 5.584624290466309, + "beta_dpo/beta_used": 0.1781310886144638, + "beta_dpo/beta_used_raw": 0.1781310886144638, + "beta_dpo/gap_mean": 15.99776840209961, + "beta_dpo/gap_std": 25.12869644165039, + "beta_dpo/loss_margin_mean": 17.373567581176758, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6303854875283447, + "grad_norm": 86.60340118408203, + "learning_rate": 1.8210807370886849e-07, + "logits/chosen": 0.559472918510437, + "logits/rejected": 0.48729389905929565, + "loss": 0.7575, + "step": 417 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4969225227832794, + "beta_dpo/beta_margin_grad_std": 0.006542589515447617, + "beta_dpo/beta_margin_mean": 0.012312895618379116, + "beta_dpo/beta_margin_std": 0.026178967207670212, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.11385427415370941, + "beta_dpo/gap_mean": 16.031269073486328, + "beta_dpo/gap_std": 25.14609146118164, + "beta_dpo/loss_margin_mean": 12.312894821166992, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6318972033257747, + "grad_norm": 0.7612284421920776, + "learning_rate": 1.8083649992336825e-07, + "logits/chosen": 0.46164387464523315, + "logits/rejected": 0.46708381175994873, + "loss": 1.374, + "step": 418 + }, + { + "beta_dpo/beta": 0.14389443397521973, + "beta_dpo/beta_margin_grad_mean": -0.34402981400489807, + "beta_dpo/beta_margin_grad_std": 0.27868032455444336, + "beta_dpo/beta_margin_mean": 3.101715326309204, + "beta_dpo/beta_margin_std": 5.610728740692139, + "beta_dpo/beta_used": 0.14389443397521973, + "beta_dpo/beta_used_raw": 0.10019838809967041, + "beta_dpo/gap_mean": 16.164939880371094, + "beta_dpo/gap_std": 25.0128173828125, + "beta_dpo/loss_margin_mean": 19.221092224121094, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6334089191232048, + "grad_norm": 75.70429992675781, + "learning_rate": 1.7956686078964255e-07, + "logits/chosen": 0.37886548042297363, + "logits/rejected": 0.3200737535953522, + "loss": 1.1798, + "step": 419 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4973110556602478, + "beta_dpo/beta_margin_grad_std": 0.007076014298945665, + "beta_dpo/beta_margin_mean": 0.010757850483059883, + "beta_dpo/beta_margin_std": 0.028311539441347122, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.1291695386171341, + "beta_dpo/gap_mean": 15.535102844238281, + "beta_dpo/gap_std": 25.324172973632812, + "beta_dpo/loss_margin_mean": 10.75784969329834, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6349206349206349, + "grad_norm": 0.7967346906661987, + "learning_rate": 1.782991918222275e-07, + "logits/chosen": 0.4360436797142029, + "logits/rejected": 0.38624030351638794, + "loss": 1.3747, + "step": 420 + }, + { + "beta_dpo/beta": 0.16308826208114624, + "beta_dpo/beta_margin_grad_mean": -0.3662463426589966, + "beta_dpo/beta_margin_grad_std": 0.26859521865844727, + "beta_dpo/beta_margin_mean": 2.6131467819213867, + "beta_dpo/beta_margin_std": 6.545647621154785, + "beta_dpo/beta_used": 0.16308826208114624, + "beta_dpo/beta_used_raw": 0.11229175329208374, + "beta_dpo/gap_mean": 15.035537719726562, + "beta_dpo/gap_std": 26.121028900146484, + "beta_dpo/loss_margin_mean": 13.687716484069824, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.636432350718065, + "grad_norm": 84.8555679321289, + "learning_rate": 1.7703352848054887e-07, + "logits/chosen": 0.436802476644516, + "logits/rejected": 0.37465423345565796, + "loss": 1.4652, + "step": 421 + }, + { + "beta_dpo/beta": 0.01910341903567314, + "beta_dpo/beta_margin_grad_mean": -0.43050575256347656, + "beta_dpo/beta_margin_grad_std": 0.13256989419460297, + "beta_dpo/beta_margin_mean": 0.33536016941070557, + "beta_dpo/beta_margin_std": 0.6859686374664307, + "beta_dpo/beta_used": 0.01910341903567314, + "beta_dpo/beta_used_raw": -0.1404467225074768, + "beta_dpo/gap_mean": 15.0646390914917, + "beta_dpo/gap_std": 26.3007869720459, + "beta_dpo/loss_margin_mean": 16.67266273498535, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6379440665154951, + "grad_norm": 17.200712203979492, + "learning_rate": 1.7576990616793137e-07, + "logits/chosen": 0.4313371777534485, + "logits/rejected": 0.40216585993766785, + "loss": 1.191, + "step": 422 + }, + { + "beta_dpo/beta": 0.2093159258365631, + "beta_dpo/beta_margin_grad_mean": -0.2503194510936737, + "beta_dpo/beta_margin_grad_std": 0.3161967694759369, + "beta_dpo/beta_margin_mean": 3.7586827278137207, + "beta_dpo/beta_margin_std": 5.329720973968506, + "beta_dpo/beta_used": 0.2093159258365631, + "beta_dpo/beta_used_raw": 0.2093159258365631, + "beta_dpo/gap_mean": 15.665881156921387, + "beta_dpo/gap_std": 26.160938262939453, + "beta_dpo/loss_margin_mean": 18.111860275268555, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6394557823129252, + "grad_norm": 83.79721069335938, + "learning_rate": 1.745083602306071e-07, + "logits/chosen": 0.46611279249191284, + "logits/rejected": 0.39604848623275757, + "loss": 0.8254, + "step": 423 + }, + { + "beta_dpo/beta": 0.14506866037845612, + "beta_dpo/beta_margin_grad_mean": -0.34038662910461426, + "beta_dpo/beta_margin_grad_std": 0.2595653831958771, + "beta_dpo/beta_margin_mean": 3.0149664878845215, + "beta_dpo/beta_margin_std": 5.455316066741943, + "beta_dpo/beta_used": 0.14506866037845612, + "beta_dpo/beta_used_raw": 0.07982266694307327, + "beta_dpo/gap_mean": 16.47106170654297, + "beta_dpo/gap_std": 25.810760498046875, + "beta_dpo/loss_margin_mean": 20.76560401916504, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6409674981103552, + "grad_norm": 75.43428039550781, + "learning_rate": 1.7324892595672804e-07, + "logits/chosen": 0.3916613757610321, + "logits/rejected": 0.3512648940086365, + "loss": 0.9367, + "step": 424 + }, + { + "beta_dpo/beta": 0.07886487990617752, + "beta_dpo/beta_margin_grad_mean": -0.3156943917274475, + "beta_dpo/beta_margin_grad_std": 0.2569364905357361, + "beta_dpo/beta_margin_mean": 1.328211784362793, + "beta_dpo/beta_margin_std": 1.9083045721054077, + "beta_dpo/beta_used": 0.07886487990617752, + "beta_dpo/beta_used_raw": 0.07886487990617752, + "beta_dpo/gap_mean": 16.746292114257812, + "beta_dpo/gap_std": 25.627155303955078, + "beta_dpo/loss_margin_mean": 16.717668533325195, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6424792139077853, + "grad_norm": 33.6092414855957, + "learning_rate": 1.7199163857537824e-07, + "logits/chosen": 0.5233234763145447, + "logits/rejected": 0.48915839195251465, + "loss": 0.8456, + "step": 425 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4974079728126526, + "beta_dpo/beta_margin_grad_std": 0.006462402641773224, + "beta_dpo/beta_margin_mean": 0.010370003059506416, + "beta_dpo/beta_margin_std": 0.02585585229098797, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.2091233730316162, + "beta_dpo/gap_mean": 15.816381454467773, + "beta_dpo/gap_std": 25.586580276489258, + "beta_dpo/loss_margin_mean": 10.370002746582031, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6439909297052154, + "grad_norm": 0.863946795463562, + "learning_rate": 1.7073653325558828e-07, + "logits/chosen": 0.4199245572090149, + "logits/rejected": 0.4237271547317505, + "loss": 1.3757, + "step": 426 + }, + { + "beta_dpo/beta": 0.16894888877868652, + "beta_dpo/beta_margin_grad_mean": -0.3107214570045471, + "beta_dpo/beta_margin_grad_std": 0.29482501745224, + "beta_dpo/beta_margin_mean": 2.7154009342193604, + "beta_dpo/beta_margin_std": 6.2772908210754395, + "beta_dpo/beta_used": 0.16894888877868652, + "beta_dpo/beta_used_raw": 0.16894888877868652, + "beta_dpo/gap_mean": 15.633672714233398, + "beta_dpo/gap_std": 26.182838439941406, + "beta_dpo/loss_margin_mean": 16.839767456054688, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6455026455026455, + "grad_norm": 59.07684326171875, + "learning_rate": 1.6948364510535218e-07, + "logits/chosen": 0.490261435508728, + "logits/rejected": 0.4341810345649719, + "loss": 0.7906, + "step": 427 + }, + { + "beta_dpo/beta": 0.206699937582016, + "beta_dpo/beta_margin_grad_mean": -0.29449787735939026, + "beta_dpo/beta_margin_grad_std": 0.3339853286743164, + "beta_dpo/beta_margin_mean": 4.745669364929199, + "beta_dpo/beta_margin_std": 8.069055557250977, + "beta_dpo/beta_used": 0.206699937582016, + "beta_dpo/beta_used_raw": 0.206699937582016, + "beta_dpo/gap_mean": 16.280973434448242, + "beta_dpo/gap_std": 26.821441650390625, + "beta_dpo/loss_margin_mean": 20.525794982910156, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6470143613000756, + "grad_norm": 159.63389587402344, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": 0.4094158709049225, + "logits/rejected": 0.35882726311683655, + "loss": 1.4005, + "step": 428 + }, + { + "beta_dpo/beta": 0.2941809594631195, + "beta_dpo/beta_margin_grad_mean": -0.2386247217655182, + "beta_dpo/beta_margin_grad_std": 0.3809661567211151, + "beta_dpo/beta_margin_mean": 5.172230243682861, + "beta_dpo/beta_margin_std": 7.734333515167236, + "beta_dpo/beta_used": 0.2941809594631195, + "beta_dpo/beta_used_raw": 0.2941809594631195, + "beta_dpo/gap_mean": 16.665008544921875, + "beta_dpo/gap_std": 26.66605567932129, + "beta_dpo/loss_margin_mean": 17.31098175048828, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6485260770975056, + "grad_norm": 149.40992736816406, + "learning_rate": 1.669846604344412e-07, + "logits/chosen": 0.38707661628723145, + "logits/rejected": 0.3977625370025635, + "loss": 1.1137, + "step": 429 + }, + { + "beta_dpo/beta": 0.271389901638031, + "beta_dpo/beta_margin_grad_mean": -0.23841898143291473, + "beta_dpo/beta_margin_grad_std": 0.2879250943660736, + "beta_dpo/beta_margin_mean": 5.876552581787109, + "beta_dpo/beta_margin_std": 9.04500961303711, + "beta_dpo/beta_used": 0.271389901638031, + "beta_dpo/beta_used_raw": 0.271389901638031, + "beta_dpo/gap_mean": 17.295337677001953, + "beta_dpo/gap_std": 26.757667541503906, + "beta_dpo/loss_margin_mean": 20.482933044433594, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6500377928949358, + "grad_norm": 124.40864562988281, + "learning_rate": 1.6573863381573954e-07, + "logits/chosen": 0.3641434907913208, + "logits/rejected": 0.3554537892341614, + "loss": 0.6021, + "step": 430 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49573636054992676, + "beta_dpo/beta_margin_grad_std": 0.00596994673833251, + "beta_dpo/beta_margin_mean": 0.017058147117495537, + "beta_dpo/beta_margin_std": 0.023886609822511673, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.14669229090213776, + "beta_dpo/gap_mean": 17.474349975585938, + "beta_dpo/gap_std": 26.309669494628906, + "beta_dpo/loss_margin_mean": 17.058147430419922, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6515495086923658, + "grad_norm": 0.8200677633285522, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 0.43957391381263733, + "logits/rejected": 0.3921339511871338, + "loss": 1.373, + "step": 431 + }, + { + "beta_dpo/beta": 0.20359541475772858, + "beta_dpo/beta_margin_grad_mean": -0.34393805265426636, + "beta_dpo/beta_margin_grad_std": 0.2853422164916992, + "beta_dpo/beta_margin_mean": 4.172194480895996, + "beta_dpo/beta_margin_std": 8.311675071716309, + "beta_dpo/beta_used": 0.20359541475772858, + "beta_dpo/beta_used_raw": 0.20359541475772858, + "beta_dpo/gap_mean": 17.662752151489258, + "beta_dpo/gap_std": 26.11996841430664, + "beta_dpo/loss_margin_mean": 18.106279373168945, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6530612244897959, + "grad_norm": 113.8149185180664, + "learning_rate": 1.632536862810844e-07, + "logits/chosen": 0.47949665784835815, + "logits/rejected": 0.4305593967437744, + "loss": 1.1397, + "step": 432 + }, + { + "beta_dpo/beta": 0.3344748020172119, + "beta_dpo/beta_margin_grad_mean": -0.22546495497226715, + "beta_dpo/beta_margin_grad_std": 0.3282316327095032, + "beta_dpo/beta_margin_mean": 7.9909210205078125, + "beta_dpo/beta_margin_std": 14.14069652557373, + "beta_dpo/beta_used": 0.3344748020172119, + "beta_dpo/beta_used_raw": 0.3344748020172119, + "beta_dpo/gap_mean": 18.123287200927734, + "beta_dpo/gap_std": 26.272716522216797, + "beta_dpo/loss_margin_mean": 22.30093765258789, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.654572940287226, + "grad_norm": 175.8500518798828, + "learning_rate": 1.6201483487445515e-07, + "logits/chosen": 0.5873138308525085, + "logits/rejected": 0.575851321220398, + "loss": 0.7079, + "step": 433 + }, + { + "beta_dpo/beta": 0.4044048488140106, + "beta_dpo/beta_margin_grad_mean": -0.24625371396541595, + "beta_dpo/beta_margin_grad_std": 0.3679701089859009, + "beta_dpo/beta_margin_mean": 9.312926292419434, + "beta_dpo/beta_margin_std": 15.837896347045898, + "beta_dpo/beta_used": 0.4044048488140106, + "beta_dpo/beta_used_raw": 0.4044048488140106, + "beta_dpo/gap_mean": 18.955612182617188, + "beta_dpo/gap_std": 27.102890014648438, + "beta_dpo/loss_margin_mean": 20.431182861328125, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.656084656084656, + "grad_norm": 217.51641845703125, + "learning_rate": 1.6077844460203204e-07, + "logits/chosen": 0.5753809213638306, + "logits/rejected": 0.5080810785293579, + "loss": 1.6548, + "step": 434 + }, + { + "beta_dpo/beta": 0.2745690941810608, + "beta_dpo/beta_margin_grad_mean": -0.36417117714881897, + "beta_dpo/beta_margin_grad_std": 0.29817262291908264, + "beta_dpo/beta_margin_mean": 5.747401714324951, + "beta_dpo/beta_margin_std": 12.350319862365723, + "beta_dpo/beta_used": 0.2745690941810608, + "beta_dpo/beta_used_raw": 0.194306880235672, + "beta_dpo/gap_mean": 18.432392120361328, + "beta_dpo/gap_std": 27.030935287475586, + "beta_dpo/loss_margin_mean": 17.350605010986328, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6575963718820862, + "grad_norm": 161.15036010742188, + "learning_rate": 1.5954455004830878e-07, + "logits/chosen": 0.5192146301269531, + "logits/rejected": 0.4788922667503357, + "loss": 1.4735, + "step": 435 + }, + { + "beta_dpo/beta": 0.13801611959934235, + "beta_dpo/beta_margin_grad_mean": -0.3944951295852661, + "beta_dpo/beta_margin_grad_std": 0.28630712628364563, + "beta_dpo/beta_margin_mean": 2.079245090484619, + "beta_dpo/beta_margin_std": 5.407124042510986, + "beta_dpo/beta_used": 0.13801611959934235, + "beta_dpo/beta_used_raw": 0.012127086520195007, + "beta_dpo/gap_mean": 18.145030975341797, + "beta_dpo/gap_std": 27.07366180419922, + "beta_dpo/loss_margin_mean": 15.945972442626953, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6591080876795162, + "grad_norm": 95.06700134277344, + "learning_rate": 1.5831318572796847e-07, + "logits/chosen": 0.4287049174308777, + "logits/rejected": 0.37227755784988403, + "loss": 1.0533, + "step": 436 + }, + { + "beta_dpo/beta": 0.08843779563903809, + "beta_dpo/beta_margin_grad_mean": -0.355763703584671, + "beta_dpo/beta_margin_grad_std": 0.27760180830955505, + "beta_dpo/beta_margin_mean": 2.1557815074920654, + "beta_dpo/beta_margin_std": 4.090998649597168, + "beta_dpo/beta_used": 0.08843779563903809, + "beta_dpo/beta_used_raw": 0.004511319100856781, + "beta_dpo/gap_mean": 18.151508331298828, + "beta_dpo/gap_std": 27.469696044921875, + "beta_dpo/loss_margin_mean": 15.952765464782715, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6606198034769464, + "grad_norm": 67.06380462646484, + "learning_rate": 1.5708438608491815e-07, + "logits/chosen": 0.4623869061470032, + "logits/rejected": 0.3321772813796997, + "loss": 1.1109, + "step": 437 + }, + { + "beta_dpo/beta": 0.16248461604118347, + "beta_dpo/beta_margin_grad_mean": -0.33106884360313416, + "beta_dpo/beta_margin_grad_std": 0.2685889005661011, + "beta_dpo/beta_margin_mean": 4.133353233337402, + "beta_dpo/beta_margin_std": 7.090116024017334, + "beta_dpo/beta_used": 0.16248461604118347, + "beta_dpo/beta_used_raw": 0.15679943561553955, + "beta_dpo/gap_mean": 18.146766662597656, + "beta_dpo/gap_std": 28.101390838623047, + "beta_dpo/loss_margin_mean": 22.395479202270508, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6621315192743764, + "grad_norm": 134.7775115966797, + "learning_rate": 1.558581854913253e-07, + "logits/chosen": 0.5341899394989014, + "logits/rejected": 0.4623284339904785, + "loss": 1.0142, + "step": 438 + }, + { + "beta_dpo/beta": 0.09724593162536621, + "beta_dpo/beta_margin_grad_mean": -0.313736230134964, + "beta_dpo/beta_margin_grad_std": 0.24954845011234283, + "beta_dpo/beta_margin_mean": 2.1920928955078125, + "beta_dpo/beta_margin_std": 3.4765686988830566, + "beta_dpo/beta_used": 0.09724593162536621, + "beta_dpo/beta_used_raw": 0.09724593162536621, + "beta_dpo/gap_mean": 18.264135360717773, + "beta_dpo/gap_std": 27.18978500366211, + "beta_dpo/loss_margin_mean": 18.61200714111328, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6636432350718064, + "grad_norm": 61.768672943115234, + "learning_rate": 1.5463461824665658e-07, + "logits/chosen": 0.39725732803344727, + "logits/rejected": 0.35775747895240784, + "loss": 0.8909, + "step": 439 + }, + { + "beta_dpo/beta": 0.22115948796272278, + "beta_dpo/beta_margin_grad_mean": -0.3153398931026459, + "beta_dpo/beta_margin_grad_std": 0.27347540855407715, + "beta_dpo/beta_margin_mean": 5.172101020812988, + "beta_dpo/beta_margin_std": 9.553386688232422, + "beta_dpo/beta_used": 0.22115948796272278, + "beta_dpo/beta_used_raw": 0.11873233318328857, + "beta_dpo/gap_mean": 18.741073608398438, + "beta_dpo/gap_std": 27.152206420898438, + "beta_dpo/loss_margin_mean": 20.439512252807617, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6651549508692366, + "grad_norm": 132.9888916015625, + "learning_rate": 1.534137185767178e-07, + "logits/chosen": 0.4396424889564514, + "logits/rejected": 0.3410155773162842, + "loss": 0.9659, + "step": 440 + }, + { + "beta_dpo/beta": 0.056231264024972916, + "beta_dpo/beta_margin_grad_mean": -0.3631954789161682, + "beta_dpo/beta_margin_grad_std": 0.2372567057609558, + "beta_dpo/beta_margin_mean": 1.306308627128601, + "beta_dpo/beta_margin_std": 2.314157009124756, + "beta_dpo/beta_used": 0.056231264024972916, + "beta_dpo/beta_used_raw": 0.014366436749696732, + "beta_dpo/gap_mean": 19.39804458618164, + "beta_dpo/gap_std": 26.627029418945312, + "beta_dpo/loss_margin_mean": 21.177230834960938, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6666666666666666, + "grad_norm": 28.244613647460938, + "learning_rate": 1.521955206326976e-07, + "logits/chosen": 0.43280029296875, + "logits/rejected": 0.3387737274169922, + "loss": 1.0871, + "step": 441 + }, + { + "beta_dpo/beta": 0.13525980710983276, + "beta_dpo/beta_margin_grad_mean": -0.3400418162345886, + "beta_dpo/beta_margin_grad_std": 0.2894745469093323, + "beta_dpo/beta_margin_mean": 2.674807071685791, + "beta_dpo/beta_margin_std": 5.317812442779541, + "beta_dpo/beta_used": 0.13525980710983276, + "beta_dpo/beta_used_raw": -0.08266353607177734, + "beta_dpo/gap_mean": 19.399829864501953, + "beta_dpo/gap_std": 26.35002899169922, + "beta_dpo/loss_margin_mean": 19.57100486755371, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6681783824640968, + "grad_norm": 101.40801239013672, + "learning_rate": 1.5098005849021078e-07, + "logits/chosen": 0.47117850184440613, + "logits/rejected": 0.4187871515750885, + "loss": 0.8993, + "step": 442 + }, + { + "beta_dpo/beta": 0.3095214366912842, + "beta_dpo/beta_margin_grad_mean": -0.1766759157180786, + "beta_dpo/beta_margin_grad_std": 0.3004312217235565, + "beta_dpo/beta_margin_mean": 7.633600234985352, + "beta_dpo/beta_margin_std": 9.593216896057129, + "beta_dpo/beta_used": 0.3095214366912842, + "beta_dpo/beta_used_raw": 0.3095214366912842, + "beta_dpo/gap_mean": 20.205463409423828, + "beta_dpo/gap_std": 26.816097259521484, + "beta_dpo/loss_margin_mean": 24.74801254272461, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6696900982615268, + "grad_norm": 115.0042953491211, + "learning_rate": 1.4976736614834662e-07, + "logits/chosen": 0.49063044786453247, + "logits/rejected": 0.41436687111854553, + "loss": 0.8238, + "step": 443 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49786376953125, + "beta_dpo/beta_margin_grad_std": 0.006280460394918919, + "beta_dpo/beta_margin_mean": 0.008546494878828526, + "beta_dpo/beta_margin_std": 0.025125892832875252, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.3049757182598114, + "beta_dpo/gap_mean": 18.85560417175293, + "beta_dpo/gap_std": 26.632326126098633, + "beta_dpo/loss_margin_mean": 8.546494483947754, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.671201814058957, + "grad_norm": 0.8523843884468079, + "learning_rate": 1.4855747752871654e-07, + "logits/chosen": 0.4893801212310791, + "logits/rejected": 0.40110495686531067, + "loss": 1.3743, + "step": 444 + }, + { + "beta_dpo/beta": 0.15167614817619324, + "beta_dpo/beta_margin_grad_mean": -0.24699601531028748, + "beta_dpo/beta_margin_grad_std": 0.29544878005981445, + "beta_dpo/beta_margin_mean": 3.1027376651763916, + "beta_dpo/beta_margin_std": 4.867432594299316, + "beta_dpo/beta_used": 0.15167614817619324, + "beta_dpo/beta_used_raw": 0.15167614817619324, + "beta_dpo/gap_mean": 18.524452209472656, + "beta_dpo/gap_std": 26.606185913085938, + "beta_dpo/loss_margin_mean": 21.157268524169922, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.672713529856387, + "grad_norm": 58.71397018432617, + "learning_rate": 1.473504264745062e-07, + "logits/chosen": 0.453709214925766, + "logits/rejected": 0.43405085802078247, + "loss": 0.7044, + "step": 445 + }, + { + "beta_dpo/beta": 0.2611894905567169, + "beta_dpo/beta_margin_grad_mean": -0.2053774893283844, + "beta_dpo/beta_margin_grad_std": 0.25838059186935425, + "beta_dpo/beta_margin_mean": 7.894767761230469, + "beta_dpo/beta_margin_std": 10.342389106750488, + "beta_dpo/beta_used": 0.2611894905567169, + "beta_dpo/beta_used_raw": 0.2611894905567169, + "beta_dpo/gap_mean": 19.257843017578125, + "beta_dpo/gap_std": 26.065818786621094, + "beta_dpo/loss_margin_mean": 24.705129623413086, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.674225245653817, + "grad_norm": 105.36664581298828, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": 0.5040090084075928, + "logits/rejected": 0.42126449942588806, + "loss": 0.6766, + "step": 446 + }, + { + "beta_dpo/beta": 0.23961615562438965, + "beta_dpo/beta_margin_grad_mean": -0.19545207917690277, + "beta_dpo/beta_margin_grad_std": 0.33320990204811096, + "beta_dpo/beta_margin_mean": 5.5971269607543945, + "beta_dpo/beta_margin_std": 6.4180378913879395, + "beta_dpo/beta_used": 0.23961615562438965, + "beta_dpo/beta_used_raw": 0.23961615562438965, + "beta_dpo/gap_mean": 20.464557647705078, + "beta_dpo/gap_std": 26.064769744873047, + "beta_dpo/loss_margin_mean": 23.23582649230957, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6757369614512472, + "grad_norm": 118.35011291503906, + "learning_rate": 1.4494497203727843e-07, + "logits/chosen": 0.4212689995765686, + "logits/rejected": 0.3177017867565155, + "loss": 1.0214, + "step": 447 + }, + { + "beta_dpo/beta": 0.006910950411111116, + "beta_dpo/beta_margin_grad_mean": -0.4693349301815033, + "beta_dpo/beta_margin_grad_std": 0.048872094601392746, + "beta_dpo/beta_margin_mean": 0.12464610487222672, + "beta_dpo/beta_margin_std": 0.20036308467388153, + "beta_dpo/beta_used": 0.006910950411111116, + "beta_dpo/beta_used_raw": 0.006910950411111116, + "beta_dpo/gap_mean": 20.188140869140625, + "beta_dpo/gap_std": 26.024574279785156, + "beta_dpo/loss_margin_mean": 18.08507537841797, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6772486772486772, + "grad_norm": 5.197481632232666, + "learning_rate": 1.4374663593999256e-07, + "logits/chosen": 0.4766013026237488, + "logits/rejected": 0.4210081100463867, + "loss": 1.2689, + "step": 448 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49735549092292786, + "beta_dpo/beta_margin_grad_std": 0.005208996124565601, + "beta_dpo/beta_margin_mean": 0.010579775087535381, + "beta_dpo/beta_margin_std": 0.020839868113398552, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.35340794920921326, + "beta_dpo/gap_mean": 18.90911865234375, + "beta_dpo/gap_std": 25.33905029296875, + "beta_dpo/loss_margin_mean": 10.579774856567383, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6787603930461074, + "grad_norm": 0.742425262928009, + "learning_rate": 1.4255127197770707e-07, + "logits/chosen": 0.3229493200778961, + "logits/rejected": 0.3151504397392273, + "loss": 1.3751, + "step": 449 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49666455388069153, + "beta_dpo/beta_margin_grad_std": 0.0058851479552686214, + "beta_dpo/beta_margin_mean": 0.013344475999474525, + "beta_dpo/beta_margin_std": 0.023545950651168823, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.10140938311815262, + "beta_dpo/gap_mean": 17.542179107666016, + "beta_dpo/gap_std": 24.803512573242188, + "beta_dpo/loss_margin_mean": 13.344475746154785, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6802721088435374, + "grad_norm": 0.8854034543037415, + "learning_rate": 1.4135891358732205e-07, + "logits/chosen": 0.5489107370376587, + "logits/rejected": 0.43710023164749146, + "loss": 1.3723, + "step": 450 + }, + { + "beta_dpo/beta": 0.0172061026096344, + "beta_dpo/beta_margin_grad_mean": -0.4475163519382477, + "beta_dpo/beta_margin_grad_std": 0.12823042273521423, + "beta_dpo/beta_margin_mean": 0.24266105890274048, + "beta_dpo/beta_margin_std": 0.6130807399749756, + "beta_dpo/beta_used": 0.0172061026096344, + "beta_dpo/beta_used_raw": -0.08448895812034607, + "beta_dpo/gap_mean": 16.902774810791016, + "beta_dpo/gap_std": 24.68649673461914, + "beta_dpo/loss_margin_mean": 14.798531532287598, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6817838246409675, + "grad_norm": 13.051300048828125, + "learning_rate": 1.4016959412166437e-07, + "logits/chosen": 0.42658311128616333, + "logits/rejected": 0.37267670035362244, + "loss": 1.1849, + "step": 451 + }, + { + "beta_dpo/beta": 0.13525496423244476, + "beta_dpo/beta_margin_grad_mean": -0.29088878631591797, + "beta_dpo/beta_margin_grad_std": 0.3417208194732666, + "beta_dpo/beta_margin_mean": 2.171989917755127, + "beta_dpo/beta_margin_std": 3.486285924911499, + "beta_dpo/beta_used": 0.13525496423244476, + "beta_dpo/beta_used_raw": 0.13525496423244476, + "beta_dpo/gap_mean": 16.710304260253906, + "beta_dpo/gap_std": 24.817352294921875, + "beta_dpo/loss_margin_mean": 16.379568099975586, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6832955404383976, + "grad_norm": 70.75874328613281, + "learning_rate": 1.3898334684855645e-07, + "logits/chosen": 0.4217544198036194, + "logits/rejected": 0.34103256464004517, + "loss": 1.1461, + "step": 452 + }, + { + "beta_dpo/beta": 0.04890910163521767, + "beta_dpo/beta_margin_grad_mean": -0.4136413037776947, + "beta_dpo/beta_margin_grad_std": 0.24708783626556396, + "beta_dpo/beta_margin_mean": 0.8333736658096313, + "beta_dpo/beta_margin_std": 2.1311116218566895, + "beta_dpo/beta_used": 0.04890910163521767, + "beta_dpo/beta_used_raw": 0.04254155978560448, + "beta_dpo/gap_mean": 16.893192291259766, + "beta_dpo/gap_std": 25.19158172607422, + "beta_dpo/loss_margin_mean": 17.536466598510742, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6848072562358276, + "grad_norm": 37.44161605834961, + "learning_rate": 1.3780020494988445e-07, + "logits/chosen": 0.41655856370925903, + "logits/rejected": 0.381106436252594, + "loss": 1.2468, + "step": 453 + }, + { + "beta_dpo/beta": 0.16885149478912354, + "beta_dpo/beta_margin_grad_mean": -0.22665803134441376, + "beta_dpo/beta_margin_grad_std": 0.304115355014801, + "beta_dpo/beta_margin_mean": 3.1320598125457764, + "beta_dpo/beta_margin_std": 4.471872806549072, + "beta_dpo/beta_used": 0.16885149478912354, + "beta_dpo/beta_used_raw": 0.16885149478912354, + "beta_dpo/gap_mean": 17.158390045166016, + "beta_dpo/gap_std": 25.75613021850586, + "beta_dpo/loss_margin_mean": 18.559307098388672, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6863189720332578, + "grad_norm": 48.60162353515625, + "learning_rate": 1.366202015206706e-07, + "logits/chosen": 0.4805641770362854, + "logits/rejected": 0.43357783555984497, + "loss": 0.6251, + "step": 454 + }, + { + "beta_dpo/beta": 0.15867355465888977, + "beta_dpo/beta_margin_grad_mean": -0.3603483736515045, + "beta_dpo/beta_margin_grad_std": 0.27238547801971436, + "beta_dpo/beta_margin_mean": 3.664536952972412, + "beta_dpo/beta_margin_std": 6.963903427124023, + "beta_dpo/beta_used": 0.15867355465888977, + "beta_dpo/beta_used_raw": 0.13864237070083618, + "beta_dpo/gap_mean": 17.709564208984375, + "beta_dpo/gap_std": 25.734886169433594, + "beta_dpo/loss_margin_mean": 19.53325080871582, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6878306878306878, + "grad_norm": 107.5218276977539, + "learning_rate": 1.354433695681474e-07, + "logits/chosen": 0.33997493982315063, + "logits/rejected": 0.29935604333877563, + "loss": 1.3453, + "step": 455 + }, + { + "beta_dpo/beta": 0.11797763407230377, + "beta_dpo/beta_margin_grad_mean": -0.2854386270046234, + "beta_dpo/beta_margin_grad_std": 0.28851282596588135, + "beta_dpo/beta_margin_mean": 2.0227293968200684, + "beta_dpo/beta_margin_std": 2.8564655780792236, + "beta_dpo/beta_used": 0.11797763407230377, + "beta_dpo/beta_used_raw": 0.11797763407230377, + "beta_dpo/gap_mean": 17.563323974609375, + "beta_dpo/gap_std": 25.402185440063477, + "beta_dpo/loss_margin_mean": 16.928739547729492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6893424036281179, + "grad_norm": 52.88853454589844, + "learning_rate": 1.3426974201083439e-07, + "logits/chosen": 0.39014971256256104, + "logits/rejected": 0.3266359865665436, + "loss": 0.7483, + "step": 456 + }, + { + "beta_dpo/beta": 0.1249324381351471, + "beta_dpo/beta_margin_grad_mean": -0.3382973372936249, + "beta_dpo/beta_margin_grad_std": 0.2664583921432495, + "beta_dpo/beta_margin_mean": 2.556375503540039, + "beta_dpo/beta_margin_std": 4.548144817352295, + "beta_dpo/beta_used": 0.1249324381351471, + "beta_dpo/beta_used_raw": -0.01606186479330063, + "beta_dpo/gap_mean": 17.55565643310547, + "beta_dpo/gap_std": 24.93761444091797, + "beta_dpo/loss_margin_mean": 16.916906356811523, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.690854119425548, + "grad_norm": 56.52458190917969, + "learning_rate": 1.3309935167761717e-07, + "logits/chosen": 0.5392422676086426, + "logits/rejected": 0.4641982913017273, + "loss": 0.9844, + "step": 457 + }, + { + "beta_dpo/beta": 0.24690914154052734, + "beta_dpo/beta_margin_grad_mean": -0.22146117687225342, + "beta_dpo/beta_margin_grad_std": 0.33166489005088806, + "beta_dpo/beta_margin_mean": 4.624577522277832, + "beta_dpo/beta_margin_std": 6.362880229949951, + "beta_dpo/beta_used": 0.24690914154052734, + "beta_dpo/beta_used_raw": 0.24690914154052734, + "beta_dpo/gap_mean": 17.572355270385742, + "beta_dpo/gap_std": 24.927356719970703, + "beta_dpo/loss_margin_mean": 18.480012893676758, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6923658352229781, + "grad_norm": 78.89071655273438, + "learning_rate": 1.3193223130682936e-07, + "logits/chosen": 0.45676666498184204, + "logits/rejected": 0.34059104323387146, + "loss": 0.6349, + "step": 458 + }, + { + "beta_dpo/beta": 0.2179662585258484, + "beta_dpo/beta_margin_grad_mean": -0.3185714781284332, + "beta_dpo/beta_margin_grad_std": 0.27158471941947937, + "beta_dpo/beta_margin_mean": 6.563882827758789, + "beta_dpo/beta_margin_std": 9.94521713256836, + "beta_dpo/beta_used": 0.2179662585258484, + "beta_dpo/beta_used_raw": 0.18530282378196716, + "beta_dpo/gap_mean": 17.721694946289062, + "beta_dpo/gap_std": 24.523391723632812, + "beta_dpo/loss_margin_mean": 21.503368377685547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6938775510204082, + "grad_norm": 122.18168640136719, + "learning_rate": 1.3076841354533658e-07, + "logits/chosen": 0.4888218343257904, + "logits/rejected": 0.44728928804397583, + "loss": 1.1406, + "step": 459 + }, + { + "beta_dpo/beta": 0.08090288937091827, + "beta_dpo/beta_margin_grad_mean": -0.37582793831825256, + "beta_dpo/beta_margin_grad_std": 0.2519203722476959, + "beta_dpo/beta_margin_mean": 1.7335608005523682, + "beta_dpo/beta_margin_std": 3.414593458175659, + "beta_dpo/beta_used": 0.08090288937091827, + "beta_dpo/beta_used_raw": -0.02788601815700531, + "beta_dpo/gap_mean": 18.970165252685547, + "beta_dpo/gap_std": 24.861597061157227, + "beta_dpo/loss_margin_mean": 22.223512649536133, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6953892668178382, + "grad_norm": 63.34912109375, + "learning_rate": 1.2960793094762345e-07, + "logits/chosen": 0.5079508423805237, + "logits/rejected": 0.37769049406051636, + "loss": 1.1034, + "step": 460 + }, + { + "beta_dpo/beta": 0.34863337874412537, + "beta_dpo/beta_margin_grad_mean": -0.29620790481567383, + "beta_dpo/beta_margin_grad_std": 0.2750532329082489, + "beta_dpo/beta_margin_mean": 9.606303215026855, + "beta_dpo/beta_margin_std": 15.002069473266602, + "beta_dpo/beta_used": 0.34863337874412537, + "beta_dpo/beta_used_raw": 0.053985416889190674, + "beta_dpo/gap_mean": 19.691003799438477, + "beta_dpo/gap_std": 24.841266632080078, + "beta_dpo/loss_margin_mean": 21.206148147583008, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6969009826152683, + "grad_norm": 78.56777954101562, + "learning_rate": 1.2845081597488286e-07, + "logits/chosen": 0.608617901802063, + "logits/rejected": 0.5119181275367737, + "loss": 0.7492, + "step": 461 + }, + { + "beta_dpo/beta": 0.3502820134162903, + "beta_dpo/beta_margin_grad_mean": -0.17331989109516144, + "beta_dpo/beta_margin_grad_std": 0.2825244069099426, + "beta_dpo/beta_margin_mean": 8.671442985534668, + "beta_dpo/beta_margin_std": 10.874760627746582, + "beta_dpo/beta_used": 0.3502820134162903, + "beta_dpo/beta_used_raw": 0.3502820134162903, + "beta_dpo/gap_mean": 20.119089126586914, + "beta_dpo/gap_std": 24.742603302001953, + "beta_dpo/loss_margin_mean": 23.444116592407227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6984126984126984, + "grad_norm": 105.43913269042969, + "learning_rate": 1.27297100994108e-07, + "logits/chosen": 0.4638398289680481, + "logits/rejected": 0.39704328775405884, + "loss": 0.8603, + "step": 462 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4957989454269409, + "beta_dpo/beta_margin_grad_std": 0.005161019042134285, + "beta_dpo/beta_margin_mean": 0.016806745901703835, + "beta_dpo/beta_margin_std": 0.02064814232289791, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.09972745180130005, + "beta_dpo/gap_mean": 19.76999282836914, + "beta_dpo/gap_std": 24.28656578063965, + "beta_dpo/loss_margin_mean": 16.806743621826172, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.6999244142101285, + "grad_norm": 0.8558281064033508, + "learning_rate": 1.2614681827718695e-07, + "logits/chosen": 0.44439205527305603, + "logits/rejected": 0.4367867410182953, + "loss": 1.37, + "step": 463 + }, + { + "beta_dpo/beta": 0.0761955976486206, + "beta_dpo/beta_margin_grad_mean": -0.3430159091949463, + "beta_dpo/beta_margin_grad_std": 0.2450447976589203, + "beta_dpo/beta_margin_mean": 1.6685881614685059, + "beta_dpo/beta_margin_std": 2.8676531314849854, + "beta_dpo/beta_used": 0.0761955976486206, + "beta_dpo/beta_used_raw": -0.042108893394470215, + "beta_dpo/gap_mean": 19.153217315673828, + "beta_dpo/gap_std": 24.542236328125, + "beta_dpo/loss_margin_mean": 18.18233871459961, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7014361300075586, + "grad_norm": 31.882461547851562, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 0.417372465133667, + "logits/rejected": 0.38535577058792114, + "loss": 0.9379, + "step": 464 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4959162771701813, + "beta_dpo/beta_margin_grad_std": 0.006333097815513611, + "beta_dpo/beta_margin_mean": 0.016338225454092026, + "beta_dpo/beta_margin_std": 0.02533765509724617, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.11223104596138, + "beta_dpo/gap_mean": 18.943988800048828, + "beta_dpo/gap_std": 24.53543472290039, + "beta_dpo/loss_margin_mean": 16.338224411010742, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7029478458049887, + "grad_norm": 0.9496918320655823, + "learning_rate": 1.238566782415197e-07, + "logits/chosen": 0.5403270125389099, + "logits/rejected": 0.47340455651283264, + "loss": 1.3711, + "step": 465 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4974312484264374, + "beta_dpo/beta_margin_grad_std": 0.005646795034408569, + "beta_dpo/beta_margin_mean": 0.010276666842401028, + "beta_dpo/beta_margin_std": 0.02259155735373497, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.3733450174331665, + "beta_dpo/gap_mean": 17.61815071105957, + "beta_dpo/gap_std": 24.498844146728516, + "beta_dpo/loss_margin_mean": 10.276666641235352, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7044595616024187, + "grad_norm": 0.8166243433952332, + "learning_rate": 1.2271688498291334e-07, + "logits/chosen": 0.47566479444503784, + "logits/rejected": 0.47687482833862305, + "loss": 1.3767, + "step": 466 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4960554540157318, + "beta_dpo/beta_margin_grad_std": 0.005727602168917656, + "beta_dpo/beta_margin_mean": 0.015781516209244728, + "beta_dpo/beta_margin_std": 0.022918064147233963, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.18097154796123505, + "beta_dpo/gap_mean": 16.883132934570312, + "beta_dpo/gap_std": 24.102888107299805, + "beta_dpo/loss_margin_mean": 15.781516075134277, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7059712773998488, + "grad_norm": 0.7626246213912964, + "learning_rate": 1.2158065210664848e-07, + "logits/chosen": 0.4934718608856201, + "logits/rejected": 0.3415928781032562, + "loss": 1.3742, + "step": 467 + }, + { + "beta_dpo/beta": 0.22503986954689026, + "beta_dpo/beta_margin_grad_mean": -0.2004920393228531, + "beta_dpo/beta_margin_grad_std": 0.31278786063194275, + "beta_dpo/beta_margin_mean": 5.361608505249023, + "beta_dpo/beta_margin_std": 6.801639080047607, + "beta_dpo/beta_used": 0.22503986954689026, + "beta_dpo/beta_used_raw": 0.22503986954689026, + "beta_dpo/gap_mean": 17.813079833984375, + "beta_dpo/gap_std": 24.297039031982422, + "beta_dpo/loss_margin_mean": 23.649864196777344, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7074829931972789, + "grad_norm": 142.0051727294922, + "learning_rate": 1.204480113956011e-07, + "logits/chosen": 0.42348283529281616, + "logits/rejected": 0.4103195369243622, + "loss": 0.7272, + "step": 468 + }, + { + "beta_dpo/beta": 0.3144519627094269, + "beta_dpo/beta_margin_grad_mean": -0.3136805295944214, + "beta_dpo/beta_margin_grad_std": 0.2636690139770508, + "beta_dpo/beta_margin_mean": 8.632575988769531, + "beta_dpo/beta_margin_std": 14.937712669372559, + "beta_dpo/beta_used": 0.3144519627094269, + "beta_dpo/beta_used_raw": 0.18009693920612335, + "beta_dpo/gap_mean": 18.668554306030273, + "beta_dpo/gap_std": 24.453214645385742, + "beta_dpo/loss_margin_mean": 19.205217361450195, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.708994708994709, + "grad_norm": 179.1083221435547, + "learning_rate": 1.1931899453216697e-07, + "logits/chosen": 0.5388230085372925, + "logits/rejected": 0.5226658582687378, + "loss": 0.8659, + "step": 469 + }, + { + "beta_dpo/beta": 0.18248410522937775, + "beta_dpo/beta_margin_grad_mean": -0.2651711404323578, + "beta_dpo/beta_margin_grad_std": 0.3218035399913788, + "beta_dpo/beta_margin_mean": 3.4488892555236816, + "beta_dpo/beta_margin_std": 5.133399486541748, + "beta_dpo/beta_used": 0.18248410522937775, + "beta_dpo/beta_used_raw": 0.18248410522937775, + "beta_dpo/gap_mean": 18.15283966064453, + "beta_dpo/gap_std": 24.360286712646484, + "beta_dpo/loss_margin_mean": 18.03568458557129, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7105064247921391, + "grad_norm": 71.76032257080078, + "learning_rate": 1.1819363309737438e-07, + "logits/chosen": 0.4892360270023346, + "logits/rejected": 0.4123338460922241, + "loss": 0.7933, + "step": 470 + }, + { + "beta_dpo/beta": 0.2866833209991455, + "beta_dpo/beta_margin_grad_mean": -0.19581058621406555, + "beta_dpo/beta_margin_grad_std": 0.3241235613822937, + "beta_dpo/beta_margin_mean": 6.425544261932373, + "beta_dpo/beta_margin_std": 7.508059501647949, + "beta_dpo/beta_used": 0.2866833209991455, + "beta_dpo/beta_used_raw": 0.2866833209991455, + "beta_dpo/gap_mean": 18.825267791748047, + "beta_dpo/gap_std": 24.537582397460938, + "beta_dpo/loss_margin_mean": 22.44715118408203, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7120181405895691, + "grad_norm": 131.3478240966797, + "learning_rate": 1.1707195857000215e-07, + "logits/chosen": 0.46292543411254883, + "logits/rejected": 0.3984526991844177, + "loss": 0.7009, + "step": 471 + }, + { + "beta_dpo/beta": 0.07434147596359253, + "beta_dpo/beta_margin_grad_mean": -0.38497617840766907, + "beta_dpo/beta_margin_grad_std": 0.2753157615661621, + "beta_dpo/beta_margin_mean": 1.7048362493515015, + "beta_dpo/beta_margin_std": 3.7578043937683105, + "beta_dpo/beta_used": 0.07434147596359253, + "beta_dpo/beta_used_raw": -0.11290460079908371, + "beta_dpo/gap_mean": 18.724979400634766, + "beta_dpo/gap_std": 25.190521240234375, + "beta_dpo/loss_margin_mean": 18.318096160888672, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7135298563869993, + "grad_norm": 63.933082580566406, + "learning_rate": 1.1595400232569768e-07, + "logits/chosen": 0.48472386598587036, + "logits/rejected": 0.43443506956100464, + "loss": 1.227, + "step": 472 + }, + { + "beta_dpo/beta": 0.00586696295067668, + "beta_dpo/beta_margin_grad_mean": -0.4737027585506439, + "beta_dpo/beta_margin_grad_std": 0.05226682126522064, + "beta_dpo/beta_margin_mean": 0.10760366171598434, + "beta_dpo/beta_margin_std": 0.21496045589447021, + "beta_dpo/beta_used": 0.00586696295067668, + "beta_dpo/beta_used_raw": 0.00586696295067668, + "beta_dpo/gap_mean": 18.903789520263672, + "beta_dpo/gap_std": 26.43364906311035, + "beta_dpo/loss_margin_mean": 18.546789169311523, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7150415721844293, + "grad_norm": 5.026509761810303, + "learning_rate": 1.1483979563610069e-07, + "logits/chosen": 0.5756895542144775, + "logits/rejected": 0.4601627588272095, + "loss": 1.2965, + "step": 473 + }, + { + "beta_dpo/beta": 0.06509552150964737, + "beta_dpo/beta_margin_grad_mean": -0.33907976746559143, + "beta_dpo/beta_margin_grad_std": 0.2729524075984955, + "beta_dpo/beta_margin_mean": 1.1306818723678589, + "beta_dpo/beta_margin_std": 1.8167601823806763, + "beta_dpo/beta_used": 0.06509552150964737, + "beta_dpo/beta_used_raw": 0.06509552150964737, + "beta_dpo/gap_mean": 18.619977951049805, + "beta_dpo/gap_std": 26.78903579711914, + "beta_dpo/loss_margin_mean": 17.39017105102539, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7165532879818595, + "grad_norm": 44.54093551635742, + "learning_rate": 1.1372936966796709e-07, + "logits/chosen": 0.516582727432251, + "logits/rejected": 0.4398994445800781, + "loss": 0.9809, + "step": 474 + }, + { + "beta_dpo/beta": 0.26542985439300537, + "beta_dpo/beta_margin_grad_mean": -0.17872604727745056, + "beta_dpo/beta_margin_grad_std": 0.28388726711273193, + "beta_dpo/beta_margin_mean": 7.227434158325195, + "beta_dpo/beta_margin_std": 8.638694763183594, + "beta_dpo/beta_used": 0.26542985439300537, + "beta_dpo/beta_used_raw": 0.26542985439300537, + "beta_dpo/gap_mean": 19.758541107177734, + "beta_dpo/gap_std": 26.653968811035156, + "beta_dpo/loss_margin_mean": 25.69193458557129, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7180650037792895, + "grad_norm": 104.1346435546875, + "learning_rate": 1.126227554822985e-07, + "logits/chosen": 0.45388850569725037, + "logits/rejected": 0.3974682092666626, + "loss": 0.5859, + "step": 475 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4959273040294647, + "beta_dpo/beta_margin_grad_std": 0.006980855017900467, + "beta_dpo/beta_margin_mean": 0.016294946894049644, + "beta_dpo/beta_margin_std": 0.027931276708841324, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.11508934199810028, + "beta_dpo/gap_mean": 19.323762893676758, + "beta_dpo/gap_std": 26.89731216430664, + "beta_dpo/loss_margin_mean": 16.294946670532227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7195767195767195, + "grad_norm": 0.8413127660751343, + "learning_rate": 1.1151998403347243e-07, + "logits/chosen": 0.4007229208946228, + "logits/rejected": 0.3899179995059967, + "loss": 1.3708, + "step": 476 + }, + { + "beta_dpo/beta": 0.05996621027588844, + "beta_dpo/beta_margin_grad_mean": -0.4041517376899719, + "beta_dpo/beta_margin_grad_std": 0.2603454887866974, + "beta_dpo/beta_margin_mean": 0.9991706609725952, + "beta_dpo/beta_margin_std": 2.668644905090332, + "beta_dpo/beta_used": 0.05996621027588844, + "beta_dpo/beta_used_raw": -0.10893569886684418, + "beta_dpo/gap_mean": 18.810359954833984, + "beta_dpo/gap_std": 27.02542495727539, + "beta_dpo/loss_margin_mean": 16.357866287231445, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7210884353741497, + "grad_norm": 59.0564079284668, + "learning_rate": 1.1042108616837692e-07, + "logits/chosen": 0.4921807050704956, + "logits/rejected": 0.44167813658714294, + "loss": 1.2437, + "step": 477 + }, + { + "beta_dpo/beta": 0.058396078646183014, + "beta_dpo/beta_margin_grad_mean": -0.4280129373073578, + "beta_dpo/beta_margin_grad_std": 0.24085818231105804, + "beta_dpo/beta_margin_mean": 0.7688745856285095, + "beta_dpo/beta_margin_std": 2.358423948287964, + "beta_dpo/beta_used": 0.058396078646183014, + "beta_dpo/beta_used_raw": -0.015529513359069824, + "beta_dpo/gap_mean": 18.31366729736328, + "beta_dpo/gap_std": 27.302425384521484, + "beta_dpo/loss_margin_mean": 15.358002662658691, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7226001511715797, + "grad_norm": 29.725767135620117, + "learning_rate": 1.0932609262554746e-07, + "logits/chosen": 0.3961327075958252, + "logits/rejected": 0.39685729146003723, + "loss": 1.128, + "step": 478 + }, + { + "beta_dpo/beta": 0.0391259491443634, + "beta_dpo/beta_margin_grad_mean": -0.41009682416915894, + "beta_dpo/beta_margin_grad_std": 0.1961776167154312, + "beta_dpo/beta_margin_mean": 0.6494525074958801, + "beta_dpo/beta_margin_std": 1.492968201637268, + "beta_dpo/beta_used": 0.0391259491443634, + "beta_dpo/beta_used_raw": -0.12356161326169968, + "beta_dpo/gap_mean": 17.511415481567383, + "beta_dpo/gap_std": 27.055574417114258, + "beta_dpo/loss_margin_mean": 13.357870101928711, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7241118669690099, + "grad_norm": 30.614665985107422, + "learning_rate": 1.0823503403430734e-07, + "logits/chosen": 0.37263572216033936, + "logits/rejected": 0.31845974922180176, + "loss": 1.1149, + "step": 479 + }, + { + "beta_dpo/beta": 0.29351285099983215, + "beta_dpo/beta_margin_grad_mean": -0.24419361352920532, + "beta_dpo/beta_margin_grad_std": 0.38002249598503113, + "beta_dpo/beta_margin_mean": 6.007750511169434, + "beta_dpo/beta_margin_std": 9.08305549621582, + "beta_dpo/beta_used": 0.29351285099983215, + "beta_dpo/beta_used_raw": 0.29351285099983215, + "beta_dpo/gap_mean": 17.11019515991211, + "beta_dpo/gap_std": 27.254867553710938, + "beta_dpo/loss_margin_mean": 18.857349395751953, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7256235827664399, + "grad_norm": 171.7132110595703, + "learning_rate": 1.0714794091391072e-07, + "logits/chosen": 0.3939332962036133, + "logits/rejected": 0.37893810868263245, + "loss": 1.2783, + "step": 480 + }, + { + "beta_dpo/beta": 0.19773970544338226, + "beta_dpo/beta_margin_grad_mean": -0.24991479516029358, + "beta_dpo/beta_margin_grad_std": 0.3319016098976135, + "beta_dpo/beta_margin_mean": 4.109105587005615, + "beta_dpo/beta_margin_std": 5.891486167907715, + "beta_dpo/beta_used": 0.19773970544338226, + "beta_dpo/beta_used_raw": 0.19773970544338226, + "beta_dpo/gap_mean": 17.974807739257812, + "beta_dpo/gap_std": 27.388599395751953, + "beta_dpo/loss_margin_mean": 20.404041290283203, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.72713529856387, + "grad_norm": 70.31513977050781, + "learning_rate": 1.0606484367268906e-07, + "logits/chosen": 0.37382426857948303, + "logits/rejected": 0.36571288108825684, + "loss": 0.7294, + "step": 481 + }, + { + "beta_dpo/beta": 0.0590001605451107, + "beta_dpo/beta_margin_grad_mean": -0.42749252915382385, + "beta_dpo/beta_margin_grad_std": 0.26465827226638794, + "beta_dpo/beta_margin_mean": 0.7799001336097717, + "beta_dpo/beta_margin_std": 3.059889078140259, + "beta_dpo/beta_used": 0.0590001605451107, + "beta_dpo/beta_used_raw": -0.014395486563444138, + "beta_dpo/gap_mean": 17.67986297607422, + "beta_dpo/gap_std": 28.380046844482422, + "beta_dpo/loss_margin_mean": 16.58453941345215, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7286470143613001, + "grad_norm": 52.24790954589844, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": 0.4137299656867981, + "logits/rejected": 0.26786112785339355, + "loss": 1.1551, + "step": 482 + }, + { + "beta_dpo/beta": 0.09157513827085495, + "beta_dpo/beta_margin_grad_mean": -0.3851158022880554, + "beta_dpo/beta_margin_grad_std": 0.2592366337776184, + "beta_dpo/beta_margin_mean": 1.9007309675216675, + "beta_dpo/beta_margin_std": 3.915107250213623, + "beta_dpo/beta_used": 0.09157513827085495, + "beta_dpo/beta_used_raw": 0.06725367903709412, + "beta_dpo/gap_mean": 18.100704193115234, + "beta_dpo/gap_std": 28.369171142578125, + "beta_dpo/loss_margin_mean": 20.073625564575195, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7301587301587301, + "grad_norm": 72.34899139404297, + "learning_rate": 1.0391075790138232e-07, + "logits/chosen": 0.5550791025161743, + "logits/rejected": 0.44513243436813354, + "loss": 1.1065, + "step": 483 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49605247378349304, + "beta_dpo/beta_margin_grad_std": 0.005534137133508921, + "beta_dpo/beta_margin_mean": 0.015792880207300186, + "beta_dpo/beta_margin_std": 0.022141609340906143, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.11549272388219833, + "beta_dpo/gap_mean": 17.736948013305664, + "beta_dpo/gap_std": 27.42501449584961, + "beta_dpo/loss_margin_mean": 15.792879104614258, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7316704459561603, + "grad_norm": 0.9392448663711548, + "learning_rate": 1.0283982962570681e-07, + "logits/chosen": 0.5212189555168152, + "logits/rejected": 0.47851696610450745, + "loss": 1.3723, + "step": 484 + }, + { + "beta_dpo/beta": 0.25393855571746826, + "beta_dpo/beta_margin_grad_mean": -0.3070473074913025, + "beta_dpo/beta_margin_grad_std": 0.2632943391799927, + "beta_dpo/beta_margin_mean": 6.5717573165893555, + "beta_dpo/beta_margin_std": 11.246562004089355, + "beta_dpo/beta_used": 0.25393855571746826, + "beta_dpo/beta_used_raw": 0.18657520413398743, + "beta_dpo/gap_mean": 18.274065017700195, + "beta_dpo/gap_std": 26.938018798828125, + "beta_dpo/loss_margin_mean": 18.81686019897461, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7331821617535903, + "grad_norm": 198.32473754882812, + "learning_rate": 1.0177301773633992e-07, + "logits/chosen": 0.4765240252017975, + "logits/rejected": 0.4477100670337677, + "loss": 1.1238, + "step": 485 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4957917332649231, + "beta_dpo/beta_margin_grad_std": 0.007428720127791166, + "beta_dpo/beta_margin_mean": 0.016838233917951584, + "beta_dpo/beta_margin_std": 0.029725831001996994, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.1067897230386734, + "beta_dpo/gap_mean": 17.730863571166992, + "beta_dpo/gap_std": 27.222911834716797, + "beta_dpo/loss_margin_mean": 16.838232040405273, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7346938775510204, + "grad_norm": 0.756230354309082, + "learning_rate": 1.007103520743035e-07, + "logits/chosen": 0.5120325088500977, + "logits/rejected": 0.3889530897140503, + "loss": 1.3722, + "step": 486 + }, + { + "beta_dpo/beta": 0.026900505647063255, + "beta_dpo/beta_margin_grad_mean": -0.4126649498939514, + "beta_dpo/beta_margin_grad_std": 0.15782445669174194, + "beta_dpo/beta_margin_mean": 0.46885383129119873, + "beta_dpo/beta_margin_std": 0.9157174825668335, + "beta_dpo/beta_used": 0.026900505647063255, + "beta_dpo/beta_used_raw": -0.053003422915935516, + "beta_dpo/gap_mean": 17.832361221313477, + "beta_dpo/gap_std": 27.04258155822754, + "beta_dpo/loss_margin_mean": 18.905290603637695, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7362055933484505, + "grad_norm": 22.816574096679688, + "learning_rate": 9.965186236464046e-08, + "logits/chosen": 0.5597760677337646, + "logits/rejected": 0.49492278695106506, + "loss": 1.0969, + "step": 487 + }, + { + "beta_dpo/beta": 0.2829245328903198, + "beta_dpo/beta_margin_grad_mean": -0.2355988472700119, + "beta_dpo/beta_margin_grad_std": 0.29733532667160034, + "beta_dpo/beta_margin_mean": 7.418085098266602, + "beta_dpo/beta_margin_std": 11.626276969909668, + "beta_dpo/beta_used": 0.2829245328903198, + "beta_dpo/beta_used_raw": 0.2829245328903198, + "beta_dpo/gap_mean": 18.307497024536133, + "beta_dpo/gap_std": 27.187271118164062, + "beta_dpo/loss_margin_mean": 22.07598114013672, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7377173091458806, + "grad_norm": 168.8614959716797, + "learning_rate": 9.859757821558337e-08, + "logits/chosen": 0.4678114354610443, + "logits/rejected": 0.392938494682312, + "loss": 1.3132, + "step": 488 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4974829852581024, + "beta_dpo/beta_margin_grad_std": 0.006341965869069099, + "beta_dpo/beta_margin_mean": 0.010070238262414932, + "beta_dpo/beta_margin_std": 0.025372974574565887, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.29972267150878906, + "beta_dpo/gap_mean": 17.63011932373047, + "beta_dpo/gap_std": 27.038352966308594, + "beta_dpo/loss_margin_mean": 10.070237159729004, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7392290249433107, + "grad_norm": 0.797245442867279, + "learning_rate": 9.754752911772615e-08, + "logits/chosen": 0.47051212191581726, + "logits/rejected": 0.42137840390205383, + "loss": 1.3755, + "step": 489 + }, + { + "beta_dpo/beta": 0.33493655920028687, + "beta_dpo/beta_margin_grad_mean": -0.34081602096557617, + "beta_dpo/beta_margin_grad_std": 0.3390964865684509, + "beta_dpo/beta_margin_mean": 5.45980167388916, + "beta_dpo/beta_margin_std": 16.640560150146484, + "beta_dpo/beta_used": 0.33493655920028687, + "beta_dpo/beta_used_raw": 0.33493655920028687, + "beta_dpo/gap_mean": 16.79808807373047, + "beta_dpo/gap_std": 27.53335189819336, + "beta_dpo/loss_margin_mean": 15.238967895507812, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7407407407407407, + "grad_norm": 157.81983947753906, + "learning_rate": 9.650174444319956e-08, + "logits/chosen": 0.5644317865371704, + "logits/rejected": 0.5341103672981262, + "loss": 1.108, + "step": 490 + }, + { + "beta_dpo/beta": 0.153602734208107, + "beta_dpo/beta_margin_grad_mean": -0.3420948386192322, + "beta_dpo/beta_margin_grad_std": 0.27578026056289673, + "beta_dpo/beta_margin_mean": 3.6059417724609375, + "beta_dpo/beta_margin_std": 6.100773811340332, + "beta_dpo/beta_used": 0.153602734208107, + "beta_dpo/beta_used_raw": 0.06139887124300003, + "beta_dpo/gap_mean": 17.124771118164062, + "beta_dpo/gap_std": 27.31386375427246, + "beta_dpo/loss_margin_mean": 17.191701889038086, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7422524565381708, + "grad_norm": 97.61920928955078, + "learning_rate": 9.546025344484868e-08, + "logits/chosen": 0.4223695993423462, + "logits/rejected": 0.36396944522857666, + "loss": 1.275, + "step": 491 + }, + { + "beta_dpo/beta": 0.11434933543205261, + "beta_dpo/beta_margin_grad_mean": -0.37879207730293274, + "beta_dpo/beta_margin_grad_std": 0.2923412024974823, + "beta_dpo/beta_margin_mean": 2.2623884677886963, + "beta_dpo/beta_margin_std": 4.472981929779053, + "beta_dpo/beta_used": 0.11434933543205261, + "beta_dpo/beta_used_raw": -0.050521112978458405, + "beta_dpo/gap_mean": 16.184886932373047, + "beta_dpo/gap_std": 27.43224334716797, + "beta_dpo/loss_margin_mean": 14.301737785339355, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7437641723356009, + "grad_norm": 91.39317321777344, + "learning_rate": 9.442308525541589e-08, + "logits/chosen": 0.42874184250831604, + "logits/rejected": 0.3520268201828003, + "loss": 1.368, + "step": 492 + }, + { + "beta_dpo/beta": 0.19871431589126587, + "beta_dpo/beta_margin_grad_mean": -0.23222456872463226, + "beta_dpo/beta_margin_grad_std": 0.2830079197883606, + "beta_dpo/beta_margin_mean": 4.435938358306885, + "beta_dpo/beta_margin_std": 7.169334888458252, + "beta_dpo/beta_used": 0.19871431589126587, + "beta_dpo/beta_used_raw": 0.19871431589126587, + "beta_dpo/gap_mean": 16.834693908691406, + "beta_dpo/gap_std": 27.216405868530273, + "beta_dpo/loss_margin_mean": 20.395715713500977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.745275888133031, + "grad_norm": 120.1291732788086, + "learning_rate": 9.339026888672468e-08, + "logits/chosen": 0.42790913581848145, + "logits/rejected": 0.34174275398254395, + "loss": 0.74, + "step": 493 + }, + { + "beta_dpo/beta": 0.04424438625574112, + "beta_dpo/beta_margin_grad_mean": -0.3808707594871521, + "beta_dpo/beta_margin_grad_std": 0.22228094935417175, + "beta_dpo/beta_margin_mean": 0.6535773277282715, + "beta_dpo/beta_margin_std": 1.5111050605773926, + "beta_dpo/beta_used": 0.04424438625574112, + "beta_dpo/beta_used_raw": 0.04424438625574112, + "beta_dpo/gap_mean": 17.077911376953125, + "beta_dpo/gap_std": 27.674633026123047, + "beta_dpo/loss_margin_mean": 15.817744255065918, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7467876039304611, + "grad_norm": 32.58065414428711, + "learning_rate": 9.236183322886945e-08, + "logits/chosen": 0.3619958758354187, + "logits/rejected": 0.3028886318206787, + "loss": 0.9887, + "step": 494 + }, + { + "beta_dpo/beta": 0.12880435585975647, + "beta_dpo/beta_margin_grad_mean": -0.3734891712665558, + "beta_dpo/beta_margin_grad_std": 0.2588602602481842, + "beta_dpo/beta_margin_mean": 2.2595906257629395, + "beta_dpo/beta_margin_std": 5.230336666107178, + "beta_dpo/beta_used": 0.12880435585975647, + "beta_dpo/beta_used_raw": -0.13839080929756165, + "beta_dpo/gap_mean": 16.387245178222656, + "beta_dpo/gap_std": 27.691650390625, + "beta_dpo/loss_margin_mean": 14.412410736083984, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7482993197278912, + "grad_norm": 99.71451568603516, + "learning_rate": 9.133780704940594e-08, + "logits/chosen": 0.4950551986694336, + "logits/rejected": 0.4249255061149597, + "loss": 1.0656, + "step": 495 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49576571583747864, + "beta_dpo/beta_margin_grad_std": 0.007818019017577171, + "beta_dpo/beta_margin_mean": 0.01694556325674057, + "beta_dpo/beta_margin_std": 0.0312989242374897, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.1241304874420166, + "beta_dpo/gap_mean": 16.68515968322754, + "beta_dpo/gap_std": 28.40119171142578, + "beta_dpo/loss_margin_mean": 16.9455623626709, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7498110355253212, + "grad_norm": 0.8398783802986145, + "learning_rate": 9.031821899254797e-08, + "logits/chosen": 0.4990454316139221, + "logits/rejected": 0.3839772939682007, + "loss": 1.3735, + "step": 496 + }, + { + "beta_dpo/beta": 0.19726786017417908, + "beta_dpo/beta_margin_grad_mean": -0.34216219186782837, + "beta_dpo/beta_margin_grad_std": 0.28134962916374207, + "beta_dpo/beta_margin_mean": 4.7668657302856445, + "beta_dpo/beta_margin_std": 9.81608772277832, + "beta_dpo/beta_used": 0.19726786017417908, + "beta_dpo/beta_used_raw": 0.19726786017417908, + "beta_dpo/gap_mean": 17.335676193237305, + "beta_dpo/gap_std": 28.46596908569336, + "beta_dpo/loss_margin_mean": 21.315322875976562, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7513227513227513, + "grad_norm": 126.32473754882812, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 0.4910498857498169, + "logits/rejected": 0.45661675930023193, + "loss": 1.6257, + "step": 497 + }, + { + "beta_dpo/beta": 0.1522623747587204, + "beta_dpo/beta_margin_grad_mean": -0.2758595049381256, + "beta_dpo/beta_margin_grad_std": 0.29086124897003174, + "beta_dpo/beta_margin_mean": 3.130350112915039, + "beta_dpo/beta_margin_std": 5.023598670959473, + "beta_dpo/beta_used": 0.1522623747587204, + "beta_dpo/beta_used_raw": 0.1522623747587204, + "beta_dpo/gap_mean": 17.873306274414062, + "beta_dpo/gap_std": 28.321250915527344, + "beta_dpo/loss_margin_mean": 20.654321670532227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7528344671201814, + "grad_norm": 66.43721771240234, + "learning_rate": 8.829247120198563e-08, + "logits/chosen": 0.47693729400634766, + "logits/rejected": 0.43756186962127686, + "loss": 0.7606, + "step": 498 + }, + { + "beta_dpo/beta": 0.15835049748420715, + "beta_dpo/beta_margin_grad_mean": -0.39443886280059814, + "beta_dpo/beta_margin_grad_std": 0.31629934906959534, + "beta_dpo/beta_margin_mean": 3.0014355182647705, + "beta_dpo/beta_margin_std": 8.597198486328125, + "beta_dpo/beta_used": 0.15835049748420715, + "beta_dpo/beta_used_raw": 0.027399331331253052, + "beta_dpo/gap_mean": 17.992496490478516, + "beta_dpo/gap_std": 29.04220199584961, + "beta_dpo/loss_margin_mean": 17.409122467041016, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7543461829176115, + "grad_norm": 124.17149353027344, + "learning_rate": 8.728636813280163e-08, + "logits/chosen": 0.47000688314437866, + "logits/rejected": 0.398383766412735, + "loss": 1.6238, + "step": 499 + }, + { + "beta_dpo/beta": 0.24701504409313202, + "beta_dpo/beta_margin_grad_mean": -0.2595021426677704, + "beta_dpo/beta_margin_grad_std": 0.348397821187973, + "beta_dpo/beta_margin_mean": 4.565580368041992, + "beta_dpo/beta_margin_std": 8.080002784729004, + "beta_dpo/beta_used": 0.24701504409313202, + "beta_dpo/beta_used_raw": 0.24701504409313202, + "beta_dpo/gap_mean": 17.846397399902344, + "beta_dpo/gap_std": 28.710865020751953, + "beta_dpo/loss_margin_mean": 17.975915908813477, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7558578987150416, + "grad_norm": 142.9105224609375, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": 0.384491503238678, + "logits/rejected": 0.3731864094734192, + "loss": 1.1911, + "step": 500 + }, + { + "epoch": 0.7558578987150416, + "eval_beta_dpo/beta": 0.09361638128757477, + "eval_beta_dpo/beta_margin_grad_mean": -0.38664332032203674, + "eval_beta_dpo/beta_margin_grad_std": 0.14713595807552338, + "eval_beta_dpo/beta_margin_mean": 1.9619940519332886, + "eval_beta_dpo/beta_margin_std": 2.572732925415039, + "eval_beta_dpo/beta_used": 0.09361638128757477, + "eval_beta_dpo/beta_used_raw": -0.011094304732978344, + "eval_beta_dpo/gap_mean": 17.90873146057129, + "eval_beta_dpo/gap_std": 28.716110229492188, + "eval_beta_dpo/loss_margin_mean": 16.057159423828125, + "eval_beta_dpo/mask_keep_frac": 1.0, + "eval_logits/chosen": 0.502650797367096, + "eval_logits/rejected": 0.44902658462524414, + "eval_loss": 0.6888377070426941, + "eval_runtime": 38.9533, + "eval_samples_per_second": 59.122, + "eval_steps_per_second": 1.848, + "step": 500 + }, + { + "beta_dpo/beta": 0.06464043259620667, + "beta_dpo/beta_margin_grad_mean": -0.37159132957458496, + "beta_dpo/beta_margin_grad_std": 0.2065133899450302, + "beta_dpo/beta_margin_mean": 1.2337942123413086, + "beta_dpo/beta_margin_std": 2.555989980697632, + "beta_dpo/beta_used": 0.06464043259620667, + "beta_dpo/beta_used_raw": 0.046276845037937164, + "beta_dpo/gap_mean": 17.642791748046875, + "beta_dpo/gap_std": 27.331443786621094, + "beta_dpo/loss_margin_mean": 16.796661376953125, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7573696145124716, + "grad_norm": 34.865447998046875, + "learning_rate": 8.528784436016878e-08, + "logits/chosen": 0.462974488735199, + "logits/rejected": 0.45432478189468384, + "loss": 1.0036, + "step": 501 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4961468577384949, + "beta_dpo/beta_margin_grad_std": 0.005573183763772249, + "beta_dpo/beta_margin_mean": 0.015415522269904613, + "beta_dpo/beta_margin_std": 0.02229905314743519, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.06323258578777313, + "beta_dpo/gap_mean": 17.295621871948242, + "beta_dpo/gap_std": 26.546051025390625, + "beta_dpo/loss_margin_mean": 15.415521621704102, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7588813303099018, + "grad_norm": 1.0474236011505127, + "learning_rate": 8.4295479559726e-08, + "logits/chosen": 0.4636223614215851, + "logits/rejected": 0.41382068395614624, + "loss": 1.3718, + "step": 502 + }, + { + "beta_dpo/beta": 0.22760546207427979, + "beta_dpo/beta_margin_grad_mean": -0.30316370725631714, + "beta_dpo/beta_margin_grad_std": 0.31399786472320557, + "beta_dpo/beta_margin_mean": 4.0226945877075195, + "beta_dpo/beta_margin_std": 9.152859687805176, + "beta_dpo/beta_used": 0.22760546207427979, + "beta_dpo/beta_used_raw": 0.22760546207427979, + "beta_dpo/gap_mean": 17.311302185058594, + "beta_dpo/gap_std": 26.70954132080078, + "beta_dpo/loss_margin_mean": 17.106420516967773, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7603930461073318, + "grad_norm": 170.26931762695312, + "learning_rate": 8.330774987092712e-08, + "logits/chosen": 0.474370539188385, + "logits/rejected": 0.4603527784347534, + "loss": 1.6266, + "step": 503 + }, + { + "beta_dpo/beta": 0.14955301582813263, + "beta_dpo/beta_margin_grad_mean": -0.2983316481113434, + "beta_dpo/beta_margin_grad_std": 0.24636363983154297, + "beta_dpo/beta_margin_mean": 4.377803325653076, + "beta_dpo/beta_margin_std": 6.885531425476074, + "beta_dpo/beta_used": 0.14955301582813263, + "beta_dpo/beta_used_raw": 0.1396861970424652, + "beta_dpo/gap_mean": 17.946958541870117, + "beta_dpo/gap_std": 26.08230209350586, + "beta_dpo/loss_margin_mean": 23.741315841674805, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7619047619047619, + "grad_norm": 36.053138732910156, + "learning_rate": 8.232468292269479e-08, + "logits/chosen": 0.42812907695770264, + "logits/rejected": 0.39763376116752625, + "loss": 0.8169, + "step": 504 + }, + { + "beta_dpo/beta": 0.1439221203327179, + "beta_dpo/beta_margin_grad_mean": -0.3790625035762787, + "beta_dpo/beta_margin_grad_std": 0.2830526828765869, + "beta_dpo/beta_margin_mean": 3.2106552124023438, + "beta_dpo/beta_margin_std": 7.093870639801025, + "beta_dpo/beta_used": 0.1439221203327179, + "beta_dpo/beta_used_raw": 0.010216429829597473, + "beta_dpo/gap_mean": 18.527366638183594, + "beta_dpo/gap_std": 26.727075576782227, + "beta_dpo/loss_margin_mean": 16.65737533569336, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.763416477702192, + "grad_norm": 104.89347839355469, + "learning_rate": 8.134630621352483e-08, + "logits/chosen": 0.46603691577911377, + "logits/rejected": 0.42604219913482666, + "loss": 1.4932, + "step": 505 + }, + { + "beta_dpo/beta": 0.14408646523952484, + "beta_dpo/beta_margin_grad_mean": -0.38642263412475586, + "beta_dpo/beta_margin_grad_std": 0.2999417781829834, + "beta_dpo/beta_margin_mean": 2.622800588607788, + "beta_dpo/beta_margin_std": 6.82088565826416, + "beta_dpo/beta_used": 0.14408646523952484, + "beta_dpo/beta_used_raw": 0.141511932015419, + "beta_dpo/gap_mean": 17.85563850402832, + "beta_dpo/gap_std": 27.261791229248047, + "beta_dpo/loss_margin_mean": 15.349295616149902, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.764928193499622, + "grad_norm": 146.69357299804688, + "learning_rate": 8.037264711071698e-08, + "logits/chosen": 0.5133321285247803, + "logits/rejected": 0.4881846308708191, + "loss": 2.0045, + "step": 506 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4952058494091034, + "beta_dpo/beta_margin_grad_std": 0.006824954878538847, + "beta_dpo/beta_margin_mean": 0.019181732088327408, + "beta_dpo/beta_margin_std": 0.027308695018291473, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.14111196994781494, + "beta_dpo/gap_mean": 17.88521957397461, + "beta_dpo/gap_std": 27.419790267944336, + "beta_dpo/loss_margin_mean": 19.181730270385742, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7664399092970522, + "grad_norm": 0.7970109581947327, + "learning_rate": 7.940373284960933e-08, + "logits/chosen": 0.4557326138019562, + "logits/rejected": 0.40037935972213745, + "loss": 1.3726, + "step": 507 + }, + { + "beta_dpo/beta": 0.27150821685791016, + "beta_dpo/beta_margin_grad_mean": -0.3548518717288971, + "beta_dpo/beta_margin_grad_std": 0.2952079772949219, + "beta_dpo/beta_margin_mean": 6.855192184448242, + "beta_dpo/beta_margin_std": 12.318726539611816, + "beta_dpo/beta_used": 0.27150821685791016, + "beta_dpo/beta_used_raw": 0.22873634099960327, + "beta_dpo/gap_mean": 18.09201431274414, + "beta_dpo/gap_std": 27.218570709228516, + "beta_dpo/loss_margin_mean": 20.832242965698242, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7679516250944822, + "grad_norm": 206.41529846191406, + "learning_rate": 7.843959053281663e-08, + "logits/chosen": 0.43897074460983276, + "logits/rejected": 0.3105481266975403, + "loss": 1.5294, + "step": 508 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49585264921188354, + "beta_dpo/beta_margin_grad_std": 0.006084589287638664, + "beta_dpo/beta_margin_mean": 0.01659206487238407, + "beta_dpo/beta_margin_std": 0.02434265986084938, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.05793162062764168, + "beta_dpo/gap_mean": 18.304162979125977, + "beta_dpo/gap_std": 26.860179901123047, + "beta_dpo/loss_margin_mean": 16.592063903808594, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7694633408919124, + "grad_norm": 0.9970650672912598, + "learning_rate": 7.748024712947204e-08, + "logits/chosen": 0.3953160345554352, + "logits/rejected": 0.368834912776947, + "loss": 1.3708, + "step": 509 + }, + { + "beta_dpo/beta": 0.02194332703948021, + "beta_dpo/beta_margin_grad_mean": -0.41290098428726196, + "beta_dpo/beta_margin_grad_std": 0.17370301485061646, + "beta_dpo/beta_margin_mean": 0.48397937417030334, + "beta_dpo/beta_margin_std": 1.0054277181625366, + "beta_dpo/beta_used": 0.02194332703948021, + "beta_dpo/beta_used_raw": -0.08349817991256714, + "beta_dpo/gap_mean": 18.485971450805664, + "beta_dpo/gap_std": 27.07589340209961, + "beta_dpo/loss_margin_mean": 20.233915328979492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7709750566893424, + "grad_norm": 20.405038833618164, + "learning_rate": 7.652572947447272e-08, + "logits/chosen": 0.5540110468864441, + "logits/rejected": 0.4546312093734741, + "loss": 1.1753, + "step": 510 + }, + { + "beta_dpo/beta": 0.0254156943410635, + "beta_dpo/beta_margin_grad_mean": -0.4023872911930084, + "beta_dpo/beta_margin_grad_std": 0.19661438465118408, + "beta_dpo/beta_margin_mean": 0.6091973185539246, + "beta_dpo/beta_margin_std": 1.233200192451477, + "beta_dpo/beta_used": 0.0254156943410635, + "beta_dpo/beta_used_raw": 0.0013822559267282486, + "beta_dpo/gap_mean": 19.057842254638672, + "beta_dpo/gap_std": 27.40789794921875, + "beta_dpo/loss_margin_mean": 22.84357261657715, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7724867724867724, + "grad_norm": 21.664785385131836, + "learning_rate": 7.557606426772961e-08, + "logits/chosen": 0.47340700030326843, + "logits/rejected": 0.4142376780509949, + "loss": 1.1929, + "step": 511 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4957908093929291, + "beta_dpo/beta_margin_grad_std": 0.00741164106875658, + "beta_dpo/beta_margin_mean": 0.0168411023914814, + "beta_dpo/beta_margin_std": 0.02965443953871727, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.09359487146139145, + "beta_dpo/gap_mean": 18.986900329589844, + "beta_dpo/gap_std": 27.84600257873535, + "beta_dpo/loss_margin_mean": 16.841102600097656, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7739984882842026, + "grad_norm": 2.918966770172119, + "learning_rate": 7.463127807341966e-08, + "logits/chosen": 0.31771016120910645, + "logits/rejected": 0.3009248971939087, + "loss": 1.3707, + "step": 512 + }, + { + "beta_dpo/beta": 0.08458594977855682, + "beta_dpo/beta_margin_grad_mean": -0.3096770644187927, + "beta_dpo/beta_margin_grad_std": 0.22452546656131744, + "beta_dpo/beta_margin_mean": 1.4769704341888428, + "beta_dpo/beta_margin_std": 2.3516552448272705, + "beta_dpo/beta_used": 0.08458594977855682, + "beta_dpo/beta_used_raw": 0.08458594977855682, + "beta_dpo/gap_mean": 19.021957397460938, + "beta_dpo/gap_std": 27.71468734741211, + "beta_dpo/loss_margin_mean": 19.09242057800293, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7755102040816326, + "grad_norm": 34.026527404785156, + "learning_rate": 7.369139731924401e-08, + "logits/chosen": 0.5480648279190063, + "logits/rejected": 0.4947201609611511, + "loss": 0.7569, + "step": 513 + }, + { + "beta_dpo/beta": 0.41926461458206177, + "beta_dpo/beta_margin_grad_mean": -0.21048571169376373, + "beta_dpo/beta_margin_grad_std": 0.3752746284008026, + "beta_dpo/beta_margin_mean": 9.245790481567383, + "beta_dpo/beta_margin_std": 11.778825759887695, + "beta_dpo/beta_used": 0.41926461458206177, + "beta_dpo/beta_used_raw": 0.41926461458206177, + "beta_dpo/gap_mean": 19.16301727294922, + "beta_dpo/gap_std": 27.396949768066406, + "beta_dpo/loss_margin_mean": 21.85741424560547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7770219198790628, + "grad_norm": 178.23617553710938, + "learning_rate": 7.275644829568747e-08, + "logits/chosen": 0.5184712409973145, + "logits/rejected": 0.4733770787715912, + "loss": 1.0529, + "step": 514 + }, + { + "beta_dpo/beta": 0.03268057852983475, + "beta_dpo/beta_margin_grad_mean": -0.3948572278022766, + "beta_dpo/beta_margin_grad_std": 0.1987946331501007, + "beta_dpo/beta_margin_mean": 0.6988779902458191, + "beta_dpo/beta_margin_std": 1.3777862787246704, + "beta_dpo/beta_used": 0.03268057852983475, + "beta_dpo/beta_used_raw": 0.0016238931566476822, + "beta_dpo/gap_mean": 19.57135772705078, + "beta_dpo/gap_std": 27.504430770874023, + "beta_dpo/loss_margin_mean": 19.71489143371582, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7785336356764928, + "grad_norm": 30.562400817871094, + "learning_rate": 7.182645715528435e-08, + "logits/chosen": 0.46997085213661194, + "logits/rejected": 0.3870126008987427, + "loss": 1.1548, + "step": 515 + }, + { + "beta_dpo/beta": 0.07201197743415833, + "beta_dpo/beta_margin_grad_mean": -0.38167789578437805, + "beta_dpo/beta_margin_grad_std": 0.2583937644958496, + "beta_dpo/beta_margin_mean": 1.1529293060302734, + "beta_dpo/beta_margin_std": 2.9729745388031006, + "beta_dpo/beta_used": 0.07201197743415833, + "beta_dpo/beta_used_raw": -0.1009015217423439, + "beta_dpo/gap_mean": 18.779314041137695, + "beta_dpo/gap_std": 27.408687591552734, + "beta_dpo/loss_margin_mean": 14.936943054199219, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.780045351473923, + "grad_norm": 26.425460815429688, + "learning_rate": 7.090144991188568e-08, + "logits/chosen": 0.41848045587539673, + "logits/rejected": 0.3700318932533264, + "loss": 0.9476, + "step": 516 + }, + { + "beta_dpo/beta": 0.015774274244904518, + "beta_dpo/beta_margin_grad_mean": -0.44003695249557495, + "beta_dpo/beta_margin_grad_std": 0.12989072501659393, + "beta_dpo/beta_margin_mean": 0.2813131809234619, + "beta_dpo/beta_margin_std": 0.6269446015357971, + "beta_dpo/beta_used": 0.015774274244904518, + "beta_dpo/beta_used_raw": -0.16760104894638062, + "beta_dpo/gap_mean": 18.17236328125, + "beta_dpo/gap_std": 27.5235652923584, + "beta_dpo/loss_margin_mean": 16.041845321655273, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.781557067271353, + "grad_norm": 12.773390769958496, + "learning_rate": 6.998145243993284e-08, + "logits/chosen": 0.49850425124168396, + "logits/rejected": 0.4892727732658386, + "loss": 1.2018, + "step": 517 + }, + { + "beta_dpo/beta": 0.04643746092915535, + "beta_dpo/beta_margin_grad_mean": -0.41171613335609436, + "beta_dpo/beta_margin_grad_std": 0.23207998275756836, + "beta_dpo/beta_margin_mean": 0.7455624341964722, + "beta_dpo/beta_margin_std": 1.841551423072815, + "beta_dpo/beta_used": 0.04643746092915535, + "beta_dpo/beta_used_raw": -0.065179243683815, + "beta_dpo/gap_mean": 17.777130126953125, + "beta_dpo/gap_std": 27.24643325805664, + "beta_dpo/loss_margin_mean": 15.644393920898438, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.783068783068783, + "grad_norm": 27.985763549804688, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": 0.47469550371170044, + "logits/rejected": 0.4198811650276184, + "loss": 1.1144, + "step": 518 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4969859719276428, + "beta_dpo/beta_margin_grad_std": 0.007910608313977718, + "beta_dpo/beta_margin_mean": 0.012059678323566914, + "beta_dpo/beta_margin_std": 0.03165047988295555, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.29590263962745667, + "beta_dpo/gap_mean": 16.789592742919922, + "beta_dpo/gap_std": 28.03281593322754, + "beta_dpo/loss_margin_mean": 12.059678077697754, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7845804988662132, + "grad_norm": 0.7758278250694275, + "learning_rate": 6.815658960673781e-08, + "logits/chosen": 0.5089113116264343, + "logits/rejected": 0.4466610848903656, + "loss": 1.3763, + "step": 519 + }, + { + "beta_dpo/beta": 0.0470229834318161, + "beta_dpo/beta_margin_grad_mean": -0.3845081925392151, + "beta_dpo/beta_margin_grad_std": 0.20655560493469238, + "beta_dpo/beta_margin_mean": 0.8753683567047119, + "beta_dpo/beta_margin_std": 1.7259058952331543, + "beta_dpo/beta_used": 0.0470229834318161, + "beta_dpo/beta_used_raw": 0.0470229834318161, + "beta_dpo/gap_mean": 16.70362091064453, + "beta_dpo/gap_std": 27.71854019165039, + "beta_dpo/loss_margin_mean": 16.06646156311035, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7860922146636432, + "grad_norm": 33.82258987426758, + "learning_rate": 6.725177529083209e-08, + "logits/chosen": 0.537433385848999, + "logits/rejected": 0.47597891092300415, + "loss": 1.0647, + "step": 520 + }, + { + "beta_dpo/beta": 0.1929291933774948, + "beta_dpo/beta_margin_grad_mean": -0.24177253246307373, + "beta_dpo/beta_margin_grad_std": 0.31293728947639465, + "beta_dpo/beta_margin_mean": 3.521935224533081, + "beta_dpo/beta_margin_std": 5.269829273223877, + "beta_dpo/beta_used": 0.1929291933774948, + "beta_dpo/beta_used_raw": 0.1929291933774948, + "beta_dpo/gap_mean": 16.720731735229492, + "beta_dpo/gap_std": 27.38351058959961, + "beta_dpo/loss_margin_mean": 18.661502838134766, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7876039304610734, + "grad_norm": 139.29025268554688, + "learning_rate": 6.63520728356167e-08, + "logits/chosen": 0.3622036278247833, + "logits/rejected": 0.27943283319473267, + "loss": 0.8118, + "step": 521 + }, + { + "beta_dpo/beta": 0.12439863383769989, + "beta_dpo/beta_margin_grad_mean": -0.31525975465774536, + "beta_dpo/beta_margin_grad_std": 0.33538639545440674, + "beta_dpo/beta_margin_mean": 1.9442089796066284, + "beta_dpo/beta_margin_std": 4.323269844055176, + "beta_dpo/beta_used": 0.12439863383769989, + "beta_dpo/beta_used_raw": 0.12439863383769989, + "beta_dpo/gap_mean": 16.79776382446289, + "beta_dpo/gap_std": 28.118989944458008, + "beta_dpo/loss_margin_mean": 15.861137390136719, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7891156462585034, + "grad_norm": 103.43621826171875, + "learning_rate": 6.545750740770336e-08, + "logits/chosen": 0.4541332721710205, + "logits/rejected": 0.44197672605514526, + "loss": 1.3581, + "step": 522 + }, + { + "beta_dpo/beta": 0.36119723320007324, + "beta_dpo/beta_margin_grad_mean": -0.2371506243944168, + "beta_dpo/beta_margin_grad_std": 0.37638023495674133, + "beta_dpo/beta_margin_mean": 7.449059009552002, + "beta_dpo/beta_margin_std": 13.321476936340332, + "beta_dpo/beta_used": 0.36119723320007324, + "beta_dpo/beta_used_raw": 0.36119723320007324, + "beta_dpo/gap_mean": 17.18457794189453, + "beta_dpo/gap_std": 28.612350463867188, + "beta_dpo/loss_margin_mean": 18.937599182128906, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7906273620559335, + "grad_norm": 231.09762573242188, + "learning_rate": 6.456810403001012e-08, + "logits/chosen": 0.49882420897483826, + "logits/rejected": 0.3736770749092102, + "loss": 1.9876, + "step": 523 + }, + { + "beta_dpo/beta": 0.06380782276391983, + "beta_dpo/beta_margin_grad_mean": -0.4031929075717926, + "beta_dpo/beta_margin_grad_std": 0.22261260449886322, + "beta_dpo/beta_margin_mean": 0.940835177898407, + "beta_dpo/beta_margin_std": 2.2211990356445312, + "beta_dpo/beta_used": 0.06380782276391983, + "beta_dpo/beta_used_raw": -0.033944085240364075, + "beta_dpo/gap_mean": 16.666719436645508, + "beta_dpo/gap_std": 28.064117431640625, + "beta_dpo/loss_margin_mean": 14.2322416305542, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7921390778533636, + "grad_norm": 42.84857177734375, + "learning_rate": 6.368388758106134e-08, + "logits/chosen": 0.3853232264518738, + "logits/rejected": 0.3610796332359314, + "loss": 1.0716, + "step": 524 + }, + { + "beta_dpo/beta": 0.054753709584474564, + "beta_dpo/beta_margin_grad_mean": -0.40546008944511414, + "beta_dpo/beta_margin_grad_std": 0.2202645093202591, + "beta_dpo/beta_margin_mean": 0.6783682107925415, + "beta_dpo/beta_margin_std": 1.8368420600891113, + "beta_dpo/beta_used": 0.054753709584474564, + "beta_dpo/beta_used_raw": -0.0017726905643939972, + "beta_dpo/gap_mean": 16.028484344482422, + "beta_dpo/gap_std": 27.208412170410156, + "beta_dpo/loss_margin_mean": 12.746237754821777, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7936507936507936, + "grad_norm": 24.4576473236084, + "learning_rate": 6.280488279429185e-08, + "logits/chosen": 0.29569315910339355, + "logits/rejected": 0.28484293818473816, + "loss": 1.017, + "step": 525 + }, + { + "beta_dpo/beta": 0.1961859166622162, + "beta_dpo/beta_margin_grad_mean": -0.3545135259628296, + "beta_dpo/beta_margin_grad_std": 0.2908618450164795, + "beta_dpo/beta_margin_mean": 4.208421230316162, + "beta_dpo/beta_margin_std": 7.948729991912842, + "beta_dpo/beta_used": 0.1961859166622162, + "beta_dpo/beta_used_raw": 0.06960143148899078, + "beta_dpo/gap_mean": 15.876822471618652, + "beta_dpo/gap_std": 26.591964721679688, + "beta_dpo/loss_margin_mean": 13.694323539733887, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7951625094482238, + "grad_norm": 117.64985656738281, + "learning_rate": 6.193111425735515e-08, + "logits/chosen": 0.45245057344436646, + "logits/rejected": 0.38413476943969727, + "loss": 1.1558, + "step": 526 + }, + { + "beta_dpo/beta": 0.005362469702959061, + "beta_dpo/beta_margin_grad_mean": -0.4843152165412903, + "beta_dpo/beta_margin_grad_std": 0.0441654697060585, + "beta_dpo/beta_margin_mean": 0.06408263742923737, + "beta_dpo/beta_margin_std": 0.18112891912460327, + "beta_dpo/beta_used": 0.005362469702959061, + "beta_dpo/beta_used_raw": -0.03283126652240753, + "beta_dpo/gap_mean": 14.933441162109375, + "beta_dpo/gap_std": 26.443832397460938, + "beta_dpo/loss_margin_mean": 12.786906242370605, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7966742252456538, + "grad_norm": 5.342923641204834, + "learning_rate": 6.106260641143546e-08, + "logits/chosen": 0.5494492650032043, + "logits/rejected": 0.4718747138977051, + "loss": 1.324, + "step": 527 + }, + { + "beta_dpo/beta": 0.18812526762485504, + "beta_dpo/beta_margin_grad_mean": -0.3760398030281067, + "beta_dpo/beta_margin_grad_std": 0.2963537275791168, + "beta_dpo/beta_margin_mean": 3.0523579120635986, + "beta_dpo/beta_margin_std": 9.375741958618164, + "beta_dpo/beta_used": 0.18812526762485504, + "beta_dpo/beta_used_raw": 0.13063597679138184, + "beta_dpo/gap_mean": 14.724921226501465, + "beta_dpo/gap_std": 27.01572608947754, + "beta_dpo/loss_margin_mean": 12.761770248413086, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.7981859410430839, + "grad_norm": 103.13839721679688, + "learning_rate": 6.019938355056422e-08, + "logits/chosen": 0.38711076974868774, + "logits/rejected": 0.3005455732345581, + "loss": 1.0302, + "step": 528 + }, + { + "beta_dpo/beta": 0.6986787915229797, + "beta_dpo/beta_margin_grad_mean": -0.17930427193641663, + "beta_dpo/beta_margin_grad_std": 0.34411507844924927, + "beta_dpo/beta_margin_mean": 21.36497688293457, + "beta_dpo/beta_margin_std": 25.568307876586914, + "beta_dpo/beta_used": 0.6986787915229797, + "beta_dpo/beta_used_raw": 0.6986787915229797, + "beta_dpo/gap_mean": 16.64554214477539, + "beta_dpo/gap_std": 26.962486267089844, + "beta_dpo/loss_margin_mean": 28.03153419494629, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.799697656840514, + "grad_norm": 356.9806823730469, + "learning_rate": 5.934146982094049e-08, + "logits/chosen": 0.34986603260040283, + "logits/rejected": 0.290559321641922, + "loss": 1.4516, + "step": 529 + }, + { + "beta_dpo/beta": 0.050965264439582825, + "beta_dpo/beta_margin_grad_mean": -0.3662997782230377, + "beta_dpo/beta_margin_grad_std": 0.20529140532016754, + "beta_dpo/beta_margin_mean": 0.9562637209892273, + "beta_dpo/beta_margin_std": 1.5845041275024414, + "beta_dpo/beta_used": 0.050965264439582825, + "beta_dpo/beta_used_raw": -0.019955798983573914, + "beta_dpo/gap_mean": 16.90506362915039, + "beta_dpo/gap_std": 26.45660400390625, + "beta_dpo/loss_margin_mean": 17.067411422729492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8012093726379441, + "grad_norm": 23.774160385131836, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 0.4809131622314453, + "logits/rejected": 0.4357951581478119, + "loss": 1.0238, + "step": 530 + }, + { + "beta_dpo/beta": 0.08633482456207275, + "beta_dpo/beta_margin_grad_mean": -0.36818552017211914, + "beta_dpo/beta_margin_grad_std": 0.23734231293201447, + "beta_dpo/beta_margin_mean": 1.7818925380706787, + "beta_dpo/beta_margin_std": 3.7052230834960938, + "beta_dpo/beta_used": 0.08633482456207275, + "beta_dpo/beta_used_raw": -0.010635964572429657, + "beta_dpo/gap_mean": 16.305274963378906, + "beta_dpo/gap_std": 26.19503402709961, + "beta_dpo/loss_margin_mean": 14.202505111694336, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8027210884353742, + "grad_norm": 57.3766975402832, + "learning_rate": 5.7641665597021435e-08, + "logits/chosen": 0.45231950283050537, + "logits/rejected": 0.3825005292892456, + "loss": 1.2027, + "step": 531 + }, + { + "beta_dpo/beta": 0.1528293490409851, + "beta_dpo/beta_margin_grad_mean": -0.37715035676956177, + "beta_dpo/beta_margin_grad_std": 0.29981446266174316, + "beta_dpo/beta_margin_mean": 2.9251790046691895, + "beta_dpo/beta_margin_std": 6.6156134605407715, + "beta_dpo/beta_used": 0.1528293490409851, + "beta_dpo/beta_used_raw": 0.14536309242248535, + "beta_dpo/gap_mean": 16.91998291015625, + "beta_dpo/gap_std": 26.47125244140625, + "beta_dpo/loss_margin_mean": 19.16761589050293, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8042328042328042, + "grad_norm": 119.86915588378906, + "learning_rate": 5.679982264990424e-08, + "logits/chosen": 0.38349199295043945, + "logits/rejected": 0.327578067779541, + "loss": 1.8351, + "step": 532 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4957379102706909, + "beta_dpo/beta_margin_grad_std": 0.006807847414165735, + "beta_dpo/beta_margin_mean": 0.01705196313560009, + "beta_dpo/beta_margin_std": 0.0272390004247427, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.1611635684967041, + "beta_dpo/gap_mean": 17.01636505126953, + "beta_dpo/gap_std": 26.619853973388672, + "beta_dpo/loss_margin_mean": 17.05196189880371, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8057445200302343, + "grad_norm": 0.8953373432159424, + "learning_rate": 5.596338392706076e-08, + "logits/chosen": 0.5655586123466492, + "logits/rejected": 0.4968782067298889, + "loss": 1.3738, + "step": 533 + }, + { + "beta_dpo/beta": 0.07555638998746872, + "beta_dpo/beta_margin_grad_mean": -0.3829325735569, + "beta_dpo/beta_margin_grad_std": 0.25800174474716187, + "beta_dpo/beta_margin_mean": 1.4311376810073853, + "beta_dpo/beta_margin_std": 3.2261085510253906, + "beta_dpo/beta_used": 0.07555638998746872, + "beta_dpo/beta_used_raw": 0.03285207971930504, + "beta_dpo/gap_mean": 17.048686981201172, + "beta_dpo/gap_std": 26.980222702026367, + "beta_dpo/loss_margin_mean": 16.214319229125977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8072562358276644, + "grad_norm": 61.75785827636719, + "learning_rate": 5.513237282548033e-08, + "logits/chosen": 0.47408878803253174, + "logits/rejected": 0.42821216583251953, + "loss": 1.3134, + "step": 534 + }, + { + "beta_dpo/beta": 0.019090402871370316, + "beta_dpo/beta_margin_grad_mean": -0.4496954381465912, + "beta_dpo/beta_margin_grad_std": 0.14462697505950928, + "beta_dpo/beta_margin_mean": 0.26040640473365784, + "beta_dpo/beta_margin_std": 0.7699604034423828, + "beta_dpo/beta_used": 0.019090402871370316, + "beta_dpo/beta_used_raw": -0.026568515226244926, + "beta_dpo/gap_mean": 16.582523345947266, + "beta_dpo/gap_std": 27.452091217041016, + "beta_dpo/loss_margin_mean": 15.58816146850586, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8087679516250945, + "grad_norm": 17.136873245239258, + "learning_rate": 5.430681259032957e-08, + "logits/chosen": 0.361327588558197, + "logits/rejected": 0.29467612504959106, + "loss": 1.1952, + "step": 535 + }, + { + "beta_dpo/beta": 0.20863793790340424, + "beta_dpo/beta_margin_grad_mean": -0.3422904312610626, + "beta_dpo/beta_margin_grad_std": 0.2740715444087982, + "beta_dpo/beta_margin_mean": 4.775197505950928, + "beta_dpo/beta_margin_std": 8.743983268737793, + "beta_dpo/beta_used": 0.20863793790340424, + "beta_dpo/beta_used_raw": 0.1986754983663559, + "beta_dpo/gap_mean": 17.35881805419922, + "beta_dpo/gap_std": 27.202693939208984, + "beta_dpo/loss_margin_mean": 21.026884078979492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8102796674225246, + "grad_norm": 102.63251495361328, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": 0.48901498317718506, + "logits/rejected": 0.39689117670059204, + "loss": 1.0097, + "step": 536 + }, + { + "beta_dpo/beta": 0.10196889936923981, + "beta_dpo/beta_margin_grad_mean": -0.3624553084373474, + "beta_dpo/beta_margin_grad_std": 0.2689007520675659, + "beta_dpo/beta_margin_mean": 2.305483102798462, + "beta_dpo/beta_margin_std": 4.311905860900879, + "beta_dpo/beta_used": 0.10196889936923981, + "beta_dpo/beta_used_raw": 0.002260163426399231, + "beta_dpo/gap_mean": 17.75217056274414, + "beta_dpo/gap_std": 26.952072143554688, + "beta_dpo/loss_margin_mean": 18.10002326965332, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8117913832199547, + "grad_norm": 75.69073486328125, + "learning_rate": 5.267213693697695e-08, + "logits/chosen": 0.5690730810165405, + "logits/rejected": 0.47435319423675537, + "loss": 1.1746, + "step": 537 + }, + { + "beta_dpo/beta": 0.18487028777599335, + "beta_dpo/beta_margin_grad_mean": -0.3477964401245117, + "beta_dpo/beta_margin_grad_std": 0.28940126299858093, + "beta_dpo/beta_margin_mean": 4.511460781097412, + "beta_dpo/beta_margin_std": 8.299779891967773, + "beta_dpo/beta_used": 0.18487028777599335, + "beta_dpo/beta_used_raw": 0.040582671761512756, + "beta_dpo/gap_mean": 18.115917205810547, + "beta_dpo/gap_std": 27.02817153930664, + "beta_dpo/loss_margin_mean": 22.240970611572266, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8133030990173847, + "grad_norm": 138.01597595214844, + "learning_rate": 5.1863067244167144e-08, + "logits/chosen": 0.42150217294692993, + "logits/rejected": 0.38494616746902466, + "loss": 1.141, + "step": 538 + }, + { + "beta_dpo/beta": 0.09701749682426453, + "beta_dpo/beta_margin_grad_mean": -0.36717933416366577, + "beta_dpo/beta_margin_grad_std": 0.25104501843452454, + "beta_dpo/beta_margin_mean": 1.9268014430999756, + "beta_dpo/beta_margin_std": 3.884998321533203, + "beta_dpo/beta_used": 0.09701749682426453, + "beta_dpo/beta_used_raw": 0.020983532071113586, + "beta_dpo/gap_mean": 18.433753967285156, + "beta_dpo/gap_std": 26.86570167541504, + "beta_dpo/loss_margin_mean": 17.664384841918945, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8148148148148148, + "grad_norm": 73.92316436767578, + "learning_rate": 5.105953986729195e-08, + "logits/chosen": 0.4441359043121338, + "logits/rejected": 0.3619890809059143, + "loss": 1.0482, + "step": 539 + }, + { + "beta_dpo/beta": 0.23082008957862854, + "beta_dpo/beta_margin_grad_mean": -0.32545050978660583, + "beta_dpo/beta_margin_grad_std": 0.28637760877609253, + "beta_dpo/beta_margin_mean": 6.588077068328857, + "beta_dpo/beta_margin_std": 11.03348159790039, + "beta_dpo/beta_used": 0.23082008957862854, + "beta_dpo/beta_used_raw": 0.16205689311027527, + "beta_dpo/gap_mean": 18.548377990722656, + "beta_dpo/gap_std": 26.964176177978516, + "beta_dpo/loss_margin_mean": 22.14543342590332, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8163265306122449, + "grad_norm": 177.70672607421875, + "learning_rate": 5.026157728273966e-08, + "logits/chosen": 0.4785692095756531, + "logits/rejected": 0.37851160764694214, + "loss": 1.3294, + "step": 540 + }, + { + "beta_dpo/beta": 0.1964220106601715, + "beta_dpo/beta_margin_grad_mean": -0.35370177030563354, + "beta_dpo/beta_margin_grad_std": 0.28223419189453125, + "beta_dpo/beta_margin_mean": 4.8944549560546875, + "beta_dpo/beta_margin_std": 9.397936820983887, + "beta_dpo/beta_used": 0.1964220106601715, + "beta_dpo/beta_used_raw": -0.016093730926513672, + "beta_dpo/gap_mean": 19.498260498046875, + "beta_dpo/gap_std": 27.14907455444336, + "beta_dpo/loss_margin_mean": 20.661378860473633, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.817838246409675, + "grad_norm": 104.91517639160156, + "learning_rate": 4.9469201811239035e-08, + "logits/chosen": 0.48476243019104004, + "logits/rejected": 0.49259325861930847, + "loss": 1.1139, + "step": 541 + }, + { + "beta_dpo/beta": 0.14371059834957123, + "beta_dpo/beta_margin_grad_mean": -0.24666696786880493, + "beta_dpo/beta_margin_grad_std": 0.3141605257987976, + "beta_dpo/beta_margin_mean": 3.1314268112182617, + "beta_dpo/beta_margin_std": 4.227056980133057, + "beta_dpo/beta_used": 0.14371059834957123, + "beta_dpo/beta_used_raw": 0.14371059834957123, + "beta_dpo/gap_mean": 19.874874114990234, + "beta_dpo/gap_std": 27.419788360595703, + "beta_dpo/loss_margin_mean": 22.236177444458008, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8193499622071051, + "grad_norm": 64.40750885009766, + "learning_rate": 4.868243561723534e-08, + "logits/chosen": 0.5121496915817261, + "logits/rejected": 0.45453765988349915, + "loss": 0.8592, + "step": 542 + }, + { + "beta_dpo/beta": 0.16332434117794037, + "beta_dpo/beta_margin_grad_mean": -0.23017369210720062, + "beta_dpo/beta_margin_grad_std": 0.2802915573120117, + "beta_dpo/beta_margin_mean": 3.163569211959839, + "beta_dpo/beta_margin_std": 4.976937294006348, + "beta_dpo/beta_used": 0.16332434117794037, + "beta_dpo/beta_used_raw": 0.16332434117794037, + "beta_dpo/gap_mean": 19.70780372619629, + "beta_dpo/gap_std": 27.284408569335938, + "beta_dpo/loss_margin_mean": 19.093935012817383, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8208616780045351, + "grad_norm": 49.96735382080078, + "learning_rate": 4.790130070827028e-08, + "logits/chosen": 0.47974371910095215, + "logits/rejected": 0.38592517375946045, + "loss": 0.6643, + "step": 543 + }, + { + "beta_dpo/beta": 0.252260684967041, + "beta_dpo/beta_margin_grad_mean": -0.19919231534004211, + "beta_dpo/beta_margin_grad_std": 0.34733232855796814, + "beta_dpo/beta_margin_mean": 6.332991600036621, + "beta_dpo/beta_margin_std": 7.6051411628723145, + "beta_dpo/beta_used": 0.252260684967041, + "beta_dpo/beta_used_raw": 0.252260684967041, + "beta_dpo/gap_mean": 20.517066955566406, + "beta_dpo/gap_std": 27.914066314697266, + "beta_dpo/loss_margin_mean": 25.072317123413086, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8223733938019653, + "grad_norm": 98.99584197998047, + "learning_rate": 4.7125818934366454e-08, + "logits/chosen": 0.4560331702232361, + "logits/rejected": 0.3775930404663086, + "loss": 1.0511, + "step": 544 + }, + { + "beta_dpo/beta": 0.07587642967700958, + "beta_dpo/beta_margin_grad_mean": -0.32201042771339417, + "beta_dpo/beta_margin_grad_std": 0.2806906998157501, + "beta_dpo/beta_margin_mean": 1.2491117715835571, + "beta_dpo/beta_margin_std": 2.0175347328186035, + "beta_dpo/beta_used": 0.07587642967700958, + "beta_dpo/beta_used_raw": 0.07587642967700958, + "beta_dpo/gap_mean": 20.09872055053711, + "beta_dpo/gap_std": 27.952808380126953, + "beta_dpo/loss_margin_mean": 16.42570686340332, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8238851095993953, + "grad_norm": 30.904129028320312, + "learning_rate": 4.635601198741607e-08, + "logits/chosen": 0.4076148271560669, + "logits/rejected": 0.3476223945617676, + "loss": 0.7492, + "step": 545 + }, + { + "beta_dpo/beta": 0.15581192076206207, + "beta_dpo/beta_margin_grad_mean": -0.31065890192985535, + "beta_dpo/beta_margin_grad_std": 0.3145395517349243, + "beta_dpo/beta_margin_mean": 2.969562292098999, + "beta_dpo/beta_margin_std": 5.391257286071777, + "beta_dpo/beta_used": 0.15581192076206207, + "beta_dpo/beta_used_raw": 0.15581192076206207, + "beta_dpo/gap_mean": 19.73141860961914, + "beta_dpo/gap_std": 27.47371482849121, + "beta_dpo/loss_margin_mean": 18.355119705200195, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8253968253968254, + "grad_norm": 147.27969360351562, + "learning_rate": 4.559190140057428e-08, + "logits/chosen": 0.5145200490951538, + "logits/rejected": 0.4989354610443115, + "loss": 1.3293, + "step": 546 + }, + { + "beta_dpo/beta": 0.07221545279026031, + "beta_dpo/beta_margin_grad_mean": -0.35358917713165283, + "beta_dpo/beta_margin_grad_std": 0.242741659283638, + "beta_dpo/beta_margin_mean": 1.2553969621658325, + "beta_dpo/beta_margin_std": 2.621389150619507, + "beta_dpo/beta_used": 0.07221545279026031, + "beta_dpo/beta_used_raw": 0.014809608459472656, + "beta_dpo/gap_mean": 19.883861541748047, + "beta_dpo/gap_std": 27.571674346923828, + "beta_dpo/loss_margin_mean": 20.501134872436523, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8269085411942555, + "grad_norm": 31.414352416992188, + "learning_rate": 4.483350854765672e-08, + "logits/chosen": 0.40707188844680786, + "logits/rejected": 0.3357613682746887, + "loss": 0.9218, + "step": 547 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49660223722457886, + "beta_dpo/beta_margin_grad_std": 0.006567038130015135, + "beta_dpo/beta_margin_mean": 0.013594682328402996, + "beta_dpo/beta_margin_std": 0.02627561055123806, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.22177554666996002, + "beta_dpo/gap_mean": 18.659589767456055, + "beta_dpo/gap_std": 27.11603546142578, + "beta_dpo/loss_margin_mean": 13.594681739807129, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8284202569916855, + "grad_norm": 0.8114597201347351, + "learning_rate": 4.4080854642541826e-08, + "logits/chosen": 0.3467254936695099, + "logits/rejected": 0.2756657302379608, + "loss": 1.3732, + "step": 548 + }, + { + "beta_dpo/beta": 0.09689143300056458, + "beta_dpo/beta_margin_grad_mean": -0.4047882854938507, + "beta_dpo/beta_margin_grad_std": 0.2726931869983673, + "beta_dpo/beta_margin_mean": 1.5041784048080444, + "beta_dpo/beta_margin_std": 4.353418350219727, + "beta_dpo/beta_used": 0.09689143300056458, + "beta_dpo/beta_used_raw": 0.035542529076337814, + "beta_dpo/gap_mean": 18.34221649169922, + "beta_dpo/gap_std": 27.050926208496094, + "beta_dpo/loss_margin_mean": 16.402610778808594, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8299319727891157, + "grad_norm": 81.26973724365234, + "learning_rate": 4.333396073857723e-08, + "logits/chosen": 0.5225358009338379, + "logits/rejected": 0.45289480686187744, + "loss": 1.3529, + "step": 549 + }, + { + "beta_dpo/beta": 0.07474076747894287, + "beta_dpo/beta_margin_grad_mean": -0.3902982175350189, + "beta_dpo/beta_margin_grad_std": 0.2482263296842575, + "beta_dpo/beta_margin_mean": 1.402299404144287, + "beta_dpo/beta_margin_std": 3.1106622219085693, + "beta_dpo/beta_used": 0.07474076747894287, + "beta_dpo/beta_used_raw": -0.07162999361753464, + "beta_dpo/gap_mean": 17.733795166015625, + "beta_dpo/gap_std": 26.989667892456055, + "beta_dpo/loss_margin_mean": 13.307918548583984, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8314436885865457, + "grad_norm": 45.28874206542969, + "learning_rate": 4.259284772799099e-08, + "logits/chosen": 0.4761781692504883, + "logits/rejected": 0.4413219094276428, + "loss": 1.0674, + "step": 550 + }, + { + "beta_dpo/beta": 0.03209269419312477, + "beta_dpo/beta_margin_grad_mean": -0.4065311551094055, + "beta_dpo/beta_margin_grad_std": 0.1826740801334381, + "beta_dpo/beta_margin_mean": 0.57929527759552, + "beta_dpo/beta_margin_std": 1.1693179607391357, + "beta_dpo/beta_used": 0.03209269419312477, + "beta_dpo/beta_used_raw": -0.03214040771126747, + "beta_dpo/gap_mean": 16.9639892578125, + "beta_dpo/gap_std": 26.479679107666016, + "beta_dpo/loss_margin_mean": 16.129413604736328, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8329554043839759, + "grad_norm": 24.777706146240234, + "learning_rate": 4.1857536341307176e-08, + "logits/chosen": 0.4830327033996582, + "logits/rejected": 0.4463438093662262, + "loss": 1.1575, + "step": 551 + }, + { + "beta_dpo/beta": 0.20424462854862213, + "beta_dpo/beta_margin_grad_mean": -0.3118079602718353, + "beta_dpo/beta_margin_grad_std": 0.28090307116508484, + "beta_dpo/beta_margin_mean": 4.122735023498535, + "beta_dpo/beta_margin_std": 7.375041484832764, + "beta_dpo/beta_used": 0.20424462854862213, + "beta_dpo/beta_used_raw": 0.15399131178855896, + "beta_dpo/gap_mean": 16.902812957763672, + "beta_dpo/gap_std": 26.333946228027344, + "beta_dpo/loss_margin_mean": 17.136497497558594, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8344671201814059, + "grad_norm": 26.707483291625977, + "learning_rate": 4.112804714676593e-08, + "logits/chosen": 0.4524657726287842, + "logits/rejected": 0.39445942640304565, + "loss": 0.7323, + "step": 552 + }, + { + "beta_dpo/beta": 0.15994586050510406, + "beta_dpo/beta_margin_grad_mean": -0.3966883718967438, + "beta_dpo/beta_margin_grad_std": 0.30760183930397034, + "beta_dpo/beta_margin_mean": 3.007977247238159, + "beta_dpo/beta_margin_std": 8.034646987915039, + "beta_dpo/beta_used": 0.15994586050510406, + "beta_dpo/beta_used_raw": 0.15004082024097443, + "beta_dpo/gap_mean": 17.17023468017578, + "beta_dpo/gap_std": 27.138286590576172, + "beta_dpo/loss_margin_mean": 17.232175827026367, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8359788359788359, + "grad_norm": 161.55067443847656, + "learning_rate": 4.0404400549748144e-08, + "logits/chosen": 0.43592414259910583, + "logits/rejected": 0.3231276869773865, + "loss": 1.3322, + "step": 553 + }, + { + "beta_dpo/beta": 0.023860443383455276, + "beta_dpo/beta_margin_grad_mean": -0.4225996136665344, + "beta_dpo/beta_margin_grad_std": 0.1500791311264038, + "beta_dpo/beta_margin_mean": 0.42666929960250854, + "beta_dpo/beta_margin_std": 0.8989187479019165, + "beta_dpo/beta_used": 0.023860443383455276, + "beta_dpo/beta_used_raw": -0.12928947806358337, + "beta_dpo/gap_mean": 17.246599197387695, + "beta_dpo/gap_std": 27.404417037963867, + "beta_dpo/loss_margin_mean": 18.306926727294922, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8374905517762661, + "grad_norm": 15.592290878295898, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": 0.42959821224212646, + "logits/rejected": 0.40046319365501404, + "loss": 1.1491, + "step": 554 + }, + { + "beta_dpo/beta": 0.20455916225910187, + "beta_dpo/beta_margin_grad_mean": -0.31857535243034363, + "beta_dpo/beta_margin_grad_std": 0.287540078163147, + "beta_dpo/beta_margin_mean": 4.8925042152404785, + "beta_dpo/beta_margin_std": 8.230351448059082, + "beta_dpo/beta_used": 0.20455916225910187, + "beta_dpo/beta_used_raw": 0.15680401027202606, + "beta_dpo/gap_mean": 17.05306625366211, + "beta_dpo/gap_std": 27.7065372467041, + "beta_dpo/loss_margin_mean": 17.45922088623047, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8390022675736961, + "grad_norm": 129.28990173339844, + "learning_rate": 3.89747159520904e-08, + "logits/chosen": 0.4454486072063446, + "logits/rejected": 0.4100185036659241, + "loss": 1.3243, + "step": 555 + }, + { + "beta_dpo/beta": 0.009554240852594376, + "beta_dpo/beta_margin_grad_mean": -0.46103566884994507, + "beta_dpo/beta_margin_grad_std": 0.08946454524993896, + "beta_dpo/beta_margin_mean": 0.16767998039722443, + "beta_dpo/beta_margin_std": 0.38722655177116394, + "beta_dpo/beta_used": 0.009554240852594376, + "beta_dpo/beta_used_raw": -0.12646989524364471, + "beta_dpo/gap_mean": 17.357540130615234, + "beta_dpo/gap_std": 27.795841217041016, + "beta_dpo/loss_margin_mean": 16.72756576538086, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8405139833711263, + "grad_norm": 7.336466312408447, + "learning_rate": 3.826871794280192e-08, + "logits/chosen": 0.506888747215271, + "logits/rejected": 0.4476608633995056, + "loss": 1.2708, + "step": 556 + }, + { + "beta_dpo/beta": 0.21170280873775482, + "beta_dpo/beta_margin_grad_mean": -0.33399519324302673, + "beta_dpo/beta_margin_grad_std": 0.28276923298835754, + "beta_dpo/beta_margin_mean": 5.678369522094727, + "beta_dpo/beta_margin_std": 9.621817588806152, + "beta_dpo/beta_used": 0.21170280873775482, + "beta_dpo/beta_used_raw": 0.15560570359230042, + "beta_dpo/gap_mean": 18.178462982177734, + "beta_dpo/gap_std": 27.78179931640625, + "beta_dpo/loss_margin_mean": 21.99428939819336, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8420256991685563, + "grad_norm": 101.20765686035156, + "learning_rate": 3.756864251262143e-08, + "logits/chosen": 0.571921706199646, + "logits/rejected": 0.4873775243759155, + "loss": 0.9721, + "step": 557 + }, + { + "beta_dpo/beta": 0.010930849239230156, + "beta_dpo/beta_margin_grad_mean": -0.44395628571510315, + "beta_dpo/beta_margin_grad_std": 0.10657542943954468, + "beta_dpo/beta_margin_mean": 0.24835842847824097, + "beta_dpo/beta_margin_std": 0.4842853546142578, + "beta_dpo/beta_used": 0.010930849239230156, + "beta_dpo/beta_used_raw": -0.05466505140066147, + "beta_dpo/gap_mean": 18.70376968383789, + "beta_dpo/gap_std": 27.77187728881836, + "beta_dpo/loss_margin_mean": 21.812089920043945, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8435374149659864, + "grad_norm": 9.379203796386719, + "learning_rate": 3.687450924416341e-08, + "logits/chosen": 0.5313233137130737, + "logits/rejected": 0.4695540964603424, + "loss": 1.2377, + "step": 558 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49510252475738525, + "beta_dpo/beta_margin_grad_std": 0.007558883633464575, + "beta_dpo/beta_margin_mean": 0.0195964053273201, + "beta_dpo/beta_margin_std": 0.030248643830418587, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.06615760177373886, + "beta_dpo/gap_mean": 19.156770706176758, + "beta_dpo/gap_std": 28.15302276611328, + "beta_dpo/loss_margin_mean": 19.596405029296875, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8450491307634165, + "grad_norm": 0.8944501280784607, + "learning_rate": 3.6186337553827743e-08, + "logits/chosen": 0.43583738803863525, + "logits/rejected": 0.3620571792125702, + "loss": 1.3701, + "step": 559 + }, + { + "beta_dpo/beta": 0.06273768097162247, + "beta_dpo/beta_margin_grad_mean": -0.37549564242362976, + "beta_dpo/beta_margin_grad_std": 0.24666382372379303, + "beta_dpo/beta_margin_mean": 1.3902297019958496, + "beta_dpo/beta_margin_std": 2.78605055809021, + "beta_dpo/beta_used": 0.06273768097162247, + "beta_dpo/beta_used_raw": -0.2646280825138092, + "beta_dpo/gap_mean": 18.45889663696289, + "beta_dpo/gap_std": 27.95433807373047, + "beta_dpo/loss_margin_mean": 17.313091278076172, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8465608465608465, + "grad_norm": 48.35575866699219, + "learning_rate": 3.550414669125573e-08, + "logits/chosen": 0.46553072333335876, + "logits/rejected": 0.4221952557563782, + "loss": 1.1836, + "step": 560 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4949081540107727, + "beta_dpo/beta_margin_grad_std": 0.00684754503890872, + "beta_dpo/beta_margin_mean": 0.02037295140326023, + "beta_dpo/beta_margin_std": 0.02739965170621872, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.14487004280090332, + "beta_dpo/gap_mean": 18.880569458007812, + "beta_dpo/gap_std": 27.770397186279297, + "beta_dpo/loss_margin_mean": 20.372949600219727, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8480725623582767, + "grad_norm": 0.9027082920074463, + "learning_rate": 3.482795573879241e-08, + "logits/chosen": 0.4389171004295349, + "logits/rejected": 0.4021867513656616, + "loss": 1.3717, + "step": 561 + }, + { + "beta_dpo/beta": 0.2258734554052353, + "beta_dpo/beta_margin_grad_mean": -0.3564591705799103, + "beta_dpo/beta_margin_grad_std": 0.28591641783714294, + "beta_dpo/beta_margin_mean": 5.468478679656982, + "beta_dpo/beta_margin_std": 9.84677791595459, + "beta_dpo/beta_used": 0.2258734554052353, + "beta_dpo/beta_used_raw": 0.10015764832496643, + "beta_dpo/gap_mean": 19.541290283203125, + "beta_dpo/gap_std": 27.708114624023438, + "beta_dpo/loss_margin_mean": 22.15447235107422, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8495842781557067, + "grad_norm": 134.08180236816406, + "learning_rate": 3.415778361095226e-08, + "logits/chosen": 0.46755078434944153, + "logits/rejected": 0.42174720764160156, + "loss": 1.2897, + "step": 562 + }, + { + "beta_dpo/beta": 0.10262561589479446, + "beta_dpo/beta_margin_grad_mean": -0.3006736636161804, + "beta_dpo/beta_margin_grad_std": 0.31810298562049866, + "beta_dpo/beta_margin_mean": 2.1566312313079834, + "beta_dpo/beta_margin_std": 3.6026384830474854, + "beta_dpo/beta_used": 0.10262561589479446, + "beta_dpo/beta_used_raw": 0.10262561589479446, + "beta_dpo/gap_mean": 19.626052856445312, + "beta_dpo/gap_std": 28.178102493286133, + "beta_dpo/loss_margin_mean": 20.39893913269043, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8510959939531368, + "grad_norm": 75.20328521728516, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 0.5157092809677124, + "logits/rejected": 0.4610385298728943, + "loss": 0.9912, + "step": 563 + }, + { + "beta_dpo/beta": 0.20660275220870972, + "beta_dpo/beta_margin_grad_mean": -0.24291859567165375, + "beta_dpo/beta_margin_grad_std": 0.32369184494018555, + "beta_dpo/beta_margin_mean": 4.798521041870117, + "beta_dpo/beta_margin_std": 6.981872081756592, + "beta_dpo/beta_used": 0.20660275220870972, + "beta_dpo/beta_used_raw": 0.20660275220870972, + "beta_dpo/gap_mean": 20.09206771850586, + "beta_dpo/gap_std": 28.188093185424805, + "beta_dpo/loss_margin_mean": 22.53466033935547, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8526077097505669, + "grad_norm": 111.04369354248047, + "learning_rate": 3.283557064487785e-08, + "logits/chosen": 0.4254182279109955, + "logits/rejected": 0.38883572816848755, + "loss": 0.8814, + "step": 564 + }, + { + "beta_dpo/beta": 0.1317221075296402, + "beta_dpo/beta_margin_grad_mean": -0.34109413623809814, + "beta_dpo/beta_margin_grad_std": 0.2687946856021881, + "beta_dpo/beta_margin_mean": 2.5971169471740723, + "beta_dpo/beta_margin_std": 5.390993118286133, + "beta_dpo/beta_used": 0.1317221075296402, + "beta_dpo/beta_used_raw": -0.17019467055797577, + "beta_dpo/gap_mean": 19.456600189208984, + "beta_dpo/gap_std": 28.187774658203125, + "beta_dpo/loss_margin_mean": 15.798896789550781, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.854119425547997, + "grad_norm": 95.097900390625, + "learning_rate": 3.218356679178252e-08, + "logits/chosen": 0.486441433429718, + "logits/rejected": 0.42756447196006775, + "loss": 1.1893, + "step": 565 + }, + { + "beta_dpo/beta": 0.09158875793218613, + "beta_dpo/beta_margin_grad_mean": -0.4200616478919983, + "beta_dpo/beta_margin_grad_std": 0.2922900915145874, + "beta_dpo/beta_margin_mean": 1.3433125019073486, + "beta_dpo/beta_margin_std": 3.9480140209198, + "beta_dpo/beta_used": 0.09158875793218613, + "beta_dpo/beta_used_raw": 0.06183715909719467, + "beta_dpo/gap_mean": 19.037540435791016, + "beta_dpo/gap_std": 28.31623077392578, + "beta_dpo/loss_margin_mean": 17.304000854492188, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8556311413454271, + "grad_norm": 94.29534912109375, + "learning_rate": 3.1537655732553764e-08, + "logits/chosen": 0.4620034694671631, + "logits/rejected": 0.43806010484695435, + "loss": 1.3457, + "step": 566 + }, + { + "beta_dpo/beta": 0.045104626566171646, + "beta_dpo/beta_margin_grad_mean": -0.35616597533226013, + "beta_dpo/beta_margin_grad_std": 0.21555054187774658, + "beta_dpo/beta_margin_mean": 1.1588886976242065, + "beta_dpo/beta_margin_std": 1.898647427558899, + "beta_dpo/beta_used": 0.045104626566171646, + "beta_dpo/beta_used_raw": -0.020848926156759262, + "beta_dpo/gap_mean": 19.459049224853516, + "beta_dpo/gap_std": 27.901168823242188, + "beta_dpo/loss_margin_mean": 19.77743148803711, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8571428571428571, + "grad_norm": 18.36300277709961, + "learning_rate": 3.089785553471233e-08, + "logits/chosen": 0.49507731199264526, + "logits/rejected": 0.3956683278083801, + "loss": 1.0065, + "step": 567 + }, + { + "beta_dpo/beta": 0.20853257179260254, + "beta_dpo/beta_margin_grad_mean": -0.21012690663337708, + "beta_dpo/beta_margin_grad_std": 0.313422828912735, + "beta_dpo/beta_margin_mean": 4.9643449783325195, + "beta_dpo/beta_margin_std": 6.101047992706299, + "beta_dpo/beta_used": 0.20853257179260254, + "beta_dpo/beta_used_raw": 0.20853257179260254, + "beta_dpo/gap_mean": 19.520610809326172, + "beta_dpo/gap_std": 27.042015075683594, + "beta_dpo/loss_margin_mean": 22.951581954956055, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8586545729402872, + "grad_norm": 99.84920501708984, + "learning_rate": 3.026418409484513e-08, + "logits/chosen": 0.5173642635345459, + "logits/rejected": 0.42536449432373047, + "loss": 0.8444, + "step": 568 + }, + { + "beta_dpo/beta": 0.008185813203454018, + "beta_dpo/beta_margin_grad_mean": -0.4594218134880066, + "beta_dpo/beta_margin_grad_std": 0.07736992090940475, + "beta_dpo/beta_margin_mean": 0.17122690379619598, + "beta_dpo/beta_margin_std": 0.33035361766815186, + "beta_dpo/beta_used": 0.008185813203454018, + "beta_dpo/beta_used_raw": -0.08097762614488602, + "beta_dpo/gap_mean": 19.53293228149414, + "beta_dpo/gap_std": 27.189006805419922, + "beta_dpo/loss_margin_mean": 15.246856689453125, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8601662887377173, + "grad_norm": 9.221585273742676, + "learning_rate": 2.963665913810451e-08, + "logits/chosen": 0.38372114300727844, + "logits/rejected": 0.34946101903915405, + "loss": 1.2617, + "step": 569 + }, + { + "beta_dpo/beta": 0.3106805086135864, + "beta_dpo/beta_margin_grad_mean": -0.2486473172903061, + "beta_dpo/beta_margin_grad_std": 0.23560231924057007, + "beta_dpo/beta_margin_mean": 9.827374458312988, + "beta_dpo/beta_margin_std": 15.061835289001465, + "beta_dpo/beta_used": 0.3106805086135864, + "beta_dpo/beta_used_raw": 0.3106805086135864, + "beta_dpo/gap_mean": 19.72932243347168, + "beta_dpo/gap_std": 26.982757568359375, + "beta_dpo/loss_margin_mean": 26.050617218017578, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8616780045351474, + "grad_norm": 127.13440704345703, + "learning_rate": 2.9015298217712453e-08, + "logits/chosen": 0.4091266095638275, + "logits/rejected": 0.32067325711250305, + "loss": 0.7522, + "step": 570 + }, + { + "beta_dpo/beta": 0.29264044761657715, + "beta_dpo/beta_margin_grad_mean": -0.36735960841178894, + "beta_dpo/beta_margin_grad_std": 0.31165120005607605, + "beta_dpo/beta_margin_mean": 5.108920097351074, + "beta_dpo/beta_margin_std": 12.917421340942383, + "beta_dpo/beta_used": 0.29264044761657715, + "beta_dpo/beta_used_raw": 0.1877349317073822, + "beta_dpo/gap_mean": 19.32115936279297, + "beta_dpo/gap_std": 26.78708267211914, + "beta_dpo/loss_margin_mean": 14.153389930725098, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8631897203325775, + "grad_norm": 37.17084884643555, + "learning_rate": 2.840011871446962e-08, + "logits/chosen": 0.4562925696372986, + "logits/rejected": 0.4176176190376282, + "loss": 0.8324, + "step": 571 + }, + { + "beta_dpo/beta": 0.10962247103452682, + "beta_dpo/beta_margin_grad_mean": -0.3743179142475128, + "beta_dpo/beta_margin_grad_std": 0.28637591004371643, + "beta_dpo/beta_margin_mean": 1.9382827281951904, + "beta_dpo/beta_margin_std": 4.297430992126465, + "beta_dpo/beta_used": 0.10962247103452682, + "beta_dpo/beta_used_raw": 0.01463637501001358, + "beta_dpo/gap_mean": 18.996253967285156, + "beta_dpo/gap_std": 26.56301498413086, + "beta_dpo/loss_margin_mean": 17.643159866333008, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8647014361300076, + "grad_norm": 64.77835845947266, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": 0.4863911271095276, + "logits/rejected": 0.5237973928451538, + "loss": 1.0658, + "step": 572 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49567198753356934, + "beta_dpo/beta_margin_grad_std": 0.00828883983194828, + "beta_dpo/beta_margin_mean": 0.017318209633231163, + "beta_dpo/beta_margin_std": 0.0331687331199646, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.19534537196159363, + "beta_dpo/gap_mean": 18.85485076904297, + "beta_dpo/gap_std": 27.400794982910156, + "beta_dpo/loss_margin_mean": 17.318208694458008, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8662131519274376, + "grad_norm": 0.8444698452949524, + "learning_rate": 2.718837261761528e-08, + "logits/chosen": 0.4736700654029846, + "logits/rejected": 0.4235808849334717, + "loss": 1.3725, + "step": 573 + }, + { + "beta_dpo/beta": 0.1897483617067337, + "beta_dpo/beta_margin_grad_mean": -0.3185376226902008, + "beta_dpo/beta_margin_grad_std": 0.25058743357658386, + "beta_dpo/beta_margin_mean": 4.7107343673706055, + "beta_dpo/beta_margin_std": 7.908545017242432, + "beta_dpo/beta_used": 0.1897483617067337, + "beta_dpo/beta_used_raw": 0.1897483617067337, + "beta_dpo/gap_mean": 18.79144859313965, + "beta_dpo/gap_std": 26.819414138793945, + "beta_dpo/loss_margin_mean": 21.234031677246094, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8677248677248677, + "grad_norm": 90.03785705566406, + "learning_rate": 2.659183991914696e-08, + "logits/chosen": 0.5208003520965576, + "logits/rejected": 0.44477763772010803, + "loss": 0.9521, + "step": 574 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4963507056236267, + "beta_dpo/beta_margin_grad_std": 0.007400399073958397, + "beta_dpo/beta_margin_mean": 0.014601343311369419, + "beta_dpo/beta_margin_std": 0.029610570520162582, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.428772509098053, + "beta_dpo/gap_mean": 18.696063995361328, + "beta_dpo/gap_std": 27.33431625366211, + "beta_dpo/loss_margin_mean": 14.601343154907227, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8692365835222978, + "grad_norm": 0.8435812592506409, + "learning_rate": 2.600155642716606e-08, + "logits/chosen": 0.531106173992157, + "logits/rejected": 0.44675496220588684, + "loss": 1.3766, + "step": 575 + }, + { + "beta_dpo/beta": 0.2750542461872101, + "beta_dpo/beta_margin_grad_mean": -0.34378066658973694, + "beta_dpo/beta_margin_grad_std": 0.2857505977153778, + "beta_dpo/beta_margin_mean": 6.987796306610107, + "beta_dpo/beta_margin_std": 12.458413124084473, + "beta_dpo/beta_used": 0.2750542461872101, + "beta_dpo/beta_used_raw": 0.14406917989253998, + "beta_dpo/gap_mean": 18.92401695251465, + "beta_dpo/gap_std": 27.16030502319336, + "beta_dpo/loss_margin_mean": 22.06940460205078, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8707482993197279, + "grad_norm": 107.54598236083984, + "learning_rate": 2.5417538653170754e-08, + "logits/chosen": 0.5154881477355957, + "logits/rejected": 0.4023563265800476, + "loss": 0.949, + "step": 576 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49634456634521484, + "beta_dpo/beta_margin_grad_std": 0.007100496906787157, + "beta_dpo/beta_margin_mean": 0.0146254263818264, + "beta_dpo/beta_margin_std": 0.02840968407690525, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.20439589023590088, + "beta_dpo/gap_mean": 18.303625106811523, + "beta_dpo/gap_std": 27.299203872680664, + "beta_dpo/loss_margin_mean": 14.625425338745117, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.872260015117158, + "grad_norm": 0.8384003639221191, + "learning_rate": 2.4839802933393607e-08, + "logits/chosen": 0.46008801460266113, + "logits/rejected": 0.4375826120376587, + "loss": 1.3733, + "step": 577 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4962397515773773, + "beta_dpo/beta_margin_grad_std": 0.006909025367349386, + "beta_dpo/beta_margin_mean": 0.015045084990561008, + "beta_dpo/beta_margin_std": 0.027643948793411255, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.14562451839447021, + "beta_dpo/gap_mean": 17.768327713012695, + "beta_dpo/gap_std": 27.450611114501953, + "beta_dpo/loss_margin_mean": 15.045083999633789, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.873771730914588, + "grad_norm": 0.863936185836792, + "learning_rate": 2.4268365428344733e-08, + "logits/chosen": 0.5061689019203186, + "logits/rejected": 0.4770342707633972, + "loss": 1.3728, + "step": 578 + }, + { + "beta_dpo/beta": 0.2994960844516754, + "beta_dpo/beta_margin_grad_mean": -0.3630431890487671, + "beta_dpo/beta_margin_grad_std": 0.30570077896118164, + "beta_dpo/beta_margin_mean": 6.887415409088135, + "beta_dpo/beta_margin_std": 13.675480842590332, + "beta_dpo/beta_used": 0.2994960844516754, + "beta_dpo/beta_used_raw": 0.27384522557258606, + "beta_dpo/gap_mean": 18.528472900390625, + "beta_dpo/gap_std": 27.44584083557129, + "beta_dpo/loss_margin_mean": 24.255210876464844, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8752834467120182, + "grad_norm": 176.87245178222656, + "learning_rate": 2.3703242122359357e-08, + "logits/chosen": 0.41431349515914917, + "logits/rejected": 0.3788405656814575, + "loss": 1.1633, + "step": 579 + }, + { + "beta_dpo/beta": 0.02022494003176689, + "beta_dpo/beta_margin_grad_mean": -0.4239354431629181, + "beta_dpo/beta_margin_grad_std": 0.16484029591083527, + "beta_dpo/beta_margin_mean": 0.39223089814186096, + "beta_dpo/beta_margin_std": 0.8803242444992065, + "beta_dpo/beta_used": 0.02022494003176689, + "beta_dpo/beta_used_raw": -0.11066954582929611, + "beta_dpo/gap_mean": 18.55239486694336, + "beta_dpo/gap_std": 27.90411376953125, + "beta_dpo/loss_margin_mean": 17.821359634399414, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8767951625094482, + "grad_norm": 15.0440092086792, + "learning_rate": 2.3144448823151392e-08, + "logits/chosen": 0.4520995020866394, + "logits/rejected": 0.3926962614059448, + "loss": 1.1646, + "step": 580 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49613216519355774, + "beta_dpo/beta_margin_grad_std": 0.006800650618970394, + "beta_dpo/beta_margin_mean": 0.015474964864552021, + "beta_dpo/beta_margin_std": 0.02721022255718708, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.04371700435876846, + "beta_dpo/gap_mean": 18.31734848022461, + "beta_dpo/gap_std": 27.919649124145508, + "beta_dpo/loss_margin_mean": 15.474964141845703, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8783068783068783, + "grad_norm": 0.9805479049682617, + "learning_rate": 2.259200116137039e-08, + "logits/chosen": 0.5199064016342163, + "logits/rejected": 0.44630610942840576, + "loss": 1.3706, + "step": 581 + }, + { + "beta_dpo/beta": 0.045170314610004425, + "beta_dpo/beta_margin_grad_mean": -0.39904969930648804, + "beta_dpo/beta_margin_grad_std": 0.2411165088415146, + "beta_dpo/beta_margin_mean": 0.7901149392127991, + "beta_dpo/beta_margin_std": 1.9573147296905518, + "beta_dpo/beta_used": 0.045170314610004425, + "beta_dpo/beta_used_raw": 0.04214519262313843, + "beta_dpo/gap_mean": 17.89398765563965, + "beta_dpo/gap_std": 27.529356002807617, + "beta_dpo/loss_margin_mean": 17.459928512573242, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8798185941043084, + "grad_norm": 32.02427291870117, + "learning_rate": 2.204591459016525e-08, + "logits/chosen": 0.49430859088897705, + "logits/rejected": 0.5163915157318115, + "loss": 1.1756, + "step": 582 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49534931778907776, + "beta_dpo/beta_margin_grad_std": 0.007267021108418703, + "beta_dpo/beta_margin_mean": 0.018607286736369133, + "beta_dpo/beta_margin_std": 0.02907698228955269, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.1370919942855835, + "beta_dpo/gap_mean": 18.084922790527344, + "beta_dpo/gap_std": 27.864459991455078, + "beta_dpo/loss_margin_mean": 18.60728645324707, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8813303099017384, + "grad_norm": 1.099493145942688, + "learning_rate": 2.1506204384751064e-08, + "logits/chosen": 0.583713173866272, + "logits/rejected": 0.46615666151046753, + "loss": 1.3723, + "step": 583 + }, + { + "beta_dpo/beta": 0.05212414637207985, + "beta_dpo/beta_margin_grad_mean": -0.4082799553871155, + "beta_dpo/beta_margin_grad_std": 0.2577343285083771, + "beta_dpo/beta_margin_mean": 1.1146053075790405, + "beta_dpo/beta_margin_std": 2.6749753952026367, + "beta_dpo/beta_used": 0.05212414637207985, + "beta_dpo/beta_used_raw": 0.048223551362752914, + "beta_dpo/gap_mean": 18.14144515991211, + "beta_dpo/gap_std": 28.521961212158203, + "beta_dpo/loss_margin_mean": 17.956157684326172, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8828420256991686, + "grad_norm": 55.47587585449219, + "learning_rate": 2.09728856419826e-08, + "logits/chosen": 0.592127799987793, + "logits/rejected": 0.482388436794281, + "loss": 1.4501, + "step": 584 + }, + { + "beta_dpo/beta": 0.18798543512821198, + "beta_dpo/beta_margin_grad_mean": -0.36346742510795593, + "beta_dpo/beta_margin_grad_std": 0.27915558218955994, + "beta_dpo/beta_margin_mean": 3.220351457595825, + "beta_dpo/beta_margin_std": 8.585843086242676, + "beta_dpo/beta_used": 0.18798543512821198, + "beta_dpo/beta_used_raw": 0.10892662405967712, + "beta_dpo/gap_mean": 17.433303833007812, + "beta_dpo/gap_std": 27.836421966552734, + "beta_dpo/loss_margin_mean": 15.049211502075195, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8843537414965986, + "grad_norm": 92.1081314086914, + "learning_rate": 2.044597327993153e-08, + "logits/chosen": 0.4187523424625397, + "logits/rejected": 0.3752346932888031, + "loss": 0.9159, + "step": 585 + }, + { + "beta_dpo/beta": 0.22254735231399536, + "beta_dpo/beta_margin_grad_mean": -0.22272846102714539, + "beta_dpo/beta_margin_grad_std": 0.298261821269989, + "beta_dpo/beta_margin_mean": 4.026047706604004, + "beta_dpo/beta_margin_std": 5.531918048858643, + "beta_dpo/beta_used": 0.22254735231399536, + "beta_dpo/beta_used_raw": 0.22254735231399536, + "beta_dpo/gap_mean": 17.71161651611328, + "beta_dpo/gap_std": 27.249780654907227, + "beta_dpo/loss_margin_mean": 18.725635528564453, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8858654572940288, + "grad_norm": 119.44193267822266, + "learning_rate": 1.9925482037469187e-08, + "logits/chosen": 0.5063425302505493, + "logits/rejected": 0.44981545209884644, + "loss": 1.0304, + "step": 586 + }, + { + "beta_dpo/beta": 0.1785348355770111, + "beta_dpo/beta_margin_grad_mean": -0.3711192309856415, + "beta_dpo/beta_margin_grad_std": 0.2906634509563446, + "beta_dpo/beta_margin_mean": 3.2333409786224365, + "beta_dpo/beta_margin_std": 8.240671157836914, + "beta_dpo/beta_used": 0.1785348355770111, + "beta_dpo/beta_used_raw": 0.0767713189125061, + "beta_dpo/gap_mean": 17.747962951660156, + "beta_dpo/gap_std": 27.803863525390625, + "beta_dpo/loss_margin_mean": 18.480619430541992, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8873771730914588, + "grad_norm": 119.20809936523438, + "learning_rate": 1.9411426473854687e-08, + "logits/chosen": 0.49700820446014404, + "logits/rejected": 0.48014259338378906, + "loss": 1.1634, + "step": 587 + }, + { + "beta_dpo/beta": 0.003681016620248556, + "beta_dpo/beta_margin_grad_mean": -0.4817616045475006, + "beta_dpo/beta_margin_grad_std": 0.033373910933732986, + "beta_dpo/beta_margin_mean": 0.0735802948474884, + "beta_dpo/beta_margin_std": 0.13508976995944977, + "beta_dpo/beta_used": 0.003681016620248556, + "beta_dpo/beta_used_raw": -0.06429736316204071, + "beta_dpo/gap_mean": 17.947223663330078, + "beta_dpo/gap_std": 28.0413875579834, + "beta_dpo/loss_margin_mean": 17.880977630615234, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8888888888888888, + "grad_norm": 3.93040132522583, + "learning_rate": 1.890382096832699e-08, + "logits/chosen": 0.5224720239639282, + "logits/rejected": 0.47010236978530884, + "loss": 1.3306, + "step": 588 + }, + { + "beta_dpo/beta": 0.2928984761238098, + "beta_dpo/beta_margin_grad_mean": -0.18171432614326477, + "beta_dpo/beta_margin_grad_std": 0.3138827085494995, + "beta_dpo/beta_margin_mean": 6.427730083465576, + "beta_dpo/beta_margin_std": 8.318559646606445, + "beta_dpo/beta_used": 0.2928984761238098, + "beta_dpo/beta_used_raw": 0.2928984761238098, + "beta_dpo/gap_mean": 18.34659194946289, + "beta_dpo/gap_std": 27.974504470825195, + "beta_dpo/loss_margin_mean": 21.777563095092773, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.890400604686319, + "grad_norm": 126.65879821777344, + "learning_rate": 1.840267971970344e-08, + "logits/chosen": 0.4884239137172699, + "logits/rejected": 0.45349234342575073, + "loss": 0.8287, + "step": 589 + }, + { + "beta_dpo/beta": 0.22133781015872955, + "beta_dpo/beta_margin_grad_mean": -0.3547823429107666, + "beta_dpo/beta_margin_grad_std": 0.28267359733581543, + "beta_dpo/beta_margin_mean": 5.317359447479248, + "beta_dpo/beta_margin_std": 10.699882507324219, + "beta_dpo/beta_used": 0.22133781015872955, + "beta_dpo/beta_used_raw": 0.1810942143201828, + "beta_dpo/gap_mean": 18.91448211669922, + "beta_dpo/gap_std": 28.036495208740234, + "beta_dpo/loss_margin_mean": 21.749927520751953, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.891912320483749, + "grad_norm": 171.58700561523438, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": 0.4372316896915436, + "logits/rejected": 0.4048703610897064, + "loss": 1.1663, + "step": 590 + }, + { + "beta_dpo/beta": 0.1789139211177826, + "beta_dpo/beta_margin_grad_mean": -0.3391687273979187, + "beta_dpo/beta_margin_grad_std": 0.28517794609069824, + "beta_dpo/beta_margin_mean": 5.130792617797852, + "beta_dpo/beta_margin_std": 8.753190994262695, + "beta_dpo/beta_used": 0.1789139211177826, + "beta_dpo/beta_used_raw": 0.0776260495185852, + "beta_dpo/gap_mean": 19.851974487304688, + "beta_dpo/gap_std": 28.40129280090332, + "beta_dpo/loss_margin_mean": 21.574308395385742, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8934240362811792, + "grad_norm": 108.95458221435547, + "learning_rate": 1.7419845883949098e-08, + "logits/chosen": 0.5855053663253784, + "logits/rejected": 0.5133456587791443, + "loss": 1.3158, + "step": 591 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4951789975166321, + "beta_dpo/beta_margin_grad_std": 0.007032149471342564, + "beta_dpo/beta_margin_mean": 0.019290054216980934, + "beta_dpo/beta_margin_std": 0.0281399916857481, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.47162604331970215, + "beta_dpo/gap_mean": 19.769153594970703, + "beta_dpo/gap_std": 28.69797134399414, + "beta_dpo/loss_margin_mean": 19.290054321289062, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8949357520786092, + "grad_norm": 0.760765790939331, + "learning_rate": 1.6938180788793556e-08, + "logits/chosen": 0.5299474000930786, + "logits/rejected": 0.41111090779304504, + "loss": 1.3762, + "step": 592 + }, + { + "beta_dpo/beta": 0.1399514377117157, + "beta_dpo/beta_margin_grad_mean": -0.2983793616294861, + "beta_dpo/beta_margin_grad_std": 0.32489582896232605, + "beta_dpo/beta_margin_mean": 2.4921958446502686, + "beta_dpo/beta_margin_std": 5.445460319519043, + "beta_dpo/beta_used": 0.1399514377117157, + "beta_dpo/beta_used_raw": 0.1399514377117157, + "beta_dpo/gap_mean": 19.371078491210938, + "beta_dpo/gap_std": 28.608142852783203, + "beta_dpo/loss_margin_mean": 18.472137451171875, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8964474678760394, + "grad_norm": 72.96538543701172, + "learning_rate": 1.6463034933723336e-08, + "logits/chosen": 0.4820663630962372, + "logits/rejected": 0.38356471061706543, + "loss": 0.9054, + "step": 593 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49589961767196655, + "beta_dpo/beta_margin_grad_std": 0.00622721528634429, + "beta_dpo/beta_margin_mean": 0.01640484668314457, + "beta_dpo/beta_margin_std": 0.024914657697081566, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.08366122096776962, + "beta_dpo/gap_mean": 18.921382904052734, + "beta_dpo/gap_std": 28.415973663330078, + "beta_dpo/loss_margin_mean": 16.40484619140625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8979591836734694, + "grad_norm": 0.9117282032966614, + "learning_rate": 1.5994421609589385e-08, + "logits/chosen": 0.40028396248817444, + "logits/rejected": 0.3776008188724518, + "loss": 1.3706, + "step": 594 + }, + { + "beta_dpo/beta": 0.35656407475471497, + "beta_dpo/beta_margin_grad_mean": -0.2003844678401947, + "beta_dpo/beta_margin_grad_std": 0.36102327704429626, + "beta_dpo/beta_margin_mean": 8.762179374694824, + "beta_dpo/beta_margin_std": 11.819833755493164, + "beta_dpo/beta_used": 0.35656407475471497, + "beta_dpo/beta_used_raw": 0.35656407475471497, + "beta_dpo/gap_mean": 19.40142822265625, + "beta_dpo/gap_std": 28.8588809967041, + "beta_dpo/loss_margin_mean": 24.271892547607422, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.8994708994708994, + "grad_norm": 206.3365936279297, + "learning_rate": 1.553235392451377e-08, + "logits/chosen": 0.4917094111442566, + "logits/rejected": 0.3987153470516205, + "loss": 1.3807, + "step": 595 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4972356855869293, + "beta_dpo/beta_margin_grad_std": 0.0067308153957128525, + "beta_dpo/beta_margin_mean": 0.01106041856110096, + "beta_dpo/beta_margin_std": 0.026931054890155792, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.40151554346084595, + "beta_dpo/gap_mean": 18.570661544799805, + "beta_dpo/gap_std": 28.856903076171875, + "beta_dpo/loss_margin_mean": 11.060418128967285, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9009826152683296, + "grad_norm": 0.7493230104446411, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 0.3801739811897278, + "logits/rejected": 0.3889135718345642, + "loss": 1.3762, + "step": 596 + }, + { + "beta_dpo/beta": 0.12935660779476166, + "beta_dpo/beta_margin_grad_mean": -0.38230597972869873, + "beta_dpo/beta_margin_grad_std": 0.24667063355445862, + "beta_dpo/beta_margin_mean": 2.488924741744995, + "beta_dpo/beta_margin_std": 5.174806118011475, + "beta_dpo/beta_used": 0.12935660779476166, + "beta_dpo/beta_used_raw": -0.017168477177619934, + "beta_dpo/gap_mean": 18.310619354248047, + "beta_dpo/gap_std": 28.702266693115234, + "beta_dpo/loss_margin_mean": 19.412736892700195, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9024943310657596, + "grad_norm": 54.9415283203125, + "learning_rate": 1.4627906988186111e-08, + "logits/chosen": 0.4415905177593231, + "logits/rejected": 0.41522032022476196, + "loss": 1.0468, + "step": 597 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49707233905792236, + "beta_dpo/beta_margin_grad_std": 0.006856715772300959, + "beta_dpo/beta_margin_mean": 0.011713477782905102, + "beta_dpo/beta_margin_std": 0.027433032169938087, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.15161052346229553, + "beta_dpo/gap_mean": 17.37384033203125, + "beta_dpo/gap_std": 28.340988159179688, + "beta_dpo/loss_margin_mean": 11.71347713470459, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9040060468631897, + "grad_norm": 0.8391776084899902, + "learning_rate": 1.4185553036259095e-08, + "logits/chosen": 0.5034236311912537, + "logits/rejected": 0.421264111995697, + "loss": 1.3733, + "step": 598 + }, + { + "beta_dpo/beta": 0.024908404797315598, + "beta_dpo/beta_margin_grad_mean": -0.43633511662483215, + "beta_dpo/beta_margin_grad_std": 0.17187337577342987, + "beta_dpo/beta_margin_mean": 0.352827787399292, + "beta_dpo/beta_margin_std": 0.9965790510177612, + "beta_dpo/beta_used": 0.024908404797315598, + "beta_dpo/beta_used_raw": -0.013875270262360573, + "beta_dpo/gap_mean": 16.874439239501953, + "beta_dpo/gap_std": 28.45179557800293, + "beta_dpo/loss_margin_mean": 16.18917465209961, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9055177626606198, + "grad_norm": 19.783573150634766, + "learning_rate": 1.3749795321332885e-08, + "logits/chosen": 0.5565443634986877, + "logits/rejected": 0.5078023076057434, + "loss": 1.1524, + "step": 599 + }, + { + "beta_dpo/beta": 0.05185613036155701, + "beta_dpo/beta_margin_grad_mean": -0.3738482892513275, + "beta_dpo/beta_margin_grad_std": 0.23426569998264313, + "beta_dpo/beta_margin_mean": 0.8810456991195679, + "beta_dpo/beta_margin_std": 1.6753695011138916, + "beta_dpo/beta_used": 0.05185613036155701, + "beta_dpo/beta_used_raw": 0.05185613036155701, + "beta_dpo/gap_mean": 16.18065071105957, + "beta_dpo/gap_std": 28.42178726196289, + "beta_dpo/loss_margin_mean": 14.054128646850586, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9070294784580499, + "grad_norm": 29.925748825073242, + "learning_rate": 1.3320646032487393e-08, + "logits/chosen": 0.5630729794502258, + "logits/rejected": 0.508116602897644, + "loss": 1.0347, + "step": 600 + }, + { + "epoch": 0.9070294784580499, + "eval_beta_dpo/beta": 0.1704542338848114, + "eval_beta_dpo/beta_margin_grad_mean": -0.33918437361717224, + "eval_beta_dpo/beta_margin_grad_std": 0.22294603288173676, + "eval_beta_dpo/beta_margin_mean": 3.5151495933532715, + "eval_beta_dpo/beta_margin_std": 4.756720542907715, + "eval_beta_dpo/beta_used": 0.1704542338848114, + "eval_beta_dpo/beta_used_raw": 0.10854478925466537, + "eval_beta_dpo/gap_mean": 16.476804733276367, + "eval_beta_dpo/gap_std": 28.413129806518555, + "eval_beta_dpo/loss_margin_mean": 16.619216918945312, + "eval_beta_dpo/mask_keep_frac": 1.0, + "eval_logits/chosen": 0.5020915269851685, + "eval_logits/rejected": 0.4486764073371887, + "eval_loss": 0.820326030254364, + "eval_runtime": 38.9183, + "eval_samples_per_second": 59.175, + "eval_steps_per_second": 1.85, + "step": 600 + }, + { + "beta_dpo/beta": 0.18654750287532806, + "beta_dpo/beta_margin_grad_mean": -0.2733524739742279, + "beta_dpo/beta_margin_grad_std": 0.34814247488975525, + "beta_dpo/beta_margin_mean": 3.743360757827759, + "beta_dpo/beta_margin_std": 5.708962917327881, + "beta_dpo/beta_used": 0.18654750287532806, + "beta_dpo/beta_used_raw": 0.18654750287532806, + "beta_dpo/gap_mean": 17.23125457763672, + "beta_dpo/gap_std": 28.533397674560547, + "beta_dpo/loss_margin_mean": 20.574644088745117, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.90854119425548, + "grad_norm": 103.95361328125, + "learning_rate": 1.2898117173950868e-08, + "logits/chosen": 0.4921482801437378, + "logits/rejected": 0.41257598996162415, + "loss": 1.2026, + "step": 601 + }, + { + "beta_dpo/beta": 0.0725310891866684, + "beta_dpo/beta_margin_grad_mean": -0.37306496500968933, + "beta_dpo/beta_margin_grad_std": 0.26036691665649414, + "beta_dpo/beta_margin_mean": 1.5764257907867432, + "beta_dpo/beta_margin_std": 3.257750988006592, + "beta_dpo/beta_used": 0.0725310891866684, + "beta_dpo/beta_used_raw": 0.0725310891866684, + "beta_dpo/gap_mean": 17.40424346923828, + "beta_dpo/gap_std": 28.048587799072266, + "beta_dpo/loss_margin_mean": 19.2706298828125, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.91005291005291, + "grad_norm": 54.13090515136719, + "learning_rate": 1.2482220564763667e-08, + "logits/chosen": 0.4925674796104431, + "logits/rejected": 0.45530757308006287, + "loss": 1.1811, + "step": 602 + }, + { + "beta_dpo/beta": 0.09298260509967804, + "beta_dpo/beta_margin_grad_mean": -0.2825284004211426, + "beta_dpo/beta_margin_grad_std": 0.25072601437568665, + "beta_dpo/beta_margin_mean": 2.0486679077148438, + "beta_dpo/beta_margin_std": 2.9732513427734375, + "beta_dpo/beta_used": 0.09298260509967804, + "beta_dpo/beta_used_raw": 0.09298260509967804, + "beta_dpo/gap_mean": 18.233320236206055, + "beta_dpo/gap_std": 27.81458282470703, + "beta_dpo/loss_margin_mean": 21.40192413330078, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9115646258503401, + "grad_norm": 40.56179428100586, + "learning_rate": 1.2072967838448051e-08, + "logits/chosen": 0.4438127875328064, + "logits/rejected": 0.3784589469432831, + "loss": 0.8606, + "step": 603 + }, + { + "beta_dpo/beta": 0.12427462637424469, + "beta_dpo/beta_margin_grad_mean": -0.4008530080318451, + "beta_dpo/beta_margin_grad_std": 0.28347763419151306, + "beta_dpo/beta_margin_mean": 2.275778293609619, + "beta_dpo/beta_margin_std": 6.116153240203857, + "beta_dpo/beta_used": 0.12427462637424469, + "beta_dpo/beta_used_raw": 0.12097650021314621, + "beta_dpo/gap_mean": 18.284557342529297, + "beta_dpo/gap_std": 28.388805389404297, + "beta_dpo/loss_margin_mean": 18.04819107055664, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9130763416477702, + "grad_norm": 97.59881591796875, + "learning_rate": 1.1670370442682459e-08, + "logits/chosen": 0.4511834383010864, + "logits/rejected": 0.44945216178894043, + "loss": 1.2998, + "step": 604 + }, + { + "beta_dpo/beta": 0.1606498509645462, + "beta_dpo/beta_margin_grad_mean": -0.372048556804657, + "beta_dpo/beta_margin_grad_std": 0.2992105782032013, + "beta_dpo/beta_margin_mean": 3.194119691848755, + "beta_dpo/beta_margin_std": 7.407918930053711, + "beta_dpo/beta_used": 0.1606498509645462, + "beta_dpo/beta_used_raw": 0.08958107978105545, + "beta_dpo/gap_mean": 18.22399139404297, + "beta_dpo/gap_std": 28.62871551513672, + "beta_dpo/loss_margin_mean": 16.938005447387695, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9145880574452003, + "grad_norm": 166.026611328125, + "learning_rate": 1.1274439638981532e-08, + "logits/chosen": 0.5186942219734192, + "logits/rejected": 0.4566429853439331, + "loss": 1.974, + "step": 605 + }, + { + "beta_dpo/beta": 0.048199739307165146, + "beta_dpo/beta_margin_grad_mean": -0.3714365065097809, + "beta_dpo/beta_margin_grad_std": 0.2303820103406906, + "beta_dpo/beta_margin_mean": 1.0771068334579468, + "beta_dpo/beta_margin_std": 2.071831226348877, + "beta_dpo/beta_used": 0.048199739307165146, + "beta_dpo/beta_used_raw": -0.15278953313827515, + "beta_dpo/gap_mean": 17.971176147460938, + "beta_dpo/gap_std": 28.285913467407227, + "beta_dpo/loss_margin_mean": 18.89348793029785, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9160997732426304, + "grad_norm": 31.8127498626709, + "learning_rate": 1.0885186502381016e-08, + "logits/chosen": 0.4706125557422638, + "logits/rejected": 0.4000948667526245, + "loss": 1.0877, + "step": 606 + }, + { + "beta_dpo/beta": 0.05985303223133087, + "beta_dpo/beta_margin_grad_mean": -0.3560808300971985, + "beta_dpo/beta_margin_grad_std": 0.24726535379886627, + "beta_dpo/beta_margin_mean": 1.5488981008529663, + "beta_dpo/beta_margin_std": 2.7780096530914307, + "beta_dpo/beta_used": 0.05985303223133087, + "beta_dpo/beta_used_raw": -0.05173212289810181, + "beta_dpo/gap_mean": 18.670974731445312, + "beta_dpo/gap_std": 28.158889770507812, + "beta_dpo/loss_margin_mean": 19.359159469604492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9176114890400605, + "grad_norm": 41.329795837402344, + "learning_rate": 1.0502621921127774e-08, + "logits/chosen": 0.4254879951477051, + "logits/rejected": 0.391355037689209, + "loss": 1.0579, + "step": 607 + }, + { + "beta_dpo/beta": 0.007975354790687561, + "beta_dpo/beta_margin_grad_mean": -0.46832314133644104, + "beta_dpo/beta_margin_grad_std": 0.06795617938041687, + "beta_dpo/beta_margin_mean": 0.1326778531074524, + "beta_dpo/beta_margin_std": 0.29037222266197205, + "beta_dpo/beta_used": 0.007975354790687561, + "beta_dpo/beta_used_raw": -0.011102650314569473, + "beta_dpo/gap_mean": 18.097671508789062, + "beta_dpo/gap_std": 27.764236450195312, + "beta_dpo/loss_margin_mean": 16.52004051208496, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9191232048374905, + "grad_norm": 9.120287895202637, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": 0.4399696886539459, + "logits/rejected": 0.3656252324581146, + "loss": 1.2708, + "step": 608 + }, + { + "beta_dpo/beta": 0.23025476932525635, + "beta_dpo/beta_margin_grad_mean": -0.19502045214176178, + "beta_dpo/beta_margin_grad_std": 0.2594439685344696, + "beta_dpo/beta_margin_mean": 5.668548583984375, + "beta_dpo/beta_margin_std": 7.694107532501221, + "beta_dpo/beta_used": 0.23025476932525635, + "beta_dpo/beta_used_raw": 0.23025476932525635, + "beta_dpo/gap_mean": 18.70809555053711, + "beta_dpo/gap_std": 27.124021530151367, + "beta_dpo/loss_margin_mean": 21.427459716796875, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9206349206349206, + "grad_norm": 77.6288070678711, + "learning_rate": 9.757601041885694e-09, + "logits/chosen": 0.5577750205993652, + "logits/rejected": 0.5173044204711914, + "loss": 0.7607, + "step": 609 + }, + { + "beta_dpo/beta": 0.060976896435022354, + "beta_dpo/beta_margin_grad_mean": -0.37375470995903015, + "beta_dpo/beta_margin_grad_std": 0.24247336387634277, + "beta_dpo/beta_margin_mean": 1.3076086044311523, + "beta_dpo/beta_margin_std": 2.8063716888427734, + "beta_dpo/beta_used": 0.060976896435022354, + "beta_dpo/beta_used_raw": -0.14492599666118622, + "beta_dpo/gap_mean": 18.970539093017578, + "beta_dpo/gap_std": 27.81329345703125, + "beta_dpo/loss_margin_mean": 20.919218063354492, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9221466364323507, + "grad_norm": 58.45539093017578, + "learning_rate": 9.395165583732379e-09, + "logits/chosen": 0.4478539228439331, + "logits/rejected": 0.44471752643585205, + "loss": 1.0937, + "step": 610 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49625667929649353, + "beta_dpo/beta_margin_grad_std": 0.0058044674806296825, + "beta_dpo/beta_margin_mean": 0.014975893311202526, + "beta_dpo/beta_margin_std": 0.02322331629693508, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.11901400983333588, + "beta_dpo/gap_mean": 18.650028228759766, + "beta_dpo/gap_std": 27.604820251464844, + "beta_dpo/loss_margin_mean": 14.975893020629883, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9236583522297808, + "grad_norm": 0.8769718408584595, + "learning_rate": 9.03946036001449e-09, + "logits/chosen": 0.5406326055526733, + "logits/rejected": 0.48837387561798096, + "loss": 1.3715, + "step": 611 + }, + { + "beta_dpo/beta": 0.1870323270559311, + "beta_dpo/beta_margin_grad_mean": -0.2257097363471985, + "beta_dpo/beta_margin_grad_std": 0.30249518156051636, + "beta_dpo/beta_margin_mean": 4.107241630554199, + "beta_dpo/beta_margin_std": 6.412012100219727, + "beta_dpo/beta_used": 0.1870323270559311, + "beta_dpo/beta_used_raw": 0.1870323270559311, + "beta_dpo/gap_mean": 18.657466888427734, + "beta_dpo/gap_std": 27.669326782226562, + "beta_dpo/loss_margin_mean": 21.523975372314453, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9251700680272109, + "grad_norm": 87.718017578125, + "learning_rate": 8.690495320571839e-09, + "logits/chosen": 0.40628379583358765, + "logits/rejected": 0.3346533179283142, + "loss": 0.5951, + "step": 612 + }, + { + "beta_dpo/beta": 0.2268044352531433, + "beta_dpo/beta_margin_grad_mean": -0.31133928894996643, + "beta_dpo/beta_margin_grad_std": 0.27967050671577454, + "beta_dpo/beta_margin_mean": 6.159829616546631, + "beta_dpo/beta_margin_std": 11.035490036010742, + "beta_dpo/beta_used": 0.2268044352531433, + "beta_dpo/beta_used_raw": 0.18576528131961823, + "beta_dpo/gap_mean": 19.324243545532227, + "beta_dpo/gap_std": 27.89671516418457, + "beta_dpo/loss_margin_mean": 23.248201370239258, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.926681783824641, + "grad_norm": 60.471290588378906, + "learning_rate": 8.348280226706722e-09, + "logits/chosen": 0.3724338114261627, + "logits/rejected": 0.3626174032688141, + "loss": 0.8819, + "step": 613 + }, + { + "beta_dpo/beta": 0.029754722490906715, + "beta_dpo/beta_margin_grad_mean": -0.4166284203529358, + "beta_dpo/beta_margin_grad_std": 0.18172357976436615, + "beta_dpo/beta_margin_mean": 0.4524066746234894, + "beta_dpo/beta_margin_std": 1.0519875288009644, + "beta_dpo/beta_used": 0.029754722490906715, + "beta_dpo/beta_used_raw": 0.029382742941379547, + "beta_dpo/gap_mean": 19.42398452758789, + "beta_dpo/gap_std": 27.316984176635742, + "beta_dpo/loss_margin_mean": 18.69666290283203, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9281934996220711, + "grad_norm": 22.923320770263672, + "learning_rate": 8.012824650910937e-09, + "logits/chosen": 0.48424768447875977, + "logits/rejected": 0.4690684676170349, + "loss": 1.0765, + "step": 614 + }, + { + "beta_dpo/beta": 0.04531731456518173, + "beta_dpo/beta_margin_grad_mean": -0.3658197224140167, + "beta_dpo/beta_margin_grad_std": 0.2268880009651184, + "beta_dpo/beta_margin_mean": 1.1839110851287842, + "beta_dpo/beta_margin_std": 2.223015546798706, + "beta_dpo/beta_used": 0.04531731456518173, + "beta_dpo/beta_used_raw": -0.08215552568435669, + "beta_dpo/gap_mean": 20.149887084960938, + "beta_dpo/gap_std": 27.636917114257812, + "beta_dpo/loss_margin_mean": 22.330238342285156, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9297052154195011, + "grad_norm": 34.31875228881836, + "learning_rate": 7.684137976598088e-09, + "logits/chosen": 0.439863383769989, + "logits/rejected": 0.39079999923706055, + "loss": 1.1883, + "step": 615 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.495390921831131, + "beta_dpo/beta_margin_grad_std": 0.006648669950664043, + "beta_dpo/beta_margin_mean": 0.018440984189510345, + "beta_dpo/beta_margin_std": 0.026603393256664276, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.3312501907348633, + "beta_dpo/gap_mean": 19.891841888427734, + "beta_dpo/gap_std": 27.807634353637695, + "beta_dpo/loss_margin_mean": 18.440982818603516, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9312169312169312, + "grad_norm": 0.8765391111373901, + "learning_rate": 7.36222939784098e-09, + "logits/chosen": 0.5462205410003662, + "logits/rejected": 0.4555407762527466, + "loss": 1.3737, + "step": 616 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.495637446641922, + "beta_dpo/beta_margin_grad_std": 0.006583395879715681, + "beta_dpo/beta_margin_mean": 0.017454272136092186, + "beta_dpo/beta_margin_std": 0.026341097429394722, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.15900275111198425, + "beta_dpo/gap_mean": 19.752906799316406, + "beta_dpo/gap_std": 27.470291137695312, + "beta_dpo/loss_margin_mean": 17.45427131652832, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9327286470143613, + "grad_norm": 0.891712486743927, + "learning_rate": 7.047107919114586e-09, + "logits/chosen": 0.4766712188720703, + "logits/rejected": 0.426116943359375, + "loss": 1.371, + "step": 617 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4955264925956726, + "beta_dpo/beta_margin_grad_std": 0.006272831000387669, + "beta_dpo/beta_margin_mean": 0.01789809949696064, + "beta_dpo/beta_margin_std": 0.025099413469433784, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.06541182100772858, + "beta_dpo/gap_mean": 19.225358963012695, + "beta_dpo/gap_std": 26.753971099853516, + "beta_dpo/loss_margin_mean": 17.898099899291992, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9342403628117913, + "grad_norm": 0.9894082546234131, + "learning_rate": 6.738782355044048e-09, + "logits/chosen": 0.44365543127059937, + "logits/rejected": 0.3375873565673828, + "loss": 1.37, + "step": 618 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4957164525985718, + "beta_dpo/beta_margin_grad_std": 0.0061353943310678005, + "beta_dpo/beta_margin_mean": 0.01713750697672367, + "beta_dpo/beta_margin_std": 0.024547090753912926, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.07072946429252625, + "beta_dpo/gap_mean": 18.825641632080078, + "beta_dpo/gap_std": 26.591014862060547, + "beta_dpo/loss_margin_mean": 17.13750648498535, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9357520786092215, + "grad_norm": 0.9369555711746216, + "learning_rate": 6.437261330158206e-09, + "logits/chosen": 0.5680551528930664, + "logits/rejected": 0.4911302328109741, + "loss": 1.3705, + "step": 619 + }, + { + "beta_dpo/beta": 0.04867827519774437, + "beta_dpo/beta_margin_grad_mean": -0.35377609729766846, + "beta_dpo/beta_margin_grad_std": 0.22725717723369598, + "beta_dpo/beta_margin_mean": 1.2456239461898804, + "beta_dpo/beta_margin_std": 2.0975840091705322, + "beta_dpo/beta_used": 0.04867827519774437, + "beta_dpo/beta_used_raw": -0.3070789873600006, + "beta_dpo/gap_mean": 17.939937591552734, + "beta_dpo/gap_std": 26.433208465576172, + "beta_dpo/loss_margin_mean": 17.016864776611328, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9372637944066515, + "grad_norm": 33.47443389892578, + "learning_rate": 6.142553278648238e-09, + "logits/chosen": 0.4851877987384796, + "logits/rejected": 0.47673359513282776, + "loss": 1.0516, + "step": 620 + }, + { + "beta_dpo/beta": 0.03595667704939842, + "beta_dpo/beta_margin_grad_mean": -0.4299541711807251, + "beta_dpo/beta_margin_grad_std": 0.20840981602668762, + "beta_dpo/beta_margin_mean": 0.38952481746673584, + "beta_dpo/beta_margin_std": 1.332067847251892, + "beta_dpo/beta_used": 0.03595667704939842, + "beta_dpo/beta_used_raw": -0.09256462752819061, + "beta_dpo/gap_mean": 17.678569793701172, + "beta_dpo/gap_std": 26.529159545898438, + "beta_dpo/loss_margin_mean": 15.055705070495605, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9387755102040817, + "grad_norm": 19.573747634887695, + "learning_rate": 5.854666444131934e-09, + "logits/chosen": 0.4968072772026062, + "logits/rejected": 0.39098191261291504, + "loss": 1.0654, + "step": 621 + }, + { + "beta_dpo/beta": 0.07677318155765533, + "beta_dpo/beta_margin_grad_mean": -0.3336784541606903, + "beta_dpo/beta_margin_grad_std": 0.21298815310001373, + "beta_dpo/beta_margin_mean": 1.1385470628738403, + "beta_dpo/beta_margin_std": 1.7350324392318726, + "beta_dpo/beta_used": 0.07677318155765533, + "beta_dpo/beta_used_raw": 0.07677318155765533, + "beta_dpo/gap_mean": 17.470783233642578, + "beta_dpo/gap_std": 25.936145782470703, + "beta_dpo/loss_margin_mean": 16.462358474731445, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9402872260015117, + "grad_norm": 35.35215759277344, + "learning_rate": 5.573608879422875e-09, + "logits/chosen": 0.45926177501678467, + "logits/rejected": 0.4160395860671997, + "loss": 0.8269, + "step": 622 + }, + { + "beta_dpo/beta": 0.23275969922542572, + "beta_dpo/beta_margin_grad_mean": -0.34063273668289185, + "beta_dpo/beta_margin_grad_std": 0.2965709865093231, + "beta_dpo/beta_margin_mean": 5.009967803955078, + "beta_dpo/beta_margin_std": 8.860584259033203, + "beta_dpo/beta_used": 0.23275969922542572, + "beta_dpo/beta_used_raw": 0.2276362031698227, + "beta_dpo/gap_mean": 18.017637252807617, + "beta_dpo/gap_std": 25.823673248291016, + "beta_dpo/loss_margin_mean": 20.478515625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9417989417989417, + "grad_norm": 65.43958282470703, + "learning_rate": 5.299388446305342e-09, + "logits/chosen": 0.4594137668609619, + "logits/rejected": 0.3915611505508423, + "loss": 0.8451, + "step": 623 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49523109197616577, + "beta_dpo/beta_margin_grad_std": 0.006707730703055859, + "beta_dpo/beta_margin_mean": 0.01908011920750141, + "beta_dpo/beta_margin_std": 0.026838213205337524, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.16606451570987701, + "beta_dpo/gap_mean": 18.317127227783203, + "beta_dpo/gap_std": 26.19288444519043, + "beta_dpo/loss_margin_mean": 19.08011817932129, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9433106575963719, + "grad_norm": 0.9514113068580627, + "learning_rate": 5.03201281531429e-09, + "logits/chosen": 0.458259642124176, + "logits/rejected": 0.3561304807662964, + "loss": 1.3726, + "step": 624 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49637553095817566, + "beta_dpo/beta_margin_grad_std": 0.006895896513015032, + "beta_dpo/beta_margin_mean": 0.014501559548079967, + "beta_dpo/beta_margin_std": 0.027591099962592125, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.17098666727542877, + "beta_dpo/gap_mean": 17.662364959716797, + "beta_dpo/gap_std": 26.34381103515625, + "beta_dpo/loss_margin_mean": 14.501558303833008, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9448223733938019, + "grad_norm": 0.7157669067382812, + "learning_rate": 4.7714894655209174e-09, + "logits/chosen": 0.5461119413375854, + "logits/rejected": 0.4578895568847656, + "loss": 1.3733, + "step": 625 + }, + { + "beta_dpo/beta": 0.23165586590766907, + "beta_dpo/beta_margin_grad_mean": -0.3843648135662079, + "beta_dpo/beta_margin_grad_std": 0.3157823085784912, + "beta_dpo/beta_margin_mean": 5.67930793762207, + "beta_dpo/beta_margin_std": 12.665205001831055, + "beta_dpo/beta_used": 0.23165586590766907, + "beta_dpo/beta_used_raw": 0.08626031875610352, + "beta_dpo/gap_mean": 18.347545623779297, + "beta_dpo/gap_std": 27.535655975341797, + "beta_dpo/loss_margin_mean": 22.30915069580078, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9463340891912321, + "grad_norm": 216.2509002685547, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": 0.5618187189102173, + "logits/rejected": 0.42893868684768677, + "loss": 2.5394, + "step": 626 + }, + { + "beta_dpo/beta": 0.22693291306495667, + "beta_dpo/beta_margin_grad_mean": -0.20085382461547852, + "beta_dpo/beta_margin_grad_std": 0.3276273012161255, + "beta_dpo/beta_margin_mean": 5.549400806427002, + "beta_dpo/beta_margin_std": 6.667171001434326, + "beta_dpo/beta_used": 0.22693291306495667, + "beta_dpo/beta_used_raw": 0.22693291306495667, + "beta_dpo/gap_mean": 19.314483642578125, + "beta_dpo/gap_std": 27.916873931884766, + "beta_dpo/loss_margin_mean": 24.403059005737305, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9478458049886621, + "grad_norm": 109.18547058105469, + "learning_rate": 4.271028567242818e-09, + "logits/chosen": 0.40109649300575256, + "logits/rejected": 0.2826465666294098, + "loss": 0.7044, + "step": 627 + }, + { + "beta_dpo/beta": 0.3724895119667053, + "beta_dpo/beta_margin_grad_mean": -0.23820850253105164, + "beta_dpo/beta_margin_grad_std": 0.38285335898399353, + "beta_dpo/beta_margin_mean": 10.399557113647461, + "beta_dpo/beta_margin_std": 13.7665376663208, + "beta_dpo/beta_used": 0.3724895119667053, + "beta_dpo/beta_used_raw": 0.3724895119667053, + "beta_dpo/gap_mean": 19.812664031982422, + "beta_dpo/gap_std": 29.181087493896484, + "beta_dpo/loss_margin_mean": 23.914892196655273, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9493575207860923, + "grad_norm": 242.51388549804688, + "learning_rate": 4.0311050177251895e-09, + "logits/chosen": 0.5061334371566772, + "logits/rejected": 0.4650648236274719, + "loss": 2.0624, + "step": 628 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49544239044189453, + "beta_dpo/beta_margin_grad_std": 0.005720792803913355, + "beta_dpo/beta_margin_mean": 0.018234064802527428, + "beta_dpo/beta_margin_std": 0.022889522835612297, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.031100261956453323, + "beta_dpo/gap_mean": 20.11130142211914, + "beta_dpo/gap_std": 28.020038604736328, + "beta_dpo/loss_margin_mean": 18.23406410217285, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9508692365835223, + "grad_norm": 1.0008560419082642, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 0.45952147245407104, + "logits/rejected": 0.44886553287506104, + "loss": 1.3686, + "step": 629 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4949391186237335, + "beta_dpo/beta_margin_grad_std": 0.006423095241189003, + "beta_dpo/beta_margin_mean": 0.02024826407432556, + "beta_dpo/beta_margin_std": 0.025700394064188004, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.11937922239303589, + "beta_dpo/gap_mean": 19.894287109375, + "beta_dpo/gap_std": 27.414846420288086, + "beta_dpo/loss_margin_mean": 20.248262405395508, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9523809523809523, + "grad_norm": 1.0278126001358032, + "learning_rate": 3.5719052736323806e-09, + "logits/chosen": 0.4336046576499939, + "logits/rejected": 0.38262441754341125, + "loss": 1.3702, + "step": 630 + }, + { + "beta_dpo/beta": 0.2094002515077591, + "beta_dpo/beta_margin_grad_mean": -0.3453434407711029, + "beta_dpo/beta_margin_grad_std": 0.31047523021698, + "beta_dpo/beta_margin_mean": 6.9932379722595215, + "beta_dpo/beta_margin_std": 11.639176368713379, + "beta_dpo/beta_used": 0.2094002515077591, + "beta_dpo/beta_used_raw": 0.11465645581483841, + "beta_dpo/gap_mean": 21.329425811767578, + "beta_dpo/gap_std": 28.016944885253906, + "beta_dpo/loss_margin_mean": 26.024810791015625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9538926681783825, + "grad_norm": 270.0311279296875, + "learning_rate": 3.352641923861144e-09, + "logits/chosen": 0.5511718988418579, + "logits/rejected": 0.44041550159454346, + "loss": 1.8247, + "step": 631 + }, + { + "beta_dpo/beta": 0.32847917079925537, + "beta_dpo/beta_margin_grad_mean": -0.3383893370628357, + "beta_dpo/beta_margin_grad_std": 0.309362530708313, + "beta_dpo/beta_margin_mean": 8.23188304901123, + "beta_dpo/beta_margin_std": 15.608662605285645, + "beta_dpo/beta_used": 0.32847917079925537, + "beta_dpo/beta_used_raw": 0.31657373905181885, + "beta_dpo/gap_mean": 21.614561080932617, + "beta_dpo/gap_std": 28.078006744384766, + "beta_dpo/loss_margin_mean": 23.72018814086914, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9554043839758125, + "grad_norm": 259.4488220214844, + "learning_rate": 3.140277830901428e-09, + "logits/chosen": 0.5116778612136841, + "logits/rejected": 0.4842451214790344, + "loss": 1.6392, + "step": 632 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4954851567745209, + "beta_dpo/beta_margin_grad_std": 0.007191209588199854, + "beta_dpo/beta_margin_mean": 0.018064936622977257, + "beta_dpo/beta_margin_std": 0.02877538837492466, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.3825916647911072, + "beta_dpo/gap_mean": 21.042991638183594, + "beta_dpo/gap_std": 27.94135856628418, + "beta_dpo/loss_margin_mean": 18.0649356842041, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9569160997732427, + "grad_norm": 0.7798741459846497, + "learning_rate": 2.9348189350335007e-09, + "logits/chosen": 0.46421927213668823, + "logits/rejected": 0.39746445417404175, + "loss": 1.3735, + "step": 633 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4972086250782013, + "beta_dpo/beta_margin_grad_std": 0.007200221065431833, + "beta_dpo/beta_margin_mean": 0.01116915326565504, + "beta_dpo/beta_margin_std": 0.02880818583071232, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.4677852690219879, + "beta_dpo/gap_mean": 19.498958587646484, + "beta_dpo/gap_std": 28.379730224609375, + "beta_dpo/loss_margin_mean": 11.169153213500977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9584278155706727, + "grad_norm": 0.7306960225105286, + "learning_rate": 2.736270983384276e-09, + "logits/chosen": 0.5405286550521851, + "logits/rejected": 0.5492910146713257, + "loss": 1.3765, + "step": 634 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.4961506724357605, + "beta_dpo/beta_margin_grad_std": 0.008018962107598782, + "beta_dpo/beta_margin_mean": 0.015402358956634998, + "beta_dpo/beta_margin_std": 0.03208545595407486, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.15202215313911438, + "beta_dpo/gap_mean": 18.54298973083496, + "beta_dpo/gap_std": 29.06603240966797, + "beta_dpo/loss_margin_mean": 15.402358055114746, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9599395313681028, + "grad_norm": 0.9664364457130432, + "learning_rate": 2.5446395297668287e-09, + "logits/chosen": 0.35498249530792236, + "logits/rejected": 0.28940173983573914, + "loss": 1.3722, + "step": 635 + }, + { + "beta_dpo/beta": 0.17797021567821503, + "beta_dpo/beta_margin_grad_mean": -0.2381211668252945, + "beta_dpo/beta_margin_grad_std": 0.2688322961330414, + "beta_dpo/beta_margin_mean": 4.206960678100586, + "beta_dpo/beta_margin_std": 6.49386739730835, + "beta_dpo/beta_used": 0.17797021567821503, + "beta_dpo/beta_used_raw": 0.17797021567821503, + "beta_dpo/gap_mean": 19.014446258544922, + "beta_dpo/gap_std": 28.835285186767578, + "beta_dpo/loss_margin_mean": 21.473487854003906, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9614512471655329, + "grad_norm": 79.36841583251953, + "learning_rate": 2.359929934524829e-09, + "logits/chosen": 0.4399072527885437, + "logits/rejected": 0.3462012708187103, + "loss": 0.6422, + "step": 636 + }, + { + "beta_dpo/beta": 0.10971579700708389, + "beta_dpo/beta_margin_grad_mean": -0.3816455602645874, + "beta_dpo/beta_margin_grad_std": 0.274883896112442, + "beta_dpo/beta_margin_mean": 2.120753526687622, + "beta_dpo/beta_margin_std": 5.4004292488098145, + "beta_dpo/beta_used": 0.10971579700708389, + "beta_dpo/beta_used_raw": -0.21972408890724182, + "beta_dpo/gap_mean": 18.41397476196289, + "beta_dpo/gap_std": 29.193927764892578, + "beta_dpo/loss_margin_mean": 16.04454803466797, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9629629629629629, + "grad_norm": 69.94818878173828, + "learning_rate": 2.1821473643827137e-09, + "logits/chosen": 0.41486144065856934, + "logits/rejected": 0.339840829372406, + "loss": 1.0826, + "step": 637 + }, + { + "beta_dpo/beta": 0.18578550219535828, + "beta_dpo/beta_margin_grad_mean": -0.28964492678642273, + "beta_dpo/beta_margin_grad_std": 0.32899391651153564, + "beta_dpo/beta_margin_mean": 3.6086082458496094, + "beta_dpo/beta_margin_std": 5.766052722930908, + "beta_dpo/beta_used": 0.18578550219535828, + "beta_dpo/beta_used_raw": 0.18578550219535828, + "beta_dpo/gap_mean": 18.60744857788086, + "beta_dpo/gap_std": 29.17364501953125, + "beta_dpo/loss_margin_mean": 19.51310920715332, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9644746787603931, + "grad_norm": 142.8914031982422, + "learning_rate": 2.0112967923011646e-09, + "logits/chosen": 0.46569958329200745, + "logits/rejected": 0.4127267599105835, + "loss": 1.1797, + "step": 638 + }, + { + "beta_dpo/beta": 0.030427690595388412, + "beta_dpo/beta_margin_grad_mean": -0.4014831483364105, + "beta_dpo/beta_margin_grad_std": 0.18178702890872955, + "beta_dpo/beta_margin_mean": 0.603205680847168, + "beta_dpo/beta_margin_std": 1.2123973369598389, + "beta_dpo/beta_used": 0.030427690595388412, + "beta_dpo/beta_used_raw": -0.04119763523340225, + "beta_dpo/gap_mean": 19.007099151611328, + "beta_dpo/gap_std": 28.996734619140625, + "beta_dpo/loss_margin_mean": 21.59126091003418, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9659863945578231, + "grad_norm": 21.713918685913086, + "learning_rate": 1.847382997337943e-09, + "logits/chosen": 0.47108232975006104, + "logits/rejected": 0.35434550046920776, + "loss": 1.1042, + "step": 639 + }, + { + "beta_dpo/beta": 0.09601413458585739, + "beta_dpo/beta_margin_grad_mean": -0.3168926239013672, + "beta_dpo/beta_margin_grad_std": 0.2539225220680237, + "beta_dpo/beta_margin_mean": 1.7108955383300781, + "beta_dpo/beta_margin_std": 3.4565632343292236, + "beta_dpo/beta_used": 0.09601413458585739, + "beta_dpo/beta_used_raw": 0.09601413458585739, + "beta_dpo/gap_mean": 18.945955276489258, + "beta_dpo/gap_std": 28.959095001220703, + "beta_dpo/loss_margin_mean": 16.798215866088867, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9674981103552532, + "grad_norm": 47.492164611816406, + "learning_rate": 1.690410564514244e-09, + "logits/chosen": 0.5608204007148743, + "logits/rejected": 0.4967699646949768, + "loss": 0.7935, + "step": 640 + }, + { + "beta_dpo/beta": 0.04548508673906326, + "beta_dpo/beta_margin_grad_mean": -0.39172929525375366, + "beta_dpo/beta_margin_grad_std": 0.21917176246643066, + "beta_dpo/beta_margin_mean": 0.8428230881690979, + "beta_dpo/beta_margin_std": 1.80697500705719, + "beta_dpo/beta_used": 0.04548508673906326, + "beta_dpo/beta_used_raw": -0.013075441122055054, + "beta_dpo/gap_mean": 18.613567352294922, + "beta_dpo/gap_std": 28.440582275390625, + "beta_dpo/loss_margin_mean": 17.370569229125977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9690098261526833, + "grad_norm": 32.292781829833984, + "learning_rate": 1.5403838846864692e-09, + "logits/chosen": 0.46294909715652466, + "logits/rejected": 0.43334758281707764, + "loss": 1.0495, + "step": 641 + }, + { + "beta_dpo/beta": 0.03415125980973244, + "beta_dpo/beta_margin_grad_mean": -0.4151100516319275, + "beta_dpo/beta_margin_grad_std": 0.2164982706308365, + "beta_dpo/beta_margin_mean": 0.6367584466934204, + "beta_dpo/beta_margin_std": 1.5332374572753906, + "beta_dpo/beta_used": 0.03415125980973244, + "beta_dpo/beta_used_raw": -0.2329845428466797, + "beta_dpo/gap_mean": 18.07403564453125, + "beta_dpo/gap_std": 28.444255828857422, + "beta_dpo/loss_margin_mean": 14.236799240112305, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9705215419501134, + "grad_norm": 34.46243667602539, + "learning_rate": 1.3973071544233218e-09, + "logits/chosen": 0.42237862944602966, + "logits/rejected": 0.42818087339401245, + "loss": 1.1772, + "step": 642 + }, + { + "beta_dpo/beta": 0.06243688613176346, + "beta_dpo/beta_margin_grad_mean": -0.3356240689754486, + "beta_dpo/beta_margin_grad_std": 0.26229581236839294, + "beta_dpo/beta_margin_mean": 1.0187599658966064, + "beta_dpo/beta_margin_std": 1.7895455360412598, + "beta_dpo/beta_used": 0.06243688613176346, + "beta_dpo/beta_used_raw": 0.06243688613176346, + "beta_dpo/gap_mean": 17.609920501708984, + "beta_dpo/gap_std": 28.21491813659668, + "beta_dpo/loss_margin_mean": 16.685531616210938, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9720332577475435, + "grad_norm": 82.64678955078125, + "learning_rate": 1.261184375888541e-09, + "logits/chosen": 0.4097246825695038, + "logits/rejected": 0.3160959482192993, + "loss": 0.9238, + "step": 643 + }, + { + "beta_dpo/beta": 0.2932557463645935, + "beta_dpo/beta_margin_grad_mean": -0.36465975642204285, + "beta_dpo/beta_margin_grad_std": 0.3044781982898712, + "beta_dpo/beta_margin_mean": 5.584909915924072, + "beta_dpo/beta_margin_std": 14.013591766357422, + "beta_dpo/beta_used": 0.2932557463645935, + "beta_dpo/beta_used_raw": 0.2932557463645935, + "beta_dpo/gap_mean": 16.896411895751953, + "beta_dpo/gap_std": 28.862873077392578, + "beta_dpo/loss_margin_mean": 15.0465669631958, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9735449735449735, + "grad_norm": 197.562255859375, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": 0.5534158945083618, + "logits/rejected": 0.5172810554504395, + "loss": 1.2636, + "step": 644 + }, + { + "beta_dpo/beta": 0.09316226840019226, + "beta_dpo/beta_margin_grad_mean": -0.37199652194976807, + "beta_dpo/beta_margin_grad_std": 0.24579140543937683, + "beta_dpo/beta_margin_mean": 1.6856802701950073, + "beta_dpo/beta_margin_std": 3.9560256004333496, + "beta_dpo/beta_used": 0.09316226840019226, + "beta_dpo/beta_used_raw": 0.059367213398218155, + "beta_dpo/gap_mean": 17.337440490722656, + "beta_dpo/gap_std": 28.79489517211914, + "beta_dpo/loss_margin_mean": 18.958377838134766, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9750566893424036, + "grad_norm": 38.84883499145508, + "learning_rate": 1.0098157099674987e-09, + "logits/chosen": 0.4112117886543274, + "logits/rejected": 0.38647520542144775, + "loss": 0.9213, + "step": 645 + }, + { + "beta_dpo/beta": 0.12865673005580902, + "beta_dpo/beta_margin_grad_mean": -0.36234015226364136, + "beta_dpo/beta_margin_grad_std": 0.2799786329269409, + "beta_dpo/beta_margin_mean": 2.2738983631134033, + "beta_dpo/beta_margin_std": 6.0258049964904785, + "beta_dpo/beta_used": 0.12865673005580902, + "beta_dpo/beta_used_raw": 0.07598783075809479, + "beta_dpo/gap_mean": 17.541095733642578, + "beta_dpo/gap_std": 28.898643493652344, + "beta_dpo/loss_margin_mean": 18.890525817871094, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9765684051398337, + "grad_norm": 96.72042846679688, + "learning_rate": 8.945768539031783e-10, + "logits/chosen": 0.5024992227554321, + "logits/rejected": 0.44613319635391235, + "loss": 1.3655, + "step": 646 + }, + { + "beta_dpo/beta": 0.11004437506198883, + "beta_dpo/beta_margin_grad_mean": -0.2987731397151947, + "beta_dpo/beta_margin_grad_std": 0.2559739351272583, + "beta_dpo/beta_margin_mean": 2.851088047027588, + "beta_dpo/beta_margin_std": 4.678904056549072, + "beta_dpo/beta_used": 0.11004437506198883, + "beta_dpo/beta_used_raw": 0.11004437506198883, + "beta_dpo/gap_mean": 18.652297973632812, + "beta_dpo/gap_std": 28.77882194519043, + "beta_dpo/loss_margin_mean": 24.79449462890625, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9780801209372638, + "grad_norm": 83.78498840332031, + "learning_rate": 7.863060120144316e-10, + "logits/chosen": 0.5124300122261047, + "logits/rejected": 0.4137851595878601, + "loss": 1.0114, + "step": 647 + }, + { + "beta_dpo/beta": 0.3000466823577881, + "beta_dpo/beta_margin_grad_mean": -0.3578697144985199, + "beta_dpo/beta_margin_grad_std": 0.3171420395374298, + "beta_dpo/beta_margin_mean": 7.148730754852295, + "beta_dpo/beta_margin_std": 16.316646575927734, + "beta_dpo/beta_used": 0.3000466823577881, + "beta_dpo/beta_used_raw": 0.24156461656093597, + "beta_dpo/gap_mean": 19.34526252746582, + "beta_dpo/gap_std": 29.065902709960938, + "beta_dpo/loss_margin_mean": 19.919885635375977, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9795918367346939, + "grad_norm": 224.33082580566406, + "learning_rate": 6.850062128694045e-10, + "logits/chosen": 0.42840665578842163, + "logits/rejected": 0.35993558168411255, + "loss": 1.7301, + "step": 648 + }, + { + "beta_dpo/beta": 0.13330677151679993, + "beta_dpo/beta_margin_grad_mean": -0.30887559056282043, + "beta_dpo/beta_margin_grad_std": 0.3698316514492035, + "beta_dpo/beta_margin_mean": 2.7245736122131348, + "beta_dpo/beta_margin_std": 4.273642063140869, + "beta_dpo/beta_used": 0.13330677151679993, + "beta_dpo/beta_used_raw": 0.13330677151679993, + "beta_dpo/gap_mean": 19.349628448486328, + "beta_dpo/gap_std": 28.986984252929688, + "beta_dpo/loss_margin_mean": 19.400482177734375, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.981103552532124, + "grad_norm": 110.32051086425781, + "learning_rate": 5.906802900412788e-10, + "logits/chosen": 0.5004416704177856, + "logits/rejected": 0.43793487548828125, + "loss": 1.2039, + "step": 649 + }, + { + "beta_dpo/beta": 0.1441710889339447, + "beta_dpo/beta_margin_grad_mean": -0.3954494297504425, + "beta_dpo/beta_margin_grad_std": 0.2960076630115509, + "beta_dpo/beta_margin_mean": 2.8221378326416016, + "beta_dpo/beta_margin_std": 7.158821105957031, + "beta_dpo/beta_used": 0.1441710889339447, + "beta_dpo/beta_used_raw": 0.09482648968696594, + "beta_dpo/gap_mean": 19.037673950195312, + "beta_dpo/gap_std": 29.18463897705078, + "beta_dpo/loss_margin_mean": 18.523643493652344, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.982615268329554, + "grad_norm": 119.91067504882812, + "learning_rate": 5.033308820289184e-10, + "logits/chosen": 0.5848698019981384, + "logits/rejected": 0.5156843066215515, + "loss": 1.3874, + "step": 650 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49571534991264343, + "beta_dpo/beta_margin_grad_std": 0.006656995974481106, + "beta_dpo/beta_margin_mean": 0.017142919823527336, + "beta_dpo/beta_margin_std": 0.026636159047484398, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.2897125482559204, + "beta_dpo/gap_mean": 18.57063865661621, + "beta_dpo/gap_std": 28.645057678222656, + "beta_dpo/loss_margin_mean": 17.142919540405273, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9841269841269841, + "grad_norm": 0.8137643337249756, + "learning_rate": 4.2296043218295606e-10, + "logits/chosen": 0.5590111017227173, + "logits/rejected": 0.4814707338809967, + "loss": 1.3744, + "step": 651 + }, + { + "beta_dpo/beta": 0.01311043556779623, + "beta_dpo/beta_margin_grad_mean": -0.44266369938850403, + "beta_dpo/beta_margin_grad_std": 0.11703706532716751, + "beta_dpo/beta_margin_mean": 0.26388391852378845, + "beta_dpo/beta_margin_std": 0.5482731461524963, + "beta_dpo/beta_used": 0.01311043556779623, + "beta_dpo/beta_used_raw": -0.11602558195590973, + "beta_dpo/gap_mean": 18.64056396484375, + "beta_dpo/gap_std": 28.707130432128906, + "beta_dpo/loss_margin_mean": 16.924619674682617, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9856386999244142, + "grad_norm": 9.936495780944824, + "learning_rate": 3.4957118863768176e-10, + "logits/chosen": 0.4921461343765259, + "logits/rejected": 0.43392133712768555, + "loss": 1.2267, + "step": 652 + }, + { + "beta_dpo/beta": 0.010960490442812443, + "beta_dpo/beta_margin_grad_mean": -0.45152518153190613, + "beta_dpo/beta_margin_grad_std": 0.0951455682516098, + "beta_dpo/beta_margin_mean": 0.21232452988624573, + "beta_dpo/beta_margin_std": 0.42660677433013916, + "beta_dpo/beta_used": 0.010960490442812443, + "beta_dpo/beta_used_raw": -0.1325518786907196, + "beta_dpo/gap_mean": 18.40667724609375, + "beta_dpo/gap_std": 28.581680297851562, + "beta_dpo/loss_margin_mean": 18.254802703857422, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9871504157218443, + "grad_norm": 10.04647445678711, + "learning_rate": 2.831652042480093e-10, + "logits/chosen": 0.4425300359725952, + "logits/rejected": 0.394487589597702, + "loss": 1.2431, + "step": 653 + }, + { + "beta_dpo/beta": 0.00953736063092947, + "beta_dpo/beta_margin_grad_mean": -0.4530160427093506, + "beta_dpo/beta_margin_grad_std": 0.09966482222080231, + "beta_dpo/beta_margin_mean": 0.2068350464105606, + "beta_dpo/beta_margin_std": 0.44756725430488586, + "beta_dpo/beta_used": 0.00953736063092947, + "beta_dpo/beta_used_raw": -0.20549921691417694, + "beta_dpo/gap_mean": 18.37057113647461, + "beta_dpo/gap_std": 28.73928451538086, + "beta_dpo/loss_margin_mean": 16.940853118896484, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9886621315192744, + "grad_norm": 10.299638748168945, + "learning_rate": 2.2374433653205016e-10, + "logits/chosen": 0.4537598490715027, + "logits/rejected": 0.3531543016433716, + "loss": 1.2612, + "step": 654 + }, + { + "beta_dpo/beta": 0.13284625113010406, + "beta_dpo/beta_margin_grad_mean": -0.29767540097236633, + "beta_dpo/beta_margin_grad_std": 0.24205495417118073, + "beta_dpo/beta_margin_mean": 3.739741563796997, + "beta_dpo/beta_margin_std": 5.718697547912598, + "beta_dpo/beta_used": 0.13284625113010406, + "beta_dpo/beta_used_raw": 0.11756931990385056, + "beta_dpo/gap_mean": 18.73858642578125, + "beta_dpo/gap_std": 27.660343170166016, + "beta_dpo/loss_margin_mean": 20.653202056884766, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9901738473167044, + "grad_norm": 36.28606033325195, + "learning_rate": 1.7131024761923852e-10, + "logits/chosen": 0.4345353841781616, + "logits/rejected": 0.3393111824989319, + "loss": 0.8738, + "step": 655 + }, + { + "beta_dpo/beta": 0.20375779271125793, + "beta_dpo/beta_margin_grad_mean": -0.25210514664649963, + "beta_dpo/beta_margin_grad_std": 0.3344373404979706, + "beta_dpo/beta_margin_mean": 4.502808570861816, + "beta_dpo/beta_margin_std": 6.9566426277160645, + "beta_dpo/beta_used": 0.20375779271125793, + "beta_dpo/beta_used_raw": 0.20375779271125793, + "beta_dpo/gap_mean": 18.90631866455078, + "beta_dpo/gap_std": 27.543325424194336, + "beta_dpo/loss_margin_mean": 20.858234405517578, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9916855631141346, + "grad_norm": 89.89404296875, + "learning_rate": 1.2586440420372934e-10, + "logits/chosen": 0.4053155183792114, + "logits/rejected": 0.34848740696907043, + "loss": 0.8027, + "step": 656 + }, + { + "beta_dpo/beta": 0.2964837849140167, + "beta_dpo/beta_margin_grad_mean": -0.21630139648914337, + "beta_dpo/beta_margin_grad_std": 0.3331965208053589, + "beta_dpo/beta_margin_mean": 7.668827056884766, + "beta_dpo/beta_margin_std": 11.465837478637695, + "beta_dpo/beta_used": 0.2964837849140167, + "beta_dpo/beta_used_raw": 0.2964837849140167, + "beta_dpo/gap_mean": 19.829242706298828, + "beta_dpo/gap_std": 28.236858367919922, + "beta_dpo/loss_margin_mean": 24.64881134033203, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9931972789115646, + "grad_norm": 171.66510009765625, + "learning_rate": 8.740807750345913e-11, + "logits/chosen": 0.5963433384895325, + "logits/rejected": 0.49983879923820496, + "loss": 0.821, + "step": 657 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49546483159065247, + "beta_dpo/beta_margin_grad_std": 0.008204931393265724, + "beta_dpo/beta_margin_mean": 0.018146011978387833, + "beta_dpo/beta_margin_std": 0.03283081203699112, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.2486192286014557, + "beta_dpo/gap_mean": 19.505714416503906, + "beta_dpo/gap_std": 28.95309066772461, + "beta_dpo/loss_margin_mean": 18.146011352539062, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9947089947089947, + "grad_norm": 0.8370410799980164, + "learning_rate": 5.594234322453539e-11, + "logits/chosen": 0.5124013423919678, + "logits/rejected": 0.4685628414154053, + "loss": 1.3728, + "step": 658 + }, + { + "beta_dpo/beta": 0.08610343188047409, + "beta_dpo/beta_margin_grad_mean": -0.372318297624588, + "beta_dpo/beta_margin_grad_std": 0.28925395011901855, + "beta_dpo/beta_margin_mean": 1.7324120998382568, + "beta_dpo/beta_margin_std": 3.9688851833343506, + "beta_dpo/beta_used": 0.08610343188047409, + "beta_dpo/beta_used_raw": -0.0352298840880394, + "beta_dpo/gap_mean": 19.216293334960938, + "beta_dpo/gap_std": 29.245624542236328, + "beta_dpo/loss_margin_mean": 15.123144149780273, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9962207105064248, + "grad_norm": 63.744686126708984, + "learning_rate": 3.146808153123293e-11, + "logits/chosen": 0.5592527389526367, + "logits/rejected": 0.48994961380958557, + "loss": 1.3127, + "step": 659 + }, + { + "beta_dpo/beta": 0.3657574951648712, + "beta_dpo/beta_margin_grad_mean": -0.19617773592472076, + "beta_dpo/beta_margin_grad_std": 0.3252088725566864, + "beta_dpo/beta_margin_mean": 8.107396125793457, + "beta_dpo/beta_margin_std": 11.35789680480957, + "beta_dpo/beta_used": 0.3657574951648712, + "beta_dpo/beta_used_raw": 0.3657574951648712, + "beta_dpo/gap_mean": 19.213069915771484, + "beta_dpo/gap_std": 28.73113250732422, + "beta_dpo/loss_margin_mean": 22.08330726623535, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.9977324263038548, + "grad_norm": 170.74034118652344, + "learning_rate": 1.3985977021235829e-11, + "logits/chosen": 0.5866272449493408, + "logits/rejected": 0.5091443061828613, + "loss": 0.9655, + "step": 660 + }, + { + "beta_dpo/beta": 0.0010000000474974513, + "beta_dpo/beta_margin_grad_mean": -0.49589696526527405, + "beta_dpo/beta_margin_grad_std": 0.007532664109021425, + "beta_dpo/beta_margin_mean": 0.016415830701589584, + "beta_dpo/beta_margin_std": 0.0301388967782259, + "beta_dpo/beta_used": 0.0010000000474974513, + "beta_dpo/beta_used_raw": -0.17030301690101624, + "beta_dpo/gap_mean": 18.86831283569336, + "beta_dpo/gap_std": 29.014659881591797, + "beta_dpo/loss_margin_mean": 16.415828704833984, + "beta_dpo/mask_keep_frac": 0.78125, + "epoch": 0.999244142101285, + "grad_norm": 0.792015016078949, + "learning_rate": 3.4965187065971735e-12, + "logits/chosen": 0.4464152455329895, + "logits/rejected": 0.3606187105178833, + "loss": 1.3721, + "step": 661 + }, + { + "epoch": 0.999244142101285, + "step": 661, + "total_flos": 0.0, + "train_loss": 1.1692889305268401, + "train_runtime": 3152.686, + "train_samples_per_second": 13.429, + "train_steps_per_second": 0.21 + } + ], + "logging_steps": 1, + "max_steps": 661, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}