{ "model_name": "Qwen/Qwen2.5-1.5B-Instruct", "resolved_model": "/home/yuxu/.cache/huggingface/hub/models--Qwen--Qwen2.5-1.5B-Instruct/snapshots/989aa7980e4cf806f80c7fef2b1adb7bc71aa306", "num_train_records": 10000, "num_validation_records_available": 0, "training_config": { "lora_rank": 16, "lora_alpha": 32, "learning_rate": 1e-05, "num_epochs": 2, "per_device_batch_size": 1, "per_device_eval_batch_size": 1, "gradient_accumulation_steps": 16, "beta": 0.1, "max_length": 512, "max_prompt_length": 384, "seed": 22, "save_steps": 250, "save_total_limit": 2, "resume_from_checkpoint": null, "precision_dtype": "torch.bfloat16", "bf16": true, "fp16": false, "tokenizer_add_bos_token": false }, "global_step": 1250, "best_metric": null, "best_model_checkpoint": null, "train_metrics": { "train_runtime": 5977.8981, "train_samples_per_second": 3.346, "train_steps_per_second": 0.209, "total_flos": 1.3612284112597402e+17, "train_loss": 0.45482119541168214 }, "log_history": [ { "loss": 0.689018440246582, "grad_norm": 0.10905654728412628, "learning_rate": 9.928e-06, "entropy": 0.8563553406624124, "num_tokens": 134743.0, "logits/chosen": -0.9788623969289567, "logits/rejected": -0.963421240675876, "mean_token_accuracy": 0.7335915770381689, "rewards/chosen": 0.003118900115805445, "rewards/rejected": -0.005566338380720026, "rewards/accuracies": 0.53125, "rewards/margins": 0.0086852383617952, "logps/chosen": -53.44429122209549, "logps/rejected": -58.05595805644989, "epoch": 0.016, "step": 10 }, { "loss": 0.6828490734100342, "grad_norm": 0.11294842511415482, "learning_rate": 9.848000000000001e-06, "entropy": 0.8075154377147555, "num_tokens": 269598.0, "logits/chosen": -0.9147015579312413, "logits/rejected": -0.9080481267141745, "mean_token_accuracy": 0.7330205589532852, "rewards/chosen": -0.009720915841876376, "rewards/rejected": -0.031031110812728004, "rewards/accuracies": 0.675, "rewards/margins": 0.02131019511289196, "logps/chosen": -55.76358859539032, "logps/rejected": -53.03535443544388, "epoch": 0.032, "step": 20 }, { "loss": 0.6677823066711426, "grad_norm": 0.11828389763832092, "learning_rate": 9.768e-06, "entropy": 0.8278283050749451, "num_tokens": 404333.0, "logits/chosen": -1.0811326675861692, "logits/rejected": -0.9070872471899184, "mean_token_accuracy": 0.7307771431282163, "rewards/chosen": -0.03384106803678151, "rewards/rejected": -0.08754932412921335, "rewards/accuracies": 0.725, "rewards/margins": 0.05370825613208581, "logps/chosen": -50.51493817567825, "logps/rejected": -61.200947272777555, "epoch": 0.048, "step": 30 }, { "loss": 0.6462172985076904, "grad_norm": 0.11367130279541016, "learning_rate": 9.688000000000001e-06, "entropy": 0.823192946088966, "num_tokens": 538701.0, "logits/chosen": -1.0191294171017014, "logits/rejected": -0.9379523297738434, "mean_token_accuracy": 0.7370131656527519, "rewards/chosen": -0.07439912984327748, "rewards/rejected": -0.17708719812508206, "rewards/accuracies": 0.76875, "rewards/margins": 0.10268806891690474, "logps/chosen": -50.990779775381085, "logps/rejected": -52.61567587852478, "epoch": 0.064, "step": 40 }, { "loss": 0.6121170997619629, "grad_norm": 0.12841418385505676, "learning_rate": 9.608e-06, "entropy": 0.8364603637135588, "num_tokens": 673524.0, "logits/chosen": -0.9610722555180565, "logits/rejected": -0.8514297521784684, "mean_token_accuracy": 0.7061145938932896, "rewards/chosen": -0.14487820940848906, "rewards/rejected": -0.332736360195122, "rewards/accuracies": 0.78125, "rewards/margins": 0.1878581509925425, "logps/chosen": -58.89807789325714, "logps/rejected": -60.78576712608337, "epoch": 0.08, "step": 50 }, { "loss": 0.5916354179382324, "grad_norm": 0.11347971856594086, "learning_rate": 9.528000000000001e-06, "entropy": 0.8045067954575643, "num_tokens": 808013.0, "logits/chosen": -1.0201351325489214, "logits/rejected": -0.8894283035358745, "mean_token_accuracy": 0.7225402432493866, "rewards/chosen": -0.24984656272717984, "rewards/rejected": -0.504135195492563, "rewards/accuracies": 0.78125, "rewards/margins": 0.25428863214328884, "logps/chosen": -52.76062165498733, "logps/rejected": -61.86190390586853, "epoch": 0.096, "step": 60 }, { "loss": 0.576406192779541, "grad_norm": 0.13050724565982819, "learning_rate": 9.448e-06, "entropy": 0.8006839740090073, "num_tokens": 942494.0, "logits/chosen": -1.0778371811063052, "logits/rejected": -0.9411381606196525, "mean_token_accuracy": 0.713543506525457, "rewards/chosen": -0.38982224147985106, "rewards/rejected": -0.7144045860972256, "rewards/accuracies": 0.76875, "rewards/margins": 0.32458234429359434, "logps/chosen": -60.53339612483978, "logps/rejected": -60.80531245470047, "epoch": 0.112, "step": 70 }, { "loss": 0.5392527103424072, "grad_norm": 0.10742519050836563, "learning_rate": 9.368e-06, "entropy": 0.8088806970277801, "num_tokens": 1076339.0, "logits/chosen": -1.1278537035014409, "logits/rejected": -0.8774969710621068, "mean_token_accuracy": 0.7182694263756275, "rewards/chosen": -0.5486542367416405, "rewards/rejected": -0.9956708228681237, "rewards/accuracies": 0.76875, "rewards/margins": 0.4470165837556124, "logps/chosen": -52.11761736869812, "logps/rejected": -67.0722449183464, "epoch": 0.128, "step": 80 }, { "loss": 0.5608308792114258, "grad_norm": 0.15674173831939697, "learning_rate": 9.288e-06, "entropy": 0.768392307573231, "num_tokens": 1210662.0, "logits/chosen": -1.1788602806606145, "logits/rejected": -1.03331966826322, "mean_token_accuracy": 0.7145220551639795, "rewards/chosen": -0.720596243354521, "rewards/rejected": -1.1825086012715473, "rewards/accuracies": 0.74375, "rewards/margins": 0.46191235817968845, "logps/chosen": -60.48848968744278, "logps/rejected": -64.15390303134919, "epoch": 0.144, "step": 90 }, { "loss": 0.555051851272583, "grad_norm": 0.13351617753505707, "learning_rate": 9.208e-06, "entropy": 0.7862678854435217, "num_tokens": 1345120.0, "logits/chosen": -1.168248595800457, "logits/rejected": -1.0325365206859007, "mean_token_accuracy": 0.708121376670897, "rewards/chosen": -0.8649011284462176, "rewards/rejected": -1.4063857643632218, "rewards/accuracies": 0.6875, "rewards/margins": 0.5414846379309892, "logps/chosen": -61.20340501070022, "logps/rejected": -66.58065437078476, "epoch": 0.16, "step": 100 }, { "loss": 0.5074045658111572, "grad_norm": 0.21781982481479645, "learning_rate": 9.128e-06, "entropy": 0.7356140260322718, "num_tokens": 1479281.0, "logits/chosen": -1.3261684939964402, "logits/rejected": -1.1306528941057346, "mean_token_accuracy": 0.7123186649754644, "rewards/chosen": -0.9123384133912623, "rewards/rejected": -1.6054315031506121, "rewards/accuracies": 0.76875, "rewards/margins": 0.6930930894799531, "logps/chosen": -59.55553774833679, "logps/rejected": -72.37696163654327, "epoch": 0.176, "step": 110 }, { "loss": 0.5216456413269043, "grad_norm": 0.19241169095039368, "learning_rate": 9.048e-06, "entropy": 0.7655054951086641, "num_tokens": 1613851.0, "logits/chosen": -1.3025066533169531, "logits/rejected": -1.143896303288123, "mean_token_accuracy": 0.7036409357562661, "rewards/chosen": -1.0751173802796985, "rewards/rejected": -1.776869938801974, "rewards/accuracies": 0.75, "rewards/margins": 0.7017525571398437, "logps/chosen": -66.2069799900055, "logps/rejected": -74.56398607492447, "epoch": 0.192, "step": 120 }, { "loss": 0.5168225765228271, "grad_norm": 0.08994148671627045, "learning_rate": 8.968000000000001e-06, "entropy": 0.7540512274950742, "num_tokens": 1748368.0, "logits/chosen": -1.391650497545763, "logits/rejected": -1.1460448347696492, "mean_token_accuracy": 0.7162771710194648, "rewards/chosen": -1.0096850864356384, "rewards/rejected": -1.827320344489999, "rewards/accuracies": 0.76875, "rewards/margins": 0.817635256703943, "logps/chosen": -61.82568025588989, "logps/rejected": -76.9469052195549, "epoch": 0.208, "step": 130 }, { "loss": 0.46869478225708006, "grad_norm": 0.13741639256477356, "learning_rate": 8.888e-06, "entropy": 0.7924248128314503, "num_tokens": 1883496.0, "logits/chosen": -1.3249019191600873, "logits/rejected": -1.1026000772151963, "mean_token_accuracy": 0.7087388038635254, "rewards/chosen": -1.0496459440822945, "rewards/rejected": -2.055653589125723, "rewards/accuracies": 0.81875, "rewards/margins": 1.0060076432302594, "logps/chosen": -64.32736076116562, "logps/rejected": -81.53705527782441, "epoch": 0.224, "step": 140 }, { "loss": 0.49356565475463865, "grad_norm": 0.13059070706367493, "learning_rate": 8.808000000000001e-06, "entropy": 0.7655621751298896, "num_tokens": 2017285.0, "logits/chosen": -1.5351193051368675, "logits/rejected": -1.235119241285585, "mean_token_accuracy": 0.7035089529119432, "rewards/chosen": -1.090340746268339, "rewards/rejected": -1.9519310850591864, "rewards/accuracies": 0.75625, "rewards/margins": 0.8615903402678668, "logps/chosen": -58.17775413990021, "logps/rejected": -74.25504211187362, "epoch": 0.24, "step": 150 }, { "loss": 0.5369758605957031, "grad_norm": 0.13786712288856506, "learning_rate": 8.728e-06, "entropy": 0.6862338791484944, "num_tokens": 2150045.0, "logits/chosen": -1.4912660164800315, "logits/rejected": -1.333143902399643, "mean_token_accuracy": 0.701057541090995, "rewards/chosen": -1.1116661975334865, "rewards/rejected": -1.7480126170441508, "rewards/accuracies": 0.775, "rewards/margins": 0.6363464183639735, "logps/chosen": -59.09992996454239, "logps/rejected": -60.96899574995041, "epoch": 0.256, "step": 160 }, { "loss": 0.5533265113830567, "grad_norm": 0.18535780906677246, "learning_rate": 8.648000000000001e-06, "entropy": 0.7963132435863371, "num_tokens": 2284628.0, "logits/chosen": -1.4200859032965045, "logits/rejected": -1.1475656571055712, "mean_token_accuracy": 0.6968366431072355, "rewards/chosen": -1.2623908873065375, "rewards/rejected": -1.9762427852023392, "rewards/accuracies": 0.70625, "rewards/margins": 0.7138518976047635, "logps/chosen": -65.92677880525589, "logps/rejected": -75.31166982650757, "epoch": 0.272, "step": 170 }, { "loss": 0.48800249099731446, "grad_norm": 0.17086467146873474, "learning_rate": 8.568e-06, "entropy": 0.7096531906281598, "num_tokens": 2418154.0, "logits/chosen": -1.4284564529605592, "logits/rejected": -1.1960109995497337, "mean_token_accuracy": 0.7045122615993022, "rewards/chosen": -1.080190175800817, "rewards/rejected": -1.9707384748850019, "rewards/accuracies": 0.71875, "rewards/margins": 0.8905482929199934, "logps/chosen": -59.75125551223755, "logps/rejected": -70.24758154153824, "epoch": 0.288, "step": 180 }, { "loss": 0.46792116165161135, "grad_norm": 0.16177140176296234, "learning_rate": 8.488e-06, "entropy": 0.7515300859929994, "num_tokens": 2551816.0, "logits/chosen": -1.2414827391616068, "logits/rejected": -1.0802036758082083, "mean_token_accuracy": 0.7072636014781892, "rewards/chosen": -1.0667701240134193, "rewards/rejected": -1.9473204111913218, "rewards/accuracies": 0.8375, "rewards/margins": 0.8805502850795165, "logps/chosen": -65.70144573450088, "logps/rejected": -68.4914104104042, "epoch": 0.304, "step": 190 }, { "loss": 0.5006054878234864, "grad_norm": 0.16638650000095367, "learning_rate": 8.408e-06, "entropy": 0.8327071964275092, "num_tokens": 2686525.0, "logits/chosen": -1.2503270258894088, "logits/rejected": -0.995497852108113, "mean_token_accuracy": 0.6908243351615966, "rewards/chosen": -1.1853100352571346, "rewards/rejected": -2.0631397599121555, "rewards/accuracies": 0.76875, "rewards/margins": 0.8778297245502472, "logps/chosen": -65.22179394960403, "logps/rejected": -76.10799474716187, "epoch": 0.32, "step": 200 }, { "loss": 0.452646541595459, "grad_norm": 0.26452815532684326, "learning_rate": 8.328e-06, "entropy": 0.8430000780150294, "num_tokens": 2821412.0, "logits/chosen": -1.1790006368335502, "logits/rejected": -0.8779570491578651, "mean_token_accuracy": 0.6723773072008044, "rewards/chosen": -1.1888155334367183, "rewards/rejected": -2.2633378646452913, "rewards/accuracies": 0.7375, "rewards/margins": 1.074522328004241, "logps/chosen": -64.86716014146805, "logps/rejected": -79.61585273742676, "epoch": 0.336, "step": 210 }, { "loss": 0.5181353569030762, "grad_norm": 0.11742082983255386, "learning_rate": 8.248e-06, "entropy": 0.8045017344535154, "num_tokens": 2956627.0, "logits/chosen": -1.2154215871610907, "logits/rejected": -0.939022998277143, "mean_token_accuracy": 0.6996388690546155, "rewards/chosen": -1.1623295453668105, "rewards/rejected": -2.163627782184631, "rewards/accuracies": 0.775, "rewards/margins": 1.00129823833704, "logps/chosen": -62.80202409029007, "logps/rejected": -81.0254952788353, "epoch": 0.352, "step": 220 }, { "loss": 0.4743824481964111, "grad_norm": 0.16635876893997192, "learning_rate": 8.168e-06, "entropy": 0.7914588764862855, "num_tokens": 3090619.0, "logits/chosen": -1.1884228506656345, "logits/rejected": -0.9203135491484804, "mean_token_accuracy": 0.6981304872781038, "rewards/chosen": -1.0808302243589423, "rewards/rejected": -2.0085678519913928, "rewards/accuracies": 0.78125, "rewards/margins": 0.9277376340702176, "logps/chosen": -60.4386828660965, "logps/rejected": -72.10626875162124, "epoch": 0.368, "step": 230 }, { "loss": 0.48973665237426756, "grad_norm": 0.1960846185684204, "learning_rate": 8.088e-06, "entropy": 0.8449207143319655, "num_tokens": 3225094.0, "logits/chosen": -1.1508734381084937, "logits/rejected": -0.9685642621415461, "mean_token_accuracy": 0.6950183667242527, "rewards/chosen": -1.2113628653900377, "rewards/rejected": -2.089532778749708, "rewards/accuracies": 0.8, "rewards/margins": 0.8781699133105576, "logps/chosen": -68.62453348636627, "logps/rejected": -75.98600549697876, "epoch": 0.384, "step": 240 }, { "loss": 0.5802257061004639, "grad_norm": 0.11632820218801498, "learning_rate": 8.008e-06, "entropy": 0.7761064321442973, "num_tokens": 3360182.0, "logits/chosen": -1.2831330545177115, "logits/rejected": -1.0997321391861137, "mean_token_accuracy": 0.6970741396769882, "rewards/chosen": -1.1727734387372037, "rewards/rejected": -1.8903911848203279, "rewards/accuracies": 0.70625, "rewards/margins": 0.7176177425310015, "logps/chosen": -66.26374975442886, "logps/rejected": -72.10199882984162, "epoch": 0.4, "step": 250 }, { "loss": 0.43405885696411134, "grad_norm": 0.14540572464466095, "learning_rate": 7.928e-06, "entropy": 0.7629542504204437, "num_tokens": 3494544.0, "logits/chosen": -1.3185474115706086, "logits/rejected": -1.0363903950282343, "mean_token_accuracy": 0.69958227686584, "rewards/chosen": -0.9348558198704268, "rewards/rejected": -1.9087367365602403, "rewards/accuracies": 0.8125, "rewards/margins": 0.9738809239119292, "logps/chosen": -61.2034912109375, "logps/rejected": -71.80750354528428, "epoch": 0.416, "step": 260 }, { "loss": 0.44985151290893555, "grad_norm": 0.14902211725711823, "learning_rate": 7.848000000000002e-06, "entropy": 0.8110592220822582, "num_tokens": 3628021.0, "logits/chosen": -1.2687084794247518, "logits/rejected": -0.9412073643616663, "mean_token_accuracy": 0.6967338371090591, "rewards/chosen": -1.0113929210696369, "rewards/rejected": -1.9841588545590638, "rewards/accuracies": 0.8, "rewards/margins": 0.9727659282274544, "logps/chosen": -58.82498153448105, "logps/rejected": -72.50857379436493, "epoch": 0.432, "step": 270 }, { "loss": 0.47422361373901367, "grad_norm": 0.14587490260601044, "learning_rate": 7.768e-06, "entropy": 0.8527051686949563, "num_tokens": 3762769.0, "logits/chosen": -1.2181775523388327, "logits/rejected": -0.8992369459063199, "mean_token_accuracy": 0.7088230043649674, "rewards/chosen": -1.1207619122928008, "rewards/rejected": -2.148672613617964, "rewards/accuracies": 0.78125, "rewards/margins": 1.0279107017442584, "logps/chosen": -61.72244974374771, "logps/rejected": -78.88542218208313, "epoch": 0.448, "step": 280 }, { "loss": 0.5027151107788086, "grad_norm": 0.09337582439184189, "learning_rate": 7.688000000000002e-06, "entropy": 0.7852856576413615, "num_tokens": 3896554.0, "logits/chosen": -1.3354775567844146, "logits/rejected": -1.1184485921419505, "mean_token_accuracy": 0.6952060368843377, "rewards/chosen": -1.1932822762464639, "rewards/rejected": -2.0500318385427816, "rewards/accuracies": 0.75, "rewards/margins": 0.8567495625931769, "logps/chosen": -66.44852304458618, "logps/rejected": -70.7834144949913, "epoch": 0.464, "step": 290 }, { "loss": 0.44806575775146484, "grad_norm": 0.1337847113609314, "learning_rate": 7.608000000000001e-06, "entropy": 0.7422546729561873, "num_tokens": 4030330.0, "logits/chosen": -1.4700220277909535, "logits/rejected": -1.1814784580674191, "mean_token_accuracy": 0.7079419396817684, "rewards/chosen": -1.1185172388635691, "rewards/rejected": -2.27528806256596, "rewards/accuracies": 0.83125, "rewards/margins": 1.1567708211019636, "logps/chosen": -63.6253160238266, "logps/rejected": -74.7424528837204, "epoch": 0.48, "step": 300 }, { "loss": 0.45131826400756836, "grad_norm": 0.16172009706497192, "learning_rate": 7.528000000000001e-06, "entropy": 0.7010916481667664, "num_tokens": 4163071.0, "logits/chosen": -1.5153129547936302, "logits/rejected": -1.209580647062671, "mean_token_accuracy": 0.720912242680788, "rewards/chosen": -1.1323760432947894, "rewards/rejected": -2.123044349125121, "rewards/accuracies": 0.825, "rewards/margins": 0.9906683029606939, "logps/chosen": -61.3173499584198, "logps/rejected": -70.07959650754928, "epoch": 0.496, "step": 310 }, { "loss": 0.4254595756530762, "grad_norm": 0.11975234746932983, "learning_rate": 7.4480000000000005e-06, "entropy": 0.7769533811253495, "num_tokens": 4297650.0, "logits/chosen": -1.3886568818841, "logits/rejected": -1.13002503495388, "mean_token_accuracy": 0.7051799762062728, "rewards/chosen": -0.9357022894022521, "rewards/rejected": -1.983945999154821, "rewards/accuracies": 0.8625, "rewards/margins": 1.048243713984266, "logps/chosen": -68.33114951848984, "logps/rejected": -71.30756684541703, "epoch": 0.512, "step": 320 }, { "loss": 0.41445064544677734, "grad_norm": 0.2776741087436676, "learning_rate": 7.3680000000000004e-06, "entropy": 0.7489449862972833, "num_tokens": 4431577.0, "logits/chosen": -1.6288736615844654, "logits/rejected": -1.2485594975833538, "mean_token_accuracy": 0.6974780206568539, "rewards/chosen": -0.9473370073334081, "rewards/rejected": -2.106761427427409, "rewards/accuracies": 0.84375, "rewards/margins": 1.1594244167208672, "logps/chosen": -62.647765040397644, "logps/rejected": -75.1746208190918, "epoch": 0.528, "step": 330 }, { "loss": 0.4307701587677002, "grad_norm": 0.13814318180084229, "learning_rate": 7.288e-06, "entropy": 0.6795995138236322, "num_tokens": 4565362.0, "logits/chosen": -1.7014978124379496, "logits/rejected": -1.360194546265395, "mean_token_accuracy": 0.7122661659494043, "rewards/chosen": -1.1425236859824508, "rewards/rejected": -2.3793784040724857, "rewards/accuracies": 0.8125, "rewards/margins": 1.236854719556868, "logps/chosen": -61.26923282146454, "logps/rejected": -76.55938069820404, "epoch": 0.544, "step": 340 }, { "loss": 0.5171660423278809, "grad_norm": 0.2421128898859024, "learning_rate": 7.208e-06, "entropy": 0.736682211542211, "num_tokens": 4699278.0, "logits/chosen": -1.5375196977562884, "logits/rejected": -1.2775026091481378, "mean_token_accuracy": 0.7100502958521246, "rewards/chosen": -1.234824466597638, "rewards/rejected": -2.2794134604977443, "rewards/accuracies": 0.78125, "rewards/margins": 1.0445889961905777, "logps/chosen": -61.170579481124875, "logps/rejected": -73.94075057506561, "epoch": 0.56, "step": 350 }, { "loss": 0.4048302173614502, "grad_norm": 0.13529057800769806, "learning_rate": 7.128e-06, "entropy": 0.7252036946330918, "num_tokens": 4834112.0, "logits/chosen": -1.5076449032651227, "logits/rejected": -1.3696309019049226, "mean_token_accuracy": 0.7079439009539783, "rewards/chosen": -1.1786035622935742, "rewards/rejected": -2.421615838177968, "rewards/accuracies": 0.825, "rewards/margins": 1.2430122738704086, "logps/chosen": -75.25101128816604, "logps/rejected": -73.58783400058746, "epoch": 0.576, "step": 360 }, { "loss": 0.4977682113647461, "grad_norm": 0.1781209260225296, "learning_rate": 7.048e-06, "entropy": 0.8224106237757951, "num_tokens": 4968874.0, "logits/chosen": -1.3960146295474423, "logits/rejected": -1.1073445296913083, "mean_token_accuracy": 0.708382755331695, "rewards/chosen": -1.115643077727873, "rewards/rejected": -2.070754229882732, "rewards/accuracies": 0.74375, "rewards/margins": 0.955111154448241, "logps/chosen": -61.408040976524354, "logps/rejected": -78.4707190990448, "epoch": 0.592, "step": 370 }, { "loss": 0.4448493480682373, "grad_norm": 0.1303374469280243, "learning_rate": 6.968e-06, "entropy": 0.7095136601157719, "num_tokens": 5102712.0, "logits/chosen": -1.550997072455129, "logits/rejected": -1.2904333606869343, "mean_token_accuracy": 0.69968516016379, "rewards/chosen": -0.9111583859747043, "rewards/rejected": -2.052812862768769, "rewards/accuracies": 0.80625, "rewards/margins": 1.141654483322054, "logps/chosen": -59.56483067274094, "logps/rejected": -70.73611218929291, "epoch": 0.608, "step": 380 }, { "loss": 0.618243932723999, "grad_norm": 0.23218761384487152, "learning_rate": 6.888e-06, "entropy": 0.7683811950089876, "num_tokens": 5236746.0, "logits/chosen": -1.4908055362391708, "logits/rejected": -1.2139118427508149, "mean_token_accuracy": 0.7114272927865386, "rewards/chosen": -1.168174527026713, "rewards/rejected": -1.9601395897567273, "rewards/accuracies": 0.71875, "rewards/margins": 0.7919650661759079, "logps/chosen": -63.611954271793365, "logps/rejected": -72.28059177398681, "epoch": 0.624, "step": 390 }, { "loss": 0.5224360942840576, "grad_norm": 0.16070543229579926, "learning_rate": 6.808e-06, "entropy": 0.8030928350694012, "num_tokens": 5371265.0, "logits/chosen": -1.3533366529032598, "logits/rejected": -1.0578829997757886, "mean_token_accuracy": 0.7191924162209034, "rewards/chosen": -0.8509922233264661, "rewards/rejected": -1.6920959531096742, "rewards/accuracies": 0.7625, "rewards/margins": 0.8411037294892594, "logps/chosen": -61.14945147037506, "logps/rejected": -68.67226406335831, "epoch": 0.64, "step": 400 }, { "loss": 0.41513900756835936, "grad_norm": 0.19860734045505524, "learning_rate": 6.728e-06, "entropy": 0.7868119461927563, "num_tokens": 5505311.0, "logits/chosen": -1.2997119180538206, "logits/rejected": -1.0947122079972385, "mean_token_accuracy": 0.7141418108716607, "rewards/chosen": -0.5277955924670096, "rewards/rejected": -1.6158742993546185, "rewards/accuracies": 0.85625, "rewards/margins": 1.0880787078291179, "logps/chosen": -58.874490237236024, "logps/rejected": -69.12621879577637, "epoch": 0.656, "step": 410 }, { "loss": 0.4617919921875, "grad_norm": 0.1614799201488495, "learning_rate": 6.648e-06, "entropy": 0.8407431220111903, "num_tokens": 5639727.0, "logits/chosen": -1.2453079399296676, "logits/rejected": -0.9694453760606313, "mean_token_accuracy": 0.6953371344134212, "rewards/chosen": -0.6480406609189231, "rewards/rejected": -1.6581713807128835, "rewards/accuracies": 0.80625, "rewards/margins": 1.0101307133561932, "logps/chosen": -58.330216348171234, "logps/rejected": -75.3308912873268, "epoch": 0.672, "step": 420 }, { "loss": 0.44162707328796386, "grad_norm": 0.15507620573043823, "learning_rate": 6.568000000000001e-06, "entropy": 0.7857385093288031, "num_tokens": 5775347.0, "logits/chosen": -1.4506047377437405, "logits/rejected": -1.1385212073298612, "mean_token_accuracy": 0.7148427126929164, "rewards/chosen": -0.9052540952921845, "rewards/rejected": -1.9813320814282633, "rewards/accuracies": 0.8, "rewards/margins": 1.0760779848322273, "logps/chosen": -70.78476246595383, "logps/rejected": -75.07093167304993, "epoch": 0.688, "step": 430 }, { "loss": 0.45162124633789064, "grad_norm": 0.23057712614536285, "learning_rate": 6.488000000000001e-06, "entropy": 0.7095841458125506, "num_tokens": 5908570.0, "logits/chosen": -1.522052635312034, "logits/rejected": -1.1914354851620277, "mean_token_accuracy": 0.7254255541600287, "rewards/chosen": -1.0367633419460618, "rewards/rejected": -2.0772957720793785, "rewards/accuracies": 0.8, "rewards/margins": 1.0405324320308864, "logps/chosen": -56.933732664585115, "logps/rejected": -71.38762845993043, "epoch": 0.704, "step": 440 }, { "loss": 0.5143694877624512, "grad_norm": 0.1373089849948883, "learning_rate": 6.408000000000001e-06, "entropy": 0.7877030725649092, "num_tokens": 6042989.0, "logits/chosen": -1.3941457694145336, "logits/rejected": -1.1642616028929327, "mean_token_accuracy": 0.7271975075826049, "rewards/chosen": -1.1276259648264386, "rewards/rejected": -2.2803055624710398, "rewards/accuracies": 0.775, "rewards/margins": 1.1526795887388288, "logps/chosen": -62.81108283996582, "logps/rejected": -78.13692320585251, "epoch": 0.72, "step": 450 }, { "loss": 0.4237183094024658, "grad_norm": 0.13749825954437256, "learning_rate": 6.328000000000001e-06, "entropy": 0.7828766799415462, "num_tokens": 6177897.0, "logits/chosen": -1.3374319642690895, "logits/rejected": -1.1246487231005464, "mean_token_accuracy": 0.7126569332554936, "rewards/chosen": -0.9298112412194314, "rewards/rejected": -2.1673050606746984, "rewards/accuracies": 0.81875, "rewards/margins": 1.2374938178574666, "logps/chosen": -63.089643478393555, "logps/rejected": -75.59077807664872, "epoch": 0.736, "step": 460 }, { "loss": 0.4884624004364014, "grad_norm": 0.1415245234966278, "learning_rate": 6.248000000000001e-06, "entropy": 0.8562492666591425, "num_tokens": 6312714.0, "logits/chosen": -1.126100791332234, "logits/rejected": -0.8647624546845234, "mean_token_accuracy": 0.6987247484736144, "rewards/chosen": -0.808612387260655, "rewards/rejected": -1.6612209561048075, "rewards/accuracies": 0.80625, "rewards/margins": 0.8526085724122823, "logps/chosen": -67.91057009696961, "logps/rejected": -70.94221692085266, "epoch": 0.752, "step": 470 }, { "loss": 0.44064011573791506, "grad_norm": 0.20672255754470825, "learning_rate": 6.168000000000001e-06, "entropy": 0.7785831509012496, "num_tokens": 6446811.0, "logits/chosen": -1.3451005530773972, "logits/rejected": -1.0701528376820553, "mean_token_accuracy": 0.7139140725135803, "rewards/chosen": -0.8083824556204491, "rewards/rejected": -1.8517554196994752, "rewards/accuracies": 0.81875, "rewards/margins": 1.0433729680255055, "logps/chosen": -58.11244525909424, "logps/rejected": -70.4776518702507, "epoch": 0.768, "step": 480 }, { "loss": 0.4577657222747803, "grad_norm": 0.20929287374019623, "learning_rate": 6.088000000000001e-06, "entropy": 0.8531491419766098, "num_tokens": 6581587.0, "logits/chosen": -1.2832854161235994, "logits/rejected": -1.1052666904723358, "mean_token_accuracy": 0.7067524623125792, "rewards/chosen": -0.9194631451624445, "rewards/rejected": -1.912933972803876, "rewards/accuracies": 0.7875, "rewards/margins": 0.9934708263725043, "logps/chosen": -62.308107471466066, "logps/rejected": -74.89246033430099, "epoch": 0.784, "step": 490 }, { "loss": 0.4375007629394531, "grad_norm": 0.20060019195079803, "learning_rate": 6.008000000000001e-06, "entropy": 0.7929941387919826, "num_tokens": 6716289.0, "logits/chosen": -1.3354546779997054, "logits/rejected": -1.0329545787740195, "mean_token_accuracy": 0.7229124492034316, "rewards/chosen": -1.0229379917611368, "rewards/rejected": -2.1298453632509338, "rewards/accuracies": 0.8, "rewards/margins": 1.1069073686376214, "logps/chosen": -63.658102416992186, "logps/rejected": -75.60895298719406, "epoch": 0.8, "step": 500 }, { "loss": 0.44527058601379393, "grad_norm": 0.16677559912204742, "learning_rate": 5.928000000000001e-06, "entropy": 0.8354968667088543, "num_tokens": 6850716.0, "logits/chosen": -1.4591034667577056, "logits/rejected": -1.0641008106016938, "mean_token_accuracy": 0.705034868977964, "rewards/chosen": -0.8949406793020899, "rewards/rejected": -2.185808292112779, "rewards/accuracies": 0.81875, "rewards/margins": 1.2908676087856292, "logps/chosen": -58.768228101730344, "logps/rejected": -78.82726471424102, "epoch": 0.816, "step": 510 }, { "loss": 0.3973827600479126, "grad_norm": 0.15284304320812225, "learning_rate": 5.848000000000001e-06, "entropy": 0.8099856940709287, "num_tokens": 6984942.0, "logits/chosen": -1.3746344108751551, "logits/rejected": -0.9748087908165729, "mean_token_accuracy": 0.7168531034141779, "rewards/chosen": -0.8542922898486722, "rewards/rejected": -2.102445878717117, "rewards/accuracies": 0.85625, "rewards/margins": 1.2481535905040801, "logps/chosen": -63.956273436546326, "logps/rejected": -74.7093752503395, "epoch": 0.832, "step": 520 }, { "loss": 0.39521021842956544, "grad_norm": 0.19803296029567719, "learning_rate": 5.7680000000000005e-06, "entropy": 0.819463662133785, "num_tokens": 7118390.0, "logits/chosen": -1.4278405835062675, "logits/rejected": -1.1354102461541689, "mean_token_accuracy": 0.7017366614192724, "rewards/chosen": -0.8539736664190286, "rewards/rejected": -2.189559509139508, "rewards/accuracies": 0.81875, "rewards/margins": 1.3355858475901186, "logps/chosen": -59.81350688934326, "logps/rejected": -72.4623057961464, "epoch": 0.848, "step": 530 }, { "loss": 0.4910752773284912, "grad_norm": 0.17725913226604462, "learning_rate": 5.6880000000000004e-06, "entropy": 0.8000208166384255, "num_tokens": 7252607.0, "logits/chosen": -1.4821630663306287, "logits/rejected": -1.1388486199317076, "mean_token_accuracy": 0.7186640851199627, "rewards/chosen": -1.0058101782691664, "rewards/rejected": -2.2602031591522973, "rewards/accuracies": 0.78125, "rewards/margins": 1.2543929865583778, "logps/chosen": -59.514958798885345, "logps/rejected": -78.18541886806489, "epoch": 0.864, "step": 540 }, { "loss": 0.4494024276733398, "grad_norm": 0.18910986185073853, "learning_rate": 5.608e-06, "entropy": 0.7143417345014313, "num_tokens": 7386208.0, "logits/chosen": -1.621721678971833, "logits/rejected": -1.4079677960636496, "mean_token_accuracy": 0.6997427928261459, "rewards/chosen": -1.0504559374399833, "rewards/rejected": -2.317046788427979, "rewards/accuracies": 0.7875, "rewards/margins": 1.2665908511728048, "logps/chosen": -59.609275901317595, "logps/rejected": -73.99389593601227, "epoch": 0.88, "step": 550 }, { "loss": 0.3651487588882446, "grad_norm": 0.14056329429149628, "learning_rate": 5.528e-06, "entropy": 0.814640334653086, "num_tokens": 7521519.0, "logits/chosen": -1.6321557376252653, "logits/rejected": -1.287688344637761, "mean_token_accuracy": 0.7133785426616669, "rewards/chosen": -0.9613554299203543, "rewards/rejected": -2.597287955011416, "rewards/accuracies": 0.83125, "rewards/margins": 1.6359325245954097, "logps/chosen": -64.85036475658417, "logps/rejected": -87.75885683298111, "epoch": 0.896, "step": 560 }, { "loss": 0.47167210578918456, "grad_norm": 0.15875674784183502, "learning_rate": 5.448e-06, "entropy": 0.8689434929285198, "num_tokens": 7656864.0, "logits/chosen": -1.484541877747767, "logits/rejected": -1.210293212753148, "mean_token_accuracy": 0.6996884623542428, "rewards/chosen": -1.097544879911584, "rewards/rejected": -2.3577645811019465, "rewards/accuracies": 0.78125, "rewards/margins": 1.260219706967473, "logps/chosen": -71.07291498184205, "logps/rejected": -80.84261965751648, "epoch": 0.912, "step": 570 }, { "loss": 0.4431413173675537, "grad_norm": 0.08832120895385742, "learning_rate": 5.368000000000001e-06, "entropy": 0.8448504954460077, "num_tokens": 7792899.0, "logits/chosen": -1.3894132053730714, "logits/rejected": -1.1916056589936446, "mean_token_accuracy": 0.6817449929192663, "rewards/chosen": -1.1424140176386572, "rewards/rejected": -2.407927218545228, "rewards/accuracies": 0.8375, "rewards/margins": 1.2655131912790238, "logps/chosen": -75.6745502948761, "logps/rejected": -80.21998720169067, "epoch": 0.928, "step": 580 }, { "loss": 0.4135700225830078, "grad_norm": 0.2704945206642151, "learning_rate": 5.288000000000001e-06, "entropy": 0.7824862065215712, "num_tokens": 7926779.0, "logits/chosen": -1.6528558851348272, "logits/rejected": -1.292456634430272, "mean_token_accuracy": 0.705788871459663, "rewards/chosen": -0.8465097412496106, "rewards/rejected": -2.368203252152307, "rewards/accuracies": 0.825, "rewards/margins": 1.5216935152653606, "logps/chosen": -60.099838173389436, "logps/rejected": -76.38954356908798, "epoch": 0.944, "step": 590 }, { "loss": 0.44308009147644045, "grad_norm": 0.24440231919288635, "learning_rate": 5.208000000000001e-06, "entropy": 0.6905676309019327, "num_tokens": 8060079.0, "logits/chosen": -1.726859618494725, "logits/rejected": -1.3902765776081965, "mean_token_accuracy": 0.7324171539396047, "rewards/chosen": -0.8706245078108623, "rewards/rejected": -2.1242041391320527, "rewards/accuracies": 0.84375, "rewards/margins": 1.253579631447792, "logps/chosen": -54.19212522506714, "logps/rejected": -68.28248654603958, "epoch": 0.96, "step": 600 }, { "loss": 0.4900795936584473, "grad_norm": 0.1992669254541397, "learning_rate": 5.128000000000001e-06, "entropy": 0.7946285988775343, "num_tokens": 8194850.0, "logits/chosen": -1.432810063268619, "logits/rejected": -1.1435234217692902, "mean_token_accuracy": 0.7105734800919891, "rewards/chosen": -0.8311952710559126, "rewards/rejected": -1.92433615202317, "rewards/accuracies": 0.80625, "rewards/margins": 1.0931408811360597, "logps/chosen": -59.344405829906464, "logps/rejected": -72.60044294595718, "epoch": 0.976, "step": 610 }, { "loss": 0.4234670639038086, "grad_norm": 0.26938584446907043, "learning_rate": 5.048000000000001e-06, "entropy": 0.846720263955649, "num_tokens": 8329816.0, "logits/chosen": -1.482459889192245, "logits/rejected": -1.2096144698576996, "mean_token_accuracy": 0.7066002277657389, "rewards/chosen": -0.7947340043901931, "rewards/rejected": -2.0874571030028166, "rewards/accuracies": 0.81875, "rewards/margins": 1.292723097000271, "logps/chosen": -61.47416917085648, "logps/rejected": -79.54253326654434, "epoch": 0.992, "step": 620 }, { "loss": 0.42821288108825684, "grad_norm": 0.20475246012210846, "learning_rate": 4.9680000000000005e-06, "entropy": 0.8765991456399206, "num_tokens": 8464146.0, "logits/chosen": -1.3660770117134349, "logits/rejected": -1.1283678471483534, "mean_token_accuracy": 0.6944115529768169, "rewards/chosen": -0.758866243439843, "rewards/rejected": -1.9606408149003982, "rewards/accuracies": 0.8, "rewards/margins": 1.2017745693214237, "logps/chosen": -59.6711128950119, "logps/rejected": -75.17260210514068, "epoch": 1.008, "step": 630 }, { "loss": 0.4121402740478516, "grad_norm": 0.19181285798549652, "learning_rate": 4.8880000000000005e-06, "entropy": 0.8112090851645917, "num_tokens": 8598165.0, "logits/chosen": -1.6396478070003788, "logits/rejected": -1.201064509639671, "mean_token_accuracy": 0.7071607926860451, "rewards/chosen": -0.8190987646230496, "rewards/rejected": -2.1065064918162535, "rewards/accuracies": 0.85, "rewards/margins": 1.2874077258165926, "logps/chosen": -58.56870514154434, "logps/rejected": -75.47631640434265, "epoch": 1.024, "step": 640 }, { "loss": 0.3832432746887207, "grad_norm": 0.16693222522735596, "learning_rate": 4.808e-06, "entropy": 0.8202884538564831, "num_tokens": 8733182.0, "logits/chosen": -1.4093097295310522, "logits/rejected": -1.0764184005015662, "mean_token_accuracy": 0.7213250549510122, "rewards/chosen": -0.8947586458227306, "rewards/rejected": -2.217220963910222, "rewards/accuracies": 0.85, "rewards/margins": 1.3224623166024685, "logps/chosen": -63.64674232006073, "logps/rejected": -79.54431369304658, "epoch": 1.04, "step": 650 }, { "loss": 0.467090368270874, "grad_norm": 0.2820304334163666, "learning_rate": 4.728e-06, "entropy": 0.8175160668091849, "num_tokens": 8866532.0, "logits/chosen": -1.4113409272988666, "logits/rejected": -1.0253050648283575, "mean_token_accuracy": 0.7056375283747911, "rewards/chosen": -1.1019376051408472, "rewards/rejected": -2.4279107422335073, "rewards/accuracies": 0.83125, "rewards/margins": 1.3259731331840157, "logps/chosen": -62.17448818683624, "logps/rejected": -77.10730903148651, "epoch": 1.056, "step": 660 }, { "loss": 0.4856276512145996, "grad_norm": 0.1508263647556305, "learning_rate": 4.648e-06, "entropy": 0.8777147593849804, "num_tokens": 9002058.0, "logits/chosen": -1.237333422062607, "logits/rejected": -0.9867713449001874, "mean_token_accuracy": 0.6980989784002304, "rewards/chosen": -1.102817886834964, "rewards/rejected": -2.2164312085602433, "rewards/accuracies": 0.7875, "rewards/margins": 1.113613315252587, "logps/chosen": -72.5415987610817, "logps/rejected": -79.59271297454833, "epoch": 1.072, "step": 670 }, { "loss": 0.40852723121643064, "grad_norm": 0.1356440633535385, "learning_rate": 4.568e-06, "entropy": 0.8335771631682292, "num_tokens": 9136559.0, "logits/chosen": -1.4357955653994279, "logits/rejected": -1.0632368045892524, "mean_token_accuracy": 0.7067163791507483, "rewards/chosen": -0.8871819378109649, "rewards/rejected": -2.2391563730299824, "rewards/accuracies": 0.8, "rewards/margins": 1.3519744293764233, "logps/chosen": -65.51536420583724, "logps/rejected": -74.92358832359314, "epoch": 1.088, "step": 680 }, { "loss": 0.42632465362548827, "grad_norm": 0.2446223795413971, "learning_rate": 4.488e-06, "entropy": 0.7106906755536329, "num_tokens": 9269437.0, "logits/chosen": -1.7027034766334481, "logits/rejected": -1.3692697426790565, "mean_token_accuracy": 0.7298609726130962, "rewards/chosen": -0.9662762339459732, "rewards/rejected": -2.2450467050541194, "rewards/accuracies": 0.825, "rewards/margins": 1.2787704736925662, "logps/chosen": -57.37728985548019, "logps/rejected": -68.59222289323807, "epoch": 1.104, "step": 690 }, { "loss": 0.35138611793518065, "grad_norm": 0.12836965918540955, "learning_rate": 4.408000000000001e-06, "entropy": 0.8666601853678003, "num_tokens": 9404559.0, "logits/chosen": -1.3400690205304637, "logits/rejected": -1.0684046347991027, "mean_token_accuracy": 0.6958472795784474, "rewards/chosen": -0.7967793260220788, "rewards/rejected": -2.3613458889187315, "rewards/accuracies": 0.84375, "rewards/margins": 1.5645665610209107, "logps/chosen": -66.0351133108139, "logps/rejected": -81.51442708969117, "epoch": 1.12, "step": 700 }, { "loss": 0.4332468032836914, "grad_norm": 0.20934900641441345, "learning_rate": 4.328000000000001e-06, "entropy": 0.854771285172319, "num_tokens": 9539800.0, "logits/chosen": -1.3352951022130157, "logits/rejected": -0.9464862926160796, "mean_token_accuracy": 0.6998173886910081, "rewards/chosen": -0.8727404756122269, "rewards/rejected": -2.1966246593976395, "rewards/accuracies": 0.85, "rewards/margins": 1.3238841853104533, "logps/chosen": -61.889275419712064, "logps/rejected": -78.38478618860245, "epoch": 1.1360000000000001, "step": 710 }, { "loss": 0.4375255584716797, "grad_norm": 0.14867250621318817, "learning_rate": 4.248000000000001e-06, "entropy": 0.7838106972893002, "num_tokens": 9673582.0, "logits/chosen": -1.6430084008837227, "logits/rejected": -1.178061142015777, "mean_token_accuracy": 0.7228047780692577, "rewards/chosen": -0.8893449913855875, "rewards/rejected": -2.1319361824076624, "rewards/accuracies": 0.78125, "rewards/margins": 1.2425911880563945, "logps/chosen": -53.7250174164772, "logps/rejected": -75.55973731279373, "epoch": 1.152, "step": 720 }, { "loss": 0.4083255767822266, "grad_norm": 0.15854252874851227, "learning_rate": 4.168000000000001e-06, "entropy": 0.8028846303815953, "num_tokens": 9807723.0, "logits/chosen": -1.5796788729741613, "logits/rejected": -1.2279314483195203, "mean_token_accuracy": 0.7037102231755853, "rewards/chosen": -0.8733749952283688, "rewards/rejected": -2.346686244173907, "rewards/accuracies": 0.79375, "rewards/margins": 1.47331124804914, "logps/chosen": -57.2558536529541, "logps/rejected": -78.73293989896774, "epoch": 1.168, "step": 730 }, { "loss": 0.40847029685974123, "grad_norm": 0.2271832525730133, "learning_rate": 4.0880000000000005e-06, "entropy": 0.7124190992151853, "num_tokens": 9940521.0, "logits/chosen": -1.7263337940860637, "logits/rejected": -1.2993630722529041, "mean_token_accuracy": 0.7212265940383077, "rewards/chosen": -0.749589913454838, "rewards/rejected": -2.0937022533995333, "rewards/accuracies": 0.81875, "rewards/margins": 1.3441123379394413, "logps/chosen": -54.285783529281616, "logps/rejected": -67.44227703809739, "epoch": 1.184, "step": 740 }, { "loss": 0.4263965129852295, "grad_norm": 0.16508913040161133, "learning_rate": 4.008e-06, "entropy": 0.8771127343410626, "num_tokens": 10075349.0, "logits/chosen": -1.3213851460192345, "logits/rejected": -1.057589043568979, "mean_token_accuracy": 0.6872733032330871, "rewards/chosen": -0.9163320093342918, "rewards/rejected": -2.138802810528432, "rewards/accuracies": 0.80625, "rewards/margins": 1.2224708050955087, "logps/chosen": -65.35894116163254, "logps/rejected": -74.31111829280853, "epoch": 1.2, "step": 750 }, { "loss": 0.3466779708862305, "grad_norm": 0.14594385027885437, "learning_rate": 3.928e-06, "entropy": 0.8973389053368009, "num_tokens": 10210309.0, "logits/chosen": -1.429195333570735, "logits/rejected": -1.0293746532892192, "mean_token_accuracy": 0.6843982387334109, "rewards/chosen": -0.790108532004524, "rewards/rejected": -2.4523484482313505, "rewards/accuracies": 0.88125, "rewards/margins": 1.6622399202547968, "logps/chosen": -64.03692282438278, "logps/rejected": -83.50145937204361, "epoch": 1.216, "step": 760 }, { "loss": 0.3902191400527954, "grad_norm": 0.21555303037166595, "learning_rate": 3.848e-06, "entropy": 0.8176256978302263, "num_tokens": 10344571.0, "logits/chosen": -1.571296518421704, "logits/rejected": -1.1463247268095662, "mean_token_accuracy": 0.7237087743356824, "rewards/chosen": -0.9857207721681334, "rewards/rejected": -2.467484907130711, "rewards/accuracies": 0.85, "rewards/margins": 1.4817641287110745, "logps/chosen": -55.41268019676208, "logps/rejected": -83.52744359970093, "epoch": 1.232, "step": 770 }, { "loss": 0.4406851291656494, "grad_norm": 0.21594573557376862, "learning_rate": 3.7680000000000006e-06, "entropy": 0.7303940998070175, "num_tokens": 10478286.0, "logits/chosen": -1.8663801540109504, "logits/rejected": -1.4280047600468566, "mean_token_accuracy": 0.7173186991363764, "rewards/chosen": -1.1608861902008356, "rewards/rejected": -2.671035580892203, "rewards/accuracies": 0.79375, "rewards/margins": 1.51014938436565, "logps/chosen": -60.46502422094345, "logps/rejected": -77.58741216659546, "epoch": 1.248, "step": 780 }, { "loss": 0.4125234127044678, "grad_norm": 0.2962269186973572, "learning_rate": 3.6880000000000005e-06, "entropy": 0.8331925821723416, "num_tokens": 10612286.0, "logits/chosen": -1.5659800443469807, "logits/rejected": -1.2150587345047403, "mean_token_accuracy": 0.7073991242796183, "rewards/chosen": -1.1026862843311391, "rewards/rejected": -2.5605769436224364, "rewards/accuracies": 0.78125, "rewards/margins": 1.4578906406648457, "logps/chosen": -62.54402623176575, "logps/rejected": -79.81341508626937, "epoch": 1.264, "step": 790 }, { "loss": 0.49402365684509275, "grad_norm": 0.155300110578537, "learning_rate": 3.6080000000000004e-06, "entropy": 0.7988151058845687, "num_tokens": 10747462.0, "logits/chosen": -1.605360317746176, "logits/rejected": -1.2881572925592417, "mean_token_accuracy": 0.7002569252625108, "rewards/chosen": -1.0135203968995483, "rewards/rejected": -2.2969876725808716, "rewards/accuracies": 0.79375, "rewards/margins": 1.2834672711789608, "logps/chosen": -67.37470293045044, "logps/rejected": -79.85493696928025, "epoch": 1.28, "step": 800 }, { "loss": 0.4200831413269043, "grad_norm": 0.14519786834716797, "learning_rate": 3.5280000000000004e-06, "entropy": 0.8120490956622234, "num_tokens": 10882594.0, "logits/chosen": -1.5574423493502165, "logits/rejected": -1.3764162162150948, "mean_token_accuracy": 0.702505898848176, "rewards/chosen": -0.979388279729028, "rewards/rejected": -2.461002457328141, "rewards/accuracies": 0.8, "rewards/margins": 1.4816141836112364, "logps/chosen": -66.68434045314788, "logps/rejected": -83.09649764299392, "epoch": 1.296, "step": 810 }, { "loss": 0.4704907894134521, "grad_norm": 0.22803907096385956, "learning_rate": 3.4480000000000003e-06, "entropy": 0.8039654018590227, "num_tokens": 11016931.0, "logits/chosen": -1.5012306281869234, "logits/rejected": -1.2876392053232184, "mean_token_accuracy": 0.7034664511680603, "rewards/chosen": -1.1309354332945076, "rewards/rejected": -2.3758349375304535, "rewards/accuracies": 0.76875, "rewards/margins": 1.2448995084967465, "logps/chosen": -67.47772282361984, "logps/rejected": -78.3834860920906, "epoch": 1.312, "step": 820 }, { "loss": 0.46920132637023926, "grad_norm": 0.2387949824333191, "learning_rate": 3.368e-06, "entropy": 0.732459101951099, "num_tokens": 11150609.0, "logits/chosen": -1.7111103661903264, "logits/rejected": -1.3267267526306643, "mean_token_accuracy": 0.7098248641937971, "rewards/chosen": -0.9866411694383714, "rewards/rejected": -2.3024006365798413, "rewards/accuracies": 0.8, "rewards/margins": 1.3157594701158815, "logps/chosen": -60.04266767501831, "logps/rejected": -73.96700737476348, "epoch": 1.328, "step": 830 }, { "loss": 0.3219926357269287, "grad_norm": 0.13847558200359344, "learning_rate": 3.288e-06, "entropy": 0.7705854054656811, "num_tokens": 11284845.0, "logits/chosen": -1.7253091998162051, "logits/rejected": -1.305342065365432, "mean_token_accuracy": 0.7240345129743219, "rewards/chosen": -0.9102539952356892, "rewards/rejected": -2.657541433814913, "rewards/accuracies": 0.875, "rewards/margins": 1.7472874362953008, "logps/chosen": -56.32816154956818, "logps/rejected": -82.49419674873351, "epoch": 1.3439999999999999, "step": 840 }, { "loss": 0.39888598918914797, "grad_norm": 0.22760330140590668, "learning_rate": 3.208e-06, "entropy": 0.6883504351309966, "num_tokens": 11418114.0, "logits/chosen": -1.8767877806339819, "logits/rejected": -1.502016394931313, "mean_token_accuracy": 0.7253359684720635, "rewards/chosen": -0.836689239833504, "rewards/rejected": -2.3422608138527723, "rewards/accuracies": 0.80625, "rewards/margins": 1.5055715713184328, "logps/chosen": -53.072984755039215, "logps/rejected": -72.80904539823533, "epoch": 1.3599999999999999, "step": 850 }, { "loss": 0.40503392219543455, "grad_norm": 0.18328320980072021, "learning_rate": 3.1280000000000004e-06, "entropy": 0.7598929285246413, "num_tokens": 11552146.0, "logits/chosen": -1.644595842548687, "logits/rejected": -1.378459610872036, "mean_token_accuracy": 0.7259372064843774, "rewards/chosen": -0.9007043580990285, "rewards/rejected": -2.372628803132102, "rewards/accuracies": 0.8, "rewards/margins": 1.4719244507141411, "logps/chosen": -55.648898458480836, "logps/rejected": -78.23955315351486, "epoch": 1.376, "step": 860 }, { "loss": 0.3458548545837402, "grad_norm": 0.25410881638526917, "learning_rate": 3.0480000000000003e-06, "entropy": 0.7665736552706222, "num_tokens": 11686051.0, "logits/chosen": -1.578677616557072, "logits/rejected": -1.3612439345571947, "mean_token_accuracy": 0.713687221519649, "rewards/chosen": -0.9132042807443213, "rewards/rejected": -2.5326640743296593, "rewards/accuracies": 0.875, "rewards/margins": 1.6194597949273883, "logps/chosen": -62.23388094902039, "logps/rejected": -74.41176266670227, "epoch": 1.392, "step": 870 }, { "loss": 0.45064554214477537, "grad_norm": 0.31622838973999023, "learning_rate": 2.9680000000000002e-06, "entropy": 0.8155602383427322, "num_tokens": 11820419.0, "logits/chosen": -1.5256738113106172, "logits/rejected": -1.2444007046615775, "mean_token_accuracy": 0.7064365288242698, "rewards/chosen": -0.8790285978000612, "rewards/rejected": -2.279954434429237, "rewards/accuracies": 0.81875, "rewards/margins": 1.400925842532888, "logps/chosen": -61.98214646577835, "logps/rejected": -77.71078071594238, "epoch": 1.408, "step": 880 }, { "loss": 0.3549015045166016, "grad_norm": 0.11679533123970032, "learning_rate": 2.888e-06, "entropy": 0.8460370196611621, "num_tokens": 11954898.0, "logits/chosen": -1.501683007594535, "logits/rejected": -1.153882464001247, "mean_token_accuracy": 0.7017565036192537, "rewards/chosen": -0.9793404275318608, "rewards/rejected": -2.5882866755331633, "rewards/accuracies": 0.89375, "rewards/margins": 1.6089462437666953, "logps/chosen": -62.671991884708405, "logps/rejected": -85.92449575662613, "epoch": 1.424, "step": 890 }, { "loss": 0.32079691886901857, "grad_norm": 0.17164677381515503, "learning_rate": 2.808e-06, "entropy": 0.7836568029015325, "num_tokens": 12089403.0, "logits/chosen": -1.5902203304330556, "logits/rejected": -1.348343018569439, "mean_token_accuracy": 0.7099103134125471, "rewards/chosen": -0.906367156367196, "rewards/rejected": -2.5413650372705887, "rewards/accuracies": 0.88125, "rewards/margins": 1.634997878689319, "logps/chosen": -65.62818305492401, "logps/rejected": -79.59799456596375, "epoch": 1.44, "step": 900 }, { "loss": 0.3879056215286255, "grad_norm": 0.15152597427368164, "learning_rate": 2.728e-06, "entropy": 0.7635506895370782, "num_tokens": 12223715.0, "logits/chosen": -1.808664480653039, "logits/rejected": -1.4436211706139415, "mean_token_accuracy": 0.716391022503376, "rewards/chosen": -1.0626204300948303, "rewards/rejected": -2.617245429044124, "rewards/accuracies": 0.8125, "rewards/margins": 1.5546250022249297, "logps/chosen": -59.04966660737991, "logps/rejected": -79.71140024662017, "epoch": 1.456, "step": 910 }, { "loss": 0.5003737926483154, "grad_norm": 0.23257283866405487, "learning_rate": 2.648e-06, "entropy": 0.7805526316573378, "num_tokens": 12357865.0, "logits/chosen": -1.7092802822631128, "logits/rejected": -1.3571210243590142, "mean_token_accuracy": 0.7101537603884935, "rewards/chosen": -1.1259239912003978, "rewards/rejected": -2.455858718138188, "rewards/accuracies": 0.7875, "rewards/margins": 1.3299347181804477, "logps/chosen": -62.12466698884964, "logps/rejected": -79.51555901765823, "epoch": 1.472, "step": 920 }, { "loss": 0.4326589107513428, "grad_norm": 0.22872693836688995, "learning_rate": 2.568e-06, "entropy": 0.7554121573979501, "num_tokens": 12492128.0, "logits/chosen": -1.6719579694003723, "logits/rejected": -1.3663122032286132, "mean_token_accuracy": 0.6942645735107362, "rewards/chosen": -1.0867409846643568, "rewards/rejected": -2.5454429023200644, "rewards/accuracies": 0.79375, "rewards/margins": 1.4587019056081771, "logps/chosen": -63.67592179775238, "logps/rejected": -80.17804374694825, "epoch": 1.488, "step": 930 }, { "loss": 0.4602513790130615, "grad_norm": 0.23694008588790894, "learning_rate": 2.488e-06, "entropy": 0.7785145582747646, "num_tokens": 12626077.0, "logits/chosen": -1.5935042309292489, "logits/rejected": -1.3624577984140387, "mean_token_accuracy": 0.6916681522503495, "rewards/chosen": -1.1778735827072524, "rewards/rejected": -2.502696030540392, "rewards/accuracies": 0.8125, "rewards/margins": 1.3248224433511495, "logps/chosen": -63.580693411827085, "logps/rejected": -74.72070835828781, "epoch": 1.504, "step": 940 }, { "loss": 0.4067554473876953, "grad_norm": 0.14756956696510315, "learning_rate": 2.408e-06, "entropy": 0.8026386831275886, "num_tokens": 12760098.0, "logits/chosen": -1.5458875154452285, "logits/rejected": -1.2564218268845326, "mean_token_accuracy": 0.7152364812791348, "rewards/chosen": -1.2055871986289275, "rewards/rejected": -2.6404173804068707, "rewards/accuracies": 0.85, "rewards/margins": 1.4348301694728434, "logps/chosen": -66.13761007785797, "logps/rejected": -80.10225908756256, "epoch": 1.52, "step": 950 }, { "loss": 0.42372560501098633, "grad_norm": 0.14788998663425446, "learning_rate": 2.3280000000000004e-06, "entropy": 0.8012942865549121, "num_tokens": 12894652.0, "logits/chosen": -1.635073784869764, "logits/rejected": -1.3146647893445076, "mean_token_accuracy": 0.7114136775955557, "rewards/chosen": -1.1332746736516128, "rewards/rejected": -2.6066967224120163, "rewards/accuracies": 0.775, "rewards/margins": 1.4734220502898097, "logps/chosen": -63.05048147439957, "logps/rejected": -82.07224049568177, "epoch": 1.536, "step": 960 }, { "loss": 0.412662935256958, "grad_norm": 0.14732256531715393, "learning_rate": 2.2480000000000003e-06, "entropy": 0.8334445571847027, "num_tokens": 13030162.0, "logits/chosen": -1.6011668878902328, "logits/rejected": -1.2587357097834815, "mean_token_accuracy": 0.702854485437274, "rewards/chosen": -1.0372710642812308, "rewards/rejected": -2.6535557047463953, "rewards/accuracies": 0.8375, "rewards/margins": 1.6162846368737518, "logps/chosen": -65.74512588381768, "logps/rejected": -88.88693373203277, "epoch": 1.552, "step": 970 }, { "loss": 0.3407352209091187, "grad_norm": 0.2886792719364166, "learning_rate": 2.1680000000000002e-06, "entropy": 0.8762669585994445, "num_tokens": 13165409.0, "logits/chosen": -1.383209431558544, "logits/rejected": -1.0744050889575703, "mean_token_accuracy": 0.6733901240862906, "rewards/chosen": -1.008211946907977, "rewards/rejected": -2.624329062405741, "rewards/accuracies": 0.86875, "rewards/margins": 1.6161171085201205, "logps/chosen": -69.2023845911026, "logps/rejected": -83.00953713655471, "epoch": 1.568, "step": 980 }, { "loss": 0.4386160850524902, "grad_norm": 0.2656986713409424, "learning_rate": 2.088e-06, "entropy": 0.8209956398721261, "num_tokens": 13299607.0, "logits/chosen": -1.4395911330145508, "logits/rejected": -1.2677628081064927, "mean_token_accuracy": 0.6904863499104976, "rewards/chosen": -1.0870793154346756, "rewards/rejected": -2.4575889983447268, "rewards/accuracies": 0.78125, "rewards/margins": 1.3705096821766347, "logps/chosen": -65.42525545358657, "logps/rejected": -74.03723225593566, "epoch": 1.584, "step": 990 }, { "loss": 0.35819923877716064, "grad_norm": 0.35382354259490967, "learning_rate": 2.008e-06, "entropy": 0.7399830836133333, "num_tokens": 13434502.0, "logits/chosen": -1.6976982361606354, "logits/rejected": -1.3072192337860213, "mean_token_accuracy": 0.7421066265553236, "rewards/chosen": -1.0984582830686123, "rewards/rejected": -2.7966011623386295, "rewards/accuracies": 0.875, "rewards/margins": 1.6981428703293204, "logps/chosen": -62.93297493457794, "logps/rejected": -82.4026222705841, "epoch": 1.6, "step": 1000 }, { "loss": 0.4160293102264404, "grad_norm": 0.11767679452896118, "learning_rate": 1.928e-06, "entropy": 0.7990678637885139, "num_tokens": 13568600.0, "logits/chosen": -1.5975146089430179, "logits/rejected": -1.2924786059115378, "mean_token_accuracy": 0.7110372580587864, "rewards/chosen": -0.9529279105423484, "rewards/rejected": -2.457513489993289, "rewards/accuracies": 0.83125, "rewards/margins": 1.504585579968989, "logps/chosen": -63.974093568325046, "logps/rejected": -74.71904839277268, "epoch": 1.616, "step": 1010 }, { "loss": 0.3250428199768066, "grad_norm": 0.12981919944286346, "learning_rate": 1.8480000000000001e-06, "entropy": 0.7811902783811092, "num_tokens": 13702787.0, "logits/chosen": -1.587494817410056, "logits/rejected": -1.1941186838362041, "mean_token_accuracy": 0.7234507920220494, "rewards/chosen": -0.8217556012241403, "rewards/rejected": -2.597050206689164, "rewards/accuracies": 0.86875, "rewards/margins": 1.7752946069464088, "logps/chosen": -61.340254080295566, "logps/rejected": -79.60141725540161, "epoch": 1.6320000000000001, "step": 1020 }, { "loss": 0.4002103805541992, "grad_norm": 0.2534341812133789, "learning_rate": 1.7680000000000003e-06, "entropy": 0.8721957165267668, "num_tokens": 13837799.0, "logits/chosen": -1.4787466162956828, "logits/rejected": -1.1347675946257396, "mean_token_accuracy": 0.7160949306562543, "rewards/chosen": -0.9889563272474333, "rewards/rejected": -2.5054163753055034, "rewards/accuracies": 0.81875, "rewards/margins": 1.5164600439369678, "logps/chosen": -69.03193366527557, "logps/rejected": -80.24655103683472, "epoch": 1.6480000000000001, "step": 1030 }, { "loss": 0.4034124374389648, "grad_norm": 0.23815952241420746, "learning_rate": 1.6880000000000002e-06, "entropy": 0.7898992500049644, "num_tokens": 13972529.0, "logits/chosen": -1.6163878058701155, "logits/rejected": -1.2316415786069164, "mean_token_accuracy": 0.7335911913774907, "rewards/chosen": -0.9068507085920828, "rewards/rejected": -2.4930222455412148, "rewards/accuracies": 0.84375, "rewards/margins": 1.5861715397797524, "logps/chosen": -62.6472428560257, "logps/rejected": -78.13663581609725, "epoch": 1.6640000000000001, "step": 1040 }, { "loss": 0.4241136074066162, "grad_norm": 0.1419256180524826, "learning_rate": 1.608e-06, "entropy": 0.9256169050706375, "num_tokens": 14107336.0, "logits/chosen": -1.2789081893064282, "logits/rejected": -1.098782251423331, "mean_token_accuracy": 0.696486575063318, "rewards/chosen": -0.8607778700679773, "rewards/rejected": -2.315709656581748, "rewards/accuracies": 0.8125, "rewards/margins": 1.454931782837957, "logps/chosen": -74.47855192422867, "logps/rejected": -76.75005420446396, "epoch": 1.6800000000000002, "step": 1050 }, { "loss": 0.48003559112548827, "grad_norm": 0.14790725708007812, "learning_rate": 1.528e-06, "entropy": 0.7936980344704352, "num_tokens": 14241540.0, "logits/chosen": -1.5867842467297604, "logits/rejected": -1.1837827649291766, "mean_token_accuracy": 0.7259030997753143, "rewards/chosen": -0.8804566722927121, "rewards/rejected": -2.2063653921242805, "rewards/accuracies": 0.79375, "rewards/margins": 1.3259087240789085, "logps/chosen": -58.475341248512265, "logps/rejected": -74.59980441331864, "epoch": 1.696, "step": 1060 }, { "loss": 0.3922820329666138, "grad_norm": 0.1496279537677765, "learning_rate": 1.4480000000000002e-06, "entropy": 0.8726246880483813, "num_tokens": 14376121.0, "logits/chosen": -1.4914376459842063, "logits/rejected": -1.1848033257275015, "mean_token_accuracy": 0.7143584050238132, "rewards/chosen": -0.9215551663655788, "rewards/rejected": -2.562272682785988, "rewards/accuracies": 0.84375, "rewards/margins": 1.6407175094820559, "logps/chosen": -67.5427442073822, "logps/rejected": -79.30532623529434, "epoch": 1.712, "step": 1070 }, { "loss": 0.44224209785461427, "grad_norm": 0.2878839075565338, "learning_rate": 1.368e-06, "entropy": 0.7888671966618859, "num_tokens": 14510553.0, "logits/chosen": -1.6546901899116329, "logits/rejected": -1.3682432627615229, "mean_token_accuracy": 0.6996359150856734, "rewards/chosen": -0.8950480898056412, "rewards/rejected": -2.3655205052462405, "rewards/accuracies": 0.80625, "rewards/margins": 1.4704724150244146, "logps/chosen": -63.91799589395523, "logps/rejected": -74.64101424217225, "epoch": 1.728, "step": 1080 }, { "loss": 0.3927299499511719, "grad_norm": 0.1370965987443924, "learning_rate": 1.288e-06, "entropy": 0.8091648026223993, "num_tokens": 14644507.0, "logits/chosen": -1.687283619747059, "logits/rejected": -1.3548523346162953, "mean_token_accuracy": 0.7228656150400639, "rewards/chosen": -0.7743405980079843, "rewards/rejected": -2.3230774260126053, "rewards/accuracies": 0.7875, "rewards/margins": 1.5487368311733007, "logps/chosen": -56.80163584947586, "logps/rejected": -80.1871037721634, "epoch": 1.744, "step": 1090 }, { "loss": 0.36075007915496826, "grad_norm": 0.12493802607059479, "learning_rate": 1.2080000000000001e-06, "entropy": 0.8516323209594703, "num_tokens": 14779558.0, "logits/chosen": -1.5479666328178479, "logits/rejected": -1.1764522167627494, "mean_token_accuracy": 0.7081021483056247, "rewards/chosen": -0.8559462582834385, "rewards/rejected": -2.430596137570683, "rewards/accuracies": 0.84375, "rewards/margins": 1.5746498768217863, "logps/chosen": -63.79940243959427, "logps/rejected": -79.56198363304138, "epoch": 1.76, "step": 1100 }, { "loss": 0.49372134208679197, "grad_norm": 0.24344250559806824, "learning_rate": 1.128e-06, "entropy": 0.8455155969480984, "num_tokens": 14914286.0, "logits/chosen": -1.498396463441885, "logits/rejected": -1.1425197974463714, "mean_token_accuracy": 0.7100071370601654, "rewards/chosen": -0.8169066427948565, "rewards/rejected": -2.166107503604144, "rewards/accuracies": 0.76875, "rewards/margins": 1.3492008646018803, "logps/chosen": -61.403113543987274, "logps/rejected": -75.34919095039368, "epoch": 1.776, "step": 1110 }, { "loss": 0.4005413055419922, "grad_norm": 0.20426344871520996, "learning_rate": 1.0480000000000002e-06, "entropy": 0.8211736791708972, "num_tokens": 15048840.0, "logits/chosen": -1.582013316231499, "logits/rejected": -1.3164341995727113, "mean_token_accuracy": 0.7223803894594312, "rewards/chosen": -0.6959973029966932, "rewards/rejected": -2.1918047105107687, "rewards/accuracies": 0.84375, "rewards/margins": 1.4958074030466377, "logps/chosen": -59.653766822814944, "logps/rejected": -75.93207306861878, "epoch": 1.792, "step": 1120 }, { "loss": 0.5007998466491699, "grad_norm": 0.1154891774058342, "learning_rate": 9.68e-07, "entropy": 0.8647114810533821, "num_tokens": 15184276.0, "logits/chosen": -1.356775653423806, "logits/rejected": -1.1500322818720279, "mean_token_accuracy": 0.7039438035339117, "rewards/chosen": -0.8606041681859097, "rewards/rejected": -2.0742746030678973, "rewards/accuracies": 0.7375, "rewards/margins": 1.2136704298667609, "logps/chosen": -64.9673285484314, "logps/rejected": -80.83175637722016, "epoch": 1.808, "step": 1130 }, { "loss": 0.4986130714416504, "grad_norm": 0.6216063499450684, "learning_rate": 8.880000000000001e-07, "entropy": 0.774267910355411, "num_tokens": 15317721.0, "logits/chosen": -1.4605975439966772, "logits/rejected": -1.3212052490062418, "mean_token_accuracy": 0.7165707518346608, "rewards/chosen": -0.8740379733324517, "rewards/rejected": -1.9363951487233861, "rewards/accuracies": 0.76875, "rewards/margins": 1.0623571707867085, "logps/chosen": -58.174967527389526, "logps/rejected": -67.10608189105987, "epoch": 1.8239999999999998, "step": 1140 }, { "loss": 0.39597718715667723, "grad_norm": 0.1611914187669754, "learning_rate": 8.08e-07, "entropy": 0.8271302699897205, "num_tokens": 15452553.0, "logits/chosen": -1.50679093223041, "logits/rejected": -1.147970850967657, "mean_token_accuracy": 0.7078391901217401, "rewards/chosen": -0.8506218325230293, "rewards/rejected": -2.2930383594852173, "rewards/accuracies": 0.8375, "rewards/margins": 1.4424165301956235, "logps/chosen": -60.48249053955078, "logps/rejected": -79.1380724787712, "epoch": 1.8399999999999999, "step": 1150 }, { "loss": 0.3954659700393677, "grad_norm": 0.19070428609848022, "learning_rate": 7.280000000000001e-07, "entropy": 0.7992117294867057, "num_tokens": 15586655.0, "logits/chosen": -1.6674989181953035, "logits/rejected": -1.2958623246252912, "mean_token_accuracy": 0.7274953337386251, "rewards/chosen": -0.897261195579631, "rewards/rejected": -2.37544046624098, "rewards/accuracies": 0.8, "rewards/margins": 1.4781792684458197, "logps/chosen": -55.330790144205096, "logps/rejected": -79.54153176546097, "epoch": 1.8559999999999999, "step": 1160 }, { "loss": 0.5150039196014404, "grad_norm": 0.2727234959602356, "learning_rate": 6.48e-07, "entropy": 0.7774595006601885, "num_tokens": 15720172.0, "logits/chosen": -1.673701828468026, "logits/rejected": -1.2861514487494707, "mean_token_accuracy": 0.7126229584217072, "rewards/chosen": -0.9092265904924716, "rewards/rejected": -2.033220373163931, "rewards/accuracies": 0.71875, "rewards/margins": 1.1239937825594097, "logps/chosen": -54.55541696548462, "logps/rejected": -76.10742880105973, "epoch": 1.8719999999999999, "step": 1170 }, { "loss": 0.4881202220916748, "grad_norm": 0.21918685734272003, "learning_rate": 5.680000000000001e-07, "entropy": 0.7866302890935912, "num_tokens": 15853755.0, "logits/chosen": -1.4762931082547661, "logits/rejected": -1.2942548537923684, "mean_token_accuracy": 0.7225816631689668, "rewards/chosen": -0.9264017570319993, "rewards/rejected": -1.9995961244334466, "rewards/accuracies": 0.7875, "rewards/margins": 1.0731943636201322, "logps/chosen": -60.705457758903506, "logps/rejected": -66.6190501689911, "epoch": 1.888, "step": 1180 }, { "loss": 0.45177521705627444, "grad_norm": 0.23347221314907074, "learning_rate": 4.88e-07, "entropy": 0.8309914332116023, "num_tokens": 15988701.0, "logits/chosen": -1.4661994627051458, "logits/rejected": -1.2287666153016743, "mean_token_accuracy": 0.7092148935422301, "rewards/chosen": -0.8807039022503886, "rewards/rejected": -2.185251401356072, "rewards/accuracies": 0.80625, "rewards/margins": 1.304547501122579, "logps/chosen": -62.431247127056125, "logps/rejected": -74.59568692445755, "epoch": 1.904, "step": 1190 }, { "loss": 0.44033398628234866, "grad_norm": 0.13558809459209442, "learning_rate": 4.0800000000000005e-07, "entropy": 0.8481319433776662, "num_tokens": 16123227.0, "logits/chosen": -1.4890639792428868, "logits/rejected": -1.1712801983612529, "mean_token_accuracy": 0.7045220224186778, "rewards/chosen": -0.8116515700123272, "rewards/rejected": -2.0684703564504163, "rewards/accuracies": 0.83125, "rewards/margins": 1.2568187874741852, "logps/chosen": -63.18868860006332, "logps/rejected": -74.17321823835373, "epoch": 1.92, "step": 1200 }, { "loss": 0.4489591598510742, "grad_norm": 0.2724771499633789, "learning_rate": 3.280000000000001e-07, "entropy": 0.8390395241905935, "num_tokens": 16257799.0, "logits/chosen": -1.3757150913324974, "logits/rejected": -1.180797201944608, "mean_token_accuracy": 0.6906995047815144, "rewards/chosen": -0.8883007764234208, "rewards/rejected": -2.3104404117213564, "rewards/accuracies": 0.8125, "rewards/margins": 1.422139625577256, "logps/chosen": -64.57728606462479, "logps/rejected": -76.76991089582444, "epoch": 1.936, "step": 1210 }, { "loss": 0.3736592769622803, "grad_norm": 0.23324978351593018, "learning_rate": 2.48e-07, "entropy": 0.7958670913358219, "num_tokens": 16392755.0, "logits/chosen": -1.5214667053964295, "logits/rejected": -1.1308945168158175, "mean_token_accuracy": 0.7339793419465422, "rewards/chosen": -0.8573693673475645, "rewards/rejected": -2.422400009748526, "rewards/accuracies": 0.84375, "rewards/margins": 1.565030633006245, "logps/chosen": -62.808938360214235, "logps/rejected": -81.37407802343368, "epoch": 1.952, "step": 1220 }, { "loss": 0.33856496810913084, "grad_norm": 0.2365734577178955, "learning_rate": 1.68e-07, "entropy": 0.7709657683037221, "num_tokens": 16526347.0, "logits/chosen": -1.5951650245485083, "logits/rejected": -1.2316822978483537, "mean_token_accuracy": 0.7203758641146123, "rewards/chosen": -0.7306884591787821, "rewards/rejected": -2.310912328850827, "rewards/accuracies": 0.8625, "rewards/margins": 1.5802238748408854, "logps/chosen": -53.69367561340332, "logps/rejected": -75.45296376943588, "epoch": 1.968, "step": 1230 }, { "loss": 0.47231736183166506, "grad_norm": 0.1407570242881775, "learning_rate": 8.800000000000001e-08, "entropy": 0.9006856581720057, "num_tokens": 16660816.0, "logits/chosen": -1.4061471100194554, "logits/rejected": -1.0629769642593176, "mean_token_accuracy": 0.697626062296331, "rewards/chosen": -0.7163842913665576, "rewards/rejected": -1.9068784552206126, "rewards/accuracies": 0.8, "rewards/margins": 1.1904941664426587, "logps/chosen": -60.04284707307816, "logps/rejected": -76.30784704685212, "epoch": 1.984, "step": 1240 }, { "loss": 0.4329345226287842, "grad_norm": 0.20022091269493103, "learning_rate": 8e-09, "entropy": 0.7137106273818062, "num_tokens": 16794078.0, "logits/chosen": -1.7487986939590061, "logits/rejected": -1.4640005781460887, "mean_token_accuracy": 0.7085168479010463, "rewards/chosen": -0.7161212222184986, "rewards/rejected": -2.086236947233556, "rewards/accuracies": 0.80625, "rewards/margins": 1.3701157211326063, "logps/chosen": -54.929140663146974, "logps/rejected": -69.96422597169877, "epoch": 2.0, "step": 1250 }, { "train_runtime": 5977.8981, "train_samples_per_second": 3.346, "train_steps_per_second": 0.209, "total_flos": 1.3612284112597402e+17, "train_loss": 0.45482119541168214, "epoch": 2.0, "step": 1250 } ], "validation_monitor_size": 0, "validation_monitor_selection": "fixed_seed_random_without_replacement", "validation_monitor_seed": 22, "validation_monitor_indices": [], "early_stopping_patience": null }