| { |
| "best_global_step": 400, |
| "best_metric": 0.8793772459030151, |
| "best_model_checkpoint": "/data6/jongbinwon/evolving_MoE/checkpoints/sft_llama31_8b_sni_ours-c_46890_ffn_only_5ep/checkpoint-400", |
| "epoch": 5.0, |
| "eval_steps": 100, |
| "global_step": 2350, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.6614337407052517, |
| "epoch": 0.021316280309086066, |
| "grad_norm": 5.885494709014893, |
| "learning_rate": 2.535211267605634e-06, |
| "loss": 1.9121068954467773, |
| "mean_token_accuracy": 0.4677846459671855, |
| "num_tokens": 31378.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.6881026968359947, |
| "epoch": 0.04263256061817213, |
| "grad_norm": 3.2283990383148193, |
| "learning_rate": 5.352112676056338e-06, |
| "loss": 1.5715059280395507, |
| "mean_token_accuracy": 0.44802144076675177, |
| "num_tokens": 65329.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 1.7457253947854041, |
| "epoch": 0.0639488409272582, |
| "grad_norm": 5.933238983154297, |
| "learning_rate": 8.169014084507043e-06, |
| "loss": 1.478184700012207, |
| "mean_token_accuracy": 0.5721993442624808, |
| "num_tokens": 96853.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 1.8166336372494698, |
| "epoch": 0.08526512123634426, |
| "grad_norm": 6.201107501983643, |
| "learning_rate": 1.0985915492957748e-05, |
| "loss": 1.5585371017456056, |
| "mean_token_accuracy": 0.6381208458915353, |
| "num_tokens": 135861.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 1.960172240436077, |
| "epoch": 0.10658140154543032, |
| "grad_norm": 4.265772819519043, |
| "learning_rate": 1.380281690140845e-05, |
| "loss": 1.2981000900268556, |
| "mean_token_accuracy": 0.7017534904181957, |
| "num_tokens": 165735.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 2.0925094671547413, |
| "epoch": 0.1278976818545164, |
| "grad_norm": 4.701080799102783, |
| "learning_rate": 1.6619718309859155e-05, |
| "loss": 0.8457387924194336, |
| "mean_token_accuracy": 0.7487788364291191, |
| "num_tokens": 198750.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 2.1667631298303602, |
| "epoch": 0.14921396216360244, |
| "grad_norm": 5.391711711883545, |
| "learning_rate": 1.943661971830986e-05, |
| "loss": 1.1811514854431153, |
| "mean_token_accuracy": 0.7028381442651153, |
| "num_tokens": 233077.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 2.2530563578009604, |
| "epoch": 0.17053024247268853, |
| "grad_norm": 1.7958399057388306, |
| "learning_rate": 1.9999391925667267e-05, |
| "loss": 0.9082818031311035, |
| "mean_token_accuracy": 0.7205365922302007, |
| "num_tokens": 260650.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 2.1963915079832077, |
| "epoch": 0.19184652278177458, |
| "grad_norm": 2.5990638732910156, |
| "learning_rate": 1.9996921750431973e-05, |
| "loss": 1.1263222694396973, |
| "mean_token_accuracy": 0.7047741265967489, |
| "num_tokens": 290183.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 2.2172813221812246, |
| "epoch": 0.21316280309086064, |
| "grad_norm": 2.4212396144866943, |
| "learning_rate": 1.9992551938675226e-05, |
| "loss": 0.8470280647277832, |
| "mean_token_accuracy": 0.7515074783936143, |
| "num_tokens": 324143.0, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.21316280309086064, |
| "eval_entropy": 2.227590183261782, |
| "eval_loss": 0.9589676260948181, |
| "eval_mean_token_accuracy": 0.7676837602630258, |
| "eval_num_tokens": 324143.0, |
| "eval_runtime": 25.1983, |
| "eval_samples_per_second": 20.319, |
| "eval_steps_per_second": 10.159, |
| "step": 100 |
| }, |
| { |
| "entropy": 2.2125924050807955, |
| "epoch": 0.23447908339994672, |
| "grad_norm": 6.292978286743164, |
| "learning_rate": 1.998628332075803e-05, |
| "loss": 1.0414030075073242, |
| "mean_token_accuracy": 0.741944869607687, |
| "num_tokens": 356326.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 2.058356484770775, |
| "epoch": 0.2557953637090328, |
| "grad_norm": 4.461580276489258, |
| "learning_rate": 1.997811708785662e-05, |
| "loss": 0.9603706359863281, |
| "mean_token_accuracy": 0.6898831229656934, |
| "num_tokens": 390362.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 2.074503220617771, |
| "epoch": 0.27711164401811883, |
| "grad_norm": 5.1775593757629395, |
| "learning_rate": 1.996805479173609e-05, |
| "loss": 1.051582431793213, |
| "mean_token_accuracy": 0.7262888364493847, |
| "num_tokens": 417801.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 2.088790734112263, |
| "epoch": 0.2984279243272049, |
| "grad_norm": 4.234198570251465, |
| "learning_rate": 1.9956098344455538e-05, |
| "loss": 1.0677826881408692, |
| "mean_token_accuracy": 0.7328823814168572, |
| "num_tokens": 446130.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 2.179301120340824, |
| "epoch": 0.31974420463629094, |
| "grad_norm": 6.005627155303955, |
| "learning_rate": 1.9942250018004733e-05, |
| "loss": 0.9156723022460938, |
| "mean_token_accuracy": 0.7256842903792858, |
| "num_tokens": 475747.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 2.136591888964176, |
| "epoch": 0.34106048494537705, |
| "grad_norm": 4.946103096008301, |
| "learning_rate": 1.992651244387238e-05, |
| "loss": 0.8417132377624512, |
| "mean_token_accuracy": 0.7636397389695049, |
| "num_tokens": 505259.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 2.1515943363308905, |
| "epoch": 0.3623767652544631, |
| "grad_norm": 4.192586421966553, |
| "learning_rate": 1.9908888612546085e-05, |
| "loss": 1.0462213516235352, |
| "mean_token_accuracy": 0.723000418767333, |
| "num_tokens": 545011.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 2.183237124979496, |
| "epoch": 0.38369304556354916, |
| "grad_norm": 7.268803596496582, |
| "learning_rate": 1.9889381872944093e-05, |
| "loss": 1.0463459968566895, |
| "mean_token_accuracy": 0.7170630015432835, |
| "num_tokens": 575722.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 2.251120628416538, |
| "epoch": 0.4050093258726352, |
| "grad_norm": 7.779407501220703, |
| "learning_rate": 1.9867995931778915e-05, |
| "loss": 1.1224423408508302, |
| "mean_token_accuracy": 0.7369851667433978, |
| "num_tokens": 606747.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 2.1053356930613516, |
| "epoch": 0.4263256061817213, |
| "grad_norm": 5.202029228210449, |
| "learning_rate": 1.9844734852852988e-05, |
| "loss": 0.7484265327453613, |
| "mean_token_accuracy": 0.7559255946427583, |
| "num_tokens": 638567.0, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.4263256061817213, |
| "eval_entropy": 2.152409736532718, |
| "eval_loss": 0.9078466296195984, |
| "eval_mean_token_accuracy": 0.7723528936039656, |
| "eval_num_tokens": 638567.0, |
| "eval_runtime": 25.1941, |
| "eval_samples_per_second": 20.322, |
| "eval_steps_per_second": 10.161, |
| "step": 200 |
| }, |
| { |
| "entropy": 2.229059624671936, |
| "epoch": 0.44764188649080733, |
| "grad_norm": 4.298992156982422, |
| "learning_rate": 1.981960305628643e-05, |
| "loss": 0.9050834655761719, |
| "mean_token_accuracy": 0.7835996920242906, |
| "num_tokens": 667695.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 2.090072526037693, |
| "epoch": 0.46895816679989344, |
| "grad_norm": 3.323593854904175, |
| "learning_rate": 1.9792605317677153e-05, |
| "loss": 0.7545342922210694, |
| "mean_token_accuracy": 0.7571007736027241, |
| "num_tokens": 702195.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 2.0942522257566454, |
| "epoch": 0.4902744471089795, |
| "grad_norm": 2.2974536418914795, |
| "learning_rate": 1.9763746767193385e-05, |
| "loss": 0.9541402816772461, |
| "mean_token_accuracy": 0.7272845700383186, |
| "num_tokens": 732846.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 2.134460535645485, |
| "epoch": 0.5115907274180655, |
| "grad_norm": 4.181608200073242, |
| "learning_rate": 1.9733032888598802e-05, |
| "loss": 0.7861089706420898, |
| "mean_token_accuracy": 0.7417262677103281, |
| "num_tokens": 764477.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 2.107550011575222, |
| "epoch": 0.5329070077271516, |
| "grad_norm": 3.211021661758423, |
| "learning_rate": 1.970046951821052e-05, |
| "loss": 0.7543912887573242, |
| "mean_token_accuracy": 0.748434409685433, |
| "num_tokens": 790900.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 2.0784494683146475, |
| "epoch": 0.5542232880362377, |
| "grad_norm": 3.4665517807006836, |
| "learning_rate": 1.966606284379005e-05, |
| "loss": 0.8785683631896972, |
| "mean_token_accuracy": 0.7546398999169469, |
| "num_tokens": 821115.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 2.0476069271564485, |
| "epoch": 0.5755395683453237, |
| "grad_norm": 3.741456985473633, |
| "learning_rate": 1.9629819403367493e-05, |
| "loss": 0.940821647644043, |
| "mean_token_accuracy": 0.7150841023772955, |
| "num_tokens": 852809.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 2.0643599420785903, |
| "epoch": 0.5968558486544098, |
| "grad_norm": 8.063089370727539, |
| "learning_rate": 1.9591746083999166e-05, |
| "loss": 0.6063175678253174, |
| "mean_token_accuracy": 0.7969223730266094, |
| "num_tokens": 881864.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 2.063040444254875, |
| "epoch": 0.6181721289634958, |
| "grad_norm": 2.7161366939544678, |
| "learning_rate": 1.9551850120458912e-05, |
| "loss": 0.9077709197998047, |
| "mean_token_accuracy": 0.768440979719162, |
| "num_tokens": 911305.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 2.047479325532913, |
| "epoch": 0.6394884092725819, |
| "grad_norm": 2.315603256225586, |
| "learning_rate": 1.9510139093863324e-05, |
| "loss": 0.8123476028442382, |
| "mean_token_accuracy": 0.7939551591873169, |
| "num_tokens": 946457.0, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6394884092725819, |
| "eval_entropy": 2.0706750750541687, |
| "eval_loss": 0.9093313217163086, |
| "eval_mean_token_accuracy": 0.7773077270248905, |
| "eval_num_tokens": 946457.0, |
| "eval_runtime": 25.2221, |
| "eval_samples_per_second": 20.3, |
| "eval_steps_per_second": 10.15, |
| "step": 300 |
| }, |
| { |
| "entropy": 2.098699188232422, |
| "epoch": 0.660804689581668, |
| "grad_norm": 5.960000991821289, |
| "learning_rate": 1.946662093023118e-05, |
| "loss": 0.653551721572876, |
| "mean_token_accuracy": 0.7601731464266777, |
| "num_tokens": 975185.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 2.055957815051079, |
| "epoch": 0.6821209698907541, |
| "grad_norm": 4.677475929260254, |
| "learning_rate": 1.94213038989773e-05, |
| "loss": 0.6271010875701905, |
| "mean_token_accuracy": 0.7835832599550485, |
| "num_tokens": 1013462.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 2.0195901483297347, |
| "epoch": 0.7034372501998402, |
| "grad_norm": 7.502997398376465, |
| "learning_rate": 1.9374196611341212e-05, |
| "loss": 0.7580204010009766, |
| "mean_token_accuracy": 0.73177395388484, |
| "num_tokens": 1039374.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 2.1021947026252747, |
| "epoch": 0.7247535305089262, |
| "grad_norm": 2.8251311779022217, |
| "learning_rate": 1.9325308018750783e-05, |
| "loss": 0.7280273914337159, |
| "mean_token_accuracy": 0.7582205191254616, |
| "num_tokens": 1070150.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 2.0586804650723933, |
| "epoch": 0.7460698108180123, |
| "grad_norm": 3.429640531539917, |
| "learning_rate": 1.9274647411121283e-05, |
| "loss": 0.732785987854004, |
| "mean_token_accuracy": 0.7683163188397885, |
| "num_tokens": 1102723.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 2.04990341514349, |
| "epoch": 0.7673860911270983, |
| "grad_norm": 7.419450759887695, |
| "learning_rate": 1.922222441509007e-05, |
| "loss": 0.7287397861480713, |
| "mean_token_accuracy": 0.7813404571264982, |
| "num_tokens": 1132601.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 2.0948241263628007, |
| "epoch": 0.7887023714361844, |
| "grad_norm": 7.239919662475586, |
| "learning_rate": 1.916804899218734e-05, |
| "loss": 0.8115198135375976, |
| "mean_token_accuracy": 0.7400928113609553, |
| "num_tokens": 1162450.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 2.035434368252754, |
| "epoch": 0.8100186517452704, |
| "grad_norm": 5.840376853942871, |
| "learning_rate": 1.9112131436943197e-05, |
| "loss": 1.0656434059143067, |
| "mean_token_accuracy": 0.7546941698528826, |
| "num_tokens": 1195541.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 2.0758760675787924, |
| "epoch": 0.8313349320543565, |
| "grad_norm": 3.7342708110809326, |
| "learning_rate": 1.905448237493147e-05, |
| "loss": 0.9137516975402832, |
| "mean_token_accuracy": 0.7729586403816938, |
| "num_tokens": 1229474.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 2.006519891321659, |
| "epoch": 0.8526512123634425, |
| "grad_norm": 3.760019063949585, |
| "learning_rate": 1.8995112760750617e-05, |
| "loss": 0.6922917366027832, |
| "mean_token_accuracy": 0.7573013998568058, |
| "num_tokens": 1254195.0, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.8526512123634425, |
| "eval_entropy": 2.065651156939566, |
| "eval_loss": 0.8793772459030151, |
| "eval_mean_token_accuracy": 0.7864483034936711, |
| "eval_num_tokens": 1254195.0, |
| "eval_runtime": 25.2097, |
| "eval_samples_per_second": 20.31, |
| "eval_steps_per_second": 10.155, |
| "step": 400 |
| }, |
| { |
| "entropy": 1.966291381418705, |
| "epoch": 0.8739674926725286, |
| "grad_norm": 2.51716685295105, |
| "learning_rate": 1.8934033875942116e-05, |
| "loss": 0.6624742984771729, |
| "mean_token_accuracy": 0.7641348399221897, |
| "num_tokens": 1286799.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 1.9988927505910397, |
| "epoch": 0.8952837729816147, |
| "grad_norm": 4.811957836151123, |
| "learning_rate": 1.8871257326846713e-05, |
| "loss": 0.7704909801483154, |
| "mean_token_accuracy": 0.7670131400227547, |
| "num_tokens": 1319439.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 2.028906521201134, |
| "epoch": 0.9166000532907008, |
| "grad_norm": 3.0249269008636475, |
| "learning_rate": 1.8806795042398977e-05, |
| "loss": 0.8749347686767578, |
| "mean_token_accuracy": 0.7590735428035259, |
| "num_tokens": 1350004.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 2.070059296488762, |
| "epoch": 0.9379163335997869, |
| "grad_norm": 3.1602978706359863, |
| "learning_rate": 1.8740659271860506e-05, |
| "loss": 0.8998587608337403, |
| "mean_token_accuracy": 0.7411337062716484, |
| "num_tokens": 1384114.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 2.069587531685829, |
| "epoch": 0.9592326139088729, |
| "grad_norm": 6.083928108215332, |
| "learning_rate": 1.867286258249233e-05, |
| "loss": 0.7605265617370606, |
| "mean_token_accuracy": 0.7560267366468907, |
| "num_tokens": 1413930.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 2.0192495703697206, |
| "epoch": 0.980548894217959, |
| "grad_norm": 6.181666851043701, |
| "learning_rate": 1.860341785716683e-05, |
| "loss": 0.867928409576416, |
| "mean_token_accuracy": 0.7474079862236976, |
| "num_tokens": 1449322.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 2.029691263420941, |
| "epoch": 1.0, |
| "grad_norm": 4.97036600112915, |
| "learning_rate": 1.8532338291919727e-05, |
| "loss": 0.6732425212860107, |
| "mean_token_accuracy": 0.8109060656534482, |
| "num_tokens": 1476801.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 2.0917655169963836, |
| "epoch": 1.021316280309086, |
| "grad_norm": 3.287158966064453, |
| "learning_rate": 1.845963739344252e-05, |
| "loss": 0.675209379196167, |
| "mean_token_accuracy": 0.813974260725081, |
| "num_tokens": 1509544.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 2.0726554676890374, |
| "epoch": 1.042632560618172, |
| "grad_norm": 3.989821434020996, |
| "learning_rate": 1.838532897651593e-05, |
| "loss": 0.7701676368713379, |
| "mean_token_accuracy": 0.8190607488155365, |
| "num_tokens": 1541139.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 1.9738347977399826, |
| "epoch": 1.0639488409272582, |
| "grad_norm": 2.9076340198516846, |
| "learning_rate": 1.8309427161384793e-05, |
| "loss": 0.7476935386657715, |
| "mean_token_accuracy": 0.7427890259772539, |
| "num_tokens": 1573047.0, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.0639488409272582, |
| "eval_entropy": 1.9950207681395113, |
| "eval_loss": 0.8855671882629395, |
| "eval_mean_token_accuracy": 0.7921363326022401, |
| "eval_num_tokens": 1573047.0, |
| "eval_runtime": 25.3352, |
| "eval_samples_per_second": 20.209, |
| "eval_steps_per_second": 10.105, |
| "step": 500 |
| }, |
| { |
| "entropy": 1.9383097663521767, |
| "epoch": 1.0852651212363442, |
| "grad_norm": 3.2081310749053955, |
| "learning_rate": 1.8231946371074875e-05, |
| "loss": 0.6379514694213867, |
| "mean_token_accuracy": 0.7919592924416066, |
| "num_tokens": 1606929.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 1.882134698331356, |
| "epoch": 1.1065814015454303, |
| "grad_norm": 3.630250930786133, |
| "learning_rate": 1.815290132865221e-05, |
| "loss": 0.690134859085083, |
| "mean_token_accuracy": 0.830422455444932, |
| "num_tokens": 1639202.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 1.9214300945401193, |
| "epoch": 1.1278976818545163, |
| "grad_norm": 9.261590957641602, |
| "learning_rate": 1.8072307054425356e-05, |
| "loss": 0.9311601638793945, |
| "mean_token_accuracy": 0.7941673217341304, |
| "num_tokens": 1670654.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 1.9036248601973056, |
| "epoch": 1.1492139621636024, |
| "grad_norm": 5.370859146118164, |
| "learning_rate": 1.799017886309124e-05, |
| "loss": 0.6138679504394531, |
| "mean_token_accuracy": 0.7961133386939764, |
| "num_tokens": 1700516.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 1.9780284225940705, |
| "epoch": 1.1705302424726884, |
| "grad_norm": 4.6354660987854, |
| "learning_rate": 1.7906532360825003e-05, |
| "loss": 0.6264813899993896, |
| "mean_token_accuracy": 0.8231249656528234, |
| "num_tokens": 1733802.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 1.9119783863425255, |
| "epoch": 1.1918465227817745, |
| "grad_norm": 5.686795711517334, |
| "learning_rate": 1.782138344231448e-05, |
| "loss": 0.5910276412963867, |
| "mean_token_accuracy": 0.7942916708067059, |
| "num_tokens": 1762524.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 1.967802731692791, |
| "epoch": 1.2131628030908606, |
| "grad_norm": 3.1755685806274414, |
| "learning_rate": 1.7734748287739877e-05, |
| "loss": 0.4959968090057373, |
| "mean_token_accuracy": 0.7778273839503527, |
| "num_tokens": 1793060.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 1.8761932864785194, |
| "epoch": 1.2344790833999468, |
| "grad_norm": 3.783674478530884, |
| "learning_rate": 1.7646643359699157e-05, |
| "loss": 0.7227077960968018, |
| "mean_token_accuracy": 0.7871355729177594, |
| "num_tokens": 1820274.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 1.915278497338295, |
| "epoch": 1.2557953637090327, |
| "grad_norm": 5.4867119789123535, |
| "learning_rate": 1.7557085400079798e-05, |
| "loss": 0.6624124050140381, |
| "mean_token_accuracy": 0.8309637935832143, |
| "num_tokens": 1848818.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 1.8896379426121712, |
| "epoch": 1.277111644018119, |
| "grad_norm": 3.290586471557617, |
| "learning_rate": 1.746609142687744e-05, |
| "loss": 0.6076981544494628, |
| "mean_token_accuracy": 0.8211023930460215, |
| "num_tokens": 1880413.0, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.277111644018119, |
| "eval_entropy": 1.905547400470823, |
| "eval_loss": 0.9051699638366699, |
| "eval_mean_token_accuracy": 0.7929422339657322, |
| "eval_num_tokens": 1880413.0, |
| "eval_runtime": 25.1414, |
| "eval_samples_per_second": 20.365, |
| "eval_steps_per_second": 10.182, |
| "step": 600 |
| }, |
| { |
| "entropy": 1.803177796304226, |
| "epoch": 1.2984279243272048, |
| "grad_norm": 2.2047650814056396, |
| "learning_rate": 1.7373678730962128e-05, |
| "loss": 0.6381420612335205, |
| "mean_token_accuracy": 0.8127276424318552, |
| "num_tokens": 1913263.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 1.8625598490238189, |
| "epoch": 1.319744204636291, |
| "grad_norm": 6.565426826477051, |
| "learning_rate": 1.7279864872792627e-05, |
| "loss": 0.5850165843963623, |
| "mean_token_accuracy": 0.8355351146310568, |
| "num_tokens": 1943248.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 1.8649341762065887, |
| "epoch": 1.341060484945377, |
| "grad_norm": 1.9119219779968262, |
| "learning_rate": 1.7184667679079563e-05, |
| "loss": 0.6439106464385986, |
| "mean_token_accuracy": 0.7754333117976785, |
| "num_tokens": 1978455.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 1.8297289326786994, |
| "epoch": 1.3623767652544632, |
| "grad_norm": 5.278177261352539, |
| "learning_rate": 1.708810523939795e-05, |
| "loss": 0.7910201072692871, |
| "mean_token_accuracy": 0.78203096203506, |
| "num_tokens": 2013189.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 1.802287730574608, |
| "epoch": 1.3836930455635492, |
| "grad_norm": 10.827245712280273, |
| "learning_rate": 1.699019590274977e-05, |
| "loss": 0.730878210067749, |
| "mean_token_accuracy": 0.7671418067067861, |
| "num_tokens": 2045951.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 1.942077250778675, |
| "epoch": 1.4050093258726353, |
| "grad_norm": 4.506467819213867, |
| "learning_rate": 1.689095827407726e-05, |
| "loss": 0.6298671722412109, |
| "mean_token_accuracy": 0.7961704637855291, |
| "num_tokens": 2075904.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 1.8380796499550343, |
| "epoch": 1.4263256061817213, |
| "grad_norm": 6.3779683113098145, |
| "learning_rate": 1.679041121072755e-05, |
| "loss": 0.7520051956176758, |
| "mean_token_accuracy": 0.7579609978944063, |
| "num_tokens": 2112521.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 1.9149467781186105, |
| "epoch": 1.4476418864908074, |
| "grad_norm": 8.926115989685059, |
| "learning_rate": 1.6688573818869367e-05, |
| "loss": 0.9066313743591309, |
| "mean_token_accuracy": 0.8224626399576664, |
| "num_tokens": 2148171.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 1.9482211485505103, |
| "epoch": 1.4689581667998934, |
| "grad_norm": 10.81765365600586, |
| "learning_rate": 1.6585465449862424e-05, |
| "loss": 0.8869404792785645, |
| "mean_token_accuracy": 0.7676707118749618, |
| "num_tokens": 2179138.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 1.956870225071907, |
| "epoch": 1.4902744471089795, |
| "grad_norm": 6.614859104156494, |
| "learning_rate": 1.6481105696580235e-05, |
| "loss": 0.610249376296997, |
| "mean_token_accuracy": 0.803439411893487, |
| "num_tokens": 2207861.0, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.4902744471089795, |
| "eval_entropy": 1.9337850506417453, |
| "eval_loss": 0.8839167356491089, |
| "eval_mean_token_accuracy": 0.7920684967539273, |
| "eval_num_tokens": 2207861.0, |
| "eval_runtime": 25.2077, |
| "eval_samples_per_second": 20.311, |
| "eval_steps_per_second": 10.156, |
| "step": 700 |
| }, |
| { |
| "entropy": 1.9258531674742698, |
| "epoch": 1.5115907274180655, |
| "grad_norm": 6.887459754943848, |
| "learning_rate": 1.6375514389687053e-05, |
| "loss": 0.47269725799560547, |
| "mean_token_accuracy": 0.8237069369293749, |
| "num_tokens": 2241260.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 1.8748144969344138, |
| "epoch": 1.5329070077271516, |
| "grad_norm": 6.9224090576171875, |
| "learning_rate": 1.626871159386957e-05, |
| "loss": 0.6628749370574951, |
| "mean_token_accuracy": 0.8672334033995867, |
| "num_tokens": 2268034.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 1.927764856815338, |
| "epoch": 1.5542232880362377, |
| "grad_norm": 6.973414897918701, |
| "learning_rate": 1.6160717604024222e-05, |
| "loss": 0.688857889175415, |
| "mean_token_accuracy": 0.8388834275305271, |
| "num_tokens": 2298017.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 1.8214979842305183, |
| "epoch": 1.5755395683453237, |
| "grad_norm": 5.830054759979248, |
| "learning_rate": 1.6051552941400684e-05, |
| "loss": 0.6109556674957275, |
| "mean_token_accuracy": 0.8256007127463818, |
| "num_tokens": 2327012.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 1.8353620037436484, |
| "epoch": 1.5968558486544098, |
| "grad_norm": 9.805107116699219, |
| "learning_rate": 1.5941238349702377e-05, |
| "loss": 0.758075761795044, |
| "mean_token_accuracy": 0.7971353657543659, |
| "num_tokens": 2355566.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 1.9207229495048523, |
| "epoch": 1.6181721289634958, |
| "grad_norm": 12.170331954956055, |
| "learning_rate": 1.5829794791144723e-05, |
| "loss": 0.47644672393798826, |
| "mean_token_accuracy": 0.8686064530164004, |
| "num_tokens": 2388668.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 1.9532766610383987, |
| "epoch": 1.6394884092725819, |
| "grad_norm": 6.61758279800415, |
| "learning_rate": 1.5717243442471838e-05, |
| "loss": 0.6267609596252441, |
| "mean_token_accuracy": 0.8059493366628885, |
| "num_tokens": 2423851.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 1.8787624850869178, |
| "epoch": 1.660804689581668, |
| "grad_norm": 5.785323619842529, |
| "learning_rate": 1.5603605690932505e-05, |
| "loss": 0.6736733436584472, |
| "mean_token_accuracy": 0.8093659289181232, |
| "num_tokens": 2453844.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 1.8224774688482284, |
| "epoch": 1.6821209698907542, |
| "grad_norm": 3.5316760540008545, |
| "learning_rate": 1.54889031302161e-05, |
| "loss": 0.6565414428710937, |
| "mean_token_accuracy": 0.8169119991362095, |
| "num_tokens": 2484991.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 1.906269972026348, |
| "epoch": 1.70343725019984, |
| "grad_norm": 2.6745364665985107, |
| "learning_rate": 1.5373157556349335e-05, |
| "loss": 0.6277201652526856, |
| "mean_token_accuracy": 0.8166821744292975, |
| "num_tokens": 2515290.0, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.70343725019984, |
| "eval_entropy": 1.8975705276243389, |
| "eval_loss": 0.8847202658653259, |
| "eval_mean_token_accuracy": 0.7936569009325467, |
| "eval_num_tokens": 2515290.0, |
| "eval_runtime": 25.2245, |
| "eval_samples_per_second": 20.298, |
| "eval_steps_per_second": 10.149, |
| "step": 800 |
| }, |
| { |
| "entropy": 1.8205670893192292, |
| "epoch": 1.7247535305089263, |
| "grad_norm": 5.146493434906006, |
| "learning_rate": 1.5256390963554504e-05, |
| "loss": 0.6863347053527832, |
| "mean_token_accuracy": 0.7854361042380333, |
| "num_tokens": 2544139.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 1.8612893477082253, |
| "epoch": 1.7460698108180122, |
| "grad_norm": 7.076154708862305, |
| "learning_rate": 1.5138625540070117e-05, |
| "loss": 0.7201765537261963, |
| "mean_token_accuracy": 0.7712966155260801, |
| "num_tokens": 2575559.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 1.856431671977043, |
| "epoch": 1.7673860911270984, |
| "grad_norm": 3.1882359981536865, |
| "learning_rate": 1.501988366393464e-05, |
| "loss": 0.5714828968048096, |
| "mean_token_accuracy": 0.8167318470776082, |
| "num_tokens": 2605721.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 1.876603300869465, |
| "epoch": 1.7887023714361843, |
| "grad_norm": 2.653169870376587, |
| "learning_rate": 1.4900187898734152e-05, |
| "loss": 0.5608267307281494, |
| "mean_token_accuracy": 0.8415903944522143, |
| "num_tokens": 2640832.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 1.8057570911943912, |
| "epoch": 1.8100186517452705, |
| "grad_norm": 4.1108174324035645, |
| "learning_rate": 1.4779560989314798e-05, |
| "loss": 0.5947935581207275, |
| "mean_token_accuracy": 0.7923301249742508, |
| "num_tokens": 2675935.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 1.831426490843296, |
| "epoch": 1.8313349320543564, |
| "grad_norm": 1.4520552158355713, |
| "learning_rate": 1.4658025857460732e-05, |
| "loss": 0.5294552326202393, |
| "mean_token_accuracy": 0.823292363807559, |
| "num_tokens": 2708774.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 1.8851005636155604, |
| "epoch": 1.8526512123634427, |
| "grad_norm": 8.170310974121094, |
| "learning_rate": 1.4535605597538487e-05, |
| "loss": 0.7207472324371338, |
| "mean_token_accuracy": 0.801678966358304, |
| "num_tokens": 2736243.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 1.8832394301891326, |
| "epoch": 1.8739674926725285, |
| "grad_norm": 12.349190711975098, |
| "learning_rate": 1.4412323472108525e-05, |
| "loss": 0.7749912738800049, |
| "mean_token_accuracy": 0.7753736047074199, |
| "num_tokens": 2763867.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 1.900235815346241, |
| "epoch": 1.8952837729816148, |
| "grad_norm": 4.599183082580566, |
| "learning_rate": 1.4288202907504857e-05, |
| "loss": 0.47832794189453126, |
| "mean_token_accuracy": 0.8412384796887636, |
| "num_tokens": 2792070.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 1.8908123061060906, |
| "epoch": 1.9166000532907008, |
| "grad_norm": 11.037459373474121, |
| "learning_rate": 1.4163267489383492e-05, |
| "loss": 0.600672721862793, |
| "mean_token_accuracy": 0.8004915975034237, |
| "num_tokens": 2823917.0, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.9166000532907008, |
| "eval_entropy": 1.8657898297533393, |
| "eval_loss": 0.8825092911720276, |
| "eval_mean_token_accuracy": 0.7915659121936187, |
| "eval_num_tokens": 2823917.0, |
| "eval_runtime": 25.2256, |
| "eval_samples_per_second": 20.297, |
| "eval_steps_per_second": 10.148, |
| "step": 900 |
| }, |
| { |
| "entropy": 1.9238350823521615, |
| "epoch": 1.9379163335997869, |
| "grad_norm": 3.057535409927368, |
| "learning_rate": 1.4037540958240652e-05, |
| "loss": 0.8191390991210937, |
| "mean_token_accuracy": 0.7847225772216916, |
| "num_tokens": 2857949.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 1.8374909654259681, |
| "epoch": 1.959232613908873, |
| "grad_norm": 5.428546905517578, |
| "learning_rate": 1.391104720490156e-05, |
| "loss": 0.6227903842926026, |
| "mean_token_accuracy": 0.8255210720002651, |
| "num_tokens": 2892934.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 1.8489774838089943, |
| "epoch": 1.980548894217959, |
| "grad_norm": 4.659483432769775, |
| "learning_rate": 1.3783810265980643e-05, |
| "loss": 0.6746618270874023, |
| "mean_token_accuracy": 0.7706679496914148, |
| "num_tokens": 2922645.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 1.82360654987701, |
| "epoch": 2.0, |
| "grad_norm": 0.5909605026245117, |
| "learning_rate": 1.365585431931405e-05, |
| "loss": 0.6041478157043457, |
| "mean_token_accuracy": 0.8070473491329037, |
| "num_tokens": 2953602.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 1.8853100061416626, |
| "epoch": 2.0213162803090863, |
| "grad_norm": 4.752373695373535, |
| "learning_rate": 1.3527203679365318e-05, |
| "loss": 0.5585506916046142, |
| "mean_token_accuracy": 0.8603952780365944, |
| "num_tokens": 2985861.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 1.6949048675596714, |
| "epoch": 2.042632560618172, |
| "grad_norm": 3.7082631587982178, |
| "learning_rate": 1.3397882792605097e-05, |
| "loss": 0.5226590156555175, |
| "mean_token_accuracy": 0.8422632806003094, |
| "num_tokens": 3019608.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 1.7091815680265428, |
| "epoch": 2.0639488409272584, |
| "grad_norm": 6.324832916259766, |
| "learning_rate": 1.3267916232865777e-05, |
| "loss": 0.3531376123428345, |
| "mean_token_accuracy": 0.8930057939141989, |
| "num_tokens": 3055541.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 1.6739458978176116, |
| "epoch": 2.085265121236344, |
| "grad_norm": 4.618046283721924, |
| "learning_rate": 1.3137328696671904e-05, |
| "loss": 0.576710557937622, |
| "mean_token_accuracy": 0.8848785009235144, |
| "num_tokens": 3084555.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 1.7249003022909164, |
| "epoch": 2.1065814015454305, |
| "grad_norm": 5.561560153961182, |
| "learning_rate": 1.3006144998547314e-05, |
| "loss": 0.5280230045318604, |
| "mean_token_accuracy": 0.8864647958427667, |
| "num_tokens": 3112648.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 1.7153487108647822, |
| "epoch": 2.1278976818545163, |
| "grad_norm": 3.626471996307373, |
| "learning_rate": 1.2874390066299797e-05, |
| "loss": 0.3527723550796509, |
| "mean_token_accuracy": 0.8574657022953034, |
| "num_tokens": 3144049.0, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.1278976818545163, |
| "eval_entropy": 1.6961151584982872, |
| "eval_loss": 0.9473029375076294, |
| "eval_mean_token_accuracy": 0.7928107368061319, |
| "eval_num_tokens": 3144049.0, |
| "eval_runtime": 25.2864, |
| "eval_samples_per_second": 20.248, |
| "eval_steps_per_second": 10.124, |
| "step": 1000 |
| }, |
| { |
| "entropy": 1.6851068414747714, |
| "epoch": 2.1492139621636026, |
| "grad_norm": 6.618740081787109, |
| "learning_rate": 1.2742088936284296e-05, |
| "loss": 0.47363767623901365, |
| "mean_token_accuracy": 0.8599901650100946, |
| "num_tokens": 3175723.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 1.665366891026497, |
| "epoch": 2.1705302424726884, |
| "grad_norm": 7.713161468505859, |
| "learning_rate": 1.2609266748645412e-05, |
| "loss": 0.37115654945373533, |
| "mean_token_accuracy": 0.8425170484930277, |
| "num_tokens": 3206132.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 1.7072678461670876, |
| "epoch": 2.1918465227817747, |
| "grad_norm": 4.482607841491699, |
| "learning_rate": 1.2475948742540234e-05, |
| "loss": 0.40703649520874025, |
| "mean_token_accuracy": 0.8795785024762154, |
| "num_tokens": 3235673.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 1.6835453681647778, |
| "epoch": 2.2131628030908606, |
| "grad_norm": 5.588048458099365, |
| "learning_rate": 1.2342160251342345e-05, |
| "loss": 0.4783912181854248, |
| "mean_token_accuracy": 0.8529447130858898, |
| "num_tokens": 3268390.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 1.6581077799201012, |
| "epoch": 2.234479083399947, |
| "grad_norm": 3.2018959522247314, |
| "learning_rate": 1.2207926697827917e-05, |
| "loss": 0.5440057754516602, |
| "mean_token_accuracy": 0.8609105728566646, |
| "num_tokens": 3295498.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 1.6508349932730197, |
| "epoch": 2.2557953637090327, |
| "grad_norm": 8.872872352600098, |
| "learning_rate": 1.2073273589344804e-05, |
| "loss": 0.4584911346435547, |
| "mean_token_accuracy": 0.8255412630736828, |
| "num_tokens": 3328122.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 1.7553797572851182, |
| "epoch": 2.277111644018119, |
| "grad_norm": 4.401421070098877, |
| "learning_rate": 1.1938226512965617e-05, |
| "loss": 0.48421125411987304, |
| "mean_token_accuracy": 0.867019722610712, |
| "num_tokens": 3358628.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 1.7421697050333023, |
| "epoch": 2.2984279243272048, |
| "grad_norm": 10.023664474487305, |
| "learning_rate": 1.1802811130625576e-05, |
| "loss": 0.4822258472442627, |
| "mean_token_accuracy": 0.868565783649683, |
| "num_tokens": 3391789.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 1.7094722345471383, |
| "epoch": 2.319744204636291, |
| "grad_norm": 8.404623031616211, |
| "learning_rate": 1.1667053174246194e-05, |
| "loss": 0.5049400329589844, |
| "mean_token_accuracy": 0.8638756167143583, |
| "num_tokens": 3421244.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 1.6783940196037292, |
| "epoch": 2.341060484945377, |
| "grad_norm": 7.203639030456543, |
| "learning_rate": 1.153097844084564e-05, |
| "loss": 0.49939761161804197, |
| "mean_token_accuracy": 0.8506023991852999, |
| "num_tokens": 3454311.0, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.341060484945377, |
| "eval_entropy": 1.7317818785086274, |
| "eval_loss": 0.9347745776176453, |
| "eval_mean_token_accuracy": 0.7876530985231511, |
| "eval_num_tokens": 3454311.0, |
| "eval_runtime": 25.1804, |
| "eval_samples_per_second": 20.333, |
| "eval_steps_per_second": 10.167, |
| "step": 1100 |
| }, |
| { |
| "entropy": 1.719636818766594, |
| "epoch": 2.362376765254463, |
| "grad_norm": 7.6285905838012695, |
| "learning_rate": 1.1394612787636726e-05, |
| "loss": 0.37557549476623536, |
| "mean_token_accuracy": 0.8724950306117535, |
| "num_tokens": 3488090.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 1.6512885347008706, |
| "epoch": 2.383693045563549, |
| "grad_norm": 3.9734034538269043, |
| "learning_rate": 1.1257982127113481e-05, |
| "loss": 0.2912411689758301, |
| "mean_token_accuracy": 0.8837580941617489, |
| "num_tokens": 3518189.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 1.662961296737194, |
| "epoch": 2.4050093258726353, |
| "grad_norm": 11.78700065612793, |
| "learning_rate": 1.1121112422127184e-05, |
| "loss": 0.6317928791046142, |
| "mean_token_accuracy": 0.8458453085273504, |
| "num_tokens": 3548885.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 1.6277989491820335, |
| "epoch": 2.426325606181721, |
| "grad_norm": 6.840092182159424, |
| "learning_rate": 1.098402968095286e-05, |
| "loss": 0.35914084911346433, |
| "mean_token_accuracy": 0.8892543137073516, |
| "num_tokens": 3581285.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 1.6398253336548805, |
| "epoch": 2.4476418864908074, |
| "grad_norm": 8.036883354187012, |
| "learning_rate": 1.0846759952347127e-05, |
| "loss": 0.5016361236572265, |
| "mean_token_accuracy": 0.8925760805606842, |
| "num_tokens": 3611916.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 1.6403828382492065, |
| "epoch": 2.4689581667998937, |
| "grad_norm": 5.119592189788818, |
| "learning_rate": 1.0709329320598352e-05, |
| "loss": 0.4121875762939453, |
| "mean_token_accuracy": 0.8585600268095732, |
| "num_tokens": 3641283.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 1.5923970475792886, |
| "epoch": 2.4902744471089795, |
| "grad_norm": 7.016224384307861, |
| "learning_rate": 1.0571763900570063e-05, |
| "loss": 0.37267417907714845, |
| "mean_token_accuracy": 0.9020616315305233, |
| "num_tokens": 3672571.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 1.602854534983635, |
| "epoch": 2.5115907274180653, |
| "grad_norm": 8.555669784545898, |
| "learning_rate": 1.0434089832738554e-05, |
| "loss": 0.4383026123046875, |
| "mean_token_accuracy": 0.8404763884842396, |
| "num_tokens": 3711555.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 1.5632836624979973, |
| "epoch": 2.5329070077271516, |
| "grad_norm": 11.179162979125977, |
| "learning_rate": 1.0296333278225599e-05, |
| "loss": 0.37712688446044923, |
| "mean_token_accuracy": 0.8969831667840481, |
| "num_tokens": 3740886.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 1.542743168771267, |
| "epoch": 2.554223288036238, |
| "grad_norm": 13.297327041625977, |
| "learning_rate": 1.0158520413827273e-05, |
| "loss": 0.34956722259521483, |
| "mean_token_accuracy": 0.8721455127000809, |
| "num_tokens": 3770488.0, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.554223288036238, |
| "eval_entropy": 1.569909728365019, |
| "eval_loss": 0.984853982925415, |
| "eval_mean_token_accuracy": 0.7946027047000825, |
| "eval_num_tokens": 3770488.0, |
| "eval_runtime": 25.2172, |
| "eval_samples_per_second": 20.304, |
| "eval_steps_per_second": 10.152, |
| "step": 1200 |
| }, |
| { |
| "entropy": 1.5610544055700302, |
| "epoch": 2.5755395683453237, |
| "grad_norm": 9.166057586669922, |
| "learning_rate": 1.002067742703977e-05, |
| "loss": 0.3869703531265259, |
| "mean_token_accuracy": 0.8965044122189283, |
| "num_tokens": 3800893.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 1.5860440082848073, |
| "epoch": 2.5968558486544095, |
| "grad_norm": 3.7810888290405273, |
| "learning_rate": 9.882830511083217e-06, |
| "loss": 0.6022892951965332, |
| "mean_token_accuracy": 0.8423100596293807, |
| "num_tokens": 3831478.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 1.5593753769993781, |
| "epoch": 2.618172128963496, |
| "grad_norm": 7.3957109451293945, |
| "learning_rate": 9.74500585992436e-06, |
| "loss": 0.5377386569976806, |
| "mean_token_accuracy": 0.8421918705105782, |
| "num_tokens": 3863456.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 1.6092627555131913, |
| "epoch": 2.639488409272582, |
| "grad_norm": 11.163238525390625, |
| "learning_rate": 9.607229663299139e-06, |
| "loss": 0.44234743118286135, |
| "mean_token_accuracy": 0.8910749081522227, |
| "num_tokens": 3898348.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 1.6111317679286004, |
| "epoch": 2.660804689581668, |
| "grad_norm": 9.182315826416016, |
| "learning_rate": 9.469528101736068e-06, |
| "loss": 0.48477892875671386, |
| "mean_token_accuracy": 0.8321116678416729, |
| "num_tokens": 3926316.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 1.5978579953312875, |
| "epoch": 2.682120969890754, |
| "grad_norm": 6.697749614715576, |
| "learning_rate": 9.331927341581342e-06, |
| "loss": 0.40438008308410645, |
| "mean_token_accuracy": 0.8784059301018715, |
| "num_tokens": 3954526.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 1.5626994654536248, |
| "epoch": 2.70343725019984, |
| "grad_norm": 6.133303642272949, |
| "learning_rate": 9.194453530026651e-06, |
| "loss": 0.47856831550598145, |
| "mean_token_accuracy": 0.8131281891837716, |
| "num_tokens": 3990136.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 1.610917054116726, |
| "epoch": 2.7247535305089263, |
| "grad_norm": 6.997807502746582, |
| "learning_rate": 9.057132790140656e-06, |
| "loss": 0.42852191925048827, |
| "mean_token_accuracy": 0.9010261096060276, |
| "num_tokens": 4021182.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 1.636612831056118, |
| "epoch": 2.746069810818012, |
| "grad_norm": 5.9889235496521, |
| "learning_rate": 8.919991215904985e-06, |
| "loss": 0.38060717582702636, |
| "mean_token_accuracy": 0.8902203716337681, |
| "num_tokens": 4056483.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 1.5953127652406693, |
| "epoch": 2.7673860911270984, |
| "grad_norm": 5.638936519622803, |
| "learning_rate": 8.783054867255852e-06, |
| "loss": 0.39396817684173585, |
| "mean_token_accuracy": 0.8820293191820383, |
| "num_tokens": 4086632.0, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.7673860911270984, |
| "eval_entropy": 1.6087646232917905, |
| "eval_loss": 0.9700126647949219, |
| "eval_mean_token_accuracy": 0.7921096142963506, |
| "eval_num_tokens": 4086632.0, |
| "eval_runtime": 25.2355, |
| "eval_samples_per_second": 20.289, |
| "eval_steps_per_second": 10.144, |
| "step": 1300 |
| }, |
| { |
| "entropy": 1.5998306781053544, |
| "epoch": 2.7887023714361843, |
| "grad_norm": 6.4997148513793945, |
| "learning_rate": 8.646349765132042e-06, |
| "loss": 0.3382676124572754, |
| "mean_token_accuracy": 0.8778775539249182, |
| "num_tokens": 4120102.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 1.5056429199874402, |
| "epoch": 2.8100186517452705, |
| "grad_norm": 8.328359603881836, |
| "learning_rate": 8.509901886530396e-06, |
| "loss": 0.47609686851501465, |
| "mean_token_accuracy": 0.8344728346914053, |
| "num_tokens": 4154142.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 1.5405534602701665, |
| "epoch": 2.8313349320543564, |
| "grad_norm": 7.813889980316162, |
| "learning_rate": 8.373737159569586e-06, |
| "loss": 0.495313835144043, |
| "mean_token_accuracy": 0.861658050864935, |
| "num_tokens": 4184216.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 1.592079722136259, |
| "epoch": 2.8526512123634427, |
| "grad_norm": 6.749233245849609, |
| "learning_rate": 8.237881458563188e-06, |
| "loss": 0.4460284233093262, |
| "mean_token_accuracy": 0.8668980307877063, |
| "num_tokens": 4213070.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 1.6304010152816772, |
| "epoch": 2.8739674926725285, |
| "grad_norm": 4.704800605773926, |
| "learning_rate": 8.102360599103022e-06, |
| "loss": 0.40764584541320803, |
| "mean_token_accuracy": 0.872374988719821, |
| "num_tokens": 4248720.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 1.645412190258503, |
| "epoch": 2.8952837729816148, |
| "grad_norm": 7.446371555328369, |
| "learning_rate": 7.967200333153583e-06, |
| "loss": 0.34721221923828127, |
| "mean_token_accuracy": 0.90929289534688, |
| "num_tokens": 4278355.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 1.5853713877499103, |
| "epoch": 2.9166000532907006, |
| "grad_norm": 8.756217002868652, |
| "learning_rate": 7.832426344158625e-06, |
| "loss": 0.5081141948699951, |
| "mean_token_accuracy": 0.8653258476406336, |
| "num_tokens": 4306965.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 1.607254397124052, |
| "epoch": 2.937916333599787, |
| "grad_norm": 9.918863296508789, |
| "learning_rate": 7.698064242160726e-06, |
| "loss": 0.5225647449493408, |
| "mean_token_accuracy": 0.8322745427489281, |
| "num_tokens": 4339529.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 1.646574142575264, |
| "epoch": 2.959232613908873, |
| "grad_norm": 5.501335144042969, |
| "learning_rate": 7.564139558934799e-06, |
| "loss": 0.40751066207885744, |
| "mean_token_accuracy": 0.8290361508727073, |
| "num_tokens": 4368311.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 1.610975767672062, |
| "epoch": 2.980548894217959, |
| "grad_norm": 6.7775750160217285, |
| "learning_rate": 7.430677743136523e-06, |
| "loss": 0.3802893400192261, |
| "mean_token_accuracy": 0.8421619407832622, |
| "num_tokens": 4398838.0, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.980548894217959, |
| "eval_entropy": 1.6564679476432502, |
| "eval_loss": 0.9532321691513062, |
| "eval_mean_token_accuracy": 0.7908860681927763, |
| "eval_num_tokens": 4398838.0, |
| "eval_runtime": 25.2927, |
| "eval_samples_per_second": 20.243, |
| "eval_steps_per_second": 10.121, |
| "step": 1400 |
| }, |
| { |
| "entropy": 1.6621201969172856, |
| "epoch": 3.0, |
| "grad_norm": 52.18832015991211, |
| "learning_rate": 7.297704155466499e-06, |
| "loss": 0.6414576053619385, |
| "mean_token_accuracy": 0.8645402229812047, |
| "num_tokens": 4430403.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 1.6340646058321, |
| "epoch": 3.0213162803090863, |
| "grad_norm": 5.4931254386901855, |
| "learning_rate": 7.165244063851177e-06, |
| "loss": 0.43079633712768556, |
| "mean_token_accuracy": 0.9154900871217251, |
| "num_tokens": 4462302.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 1.5907170630991458, |
| "epoch": 3.042632560618172, |
| "grad_norm": 2.4832773208618164, |
| "learning_rate": 7.033322638641396e-06, |
| "loss": 0.2773794174194336, |
| "mean_token_accuracy": 0.8983474470674991, |
| "num_tokens": 4490390.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 1.4824031606316566, |
| "epoch": 3.0639488409272584, |
| "grad_norm": 14.030230522155762, |
| "learning_rate": 6.901964947829427e-06, |
| "loss": 0.3276866674423218, |
| "mean_token_accuracy": 0.9273385688662529, |
| "num_tokens": 4523458.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 1.4728427067399026, |
| "epoch": 3.085265121236344, |
| "grad_norm": 6.50776481628418, |
| "learning_rate": 6.771195952285541e-06, |
| "loss": 0.34999635219573977, |
| "mean_token_accuracy": 0.8883909173309803, |
| "num_tokens": 4551511.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 1.5375451922416687, |
| "epoch": 3.1065814015454305, |
| "grad_norm": 3.3505122661590576, |
| "learning_rate": 6.641040501014856e-06, |
| "loss": 0.26920404434204104, |
| "mean_token_accuracy": 0.9332471784204245, |
| "num_tokens": 4580674.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 1.452385664731264, |
| "epoch": 3.1278976818545163, |
| "grad_norm": 6.152368068695068, |
| "learning_rate": 6.511523326435501e-06, |
| "loss": 0.2656646490097046, |
| "mean_token_accuracy": 0.9215638756752014, |
| "num_tokens": 4610859.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 1.4217439875006677, |
| "epoch": 3.1492139621636026, |
| "grad_norm": 8.59308910369873, |
| "learning_rate": 6.382669039678899e-06, |
| "loss": 0.24169042110443115, |
| "mean_token_accuracy": 0.9385403782129288, |
| "num_tokens": 4638835.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 1.4209895387291909, |
| "epoch": 3.1705302424726884, |
| "grad_norm": 7.641168594360352, |
| "learning_rate": 6.254502125913095e-06, |
| "loss": 0.22674014568328857, |
| "mean_token_accuracy": 0.911290580779314, |
| "num_tokens": 4674846.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 1.3677985310554504, |
| "epoch": 3.1918465227817747, |
| "grad_norm": 9.29020881652832, |
| "learning_rate": 6.127046939690056e-06, |
| "loss": 0.2984227895736694, |
| "mean_token_accuracy": 0.9373271211981773, |
| "num_tokens": 4705085.0, |
| "step": 1500 |
| }, |
| { |
| "epoch": 3.1918465227817747, |
| "eval_entropy": 1.444472799776122, |
| "eval_loss": 1.1713542938232422, |
| "eval_mean_token_accuracy": 0.7895977763691917, |
| "eval_num_tokens": 4705085.0, |
| "eval_runtime": 25.1995, |
| "eval_samples_per_second": 20.318, |
| "eval_steps_per_second": 10.159, |
| "step": 1500 |
| }, |
| { |
| "entropy": 1.4269857950508595, |
| "epoch": 3.2131628030908606, |
| "grad_norm": 11.469884872436523, |
| "learning_rate": 6.000327700317748e-06, |
| "loss": 0.2425680637359619, |
| "mean_token_accuracy": 0.9259982876479625, |
| "num_tokens": 4739190.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 1.4209635138511658, |
| "epoch": 3.234479083399947, |
| "grad_norm": 5.315972805023193, |
| "learning_rate": 5.87436848725794e-06, |
| "loss": 0.2542020082473755, |
| "mean_token_accuracy": 0.9073813389986753, |
| "num_tokens": 4772098.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 1.4560465589165688, |
| "epoch": 3.2557953637090327, |
| "grad_norm": 24.200057983398438, |
| "learning_rate": 5.7491932355505656e-06, |
| "loss": 0.30522403717041013, |
| "mean_token_accuracy": 0.9136910080909729, |
| "num_tokens": 4800090.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 1.4656708396971225, |
| "epoch": 3.277111644018119, |
| "grad_norm": 10.271306991577148, |
| "learning_rate": 5.624825731265543e-06, |
| "loss": 0.2127098798751831, |
| "mean_token_accuracy": 0.931862723827362, |
| "num_tokens": 4834524.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 1.4886308126151562, |
| "epoch": 3.2984279243272048, |
| "grad_norm": 12.6162691116333, |
| "learning_rate": 5.501289606982898e-06, |
| "loss": 0.2684090375900269, |
| "mean_token_accuracy": 0.9222205139696598, |
| "num_tokens": 4863468.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 1.387893059849739, |
| "epoch": 3.319744204636291, |
| "grad_norm": 9.085896492004395, |
| "learning_rate": 5.378608337302032e-06, |
| "loss": 0.2731961250305176, |
| "mean_token_accuracy": 0.9436910226941109, |
| "num_tokens": 4896467.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 1.4491034008562564, |
| "epoch": 3.341060484945377, |
| "grad_norm": 7.388673782348633, |
| "learning_rate": 5.256805234381055e-06, |
| "loss": 0.24700915813446045, |
| "mean_token_accuracy": 0.8964447133243084, |
| "num_tokens": 4925719.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 1.4569350488483905, |
| "epoch": 3.362376765254463, |
| "grad_norm": 7.207291126251221, |
| "learning_rate": 5.135903443506927e-06, |
| "loss": 0.3276431322097778, |
| "mean_token_accuracy": 0.8873641312122345, |
| "num_tokens": 4954162.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 1.4607440739870072, |
| "epoch": 3.383693045563549, |
| "grad_norm": 4.589608669281006, |
| "learning_rate": 5.015925938697377e-06, |
| "loss": 0.23540747165679932, |
| "mean_token_accuracy": 0.9337049081921578, |
| "num_tokens": 4988402.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 1.4307367473840713, |
| "epoch": 3.4050093258726353, |
| "grad_norm": 5.884532928466797, |
| "learning_rate": 4.896895518335306e-06, |
| "loss": 0.2338495969772339, |
| "mean_token_accuracy": 0.9359182961285114, |
| "num_tokens": 5022816.0, |
| "step": 1600 |
| }, |
| { |
| "epoch": 3.4050093258726353, |
| "eval_entropy": 1.4896407034248114, |
| "eval_loss": 1.1274768114089966, |
| "eval_mean_token_accuracy": 0.7863516705692746, |
| "eval_num_tokens": 5022816.0, |
| "eval_runtime": 25.2716, |
| "eval_samples_per_second": 20.26, |
| "eval_steps_per_second": 10.13, |
| "step": 1600 |
| }, |
| { |
| "entropy": 1.446070448309183, |
| "epoch": 3.426325606181721, |
| "grad_norm": 10.2098388671875, |
| "learning_rate": 4.778834800836591e-06, |
| "loss": 0.20406453609466552, |
| "mean_token_accuracy": 0.9318795662373305, |
| "num_tokens": 5052718.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 1.4429432988166808, |
| "epoch": 3.4476418864908074, |
| "grad_norm": 7.907771110534668, |
| "learning_rate": 4.661766220352098e-06, |
| "loss": 0.29486634731292727, |
| "mean_token_accuracy": 0.9161755211651326, |
| "num_tokens": 5084521.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 1.4223275877535344, |
| "epoch": 3.4689581667998937, |
| "grad_norm": 10.563481330871582, |
| "learning_rate": 4.545712022504679e-06, |
| "loss": 0.22321033477783203, |
| "mean_token_accuracy": 0.9142704658210278, |
| "num_tokens": 5115796.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 1.4276227876543999, |
| "epoch": 3.4902744471089795, |
| "grad_norm": 11.097084999084473, |
| "learning_rate": 4.430694260162032e-06, |
| "loss": 0.24617478847503663, |
| "mean_token_accuracy": 0.8925216607749462, |
| "num_tokens": 5150503.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 1.413447444140911, |
| "epoch": 3.5115907274180653, |
| "grad_norm": 5.738354206085205, |
| "learning_rate": 4.316734789246154e-06, |
| "loss": 0.1732957124710083, |
| "mean_token_accuracy": 0.9149338848888874, |
| "num_tokens": 5182439.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 1.4452892072498797, |
| "epoch": 3.5329070077271516, |
| "grad_norm": 15.468720436096191, |
| "learning_rate": 4.203855264580224e-06, |
| "loss": 0.2644877195358276, |
| "mean_token_accuracy": 0.914117070659995, |
| "num_tokens": 5216517.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 1.419405361264944, |
| "epoch": 3.554223288036238, |
| "grad_norm": 9.270110130310059, |
| "learning_rate": 4.092077135773731e-06, |
| "loss": 0.3064971685409546, |
| "mean_token_accuracy": 0.9066247828304768, |
| "num_tokens": 5248859.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 1.5174184702336788, |
| "epoch": 3.5755395683453237, |
| "grad_norm": 2.8754255771636963, |
| "learning_rate": 3.981421643146555e-06, |
| "loss": 0.21029229164123536, |
| "mean_token_accuracy": 0.9305562056601048, |
| "num_tokens": 5278300.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 1.388711978495121, |
| "epoch": 3.5968558486544095, |
| "grad_norm": 9.280269622802734, |
| "learning_rate": 3.8719098136928315e-06, |
| "loss": 0.2969250202178955, |
| "mean_token_accuracy": 0.9320756200700998, |
| "num_tokens": 5309092.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 1.455902987718582, |
| "epoch": 3.618172128963496, |
| "grad_norm": 5.556161403656006, |
| "learning_rate": 3.763562457085379e-06, |
| "loss": 0.2819516658782959, |
| "mean_token_accuracy": 0.9180143646895885, |
| "num_tokens": 5340126.0, |
| "step": 1700 |
| }, |
| { |
| "epoch": 3.618172128963496, |
| "eval_entropy": 1.4539805192034692, |
| "eval_loss": 1.1700488328933716, |
| "eval_mean_token_accuracy": 0.7862855003331788, |
| "eval_num_tokens": 5340126.0, |
| "eval_runtime": 25.2104, |
| "eval_samples_per_second": 20.309, |
| "eval_steps_per_second": 10.155, |
| "step": 1700 |
| }, |
| { |
| "entropy": 1.4361699119210243, |
| "epoch": 3.639488409272582, |
| "grad_norm": 20.25632095336914, |
| "learning_rate": 3.6564001617213827e-06, |
| "loss": 0.3046926498413086, |
| "mean_token_accuracy": 0.925341609120369, |
| "num_tokens": 5375346.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 1.4369515381753444, |
| "epoch": 3.660804689581668, |
| "grad_norm": 9.118430137634277, |
| "learning_rate": 3.5504432908101405e-06, |
| "loss": 0.3264621257781982, |
| "mean_token_accuracy": 0.8759976290166378, |
| "num_tokens": 5410878.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 1.4577076785266398, |
| "epoch": 3.682120969890754, |
| "grad_norm": 6.7929182052612305, |
| "learning_rate": 3.4457119785036173e-06, |
| "loss": 0.34423580169677737, |
| "mean_token_accuracy": 0.8818474646657706, |
| "num_tokens": 5438858.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 1.479407573491335, |
| "epoch": 3.70343725019984, |
| "grad_norm": 7.327031135559082, |
| "learning_rate": 3.342226126070506e-06, |
| "loss": 0.16338071823120118, |
| "mean_token_accuracy": 0.9366827972233296, |
| "num_tokens": 5470379.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 1.4941159047186374, |
| "epoch": 3.7247535305089263, |
| "grad_norm": 7.238304138183594, |
| "learning_rate": 3.2400053981145263e-06, |
| "loss": 0.22500226497650147, |
| "mean_token_accuracy": 0.9087515972554684, |
| "num_tokens": 5503410.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 1.4180281534790993, |
| "epoch": 3.746069810818012, |
| "grad_norm": 17.629175186157227, |
| "learning_rate": 3.139069218837745e-06, |
| "loss": 0.18224897384643554, |
| "mean_token_accuracy": 0.9434241332113743, |
| "num_tokens": 5537169.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 1.4403494112193584, |
| "epoch": 3.7673860911270984, |
| "grad_norm": 4.520951747894287, |
| "learning_rate": 3.0394367683495295e-06, |
| "loss": 0.3438844919204712, |
| "mean_token_accuracy": 0.9051133707165718, |
| "num_tokens": 5569219.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 1.4302593432366848, |
| "epoch": 3.7887023714361843, |
| "grad_norm": 14.042414665222168, |
| "learning_rate": 2.9411269790218965e-06, |
| "loss": 0.332825231552124, |
| "mean_token_accuracy": 0.9289621658623218, |
| "num_tokens": 5602198.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 1.4533981308341026, |
| "epoch": 3.8100186517452705, |
| "grad_norm": 7.9349260330200195, |
| "learning_rate": 2.844158531891966e-06, |
| "loss": 0.2003183603286743, |
| "mean_token_accuracy": 0.9049505375325679, |
| "num_tokens": 5629403.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 1.410759261995554, |
| "epoch": 3.8313349320543564, |
| "grad_norm": 5.083995819091797, |
| "learning_rate": 2.7485498531121204e-06, |
| "loss": 0.2712679147720337, |
| "mean_token_accuracy": 0.9239927411079407, |
| "num_tokens": 5659230.0, |
| "step": 1800 |
| }, |
| { |
| "epoch": 3.8313349320543564, |
| "eval_entropy": 1.4693333019968122, |
| "eval_loss": 1.1137001514434814, |
| "eval_mean_token_accuracy": 0.7850039067561738, |
| "eval_num_tokens": 5659230.0, |
| "eval_runtime": 25.4021, |
| "eval_samples_per_second": 20.156, |
| "eval_steps_per_second": 10.078, |
| "step": 1800 |
| }, |
| { |
| "entropy": 1.4003525048494339, |
| "epoch": 3.8526512123634427, |
| "grad_norm": 6.573414325714111, |
| "learning_rate": 2.654319110448651e-06, |
| "loss": 0.23307204246520996, |
| "mean_token_accuracy": 0.9070353828370571, |
| "num_tokens": 5689236.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 1.4843521788716316, |
| "epoch": 3.8739674926725285, |
| "grad_norm": 7.782621383666992, |
| "learning_rate": 2.5614842098294745e-06, |
| "loss": 0.3204060077667236, |
| "mean_token_accuracy": 0.9074679099023342, |
| "num_tokens": 5724536.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 1.4347909986972809, |
| "epoch": 3.8952837729816148, |
| "grad_norm": 5.542223930358887, |
| "learning_rate": 2.4700627919415966e-06, |
| "loss": 0.3621266603469849, |
| "mean_token_accuracy": 0.9036989733576775, |
| "num_tokens": 5754805.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 1.4112458311021328, |
| "epoch": 3.9166000532907006, |
| "grad_norm": 15.386390686035156, |
| "learning_rate": 2.380072228879012e-06, |
| "loss": 0.20611109733581542, |
| "mean_token_accuracy": 0.9141320556402206, |
| "num_tokens": 5785957.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 1.3910652115941047, |
| "epoch": 3.937916333599787, |
| "grad_norm": 4.969911575317383, |
| "learning_rate": 2.2915296208416038e-06, |
| "loss": 0.19969615936279297, |
| "mean_token_accuracy": 0.9188599303364754, |
| "num_tokens": 5817438.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 1.3991701498627662, |
| "epoch": 3.959232613908873, |
| "grad_norm": 14.075639724731445, |
| "learning_rate": 2.204451792885739e-06, |
| "loss": 0.2811038732528687, |
| "mean_token_accuracy": 0.9124676756560802, |
| "num_tokens": 5853028.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 1.4032354734838008, |
| "epoch": 3.980548894217959, |
| "grad_norm": 8.749677658081055, |
| "learning_rate": 2.118855291727131e-06, |
| "loss": 0.39807083606719973, |
| "mean_token_accuracy": 0.9114172287285328, |
| "num_tokens": 5879749.0, |
| "step": 1870 |
| }, |
| { |
| "entropy": 1.4793576941098252, |
| "epoch": 4.0, |
| "grad_norm": 0.40228429436683655, |
| "learning_rate": 2.03475638259659e-06, |
| "loss": 0.28619022369384767, |
| "mean_token_accuracy": 0.9326312027565421, |
| "num_tokens": 5907204.0, |
| "step": 1880 |
| }, |
| { |
| "entropy": 1.4576176889240742, |
| "epoch": 4.021316280309086, |
| "grad_norm": 3.4084649085998535, |
| "learning_rate": 1.952171046149286e-06, |
| "loss": 0.16383445262908936, |
| "mean_token_accuracy": 0.9292282827198506, |
| "num_tokens": 5937547.0, |
| "step": 1890 |
| }, |
| { |
| "entropy": 1.4545354157686234, |
| "epoch": 4.0426325606181726, |
| "grad_norm": 10.416617393493652, |
| "learning_rate": 1.871114975428049e-06, |
| "loss": 0.22887809276580812, |
| "mean_token_accuracy": 0.9353417418897152, |
| "num_tokens": 5964420.0, |
| "step": 1900 |
| }, |
| { |
| "epoch": 4.0426325606181726, |
| "eval_entropy": 1.4333079177886248, |
| "eval_loss": 1.1568964719772339, |
| "eval_mean_token_accuracy": 0.7843159851036035, |
| "eval_num_tokens": 5964420.0, |
| "eval_runtime": 25.1877, |
| "eval_samples_per_second": 20.327, |
| "eval_steps_per_second": 10.164, |
| "step": 1900 |
| }, |
| { |
| "entropy": 1.4628706477582454, |
| "epoch": 4.063948840927258, |
| "grad_norm": 2.5383825302124023, |
| "learning_rate": 1.7916035728813653e-06, |
| "loss": 0.10247511863708496, |
| "mean_token_accuracy": 0.9593492932617664, |
| "num_tokens": 5992125.0, |
| "step": 1910 |
| }, |
| { |
| "entropy": 1.3804713360965253, |
| "epoch": 4.085265121236344, |
| "grad_norm": 5.305675983428955, |
| "learning_rate": 1.7136519474365475e-06, |
| "loss": 0.24265851974487304, |
| "mean_token_accuracy": 0.9445340454578399, |
| "num_tokens": 6017516.0, |
| "step": 1920 |
| }, |
| { |
| "entropy": 1.3891515977680684, |
| "epoch": 4.10658140154543, |
| "grad_norm": 7.991399765014648, |
| "learning_rate": 1.637274911628729e-06, |
| "loss": 0.17978017330169677, |
| "mean_token_accuracy": 0.93779431656003, |
| "num_tokens": 6046058.0, |
| "step": 1930 |
| }, |
| { |
| "entropy": 1.3703414618968963, |
| "epoch": 4.127897681854517, |
| "grad_norm": 2.306601047515869, |
| "learning_rate": 1.5624869787861385e-06, |
| "loss": 0.1866108775138855, |
| "mean_token_accuracy": 0.9406570665538311, |
| "num_tokens": 6078166.0, |
| "step": 1940 |
| }, |
| { |
| "entropy": 1.3552788123488426, |
| "epoch": 4.149213962163603, |
| "grad_norm": 10.646674156188965, |
| "learning_rate": 1.4893023602722412e-06, |
| "loss": 0.17565094232559203, |
| "mean_token_accuracy": 0.9670388154685498, |
| "num_tokens": 6110728.0, |
| "step": 1950 |
| }, |
| { |
| "entropy": 1.361258938163519, |
| "epoch": 4.170530242472688, |
| "grad_norm": 2.1465964317321777, |
| "learning_rate": 1.4177349627852855e-06, |
| "loss": 0.08721579909324646, |
| "mean_token_accuracy": 0.9694276623427868, |
| "num_tokens": 6146320.0, |
| "step": 1960 |
| }, |
| { |
| "entropy": 1.292334294319153, |
| "epoch": 4.191846522781774, |
| "grad_norm": 5.472266674041748, |
| "learning_rate": 1.3477983857156996e-06, |
| "loss": 0.12929170131683348, |
| "mean_token_accuracy": 0.9474783554673195, |
| "num_tokens": 6175327.0, |
| "step": 1970 |
| }, |
| { |
| "entropy": 1.403335941582918, |
| "epoch": 4.213162803090861, |
| "grad_norm": 15.660310745239258, |
| "learning_rate": 1.279505918561923e-06, |
| "loss": 0.25771000385284426, |
| "mean_token_accuracy": 0.9202743619680405, |
| "num_tokens": 6210554.0, |
| "step": 1980 |
| }, |
| { |
| "entropy": 1.3927375204861163, |
| "epoch": 4.234479083399947, |
| "grad_norm": 9.373515129089355, |
| "learning_rate": 1.2128705384051032e-06, |
| "loss": 0.2001044273376465, |
| "mean_token_accuracy": 0.9633757442235946, |
| "num_tokens": 6244706.0, |
| "step": 1990 |
| }, |
| { |
| "entropy": 1.3265936575829982, |
| "epoch": 4.255795363709033, |
| "grad_norm": 3.4762251377105713, |
| "learning_rate": 1.1479049074431526e-06, |
| "loss": 0.20537242889404297, |
| "mean_token_accuracy": 0.9328463308513164, |
| "num_tokens": 6278954.0, |
| "step": 2000 |
| }, |
| { |
| "epoch": 4.255795363709033, |
| "eval_entropy": 1.369972410146147, |
| "eval_loss": 1.2773594856262207, |
| "eval_mean_token_accuracy": 0.7810963835800067, |
| "eval_num_tokens": 6278954.0, |
| "eval_runtime": 25.2242, |
| "eval_samples_per_second": 20.298, |
| "eval_steps_per_second": 10.149, |
| "step": 2000 |
| }, |
| { |
| "entropy": 1.3605533331632613, |
| "epoch": 4.2771116440181185, |
| "grad_norm": 9.257750511169434, |
| "learning_rate": 1.0846213705846664e-06, |
| "loss": 0.19485853910446166, |
| "mean_token_accuracy": 0.9474734902381897, |
| "num_tokens": 6309202.0, |
| "step": 2010 |
| }, |
| { |
| "entropy": 1.3613811306655408, |
| "epoch": 4.298427924327205, |
| "grad_norm": 4.7302422523498535, |
| "learning_rate": 1.0230319531031042e-06, |
| "loss": 0.1681363582611084, |
| "mean_token_accuracy": 0.9616480603814125, |
| "num_tokens": 6343103.0, |
| "step": 2020 |
| }, |
| { |
| "entropy": 1.3343191578984261, |
| "epoch": 4.319744204636291, |
| "grad_norm": 9.000184059143066, |
| "learning_rate": 9.631483583517288e-07, |
| "loss": 0.17980358600616456, |
| "mean_token_accuracy": 0.9588635042309761, |
| "num_tokens": 6375041.0, |
| "step": 2030 |
| }, |
| { |
| "entropy": 1.325505518168211, |
| "epoch": 4.341060484945377, |
| "grad_norm": 0.36019816994667053, |
| "learning_rate": 9.049819655397108e-07, |
| "loss": 0.15309940576553344, |
| "mean_token_accuracy": 0.9582446455955506, |
| "num_tokens": 6407611.0, |
| "step": 2040 |
| }, |
| { |
| "entropy": 1.3149354815483094, |
| "epoch": 4.362376765254463, |
| "grad_norm": 8.396425247192383, |
| "learning_rate": 8.485438275698154e-07, |
| "loss": 0.12012659311294556, |
| "mean_token_accuracy": 0.9287039190530777, |
| "num_tokens": 6439014.0, |
| "step": 2050 |
| }, |
| { |
| "entropy": 1.3025728821754456, |
| "epoch": 4.383693045563549, |
| "grad_norm": 13.925297737121582, |
| "learning_rate": 7.938446689381229e-07, |
| "loss": 0.16408991813659668, |
| "mean_token_accuracy": 0.9388780653476715, |
| "num_tokens": 6471556.0, |
| "step": 2060 |
| }, |
| { |
| "entropy": 1.276213175058365, |
| "epoch": 4.405009325872635, |
| "grad_norm": 3.2025303840637207, |
| "learning_rate": 7.408948836961216e-07, |
| "loss": 0.1705629587173462, |
| "mean_token_accuracy": 0.9424380153417588, |
| "num_tokens": 6504282.0, |
| "step": 2070 |
| }, |
| { |
| "entropy": 1.3470777593553067, |
| "epoch": 4.426325606181721, |
| "grad_norm": 4.128990650177002, |
| "learning_rate": 6.897045334756191e-07, |
| "loss": 0.1434122920036316, |
| "mean_token_accuracy": 0.9367260579019785, |
| "num_tokens": 6533310.0, |
| "step": 2080 |
| }, |
| { |
| "entropy": 1.3858976893126964, |
| "epoch": 4.447641886490807, |
| "grad_norm": 3.0865390300750732, |
| "learning_rate": 6.402833455768054e-07, |
| "loss": 0.17338463068008422, |
| "mean_token_accuracy": 0.9535353198647499, |
| "num_tokens": 6561833.0, |
| "step": 2090 |
| }, |
| { |
| "entropy": 1.3846996150910855, |
| "epoch": 4.468958166799894, |
| "grad_norm": 14.10164737701416, |
| "learning_rate": 5.926407111198451e-07, |
| "loss": 0.24046428203582765, |
| "mean_token_accuracy": 0.9303554680198431, |
| "num_tokens": 6592283.0, |
| "step": 2100 |
| }, |
| { |
| "epoch": 4.468958166799894, |
| "eval_entropy": 1.355904731201008, |
| "eval_loss": 1.3142094612121582, |
| "eval_mean_token_accuracy": 0.7773873422993347, |
| "eval_num_tokens": 6592283.0, |
| "eval_runtime": 25.1866, |
| "eval_samples_per_second": 20.328, |
| "eval_steps_per_second": 10.164, |
| "step": 2100 |
| }, |
| { |
| "entropy": 1.3094307713210582, |
| "epoch": 4.4902744471089795, |
| "grad_norm": 9.617660522460938, |
| "learning_rate": 5.467856832603669e-07, |
| "loss": 0.17363038063049316, |
| "mean_token_accuracy": 0.9140197165310383, |
| "num_tokens": 6624745.0, |
| "step": 2110 |
| }, |
| { |
| "entropy": 1.372648873925209, |
| "epoch": 4.511590727418065, |
| "grad_norm": 11.372284889221191, |
| "learning_rate": 5.027269754691544e-07, |
| "loss": 0.18337360620498658, |
| "mean_token_accuracy": 0.9457527458667755, |
| "num_tokens": 6657590.0, |
| "step": 2120 |
| }, |
| { |
| "entropy": 1.2979931004345417, |
| "epoch": 4.532907007727152, |
| "grad_norm": 7.741551876068115, |
| "learning_rate": 4.604729598764024e-07, |
| "loss": 0.22370004653930664, |
| "mean_token_accuracy": 0.9535742335021495, |
| "num_tokens": 6693274.0, |
| "step": 2130 |
| }, |
| { |
| "entropy": 1.2493705317378043, |
| "epoch": 4.554223288036238, |
| "grad_norm": 5.593754768371582, |
| "learning_rate": 4.200316656808134e-07, |
| "loss": 0.19300585985183716, |
| "mean_token_accuracy": 0.9324206352233887, |
| "num_tokens": 6724368.0, |
| "step": 2140 |
| }, |
| { |
| "entropy": 1.3586355127394198, |
| "epoch": 4.575539568345324, |
| "grad_norm": 6.431520462036133, |
| "learning_rate": 3.8141077762388846e-07, |
| "loss": 0.2666941165924072, |
| "mean_token_accuracy": 0.947481868416071, |
| "num_tokens": 6755479.0, |
| "step": 2150 |
| }, |
| { |
| "entropy": 1.3126740761101245, |
| "epoch": 4.5968558486544095, |
| "grad_norm": 2.596592664718628, |
| "learning_rate": 3.446176345296459e-07, |
| "loss": 0.2076418399810791, |
| "mean_token_accuracy": 0.948704682290554, |
| "num_tokens": 6793756.0, |
| "step": 2160 |
| }, |
| { |
| "entropy": 1.3612797297537327, |
| "epoch": 4.618172128963495, |
| "grad_norm": 11.598284721374512, |
| "learning_rate": 3.0965922791008183e-07, |
| "loss": 0.20219831466674804, |
| "mean_token_accuracy": 0.9383927457034588, |
| "num_tokens": 6823256.0, |
| "step": 2170 |
| }, |
| { |
| "entropy": 1.310491831600666, |
| "epoch": 4.639488409272582, |
| "grad_norm": 0.6937721371650696, |
| "learning_rate": 2.765422006366336e-07, |
| "loss": 0.1360365629196167, |
| "mean_token_accuracy": 0.9622993104159832, |
| "num_tokens": 6852587.0, |
| "step": 2180 |
| }, |
| { |
| "entropy": 1.3515724152326585, |
| "epoch": 4.660804689581668, |
| "grad_norm": 5.8835954666137695, |
| "learning_rate": 2.452728456778819e-07, |
| "loss": 0.18895021677017212, |
| "mean_token_accuracy": 0.9227117404341698, |
| "num_tokens": 6879919.0, |
| "step": 2190 |
| }, |
| { |
| "entropy": 1.319688442349434, |
| "epoch": 4.682120969890754, |
| "grad_norm": 10.677730560302734, |
| "learning_rate": 2.1585710490375168e-07, |
| "loss": 0.2697124719619751, |
| "mean_token_accuracy": 0.9373554818332195, |
| "num_tokens": 6912648.0, |
| "step": 2200 |
| }, |
| { |
| "epoch": 4.682120969890754, |
| "eval_entropy": 1.356080744881183, |
| "eval_loss": 1.3120059967041016, |
| "eval_mean_token_accuracy": 0.7787370060686953, |
| "eval_num_tokens": 6912648.0, |
| "eval_runtime": 25.2146, |
| "eval_samples_per_second": 20.306, |
| "eval_steps_per_second": 10.153, |
| "step": 2200 |
| }, |
| { |
| "entropy": 1.2888402037322522, |
| "epoch": 4.7034372501998405, |
| "grad_norm": 4.392502307891846, |
| "learning_rate": 1.8830056795642315e-07, |
| "loss": 0.17859218120574952, |
| "mean_token_accuracy": 0.9356974855065345, |
| "num_tokens": 6943210.0, |
| "step": 2210 |
| }, |
| { |
| "entropy": 1.306819212436676, |
| "epoch": 4.724753530508926, |
| "grad_norm": 9.902172088623047, |
| "learning_rate": 1.626084711881726e-07, |
| "loss": 0.12197283506393433, |
| "mean_token_accuracy": 0.9637115865945816, |
| "num_tokens": 6972906.0, |
| "step": 2220 |
| }, |
| { |
| "entropy": 1.3649342678487302, |
| "epoch": 4.746069810818012, |
| "grad_norm": 6.050726413726807, |
| "learning_rate": 1.3878569666635922e-07, |
| "loss": 0.13252955675125122, |
| "mean_token_accuracy": 0.9423332914710045, |
| "num_tokens": 7006735.0, |
| "step": 2230 |
| }, |
| { |
| "entropy": 1.344475695490837, |
| "epoch": 4.767386091127098, |
| "grad_norm": 20.14447784423828, |
| "learning_rate": 1.168367712457108e-07, |
| "loss": 0.2347338914871216, |
| "mean_token_accuracy": 0.9231791608035564, |
| "num_tokens": 7036174.0, |
| "step": 2240 |
| }, |
| { |
| "entropy": 1.313488345593214, |
| "epoch": 4.788702371436185, |
| "grad_norm": 5.92142915725708, |
| "learning_rate": 9.676586570813385e-08, |
| "loss": 0.18055379390716553, |
| "mean_token_accuracy": 0.9538333244621754, |
| "num_tokens": 7069122.0, |
| "step": 2250 |
| }, |
| { |
| "entropy": 1.3362836241722107, |
| "epoch": 4.8100186517452705, |
| "grad_norm": 5.3233513832092285, |
| "learning_rate": 7.857679397016316e-08, |
| "loss": 0.2382068157196045, |
| "mean_token_accuracy": 0.9316875658929348, |
| "num_tokens": 7098388.0, |
| "step": 2260 |
| }, |
| { |
| "entropy": 1.320764410495758, |
| "epoch": 4.831334932054356, |
| "grad_norm": 6.75054931640625, |
| "learning_rate": 6.227301235824046e-08, |
| "loss": 0.29803290367126467, |
| "mean_token_accuracy": 0.9219658993184566, |
| "num_tokens": 7127279.0, |
| "step": 2270 |
| }, |
| { |
| "entropy": 1.3429523721337318, |
| "epoch": 4.852651212363442, |
| "grad_norm": 6.420995235443115, |
| "learning_rate": 4.7857618951929754e-08, |
| "loss": 0.18056874275207518, |
| "mean_token_accuracy": 0.9634992338716983, |
| "num_tokens": 7161317.0, |
| "step": 2280 |
| }, |
| { |
| "entropy": 1.2852252803742885, |
| "epoch": 4.873967492672529, |
| "grad_norm": 8.42899227142334, |
| "learning_rate": 3.533335299521823e-08, |
| "loss": 0.16432349681854247, |
| "mean_token_accuracy": 0.9391075037419796, |
| "num_tokens": 7193312.0, |
| "step": 2290 |
| }, |
| { |
| "entropy": 1.2744086734950542, |
| "epoch": 4.895283772981615, |
| "grad_norm": 12.795449256896973, |
| "learning_rate": 2.470259437599487e-08, |
| "loss": 0.17056713104248047, |
| "mean_token_accuracy": 0.9438464976847172, |
| "num_tokens": 7225752.0, |
| "step": 2300 |
| }, |
| { |
| "epoch": 4.895283772981615, |
| "eval_entropy": 1.3555747161153704, |
| "eval_loss": 1.3144317865371704, |
| "eval_mean_token_accuracy": 0.779438130906783, |
| "eval_num_tokens": 7225752.0, |
| "eval_runtime": 25.339, |
| "eval_samples_per_second": 20.206, |
| "eval_steps_per_second": 10.103, |
| "step": 2300 |
| }, |
| { |
| "entropy": 1.3686442136764527, |
| "epoch": 4.916600053290701, |
| "grad_norm": 3.3401401042938232, |
| "learning_rate": 1.5967363173823304e-08, |
| "loss": 0.12192281484603881, |
| "mean_token_accuracy": 0.9248238526284694, |
| "num_tokens": 7261343.0, |
| "step": 2310 |
| }, |
| { |
| "entropy": 1.3582661159336566, |
| "epoch": 4.937916333599787, |
| "grad_norm": 8.621865272521973, |
| "learning_rate": 9.129319276081073e-09, |
| "loss": 0.2683550357818604, |
| "mean_token_accuracy": 0.9585993848741055, |
| "num_tokens": 7296635.0, |
| "step": 2320 |
| }, |
| { |
| "entropy": 1.31158257573843, |
| "epoch": 4.959232613908873, |
| "grad_norm": 15.862990379333496, |
| "learning_rate": 4.189762062540847e-09, |
| "loss": 0.3723548173904419, |
| "mean_token_accuracy": 0.9173178903758525, |
| "num_tokens": 7332271.0, |
| "step": 2330 |
| }, |
| { |
| "entropy": 1.2785717263817786, |
| "epoch": 4.980548894217959, |
| "grad_norm": 9.872111320495605, |
| "learning_rate": 1.1496301584634772e-09, |
| "loss": 0.10508073568344116, |
| "mean_token_accuracy": 0.9446402385830879, |
| "num_tokens": 7362022.0, |
| "step": 2340 |
| }, |
| { |
| "entropy": 1.3751475908984876, |
| "epoch": 5.0, |
| "grad_norm": 1.7823141813278198, |
| "learning_rate": 9.501256236221778e-12, |
| "loss": 0.1756437063217163, |
| "mean_token_accuracy": 0.931066878854412, |
| "num_tokens": 7384005.0, |
| "step": 2350 |
| }, |
| { |
| "epoch": 5.0, |
| "eval_entropy": 1.3568143870215863, |
| "eval_loss": 1.3119933605194092, |
| "eval_mean_token_accuracy": 0.7803392773494124, |
| "eval_num_tokens": 7384005.0, |
| "eval_runtime": 25.1433, |
| "eval_samples_per_second": 20.363, |
| "eval_steps_per_second": 10.182, |
| "step": 2350 |
| }, |
| { |
| "epoch": 5.0, |
| "step": 2350, |
| "total_flos": 3.337527236651827e+17, |
| "train_loss": 0.5025193223293791, |
| "train_runtime": 4555.1644, |
| "train_samples_per_second": 4.12, |
| "train_steps_per_second": 0.516 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 2350, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 5, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 3.337527236651827e+17, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
