| { |
| "best_global_step": 500, |
| "best_metric": 0.6567164063453674, |
| "best_model_checkpoint": "/data6/jongbinwon/evolving_MoE/checkpoints/sft_llama31_8b_lbox_legal-recidivism-analyst_ffn_only_5ep_eval500/checkpoint-500", |
| "epoch": 5.0, |
| "eval_steps": 500, |
| "global_step": 1875, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 2.4114491134881972, |
| "epoch": 0.026720106880427523, |
| "grad_norm": 5.596068859100342, |
| "learning_rate": 3.157894736842105e-06, |
| "loss": 2.2780237197875977, |
| "mean_token_accuracy": 0.6010795146226883, |
| "num_tokens": 38338.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 2.406516769528389, |
| "epoch": 0.053440213760855046, |
| "grad_norm": 10.855905532836914, |
| "learning_rate": 6.666666666666667e-06, |
| "loss": 2.1981935501098633, |
| "mean_token_accuracy": 0.6083482213318347, |
| "num_tokens": 75707.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 2.3304371386766434, |
| "epoch": 0.08016032064128256, |
| "grad_norm": 5.457886695861816, |
| "learning_rate": 1.017543859649123e-05, |
| "loss": 1.822286033630371, |
| "mean_token_accuracy": 0.6118973881006241, |
| "num_tokens": 117281.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 2.4299359738826753, |
| "epoch": 0.10688042752171009, |
| "grad_norm": 4.230496883392334, |
| "learning_rate": 1.3684210526315791e-05, |
| "loss": 1.3760682106018067, |
| "mean_token_accuracy": 0.7258752726018429, |
| "num_tokens": 154963.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 2.4413882046937943, |
| "epoch": 0.13360053440213762, |
| "grad_norm": 8.62831974029541, |
| "learning_rate": 1.719298245614035e-05, |
| "loss": 0.9998857498168945, |
| "mean_token_accuracy": 0.7697913683950901, |
| "num_tokens": 193819.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 2.392297226190567, |
| "epoch": 0.16032064128256512, |
| "grad_norm": 6.337141990661621, |
| "learning_rate": 1.9999940277008807e-05, |
| "loss": 0.7882119178771972, |
| "mean_token_accuracy": 0.8111064344644546, |
| "num_tokens": 237628.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 2.451492914557457, |
| "epoch": 0.18704074816299265, |
| "grad_norm": 4.731862545013428, |
| "learning_rate": 1.999785004721968e-05, |
| "loss": 0.6730443954467773, |
| "mean_token_accuracy": 0.8397256970405579, |
| "num_tokens": 275923.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 2.436008632183075, |
| "epoch": 0.21376085504342018, |
| "grad_norm": 5.050048828125, |
| "learning_rate": 1.999277438119978e-05, |
| "loss": 0.39104249477386477, |
| "mean_token_accuracy": 0.9022710204124451, |
| "num_tokens": 319033.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 2.356634294986725, |
| "epoch": 0.24048096192384769, |
| "grad_norm": 6.055978298187256, |
| "learning_rate": 1.9984714794582682e-05, |
| "loss": 0.37810912132263186, |
| "mean_token_accuracy": 0.8965166822075844, |
| "num_tokens": 359231.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 2.384824666380882, |
| "epoch": 0.26720106880427524, |
| "grad_norm": 9.618133544921875, |
| "learning_rate": 1.9973673694024002e-05, |
| "loss": 0.36782715320587156, |
| "mean_token_accuracy": 0.898387248814106, |
| "num_tokens": 398276.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 2.3534796088933945, |
| "epoch": 0.2939211756847027, |
| "grad_norm": 8.99193286895752, |
| "learning_rate": 1.995965437648273e-05, |
| "loss": 0.3414323806762695, |
| "mean_token_accuracy": 0.9059841021895408, |
| "num_tokens": 440225.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 2.3363087326288223, |
| "epoch": 0.32064128256513025, |
| "grad_norm": 4.923404693603516, |
| "learning_rate": 1.9942661028236746e-05, |
| "loss": 0.28258490562438965, |
| "mean_token_accuracy": 0.9170141533017159, |
| "num_tokens": 482016.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 2.301443299651146, |
| "epoch": 0.3473613894455578, |
| "grad_norm": 6.892687797546387, |
| "learning_rate": 1.992269872363277e-05, |
| "loss": 0.3212214469909668, |
| "mean_token_accuracy": 0.9211123198270798, |
| "num_tokens": 526770.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 2.2242509961128234, |
| "epoch": 0.3740814963259853, |
| "grad_norm": 5.270275592803955, |
| "learning_rate": 1.9899773423571102e-05, |
| "loss": 0.2522684812545776, |
| "mean_token_accuracy": 0.9321060806512833, |
| "num_tokens": 568279.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 2.3202496439218523, |
| "epoch": 0.40080160320641284, |
| "grad_norm": 7.142147541046143, |
| "learning_rate": 1.9873891973725673e-05, |
| "loss": 0.2672285795211792, |
| "mean_token_accuracy": 0.9259091302752495, |
| "num_tokens": 610153.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 2.3666753977537156, |
| "epoch": 0.42752171008684037, |
| "grad_norm": 4.853704452514648, |
| "learning_rate": 1.984506210249986e-05, |
| "loss": 0.2201080083847046, |
| "mean_token_accuracy": 0.9253387123346328, |
| "num_tokens": 646618.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 2.281407207250595, |
| "epoch": 0.45424181696726784, |
| "grad_norm": 7.020781517028809, |
| "learning_rate": 1.9813292418718734e-05, |
| "loss": 0.19281790256500245, |
| "mean_token_accuracy": 0.9436771214008332, |
| "num_tokens": 684692.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 2.313987892866135, |
| "epoch": 0.48096192384769537, |
| "grad_norm": 4.606362819671631, |
| "learning_rate": 1.9778592409058376e-05, |
| "loss": 0.29049561023712156, |
| "mean_token_accuracy": 0.9195127740502358, |
| "num_tokens": 721590.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 2.339491590857506, |
| "epoch": 0.5076820307281229, |
| "grad_norm": 3.5211105346679688, |
| "learning_rate": 1.9740972435213114e-05, |
| "loss": 0.20609443187713622, |
| "mean_token_accuracy": 0.9414233103394508, |
| "num_tokens": 758818.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 2.317407730221748, |
| "epoch": 0.5344021376085505, |
| "grad_norm": 4.058583736419678, |
| "learning_rate": 1.9700443730801412e-05, |
| "loss": 0.2989838123321533, |
| "mean_token_accuracy": 0.9268160626292229, |
| "num_tokens": 801682.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 2.302973747253418, |
| "epoch": 0.561122244488978, |
| "grad_norm": 4.720394134521484, |
| "learning_rate": 1.9657018398011435e-05, |
| "loss": 0.2144409418106079, |
| "mean_token_accuracy": 0.9486440360546112, |
| "num_tokens": 845221.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 2.273245298862457, |
| "epoch": 0.5878423513694054, |
| "grad_norm": 5.325322151184082, |
| "learning_rate": 1.9610709403987248e-05, |
| "loss": 0.267060375213623, |
| "mean_token_accuracy": 0.9317445114254952, |
| "num_tokens": 889428.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 2.338310980796814, |
| "epoch": 0.614562458249833, |
| "grad_norm": 6.301466464996338, |
| "learning_rate": 1.9561530576956703e-05, |
| "loss": 0.14638622999191284, |
| "mean_token_accuracy": 0.9580206617712974, |
| "num_tokens": 926982.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 2.324004775285721, |
| "epoch": 0.6412825651302605, |
| "grad_norm": 3.4183552265167236, |
| "learning_rate": 1.9509496602102253e-05, |
| "loss": 0.14707474708557128, |
| "mean_token_accuracy": 0.9522150292992592, |
| "num_tokens": 966259.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 2.2831999957561493, |
| "epoch": 0.6680026720106881, |
| "grad_norm": 2.5120954513549805, |
| "learning_rate": 1.9454623017175814e-05, |
| "loss": 0.14005672931671143, |
| "mean_token_accuracy": 0.9632398635149002, |
| "num_tokens": 1002347.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 2.277027702331543, |
| "epoch": 0.6947227788911156, |
| "grad_norm": 5.257753372192383, |
| "learning_rate": 1.9396926207859085e-05, |
| "loss": 0.1792516827583313, |
| "mean_token_accuracy": 0.9589207723736763, |
| "num_tokens": 1042742.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 2.195540490746498, |
| "epoch": 0.7214428857715431, |
| "grad_norm": 7.266210556030273, |
| "learning_rate": 1.9336423402870655e-05, |
| "loss": 0.25304622650146485, |
| "mean_token_accuracy": 0.9308184772729874, |
| "num_tokens": 1086958.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 2.2867618560791017, |
| "epoch": 0.7481629926519706, |
| "grad_norm": 3.5316925048828125, |
| "learning_rate": 1.9273132668821363e-05, |
| "loss": 0.1557828426361084, |
| "mean_token_accuracy": 0.9693289130926133, |
| "num_tokens": 1122620.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 2.290681630373001, |
| "epoch": 0.7748830995323981, |
| "grad_norm": 5.07416296005249, |
| "learning_rate": 1.9207072904819484e-05, |
| "loss": 0.24710845947265625, |
| "mean_token_accuracy": 0.9464481830596924, |
| "num_tokens": 1164211.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 2.3110738426446913, |
| "epoch": 0.8016032064128257, |
| "grad_norm": 4.62537145614624, |
| "learning_rate": 1.913826383682729e-05, |
| "loss": 0.15239956378936767, |
| "mean_token_accuracy": 0.9535225883126259, |
| "num_tokens": 1198723.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 2.236919492483139, |
| "epoch": 0.8283233132932531, |
| "grad_norm": 2.020097017288208, |
| "learning_rate": 1.9066726011770725e-05, |
| "loss": 0.11672446727752686, |
| "mean_token_accuracy": 0.9745367020368576, |
| "num_tokens": 1238573.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 2.242557743191719, |
| "epoch": 0.8550434201736807, |
| "grad_norm": 1.9286326169967651, |
| "learning_rate": 1.8992480791403957e-05, |
| "loss": 0.07258902788162232, |
| "mean_token_accuracy": 0.9772431373596191, |
| "num_tokens": 1276289.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 2.306964412331581, |
| "epoch": 0.8817635270541082, |
| "grad_norm": 6.739360332489014, |
| "learning_rate": 1.891555034593055e-05, |
| "loss": 0.1884454607963562, |
| "mean_token_accuracy": 0.9459798082709312, |
| "num_tokens": 1313086.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 2.245205116271973, |
| "epoch": 0.9084836339345357, |
| "grad_norm": 2.9247703552246094, |
| "learning_rate": 1.8835957647383304e-05, |
| "loss": 0.0888478398323059, |
| "mean_token_accuracy": 0.9747821375727653, |
| "num_tokens": 1350324.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 2.1432482481002806, |
| "epoch": 0.9352037408149633, |
| "grad_norm": 8.896561622619629, |
| "learning_rate": 1.87537264627646e-05, |
| "loss": 0.18118684291839598, |
| "mean_token_accuracy": 0.9673180118203163, |
| "num_tokens": 1397766.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 2.2885415494441985, |
| "epoch": 0.9619238476953907, |
| "grad_norm": 2.2434868812561035, |
| "learning_rate": 1.866888134694942e-05, |
| "loss": 0.1295708179473877, |
| "mean_token_accuracy": 0.9676819637417793, |
| "num_tokens": 1434463.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 2.3136557012796404, |
| "epoch": 0.9886439545758183, |
| "grad_norm": 3.9793589115142822, |
| "learning_rate": 1.858144763535302e-05, |
| "loss": 0.12864513397216798, |
| "mean_token_accuracy": 0.9653062790632247, |
| "num_tokens": 1468996.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 2.198131728816677, |
| "epoch": 1.0133600534402138, |
| "grad_norm": 2.8816585540771484, |
| "learning_rate": 1.8491451436365628e-05, |
| "loss": 0.11299718618392944, |
| "mean_token_accuracy": 0.9596015585435403, |
| "num_tokens": 1512840.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 2.263194355368614, |
| "epoch": 1.0400801603206413, |
| "grad_norm": 6.7774739265441895, |
| "learning_rate": 1.839891962355624e-05, |
| "loss": 0.12956639528274536, |
| "mean_token_accuracy": 0.9688687428832055, |
| "num_tokens": 1549259.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 2.276552340388298, |
| "epoch": 1.0668002672010688, |
| "grad_norm": 1.2939426898956299, |
| "learning_rate": 1.8303879827647977e-05, |
| "loss": 0.0762969970703125, |
| "mean_token_accuracy": 0.9799371302127838, |
| "num_tokens": 1591121.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 2.2271972447633743, |
| "epoch": 1.0935203740814963, |
| "grad_norm": 2.974682331085205, |
| "learning_rate": 1.8206360428267332e-05, |
| "loss": 0.06802645921707154, |
| "mean_token_accuracy": 0.9835173025727272, |
| "num_tokens": 1633167.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 2.2570935279130935, |
| "epoch": 1.1202404809619237, |
| "grad_norm": 3.0515291690826416, |
| "learning_rate": 1.8106390545469797e-05, |
| "loss": 0.09067635536193848, |
| "mean_token_accuracy": 0.9783770129084587, |
| "num_tokens": 1673080.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 2.2780718475580217, |
| "epoch": 1.1469605878423514, |
| "grad_norm": 4.675966262817383, |
| "learning_rate": 1.8004000031044363e-05, |
| "loss": 0.05609263181686401, |
| "mean_token_accuracy": 0.9771336480975151, |
| "num_tokens": 1710884.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 2.2877213954925537, |
| "epoch": 1.173680694722779, |
| "grad_norm": 0.942188024520874, |
| "learning_rate": 1.789921945959958e-05, |
| "loss": 0.07112202048301697, |
| "mean_token_accuracy": 0.9793166488409042, |
| "num_tokens": 1752168.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 2.2753446280956267, |
| "epoch": 1.2004008016032064, |
| "grad_norm": 2.212392568588257, |
| "learning_rate": 1.779208011943371e-05, |
| "loss": 0.06950807571411133, |
| "mean_token_accuracy": 0.9808650970458984, |
| "num_tokens": 1788732.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 2.242426198720932, |
| "epoch": 1.2271209084836339, |
| "grad_norm": 3.935898780822754, |
| "learning_rate": 1.7682614003191807e-05, |
| "loss": 0.07974758744239807, |
| "mean_token_accuracy": 0.9791070282459259, |
| "num_tokens": 1829031.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 2.361632689833641, |
| "epoch": 1.2538410153640616, |
| "grad_norm": 2.1741864681243896, |
| "learning_rate": 1.7570853798312462e-05, |
| "loss": 0.10215983390808106, |
| "mean_token_accuracy": 0.9750432461500168, |
| "num_tokens": 1862606.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 2.2773169964551925, |
| "epoch": 1.280561122244489, |
| "grad_norm": 2.8426294326782227, |
| "learning_rate": 1.7456832877267083e-05, |
| "loss": 0.07236079573631286, |
| "mean_token_accuracy": 0.9800545871257782, |
| "num_tokens": 1900331.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 2.3229851692914965, |
| "epoch": 1.3072812291249165, |
| "grad_norm": 0.7765806913375854, |
| "learning_rate": 1.7340585287594605e-05, |
| "loss": 0.07500131726264954, |
| "mean_token_accuracy": 0.9854851171374321, |
| "num_tokens": 1934641.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 2.265136310458183, |
| "epoch": 1.334001336005344, |
| "grad_norm": 2.6891231536865234, |
| "learning_rate": 1.7222145741734625e-05, |
| "loss": 0.07737380862236024, |
| "mean_token_accuracy": 0.9780729189515114, |
| "num_tokens": 1976773.0, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.334001336005344, |
| "eval_entropy": 2.00196760936466, |
| "eval_loss": 0.6567164063453674, |
| "eval_mean_token_accuracy": 0.862737625681619, |
| "eval_num_tokens": 1976773.0, |
| "eval_runtime": 1012.358, |
| "eval_samples_per_second": 7.558, |
| "eval_steps_per_second": 0.945, |
| "step": 500 |
| }, |
| { |
| "entropy": 2.2962641596794127, |
| "epoch": 1.3607214428857715, |
| "grad_norm": 1.7896814346313477, |
| "learning_rate": 1.7101549606662025e-05, |
| "loss": 0.03911572396755218, |
| "mean_token_accuracy": 0.9860339283943176, |
| "num_tokens": 2011231.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 2.337234216928482, |
| "epoch": 1.3874415497661992, |
| "grad_norm": 1.1328283548355103, |
| "learning_rate": 1.6978832893326074e-05, |
| "loss": 0.09991501569747925, |
| "mean_token_accuracy": 0.9799808233976364, |
| "num_tokens": 2048433.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 2.2558046162128447, |
| "epoch": 1.4141616566466266, |
| "grad_norm": 5.029273509979248, |
| "learning_rate": 1.685403224589731e-05, |
| "loss": 0.11410105228424072, |
| "mean_token_accuracy": 0.9652441591024399, |
| "num_tokens": 2089910.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 2.272854980826378, |
| "epoch": 1.440881763527054, |
| "grad_norm": 3.9721779823303223, |
| "learning_rate": 1.672718493082529e-05, |
| "loss": 0.07892256379127502, |
| "mean_token_accuracy": 0.9825230196118355, |
| "num_tokens": 2129164.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 2.314727634191513, |
| "epoch": 1.4676018704074816, |
| "grad_norm": 0.5437803864479065, |
| "learning_rate": 1.6598328825710536e-05, |
| "loss": 0.05945557355880737, |
| "mean_token_accuracy": 0.9851018249988556, |
| "num_tokens": 2164534.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 2.2285847306251525, |
| "epoch": 1.494321977287909, |
| "grad_norm": 3.5619757175445557, |
| "learning_rate": 1.6467502407993995e-05, |
| "loss": 0.06091769933700562, |
| "mean_token_accuracy": 0.9850202932953834, |
| "num_tokens": 2204576.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 2.270927533507347, |
| "epoch": 1.5210420841683367, |
| "grad_norm": 5.918709754943848, |
| "learning_rate": 1.6334744743467366e-05, |
| "loss": 0.04383589327335358, |
| "mean_token_accuracy": 0.9848530620336533, |
| "num_tokens": 2244844.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 2.206745645403862, |
| "epoch": 1.5477621910487642, |
| "grad_norm": 0.24053511023521423, |
| "learning_rate": 1.6200095474607753e-05, |
| "loss": 0.040359845757484435, |
| "mean_token_accuracy": 0.9904681667685509, |
| "num_tokens": 2288268.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 2.1809007704257963, |
| "epoch": 1.5744822979291917, |
| "grad_norm": 4.573062419891357, |
| "learning_rate": 1.6063594808740112e-05, |
| "loss": 0.10933890342712402, |
| "mean_token_accuracy": 0.9740328788757324, |
| "num_tokens": 2332694.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 2.243269944190979, |
| "epoch": 1.6012024048096194, |
| "grad_norm": 0.580310583114624, |
| "learning_rate": 1.592528350603103e-05, |
| "loss": 0.05813808441162109, |
| "mean_token_accuracy": 0.9884604424238205, |
| "num_tokens": 2375030.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 2.2944773554801943, |
| "epoch": 1.6279225116900466, |
| "grad_norm": 0.3347652554512024, |
| "learning_rate": 1.578520286731741e-05, |
| "loss": 0.06750264167785644, |
| "mean_token_accuracy": 0.9863917827606201, |
| "num_tokens": 2410934.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 2.2290642082691194, |
| "epoch": 1.6546426185704743, |
| "grad_norm": 6.804841995239258, |
| "learning_rate": 1.564339472177373e-05, |
| "loss": 0.08437965512275696, |
| "mean_token_accuracy": 0.9736600682139397, |
| "num_tokens": 2456175.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 2.1188001722097396, |
| "epoch": 1.6813627254509018, |
| "grad_norm": 1.9430474042892456, |
| "learning_rate": 1.549990141442153e-05, |
| "loss": 0.10908979177474976, |
| "mean_token_accuracy": 0.9731676578521729, |
| "num_tokens": 2499789.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 2.2526688307523726, |
| "epoch": 1.7080828323313293, |
| "grad_norm": 3.0555431842803955, |
| "learning_rate": 1.5354765793484834e-05, |
| "loss": 0.05227026343345642, |
| "mean_token_accuracy": 0.9853978455066681, |
| "num_tokens": 2538261.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 2.2599262297153473, |
| "epoch": 1.734802939211757, |
| "grad_norm": 2.0404698848724365, |
| "learning_rate": 1.5208031197595357e-05, |
| "loss": 0.07762071490287781, |
| "mean_token_accuracy": 0.9834817737340927, |
| "num_tokens": 2577538.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 2.2144850313663484, |
| "epoch": 1.7615230460921842, |
| "grad_norm": 1.7991482019424438, |
| "learning_rate": 1.505974144285124e-05, |
| "loss": 0.08743497729301453, |
| "mean_token_accuracy": 0.9796784415841102, |
| "num_tokens": 2615702.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 2.252047023177147, |
| "epoch": 1.788243152972612, |
| "grad_norm": 1.9210928678512573, |
| "learning_rate": 1.4909940809733223e-05, |
| "loss": 0.06988455057144165, |
| "mean_token_accuracy": 0.9823303431272506, |
| "num_tokens": 2656170.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 2.271262985467911, |
| "epoch": 1.8149632598530394, |
| "grad_norm": 5.849321365356445, |
| "learning_rate": 1.4758674029882152e-05, |
| "loss": 0.057480788230895995, |
| "mean_token_accuracy": 0.9779448106884956, |
| "num_tokens": 2695664.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 2.3097633570432663, |
| "epoch": 1.8416833667334669, |
| "grad_norm": 3.288543939590454, |
| "learning_rate": 1.4605986272741748e-05, |
| "loss": 0.06553007364273071, |
| "mean_token_accuracy": 0.9788680151104927, |
| "num_tokens": 2734705.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 2.2239883840084076, |
| "epoch": 1.8684034736138946, |
| "grad_norm": 1.9443882703781128, |
| "learning_rate": 1.445192313207067e-05, |
| "loss": 0.03168229162693024, |
| "mean_token_accuracy": 0.9910122290253639, |
| "num_tokens": 2780362.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 2.268227940797806, |
| "epoch": 1.8951235804943218, |
| "grad_norm": 2.3338937759399414, |
| "learning_rate": 1.4296530612327864e-05, |
| "loss": 0.08805855512619018, |
| "mean_token_accuracy": 0.9846275597810745, |
| "num_tokens": 2815363.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 2.2527557611465454, |
| "epoch": 1.9218436873747495, |
| "grad_norm": 2.4873032569885254, |
| "learning_rate": 1.4139855114935253e-05, |
| "loss": 0.07748764753341675, |
| "mean_token_accuracy": 0.9770905658602714, |
| "num_tokens": 2854168.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 2.2531791061162947, |
| "epoch": 1.948563794255177, |
| "grad_norm": 0.5768864154815674, |
| "learning_rate": 1.3981943424421932e-05, |
| "loss": 0.051403605937957765, |
| "mean_token_accuracy": 0.9766033187508583, |
| "num_tokens": 2893718.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 2.242309182882309, |
| "epoch": 1.9752839011356045, |
| "grad_norm": 0.6378281712532043, |
| "learning_rate": 1.3822842694453923e-05, |
| "loss": 0.052982181310653687, |
| "mean_token_accuracy": 0.9850667178630829, |
| "num_tokens": 2939147.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 2.2073887844343445, |
| "epoch": 2.0, |
| "grad_norm": 0.04015383869409561, |
| "learning_rate": 1.3662600433753746e-05, |
| "loss": 0.01695578694343567, |
| "mean_token_accuracy": 0.9941012440501032, |
| "num_tokens": 2980444.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 2.256787967681885, |
| "epoch": 2.0267201068804277, |
| "grad_norm": 1.1177358627319336, |
| "learning_rate": 1.3501264491913909e-05, |
| "loss": 0.01226435974240303, |
| "mean_token_accuracy": 0.9970220297574997, |
| "num_tokens": 3020734.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 2.252268907427788, |
| "epoch": 2.053440213760855, |
| "grad_norm": 0.0968155562877655, |
| "learning_rate": 1.3338883045108674e-05, |
| "loss": 0.04032035768032074, |
| "mean_token_accuracy": 0.9906749814748764, |
| "num_tokens": 3062528.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 2.223544543981552, |
| "epoch": 2.0801603206412826, |
| "grad_norm": 0.3498780429363251, |
| "learning_rate": 1.3175504581708261e-05, |
| "loss": 0.034950819611549375, |
| "mean_token_accuracy": 0.9915833666920661, |
| "num_tokens": 3100355.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 2.3466701090335844, |
| "epoch": 2.10688042752171, |
| "grad_norm": 0.216510608792305, |
| "learning_rate": 1.3011177887799846e-05, |
| "loss": 0.02903362512588501, |
| "mean_token_accuracy": 0.9919949501752854, |
| "num_tokens": 3135185.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 2.290475943684578, |
| "epoch": 2.1336005344021376, |
| "grad_norm": 3.714592456817627, |
| "learning_rate": 1.2845952032619651e-05, |
| "loss": 0.023531119525432586, |
| "mean_token_accuracy": 0.9948310509324074, |
| "num_tokens": 3173323.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 2.223797783255577, |
| "epoch": 2.1603206412825653, |
| "grad_norm": 0.36913734674453735, |
| "learning_rate": 1.2679876353900482e-05, |
| "loss": 0.012396839261054993, |
| "mean_token_accuracy": 0.994361761212349, |
| "num_tokens": 3215580.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 2.21057957559824, |
| "epoch": 2.1870407481629925, |
| "grad_norm": 1.5732353925704956, |
| "learning_rate": 1.2513000443139112e-05, |
| "loss": 0.036057019233703615, |
| "mean_token_accuracy": 0.9922081708908081, |
| "num_tokens": 3257736.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 2.236904078722, |
| "epoch": 2.2137608550434202, |
| "grad_norm": 4.868381977081299, |
| "learning_rate": 1.2345374130787855e-05, |
| "loss": 0.034664037823677066, |
| "mean_token_accuracy": 0.9887170046567917, |
| "num_tokens": 3298447.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 2.1934050023555756, |
| "epoch": 2.2404809619238475, |
| "grad_norm": 2.6058948040008545, |
| "learning_rate": 1.2177047471374808e-05, |
| "loss": 0.025552961230278014, |
| "mean_token_accuracy": 0.9930191978812217, |
| "num_tokens": 3342465.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 2.277759903669357, |
| "epoch": 2.267201068804275, |
| "grad_norm": 2.857268810272217, |
| "learning_rate": 1.2008070728557186e-05, |
| "loss": 0.031257528066635135, |
| "mean_token_accuracy": 0.992735068500042, |
| "num_tokens": 3387462.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 2.2034470826387405, |
| "epoch": 2.293921175684703, |
| "grad_norm": 0.26473426818847656, |
| "learning_rate": 1.1838494360112185e-05, |
| "loss": 0.02097797840833664, |
| "mean_token_accuracy": 0.9918780580163002, |
| "num_tokens": 3426953.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 2.224232539534569, |
| "epoch": 2.32064128256513, |
| "grad_norm": 1.356302261352539, |
| "learning_rate": 1.1668369002869912e-05, |
| "loss": 0.031401839852333066, |
| "mean_token_accuracy": 0.9943979844450951, |
| "num_tokens": 3466533.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 2.2016593903303145, |
| "epoch": 2.347361389445558, |
| "grad_norm": 3.808924913406372, |
| "learning_rate": 1.1497745457592817e-05, |
| "loss": 0.040309226512908934, |
| "mean_token_accuracy": 0.9896655455231667, |
| "num_tokens": 3509167.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 2.2696240931749343, |
| "epoch": 2.374081496325985, |
| "grad_norm": 0.8555730581283569, |
| "learning_rate": 1.1326674673806195e-05, |
| "loss": 0.013172776997089386, |
| "mean_token_accuracy": 0.9958177715539932, |
| "num_tokens": 3548882.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 2.2943250626325606, |
| "epoch": 2.400801603206413, |
| "grad_norm": 0.32936611771583557, |
| "learning_rate": 1.1155207734584264e-05, |
| "loss": 0.03971285223960876, |
| "mean_token_accuracy": 0.9879287019371986, |
| "num_tokens": 3585237.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 2.2017696261405946, |
| "epoch": 2.4275217100868405, |
| "grad_norm": 3.144895553588867, |
| "learning_rate": 1.0983395841296349e-05, |
| "loss": 0.017077907919883728, |
| "mean_token_accuracy": 0.9940563708543777, |
| "num_tokens": 3625995.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 2.2841670215129852, |
| "epoch": 2.4542418169672677, |
| "grad_norm": 0.028157195076346397, |
| "learning_rate": 1.0811290298317755e-05, |
| "loss": 0.02282451242208481, |
| "mean_token_accuracy": 0.994524571299553, |
| "num_tokens": 3664403.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 2.2331026285886764, |
| "epoch": 2.4809619238476954, |
| "grad_norm": 1.9096051454544067, |
| "learning_rate": 1.063894249770989e-05, |
| "loss": 0.03884054720401764, |
| "mean_token_accuracy": 0.9940769791603088, |
| "num_tokens": 3707667.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 2.2877497136592866, |
| "epoch": 2.507682030728123, |
| "grad_norm": 0.09394676983356476, |
| "learning_rate": 1.0466403903874176e-05, |
| "loss": 0.030750763416290284, |
| "mean_token_accuracy": 0.9935285225510597, |
| "num_tokens": 3743958.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 2.2662788361310957, |
| "epoch": 2.5344021376085504, |
| "grad_norm": 0.06504588574171066, |
| "learning_rate": 1.0293726038184393e-05, |
| "loss": 0.005493773892521858, |
| "mean_token_accuracy": 0.9980263158679008, |
| "num_tokens": 3781085.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 2.238060674071312, |
| "epoch": 2.561122244488978, |
| "grad_norm": 3.588862895965576, |
| "learning_rate": 1.0120960463601977e-05, |
| "loss": 0.02108605355024338, |
| "mean_token_accuracy": 0.9930532336235046, |
| "num_tokens": 3821494.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 2.226763039827347, |
| "epoch": 2.5878423513694053, |
| "grad_norm": 0.40724924206733704, |
| "learning_rate": 9.948158769278939e-06, |
| "loss": 0.038677141070365906, |
| "mean_token_accuracy": 0.9927912071347237, |
| "num_tokens": 3861528.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 2.2995820313692095, |
| "epoch": 2.614562458249833, |
| "grad_norm": 0.012613887898623943, |
| "learning_rate": 9.775372555152912e-06, |
| "loss": 0.007771652191877365, |
| "mean_token_accuracy": 0.9978968247771263, |
| "num_tokens": 3897755.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 2.2521371841430664, |
| "epoch": 2.6412825651302603, |
| "grad_norm": 0.5051451921463013, |
| "learning_rate": 9.602653416539031e-06, |
| "loss": 0.01716648042201996, |
| "mean_token_accuracy": 0.9950040057301521, |
| "num_tokens": 3936373.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 2.3034828037023543, |
| "epoch": 2.668002672010688, |
| "grad_norm": 0.0912921279668808, |
| "learning_rate": 9.430052928723153e-06, |
| "loss": 0.010478074848651885, |
| "mean_token_accuracy": 0.9974567100405693, |
| "num_tokens": 3971320.0, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.668002672010688, |
| "eval_entropy": 1.9351995090954979, |
| "eval_loss": 0.6903934478759766, |
| "eval_mean_token_accuracy": 0.8785945819836799, |
| "eval_num_tokens": 3971320.0, |
| "eval_runtime": 1012.712, |
| "eval_samples_per_second": 7.555, |
| "eval_steps_per_second": 0.945, |
| "step": 1000 |
| }, |
| { |
| "entropy": 2.1870287477970125, |
| "epoch": 2.6947227788911157, |
| "grad_norm": 0.04750761389732361, |
| "learning_rate": 9.257622631561085e-06, |
| "loss": 0.06720720529556275, |
| "mean_token_accuracy": 0.9875113531947136, |
| "num_tokens": 4012699.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 2.2189815163612367, |
| "epoch": 2.721442885771543, |
| "grad_norm": 1.9374785423278809, |
| "learning_rate": 9.085414014088368e-06, |
| "loss": 0.018031400442123414, |
| "mean_token_accuracy": 0.99626876860857, |
| "num_tokens": 4052085.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 2.2915788531303405, |
| "epoch": 2.7481629926519706, |
| "grad_norm": 4.376842021942139, |
| "learning_rate": 8.913478499145255e-06, |
| "loss": 0.04538655877113342, |
| "mean_token_accuracy": 0.9898257419466973, |
| "num_tokens": 4086010.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 2.1904806703329087, |
| "epoch": 2.7748830995323983, |
| "grad_norm": 0.10383885353803635, |
| "learning_rate": 8.741867428021447e-06, |
| "loss": 0.028022685647010805, |
| "mean_token_accuracy": 0.9921670004725456, |
| "num_tokens": 4130141.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 2.2670040905475615, |
| "epoch": 2.8016032064128256, |
| "grad_norm": 0.022271202877163887, |
| "learning_rate": 8.570632045125185e-06, |
| "loss": 0.015449178218841553, |
| "mean_token_accuracy": 0.9905866920948029, |
| "num_tokens": 4172516.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 2.25451982319355, |
| "epoch": 2.8283233132932533, |
| "grad_norm": 0.24705791473388672, |
| "learning_rate": 8.399823482681263e-06, |
| "loss": 0.040058845281600954, |
| "mean_token_accuracy": 0.9895619317889214, |
| "num_tokens": 4220191.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 2.2900613993406296, |
| "epoch": 2.855043420173681, |
| "grad_norm": 0.8130829334259033, |
| "learning_rate": 8.229492745462551e-06, |
| "loss": 0.025635138154029846, |
| "mean_token_accuracy": 0.9944342941045761, |
| "num_tokens": 4253931.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 2.1844753593206407, |
| "epoch": 2.881763527054108, |
| "grad_norm": 0.3202403485774994, |
| "learning_rate": 8.05969069555957e-06, |
| "loss": 0.022118450701236726, |
| "mean_token_accuracy": 0.9955674543976784, |
| "num_tokens": 4290879.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 2.2233424872159957, |
| "epoch": 2.9084836339345355, |
| "grad_norm": 2.2554895877838135, |
| "learning_rate": 7.89046803719267e-06, |
| "loss": 0.024461272358894347, |
| "mean_token_accuracy": 0.9931944444775581, |
| "num_tokens": 4332733.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 2.2441813826560972, |
| "epoch": 2.935203740814963, |
| "grad_norm": 0.35499951243400574, |
| "learning_rate": 7.721875301571359e-06, |
| "loss": 0.023122313618659972, |
| "mean_token_accuracy": 0.9941071420907974, |
| "num_tokens": 4369165.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 2.29788116812706, |
| "epoch": 2.961923847695391, |
| "grad_norm": 3.8682150840759277, |
| "learning_rate": 7.553962831805291e-06, |
| "loss": 0.013222594559192658, |
| "mean_token_accuracy": 0.9982142850756646, |
| "num_tokens": 4412713.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 2.22674503326416, |
| "epoch": 2.988643954575818, |
| "grad_norm": 2.1351380348205566, |
| "learning_rate": 7.3867807678713965e-06, |
| "loss": 0.015868033468723296, |
| "mean_token_accuracy": 0.995640316605568, |
| "num_tokens": 4454047.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 2.24416767584311, |
| "epoch": 3.0133600534402136, |
| "grad_norm": 0.16914494335651398, |
| "learning_rate": 7.22037903164173e-06, |
| "loss": 0.009346211701631546, |
| "mean_token_accuracy": 0.9974918107728701, |
| "num_tokens": 4492337.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 2.2533529102802277, |
| "epoch": 3.0400801603206413, |
| "grad_norm": 0.06259158253669739, |
| "learning_rate": 7.05480731197638e-06, |
| "loss": 0.0021233540028333664, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 4529479.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 2.2435650140047074, |
| "epoch": 3.066800267201069, |
| "grad_norm": 1.3215523958206177, |
| "learning_rate": 6.890115049885995e-06, |
| "loss": 0.013927657902240754, |
| "mean_token_accuracy": 0.9983769372105599, |
| "num_tokens": 4572178.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 2.3701954245567323, |
| "epoch": 3.0935203740814963, |
| "grad_norm": 0.62388014793396, |
| "learning_rate": 6.726351423768323e-06, |
| "loss": 0.011201680451631547, |
| "mean_token_accuracy": 0.9981441885232926, |
| "num_tokens": 4609157.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 2.2140144526958467, |
| "epoch": 3.120240480961924, |
| "grad_norm": 0.06543799489736557, |
| "learning_rate": 6.563565334723134e-06, |
| "loss": 0.005485345050692559, |
| "mean_token_accuracy": 0.9982142850756646, |
| "num_tokens": 4650290.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 2.251464119553566, |
| "epoch": 3.146960587842351, |
| "grad_norm": 2.703436851501465, |
| "learning_rate": 6.40180539194999e-06, |
| "loss": 0.003857447579503059, |
| "mean_token_accuracy": 0.997916667163372, |
| "num_tokens": 4687330.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 2.2594422668218614, |
| "epoch": 3.173680694722779, |
| "grad_norm": 0.06470175832509995, |
| "learning_rate": 6.2411198982331435e-06, |
| "loss": 0.009718221426010133, |
| "mean_token_accuracy": 0.9979184180498123, |
| "num_tokens": 4730354.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 2.2663645327091215, |
| "epoch": 3.2004008016032066, |
| "grad_norm": 0.0670115128159523, |
| "learning_rate": 6.081556835517955e-06, |
| "loss": 0.005593789368867874, |
| "mean_token_accuracy": 0.9973069116473198, |
| "num_tokens": 4772534.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 2.2799044013023377, |
| "epoch": 3.227120908483634, |
| "grad_norm": 0.13721637427806854, |
| "learning_rate": 5.923163850583114e-06, |
| "loss": 0.012480302900075912, |
| "mean_token_accuracy": 0.9976782888174057, |
| "num_tokens": 4809581.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 2.1988295197486876, |
| "epoch": 3.2538410153640616, |
| "grad_norm": 0.20896881818771362, |
| "learning_rate": 5.7659882408129204e-06, |
| "loss": 0.004379065707325935, |
| "mean_token_accuracy": 0.9994897961616516, |
| "num_tokens": 4849943.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 2.211800253391266, |
| "epoch": 3.280561122244489, |
| "grad_norm": 0.08681628108024597, |
| "learning_rate": 5.610076940073939e-06, |
| "loss": 0.0038055859506130217, |
| "mean_token_accuracy": 0.9961352661252022, |
| "num_tokens": 4890785.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 2.2064353942871096, |
| "epoch": 3.3072812291249165, |
| "grad_norm": 0.1481466442346573, |
| "learning_rate": 5.455476504700161e-06, |
| "loss": 0.004182736203074455, |
| "mean_token_accuracy": 0.9988095238804817, |
| "num_tokens": 4931070.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 2.2415756046772004, |
| "epoch": 3.334001336005344, |
| "grad_norm": 0.02015281654894352, |
| "learning_rate": 5.302233099590928e-06, |
| "loss": 0.002793450653553009, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 4966739.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 2.2732053816318514, |
| "epoch": 3.3607214428857715, |
| "grad_norm": 0.09282029420137405, |
| "learning_rate": 5.150392484425728e-06, |
| "loss": 0.008500372618436813, |
| "mean_token_accuracy": 0.9989999994635582, |
| "num_tokens": 5003874.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 2.229057991504669, |
| "epoch": 3.387441549766199, |
| "grad_norm": 3.0661253929138184, |
| "learning_rate": 5.000000000000003e-06, |
| "loss": 0.007586031407117844, |
| "mean_token_accuracy": 0.9971370965242385, |
| "num_tokens": 5046143.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 2.2301405310630797, |
| "epoch": 3.4141616566466264, |
| "grad_norm": 0.07116200774908066, |
| "learning_rate": 4.8511005546860214e-06, |
| "loss": 0.0049221344292163845, |
| "mean_token_accuracy": 0.9983333334326744, |
| "num_tokens": 5091038.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 2.231079003214836, |
| "epoch": 3.440881763527054, |
| "grad_norm": 0.05422259122133255, |
| "learning_rate": 4.703738611022899e-06, |
| "loss": 0.002114175260066986, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5130246.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 2.267631542682648, |
| "epoch": 3.467601870407482, |
| "grad_norm": 3.853044033050537, |
| "learning_rate": 4.557958172439726e-06, |
| "loss": 0.019249360263347625, |
| "mean_token_accuracy": 0.9916100561618805, |
| "num_tokens": 5172413.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 2.294846275448799, |
| "epoch": 3.494321977287909, |
| "grad_norm": 0.19263218343257904, |
| "learning_rate": 4.413802770115816e-06, |
| "loss": 0.01337273120880127, |
| "mean_token_accuracy": 0.9973717942833901, |
| "num_tokens": 5209147.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 2.2339998126029967, |
| "epoch": 3.5210420841683367, |
| "grad_norm": 0.0738057941198349, |
| "learning_rate": 4.2713154499819345e-06, |
| "loss": 0.010604117810726166, |
| "mean_token_accuracy": 0.9974877446889877, |
| "num_tokens": 5250516.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 2.2215969979763033, |
| "epoch": 3.547762191048764, |
| "grad_norm": 0.009852610528469086, |
| "learning_rate": 4.130538759866457e-06, |
| "loss": 0.030475294589996337, |
| "mean_token_accuracy": 0.9948631942272186, |
| "num_tokens": 5285798.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 2.2144288033246995, |
| "epoch": 3.5744822979291917, |
| "grad_norm": 0.03805363550782204, |
| "learning_rate": 3.991514736790259e-06, |
| "loss": 0.018721377849578856, |
| "mean_token_accuracy": 0.9968991339206695, |
| "num_tokens": 5322526.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 2.1923436015844344, |
| "epoch": 3.6012024048096194, |
| "grad_norm": 0.0317704901099205, |
| "learning_rate": 3.854284894414122e-06, |
| "loss": 0.002373117581009865, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5364366.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 2.219408541917801, |
| "epoch": 3.6279225116900466, |
| "grad_norm": 0.0647914856672287, |
| "learning_rate": 3.718890210642442e-06, |
| "loss": 0.009029030799865723, |
| "mean_token_accuracy": 0.9971516281366348, |
| "num_tokens": 5404587.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 2.1901919513940813, |
| "epoch": 3.6546426185704743, |
| "grad_norm": 0.0483131930232048, |
| "learning_rate": 3.5853711153868962e-06, |
| "loss": 0.008125517517328262, |
| "mean_token_accuracy": 0.9958531737327576, |
| "num_tokens": 5447924.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 2.2181854397058487, |
| "epoch": 3.681362725450902, |
| "grad_norm": 0.05130556598305702, |
| "learning_rate": 3.453767478493761e-06, |
| "loss": 0.001593584753572941, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5485390.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 2.246033489704132, |
| "epoch": 3.7080828323313293, |
| "grad_norm": 0.11879993230104446, |
| "learning_rate": 3.3241185978384636e-06, |
| "loss": 0.0033624660223722457, |
| "mean_token_accuracy": 0.9987499997019768, |
| "num_tokens": 5521206.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 2.2084401190280913, |
| "epoch": 3.734802939211757, |
| "grad_norm": 4.081003189086914, |
| "learning_rate": 3.196463187590929e-06, |
| "loss": 0.023326359689235687, |
| "mean_token_accuracy": 0.9964135706424713, |
| "num_tokens": 5560114.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 2.294724687933922, |
| "epoch": 3.7615230460921842, |
| "grad_norm": 0.3147370219230652, |
| "learning_rate": 3.070839366655215e-06, |
| "loss": 0.002202053926885128, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5594655.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 2.197516432404518, |
| "epoch": 3.788243152972612, |
| "grad_norm": 0.02722419984638691, |
| "learning_rate": 2.94728464728693e-06, |
| "loss": 0.012243854254484177, |
| "mean_token_accuracy": 0.9971879601478577, |
| "num_tokens": 5636183.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 2.225996693968773, |
| "epoch": 3.814963259853039, |
| "grad_norm": 0.11651862412691116, |
| "learning_rate": 2.8258359238917665e-06, |
| "loss": 0.0027832575142383575, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5672190.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 2.254541629552841, |
| "epoch": 3.841683366733467, |
| "grad_norm": 0.6706406474113464, |
| "learning_rate": 2.7065294620085425e-06, |
| "loss": 0.01011493280529976, |
| "mean_token_accuracy": 0.9963328331708908, |
| "num_tokens": 5709131.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 2.1990156888961794, |
| "epoch": 3.8684034736138946, |
| "grad_norm": 1.7040566205978394, |
| "learning_rate": 2.5894008874800323e-06, |
| "loss": 0.002497620694339275, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5753210.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 2.1685468345880508, |
| "epoch": 3.895123580494322, |
| "grad_norm": 0.4803442656993866, |
| "learning_rate": 2.474485175814816e-06, |
| "loss": 0.0020332710817456247, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5797540.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 2.2378750056028367, |
| "epoch": 3.9218436873747495, |
| "grad_norm": 0.9694292545318604, |
| "learning_rate": 2.361816641743303e-06, |
| "loss": 0.008235185593366622, |
| "mean_token_accuracy": 0.9973824784159661, |
| "num_tokens": 5834232.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 2.2470070898532866, |
| "epoch": 3.9485637942551772, |
| "grad_norm": 0.26679056882858276, |
| "learning_rate": 2.251428928971102e-06, |
| "loss": 0.027912315726280213, |
| "mean_token_accuracy": 0.9929819032549858, |
| "num_tokens": 5879474.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 2.1723068922758104, |
| "epoch": 3.9752839011356045, |
| "grad_norm": 0.12753742933273315, |
| "learning_rate": 2.1433550001327376e-06, |
| "loss": 0.023546719551086427, |
| "mean_token_accuracy": 0.99633309841156, |
| "num_tokens": 5921234.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 2.2046521161053634, |
| "epoch": 4.0, |
| "grad_norm": 0.03592666611075401, |
| "learning_rate": 2.037627126948751e-06, |
| "loss": 0.0020991336554288865, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 5960888.0, |
| "step": 1500 |
| }, |
| { |
| "epoch": 4.0, |
| "eval_entropy": 1.9289183919332618, |
| "eval_loss": 0.7388398051261902, |
| "eval_mean_token_accuracy": 0.879620148532692, |
| "eval_num_tokens": 5960888.0, |
| "eval_runtime": 1013.1291, |
| "eval_samples_per_second": 7.552, |
| "eval_steps_per_second": 0.945, |
| "step": 1500 |
| }, |
| { |
| "entropy": 2.2434328526258467, |
| "epoch": 4.026720106880427, |
| "grad_norm": 1.233505129814148, |
| "learning_rate": 1.9342768805891176e-06, |
| "loss": 0.014271475374698639, |
| "mean_token_accuracy": 0.997916667163372, |
| "num_tokens": 5998429.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 2.2602454483509065, |
| "epoch": 4.053440213760855, |
| "grad_norm": 0.03542209416627884, |
| "learning_rate": 1.8333351222458407e-06, |
| "loss": 0.0032291453331708907, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6038061.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 2.292672246694565, |
| "epoch": 4.080160320641283, |
| "grad_norm": 0.061248622834682465, |
| "learning_rate": 1.734831993917564e-06, |
| "loss": 0.006363029778003693, |
| "mean_token_accuracy": 0.9970833331346511, |
| "num_tokens": 6075581.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 2.281049346923828, |
| "epoch": 4.10688042752171, |
| "grad_norm": 0.05142101272940636, |
| "learning_rate": 1.6387969094089318e-06, |
| "loss": 0.0009409012272953987, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6112533.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 2.265934219956398, |
| "epoch": 4.133600534402138, |
| "grad_norm": 0.06876987218856812, |
| "learning_rate": 1.545258545547398e-06, |
| "loss": 0.0035512372851371765, |
| "mean_token_accuracy": 0.9991666659712791, |
| "num_tokens": 6148848.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 2.311371296644211, |
| "epoch": 4.160320641282565, |
| "grad_norm": 0.15699031949043274, |
| "learning_rate": 1.4542448336201021e-06, |
| "loss": 0.0016276374459266662, |
| "mean_token_accuracy": 0.9985294118523598, |
| "num_tokens": 6183351.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 2.279196488857269, |
| "epoch": 4.1870407481629925, |
| "grad_norm": 0.050407592207193375, |
| "learning_rate": 1.3657829510333653e-06, |
| "loss": 0.012745055556297302, |
| "mean_token_accuracy": 0.9976190477609634, |
| "num_tokens": 6219690.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 2.2390474647283556, |
| "epoch": 4.21376085504342, |
| "grad_norm": 0.19574671983718872, |
| "learning_rate": 1.2798993131973093e-06, |
| "loss": 0.007319384068250656, |
| "mean_token_accuracy": 0.9986842110753059, |
| "num_tokens": 6259058.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 2.2185899525880814, |
| "epoch": 4.240480961923848, |
| "grad_norm": 0.07992465049028397, |
| "learning_rate": 1.196619565638003e-06, |
| "loss": 0.0011901114135980605, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6298314.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 2.2456587731838225, |
| "epoch": 4.267201068804275, |
| "grad_norm": 0.03778492286801338, |
| "learning_rate": 1.1159685763395113e-06, |
| "loss": 0.0031911343336105346, |
| "mean_token_accuracy": 0.9985294118523598, |
| "num_tokens": 6337419.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 2.2879197150468826, |
| "epoch": 4.293921175684702, |
| "grad_norm": 0.09683331102132797, |
| "learning_rate": 1.037970428318118e-06, |
| "loss": 0.007102232426404953, |
| "mean_token_accuracy": 0.9976842105388641, |
| "num_tokens": 6376691.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 2.2439812660217284, |
| "epoch": 4.320641282565131, |
| "grad_norm": 0.15115514397621155, |
| "learning_rate": 9.62648412430951e-07, |
| "loss": 0.0011501027271151542, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6417508.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 2.25390011370182, |
| "epoch": 4.347361389445558, |
| "grad_norm": 0.07224220037460327, |
| "learning_rate": 8.900250204211513e-07, |
| "loss": 0.001615801639854908, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6456873.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 2.1351063311100007, |
| "epoch": 4.374081496325985, |
| "grad_norm": 0.0710083618760109, |
| "learning_rate": 8.201219382016556e-07, |
| "loss": 0.001980478689074516, |
| "mean_token_accuracy": 0.9989999994635582, |
| "num_tokens": 6502251.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 2.2338476330041885, |
| "epoch": 4.400801603206413, |
| "grad_norm": 0.07196642458438873, |
| "learning_rate": 7.529600393796232e-07, |
| "loss": 0.0016765512526035308, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6544249.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 2.2703020840883257, |
| "epoch": 4.4275217100868405, |
| "grad_norm": 0.033600643277168274, |
| "learning_rate": 6.885593790234057e-07, |
| "loss": 0.0008226408623158932, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6582388.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 2.2531224131584167, |
| "epoch": 4.454241816967268, |
| "grad_norm": 0.024937018752098083, |
| "learning_rate": 6.269391876739494e-07, |
| "loss": 0.004456085711717605, |
| "mean_token_accuracy": 0.9991935476660728, |
| "num_tokens": 6621389.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 2.1802454233169555, |
| "epoch": 4.480961923847695, |
| "grad_norm": 0.283072829246521, |
| "learning_rate": 5.681178656024055e-07, |
| "loss": 0.004049577564001083, |
| "mean_token_accuracy": 0.9990384608507157, |
| "num_tokens": 6662855.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 2.2954131811857224, |
| "epoch": 4.507682030728123, |
| "grad_norm": 0.03239845857024193, |
| "learning_rate": 5.121129773156663e-07, |
| "loss": 0.0011025561019778253, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6701701.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 2.172041815519333, |
| "epoch": 4.53440213760855, |
| "grad_norm": 0.3291212022304535, |
| "learning_rate": 4.58941246311464e-07, |
| "loss": 0.007027396559715271, |
| "mean_token_accuracy": 0.9991071432828903, |
| "num_tokens": 6742806.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 2.19354427754879, |
| "epoch": 4.561122244488978, |
| "grad_norm": 2.852701425552368, |
| "learning_rate": 4.0861855008460403e-07, |
| "loss": 0.008446280658245087, |
| "mean_token_accuracy": 0.9975268810987472, |
| "num_tokens": 6781034.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 2.212263357639313, |
| "epoch": 4.587842351369406, |
| "grad_norm": 0.18127447366714478, |
| "learning_rate": 3.611599153858214e-07, |
| "loss": 0.00584067553281784, |
| "mean_token_accuracy": 0.9991071432828903, |
| "num_tokens": 6824543.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 2.2566528230905534, |
| "epoch": 4.614562458249833, |
| "grad_norm": 0.04101433604955673, |
| "learning_rate": 3.16579513734675e-07, |
| "loss": 0.0048684652894735335, |
| "mean_token_accuracy": 0.9988095238804817, |
| "num_tokens": 6861456.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 2.1923379600048065, |
| "epoch": 4.64128256513026, |
| "grad_norm": 0.05011329799890518, |
| "learning_rate": 2.748906571878207e-07, |
| "loss": 0.0010127362795174122, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6899773.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 2.261983773112297, |
| "epoch": 4.668002672010688, |
| "grad_norm": 1.1403498649597168, |
| "learning_rate": 2.3610579436392999e-07, |
| "loss": 0.00359833762049675, |
| "mean_token_accuracy": 0.9993243247270585, |
| "num_tokens": 6940165.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 2.2240211933851244, |
| "epoch": 4.694722778891116, |
| "grad_norm": 0.13198979198932648, |
| "learning_rate": 2.002365067264289e-07, |
| "loss": 0.0013021196238696576, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 6985541.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 2.2223060548305513, |
| "epoch": 4.721442885771543, |
| "grad_norm": 2.598280191421509, |
| "learning_rate": 1.6729350512519006e-07, |
| "loss": 0.003232601657509804, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 7029913.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 2.2316210359334945, |
| "epoch": 4.74816299265197, |
| "grad_norm": 0.04392010718584061, |
| "learning_rate": 1.3728662659818205e-07, |
| "loss": 0.0056718744337558745, |
| "mean_token_accuracy": 0.9986752718687057, |
| "num_tokens": 7068455.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 2.2579612761735914, |
| "epoch": 4.774883099532398, |
| "grad_norm": 0.179452046751976, |
| "learning_rate": 1.1022483143405705e-07, |
| "loss": 0.0074336245656013485, |
| "mean_token_accuracy": 0.9977964743971824, |
| "num_tokens": 7105288.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 2.187806871533394, |
| "epoch": 4.801603206412826, |
| "grad_norm": 0.05380546301603317, |
| "learning_rate": 8.61162004965388e-08, |
| "loss": 0.0035678640007972716, |
| "mean_token_accuracy": 0.9990384608507157, |
| "num_tokens": 7145256.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 2.176051089167595, |
| "epoch": 4.828323313293253, |
| "grad_norm": 0.12657630443572998, |
| "learning_rate": 6.496793281141056e-08, |
| "loss": 0.0022948944941163065, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 7188952.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 2.2141434013843537, |
| "epoch": 4.855043420173681, |
| "grad_norm": 0.086207315325737, |
| "learning_rate": 4.678634341683252e-08, |
| "loss": 0.00184146948158741, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 7227402.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 2.234368768334389, |
| "epoch": 4.881763527054108, |
| "grad_norm": 0.24360403418540955, |
| "learning_rate": 3.157686147762129e-08, |
| "loss": 0.0014556343667209148, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 7267406.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 2.214269518852234, |
| "epoch": 4.9084836339345355, |
| "grad_norm": 0.313006192445755, |
| "learning_rate": 1.9344028664056715e-08, |
| "loss": 0.0012312169186770917, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 7309765.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 2.1800210982561112, |
| "epoch": 4.935203740814964, |
| "grad_norm": 0.0846979096531868, |
| "learning_rate": 1.0091497795706728e-08, |
| "loss": 0.003886619582772255, |
| "mean_token_accuracy": 0.999479167163372, |
| "num_tokens": 7355951.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 2.2378896445035936, |
| "epoch": 4.961923847695391, |
| "grad_norm": 0.021038636565208435, |
| "learning_rate": 3.8220317506654226e-09, |
| "loss": 0.0026206234470009804, |
| "mean_token_accuracy": 1.0, |
| "num_tokens": 7393991.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 2.243942254781723, |
| "epoch": 4.988643954575818, |
| "grad_norm": 0.014777406118810177, |
| "learning_rate": 5.375026405352035e-10, |
| "loss": 0.002264722436666489, |
| "mean_token_accuracy": 0.9993243247270585, |
| "num_tokens": 7434217.0, |
| "step": 1870 |
| }, |
| { |
| "epoch": 5.0, |
| "eval_entropy": 1.9315332296375447, |
| "eval_loss": 0.7543728947639465, |
| "eval_mean_token_accuracy": 0.8798603028090147, |
| "eval_num_tokens": 7451110.0, |
| "eval_runtime": 1013.2608, |
| "eval_samples_per_second": 7.551, |
| "eval_steps_per_second": 0.944, |
| "step": 1875 |
| }, |
| { |
| "epoch": 5.0, |
| "step": 1875, |
| "total_flos": 4.404638924743803e+17, |
| "train_loss": 0.11201101158509652, |
| "train_runtime": 7348.4295, |
| "train_samples_per_second": 2.036, |
| "train_steps_per_second": 0.255 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1875, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 5, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 4.404638924743803e+17, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
