{ "best_global_step": 700, "best_metric": 0.8686603307723999, "best_model_checkpoint": "/data6/jongbinwon/evolving_MoE/checkpoints/sft_llama31_8b_sni_ours-c_13393_ffn_only_5ep/checkpoint-700", "epoch": 5.0, "eval_steps": 100, "global_step": 2000, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "entropy": 1.723603368550539, "epoch": 0.025023459493274947, "grad_norm": 8.617183685302734, "learning_rate": 3e-06, "loss": 1.8632482528686523, "mean_token_accuracy": 0.45814704075455664, "num_tokens": 26675.0, "step": 10 }, { "entropy": 1.6786307826638223, "epoch": 0.05004691898654989, "grad_norm": 5.2456488609313965, "learning_rate": 6.333333333333333e-06, "loss": 1.7982595443725586, "mean_token_accuracy": 0.48966174516826866, "num_tokens": 51872.0, "step": 20 }, { "entropy": 1.7768761590123177, "epoch": 0.07507037847982484, "grad_norm": 4.166147708892822, "learning_rate": 9.666666666666667e-06, "loss": 2.042780876159668, "mean_token_accuracy": 0.5317133395001292, "num_tokens": 85745.0, "step": 30 }, { "entropy": 1.9048189118504524, "epoch": 0.10009383797309979, "grad_norm": 4.267604351043701, "learning_rate": 1.3000000000000001e-05, "loss": 1.585220718383789, "mean_token_accuracy": 0.6397202219814062, "num_tokens": 118442.0, "step": 40 }, { "entropy": 2.0537888914346696, "epoch": 0.12511729746637473, "grad_norm": 9.502376556396484, "learning_rate": 1.6333333333333335e-05, "loss": 1.1469281196594239, "mean_token_accuracy": 0.6897181114181876, "num_tokens": 147201.0, "step": 50 }, { "entropy": 2.2640592947602274, "epoch": 0.15014075695964968, "grad_norm": 2.122598171234131, "learning_rate": 1.9666666666666666e-05, "loss": 1.0125106811523437, "mean_token_accuracy": 0.7641670696437359, "num_tokens": 170569.0, "step": 60 }, { "entropy": 2.1425670742988587, "epoch": 0.17516421645292463, "grad_norm": 1.710357666015625, "learning_rate": 1.999893795328188e-05, "loss": 1.1022185325622558, "mean_token_accuracy": 0.7504967793822288, "num_tokens": 198185.0, "step": 70 }, { "entropy": 2.247272165119648, "epoch": 0.20018767594619957, "grad_norm": 2.6029555797576904, "learning_rate": 1.9995266970296856e-05, "loss": 1.1327926635742187, "mean_token_accuracy": 0.7273309765383601, "num_tokens": 221480.0, "step": 80 }, { "entropy": 2.2861083179712294, "epoch": 0.22521113543947452, "grad_norm": 2.214897632598877, "learning_rate": 1.9988974901779482e-05, "loss": 1.0936859130859375, "mean_token_accuracy": 0.744602588750422, "num_tokens": 244772.0, "step": 90 }, { "entropy": 2.250556546449661, "epoch": 0.25023459493274947, "grad_norm": 2.8186545372009277, "learning_rate": 1.9980063397715685e-05, "loss": 0.89981050491333, "mean_token_accuracy": 0.714475267007947, "num_tokens": 273805.0, "step": 100 }, { "epoch": 0.25023459493274947, "eval_entropy": 2.2342376695014536, "eval_loss": 0.9610199332237244, "eval_mean_token_accuracy": 0.7704979736590758, "eval_num_tokens": 273805.0, "eval_runtime": 25.919, "eval_samples_per_second": 19.754, "eval_steps_per_second": 9.877, "step": 100 }, { "entropy": 2.279540453851223, "epoch": 0.2752580544260244, "grad_norm": 5.038643836975098, "learning_rate": 1.9968534794992947e-05, "loss": 0.9877372741699219, "mean_token_accuracy": 0.7308646870777011, "num_tokens": 302872.0, "step": 110 }, { "entropy": 2.119027265906334, "epoch": 0.30028151391929936, "grad_norm": 4.737979412078857, "learning_rate": 1.995439211678754e-05, "loss": 0.853663444519043, "mean_token_accuracy": 0.76122229360044, "num_tokens": 330737.0, "step": 120 }, { "entropy": 2.1482249490916727, "epoch": 0.3253049734125743, "grad_norm": 2.8202335834503174, "learning_rate": 1.9937639071771704e-05, "loss": 0.889527416229248, "mean_token_accuracy": 0.6904077736660839, "num_tokens": 362657.0, "step": 130 }, { "entropy": 2.179778201878071, "epoch": 0.35032843290584925, "grad_norm": 4.295614719390869, "learning_rate": 1.9918280053141144e-05, "loss": 1.098190689086914, "mean_token_accuracy": 0.6943824682384729, "num_tokens": 390971.0, "step": 140 }, { "entropy": 2.1700742825865746, "epoch": 0.3753518923991242, "grad_norm": 2.5075595378875732, "learning_rate": 1.9896320137462984e-05, "loss": 1.0178719520568849, "mean_token_accuracy": 0.7038579940795898, "num_tokens": 417595.0, "step": 150 }, { "entropy": 2.135371221601963, "epoch": 0.40037535189239915, "grad_norm": 3.29361629486084, "learning_rate": 1.987176508334451e-05, "loss": 0.8292133331298828, "mean_token_accuracy": 0.756744677759707, "num_tokens": 442918.0, "step": 160 }, { "entropy": 2.1454797983169556, "epoch": 0.4253988113856741, "grad_norm": 2.2850234508514404, "learning_rate": 1.98446213299231e-05, "loss": 0.7698661804199218, "mean_token_accuracy": 0.7885082781314849, "num_tokens": 477921.0, "step": 170 }, { "entropy": 2.17876655459404, "epoch": 0.45042227087894904, "grad_norm": 1.6584603786468506, "learning_rate": 1.9814895995177653e-05, "loss": 0.9373016357421875, "mean_token_accuracy": 0.7417640507221221, "num_tokens": 504910.0, "step": 180 }, { "entropy": 2.213928133249283, "epoch": 0.475445730372224, "grad_norm": 3.5426175594329834, "learning_rate": 1.9782596874062028e-05, "loss": 0.9217703819274903, "mean_token_accuracy": 0.7309052364900708, "num_tokens": 531010.0, "step": 190 }, { "entropy": 2.079026885330677, "epoch": 0.5004691898654989, "grad_norm": 3.355250358581543, "learning_rate": 1.9747732436460955e-05, "loss": 0.808747673034668, "mean_token_accuracy": 0.7717655746266245, "num_tokens": 561639.0, "step": 200 }, { "epoch": 0.5004691898654989, "eval_entropy": 2.1456044730730355, "eval_loss": 0.9099652767181396, "eval_mean_token_accuracy": 0.7726191087858751, "eval_num_tokens": 561639.0, "eval_runtime": 25.922, "eval_samples_per_second": 19.752, "eval_steps_per_second": 9.876, "step": 200 }, { "entropy": 2.1017219856381417, "epoch": 0.5254926493587738, "grad_norm": 2.787653684616089, "learning_rate": 1.9710311824968942e-05, "loss": 0.8461710929870605, "mean_token_accuracy": 0.7469903081655502, "num_tokens": 589942.0, "step": 210 }, { "entropy": 2.134711952507496, "epoch": 0.5505161088520488, "grad_norm": 3.67427921295166, "learning_rate": 1.9670344852492814e-05, "loss": 0.7912054538726807, "mean_token_accuracy": 0.7155324574559927, "num_tokens": 617432.0, "step": 220 }, { "entropy": 2.1465815491974354, "epoch": 0.5755395683453237, "grad_norm": 5.175668239593506, "learning_rate": 1.9627841999678422e-05, "loss": 0.9203764915466308, "mean_token_accuracy": 0.7518326181918382, "num_tokens": 651978.0, "step": 230 }, { "entropy": 2.257516998052597, "epoch": 0.6005630278385987, "grad_norm": 8.140710830688477, "learning_rate": 1.9582814412162288e-05, "loss": 0.9473254203796386, "mean_token_accuracy": 0.7131743170320988, "num_tokens": 687677.0, "step": 240 }, { "entropy": 2.1520414032042026, "epoch": 0.6255864873318736, "grad_norm": 8.945752143859863, "learning_rate": 1.9535273897648857e-05, "loss": 0.9689723968505859, "mean_token_accuracy": 0.7218846149742604, "num_tokens": 717302.0, "step": 250 }, { "entropy": 2.2178710743784906, "epoch": 0.6506099468251486, "grad_norm": 2.2079780101776123, "learning_rate": 1.9485232922814117e-05, "loss": 0.6868520736694336, "mean_token_accuracy": 0.750518599525094, "num_tokens": 749747.0, "step": 260 }, { "entropy": 2.233424980938435, "epoch": 0.6756334063184235, "grad_norm": 2.361921548843384, "learning_rate": 1.9432704610036448e-05, "loss": 0.9331287384033203, "mean_token_accuracy": 0.7324752386659383, "num_tokens": 780201.0, "step": 270 }, { "entropy": 2.2572396978735925, "epoch": 0.7006568658116985, "grad_norm": 8.503108024597168, "learning_rate": 1.9377702733955493e-05, "loss": 0.9354991912841797, "mean_token_accuracy": 0.7744514495134354, "num_tokens": 809852.0, "step": 280 }, { "entropy": 2.2340517044067383, "epoch": 0.7256803253049734, "grad_norm": 2.9905447959899902, "learning_rate": 1.9320241717860007e-05, "loss": 0.821927833557129, "mean_token_accuracy": 0.7892053715884686, "num_tokens": 839080.0, "step": 290 }, { "entropy": 2.1411201044917108, "epoch": 0.7507037847982484, "grad_norm": 4.045820236206055, "learning_rate": 1.926033662990558e-05, "loss": 1.0363512992858888, "mean_token_accuracy": 0.695093622803688, "num_tokens": 872158.0, "step": 300 }, { "epoch": 0.7507037847982484, "eval_entropy": 2.1794460150413215, "eval_loss": 0.8942313194274902, "eval_mean_token_accuracy": 0.7791136712767184, "eval_num_tokens": 872158.0, "eval_runtime": 25.9051, "eval_samples_per_second": 19.764, "eval_steps_per_second": 9.882, "step": 300 }, { "entropy": 2.2423775270581245, "epoch": 0.7757272442915233, "grad_norm": 2.8176357746124268, "learning_rate": 1.9198003179163308e-05, "loss": 0.9462035179138184, "mean_token_accuracy": 0.7392122287303209, "num_tokens": 900576.0, "step": 310 }, { "entropy": 2.2096558839082716, "epoch": 0.8007507037847983, "grad_norm": 6.080548286437988, "learning_rate": 1.9133257711500318e-05, "loss": 0.833648681640625, "mean_token_accuracy": 0.7921354983001947, "num_tokens": 941726.0, "step": 320 }, { "entropy": 2.186213733255863, "epoch": 0.8257741632780732, "grad_norm": 2.5898144245147705, "learning_rate": 1.9066117205293393e-05, "loss": 0.7522900581359864, "mean_token_accuracy": 0.7382632024586201, "num_tokens": 970413.0, "step": 330 }, { "entropy": 2.2108209684491156, "epoch": 0.8507976227713482, "grad_norm": 4.91647481918335, "learning_rate": 1.8996599266976658e-05, "loss": 0.7787142753601074, "mean_token_accuracy": 0.7524209380149841, "num_tokens": 993790.0, "step": 340 }, { "entropy": 2.1969844341278075, "epoch": 0.8758210822646231, "grad_norm": 6.000455379486084, "learning_rate": 1.892472212642459e-05, "loss": 0.8661828994750976, "mean_token_accuracy": 0.7468080155551433, "num_tokens": 1026126.0, "step": 350 }, { "entropy": 2.1586176477372647, "epoch": 0.9008445417578981, "grad_norm": 3.4196536540985107, "learning_rate": 1.885050463217159e-05, "loss": 0.8391610145568847, "mean_token_accuracy": 0.7550970822572708, "num_tokens": 1055053.0, "step": 360 }, { "entropy": 2.106422890722752, "epoch": 0.925868001251173, "grad_norm": 2.27886962890625, "learning_rate": 1.8773966246469238e-05, "loss": 0.7569664478302002, "mean_token_accuracy": 0.7327868092805148, "num_tokens": 1086499.0, "step": 370 }, { "entropy": 2.150851938128471, "epoch": 0.950891460744448, "grad_norm": 2.702540159225464, "learning_rate": 1.8695127040182678e-05, "loss": 1.048074722290039, "mean_token_accuracy": 0.7364495120942592, "num_tokens": 1118377.0, "step": 380 }, { "entropy": 2.207886445522308, "epoch": 0.9759149202377229, "grad_norm": 6.179843902587891, "learning_rate": 1.8614007687527374e-05, "loss": 0.9187170982360839, "mean_token_accuracy": 0.7189946949481965, "num_tokens": 1146754.0, "step": 390 }, { "entropy": 2.1371198025616733, "epoch": 1.0, "grad_norm": 5.99985933303833, "learning_rate": 1.8530629460647658e-05, "loss": 0.6151233673095703, "mean_token_accuracy": 0.7796741042818341, "num_tokens": 1172570.0, "step": 400 }, { "epoch": 1.0, "eval_entropy": 2.1289648609235883, "eval_loss": 0.8714665174484253, "eval_mean_token_accuracy": 0.7819707952439785, "eval_num_tokens": 1172570.0, "eval_runtime": 25.9103, "eval_samples_per_second": 19.76, "eval_steps_per_second": 9.88, "step": 400 }, { "entropy": 2.098095750808716, "epoch": 1.025023459493275, "grad_norm": 2.6623897552490234, "learning_rate": 1.8445014224038485e-05, "loss": 0.6235037803649902, "mean_token_accuracy": 0.823400478810072, "num_tokens": 1201801.0, "step": 410 }, { "entropy": 2.1191157147288324, "epoch": 1.05004691898655, "grad_norm": 4.902485370635986, "learning_rate": 1.835718442881183e-05, "loss": 0.8132280349731446, "mean_token_accuracy": 0.7433615382760763, "num_tokens": 1229848.0, "step": 420 }, { "entropy": 2.057385340332985, "epoch": 1.0750703784798248, "grad_norm": 10.126677513122559, "learning_rate": 1.8267163106809288e-05, "loss": 0.8311259269714355, "mean_token_accuracy": 0.7856612842530012, "num_tokens": 1256935.0, "step": 430 }, { "entropy": 2.050567016005516, "epoch": 1.1000938379730998, "grad_norm": 3.812150478363037, "learning_rate": 1.817497386456238e-05, "loss": 0.7683423042297364, "mean_token_accuracy": 0.7796268314123154, "num_tokens": 1286284.0, "step": 440 }, { "entropy": 2.1574359863996504, "epoch": 1.1251172974663748, "grad_norm": 3.055018186569214, "learning_rate": 1.808064087710212e-05, "loss": 0.6059474468231201, "mean_token_accuracy": 0.8015186853706837, "num_tokens": 1312395.0, "step": 450 }, { "entropy": 2.0954012736678123, "epoch": 1.1501407569596496, "grad_norm": 1.8690502643585205, "learning_rate": 1.7984188881619563e-05, "loss": 0.7081769943237305, "mean_token_accuracy": 0.8020102433860302, "num_tokens": 1342341.0, "step": 460 }, { "entropy": 2.080510801076889, "epoch": 1.1751642164529246, "grad_norm": 5.163702964782715, "learning_rate": 1.788564317097889e-05, "loss": 0.6229144096374511, "mean_token_accuracy": 0.8259935293346643, "num_tokens": 1368026.0, "step": 470 }, { "entropy": 2.0373916894197466, "epoch": 1.2001876759461996, "grad_norm": 3.3629536628723145, "learning_rate": 1.7785029587084793e-05, "loss": 0.7988658905029297, "mean_token_accuracy": 0.836330172047019, "num_tokens": 1408554.0, "step": 480 }, { "entropy": 2.0612950578331946, "epoch": 1.2252111354394746, "grad_norm": 2.9767253398895264, "learning_rate": 1.768237451410589e-05, "loss": 0.8266899108886718, "mean_token_accuracy": 0.8041692972183228, "num_tokens": 1433615.0, "step": 490 }, { "entropy": 2.1810658618807794, "epoch": 1.2502345949327496, "grad_norm": 8.24252700805664, "learning_rate": 1.7577704871555924e-05, "loss": 0.75057373046875, "mean_token_accuracy": 0.7435132082551718, "num_tokens": 1461697.0, "step": 500 }, { "epoch": 1.2502345949327496, "eval_entropy": 2.0794698572717607, "eval_loss": 0.8950290679931641, "eval_mean_token_accuracy": 0.7889105979120359, "eval_num_tokens": 1461697.0, "eval_runtime": 25.9129, "eval_samples_per_second": 19.758, "eval_steps_per_second": 9.879, "step": 500 }, { "entropy": 1.9847739323973657, "epoch": 1.2752580544260244, "grad_norm": 3.5981249809265137, "learning_rate": 1.74710481072346e-05, "loss": 0.6473482131958008, "mean_token_accuracy": 0.8046811826527118, "num_tokens": 1492119.0, "step": 510 }, { "entropy": 2.144796669483185, "epoch": 1.3002815139192994, "grad_norm": 10.027125358581543, "learning_rate": 1.7362432190029862e-05, "loss": 1.0633076667785644, "mean_token_accuracy": 0.7895992513746023, "num_tokens": 1518884.0, "step": 520 }, { "entropy": 2.1153031289577484, "epoch": 1.3253049734125744, "grad_norm": 5.87807035446167, "learning_rate": 1.7251885602583547e-05, "loss": 0.7672069549560547, "mean_token_accuracy": 0.8115054946392775, "num_tokens": 1547318.0, "step": 530 }, { "entropy": 2.1572294265031813, "epoch": 1.3503284329058491, "grad_norm": 8.224700927734375, "learning_rate": 1.7139437333822303e-05, "loss": 0.7436501502990722, "mean_token_accuracy": 0.8101725693792104, "num_tokens": 1576089.0, "step": 540 }, { "entropy": 2.138009563088417, "epoch": 1.3753518923991241, "grad_norm": 2.2141480445861816, "learning_rate": 1.7025116871355737e-05, "loss": 0.654974365234375, "mean_token_accuracy": 0.7730139032006264, "num_tokens": 1606732.0, "step": 550 }, { "entropy": 2.139923092722893, "epoch": 1.4003753518923991, "grad_norm": 5.900820732116699, "learning_rate": 1.6908954193743816e-05, "loss": 0.7841741561889648, "mean_token_accuracy": 0.7526014145463705, "num_tokens": 1639646.0, "step": 560 }, { "entropy": 2.1373246192932127, "epoch": 1.4253988113856741, "grad_norm": 6.760275840759277, "learning_rate": 1.6790979762635497e-05, "loss": 0.7275867462158203, "mean_token_accuracy": 0.7795548062771559, "num_tokens": 1668659.0, "step": 570 }, { "entropy": 2.187282508611679, "epoch": 1.4504222708789491, "grad_norm": 4.651549339294434, "learning_rate": 1.6671224514780692e-05, "loss": 0.9761672019958496, "mean_token_accuracy": 0.7901028756052255, "num_tokens": 1695170.0, "step": 580 }, { "entropy": 2.097830060124397, "epoch": 1.475445730372224, "grad_norm": 3.098156690597534, "learning_rate": 1.654971985391764e-05, "loss": 0.7796518802642822, "mean_token_accuracy": 0.8037835378199816, "num_tokens": 1721326.0, "step": 590 }, { "entropy": 2.190341380238533, "epoch": 1.500469189865499, "grad_norm": 2.757059097290039, "learning_rate": 1.6426497642537826e-05, "loss": 0.648505973815918, "mean_token_accuracy": 0.824278862029314, "num_tokens": 1748835.0, "step": 600 }, { "epoch": 1.500469189865499, "eval_entropy": 2.1228315108455718, "eval_loss": 0.8891530632972717, "eval_mean_token_accuracy": 0.7763976352289319, "eval_num_tokens": 1748835.0, "eval_runtime": 25.9131, "eval_samples_per_second": 19.758, "eval_steps_per_second": 9.879, "step": 600 }, { "entropy": 2.1038011401891707, "epoch": 1.5254926493587737, "grad_norm": 4.028401851654053, "learning_rate": 1.6301590193530585e-05, "loss": 0.7076716423034668, "mean_token_accuracy": 0.792342029325664, "num_tokens": 1781732.0, "step": 610 }, { "entropy": 2.050699570029974, "epoch": 1.5505161088520487, "grad_norm": 2.7474255561828613, "learning_rate": 1.6175030261709615e-05, "loss": 0.8747171401977539, "mean_token_accuracy": 0.7811730474233627, "num_tokens": 1809560.0, "step": 620 }, { "entropy": 2.0647204160690307, "epoch": 1.5755395683453237, "grad_norm": 9.905837059020996, "learning_rate": 1.6046851035223594e-05, "loss": 0.7337132930755615, "mean_token_accuracy": 0.7674034627154469, "num_tokens": 1838965.0, "step": 630 }, { "entropy": 2.054245483875275, "epoch": 1.6005630278385987, "grad_norm": 3.5519328117370605, "learning_rate": 1.591708612685316e-05, "loss": 0.6100962638854981, "mean_token_accuracy": 0.833642303198576, "num_tokens": 1871841.0, "step": 640 }, { "entropy": 2.149427868425846, "epoch": 1.6255864873318737, "grad_norm": 3.8471930027008057, "learning_rate": 1.5785769565196543e-05, "loss": 0.8112316131591797, "mean_token_accuracy": 0.7824445836246013, "num_tokens": 1900058.0, "step": 650 }, { "entropy": 2.034525628387928, "epoch": 1.6506099468251487, "grad_norm": 2.7920093536376953, "learning_rate": 1.565293578574615e-05, "loss": 0.5858467102050782, "mean_token_accuracy": 0.7768620416522026, "num_tokens": 1931656.0, "step": 660 }, { "entropy": 2.1494649052619934, "epoch": 1.6756334063184235, "grad_norm": 3.6777379512786865, "learning_rate": 1.5518619621858474e-05, "loss": 0.6075996398925781, "mean_token_accuracy": 0.8004748497158289, "num_tokens": 1960003.0, "step": 670 }, { "entropy": 2.056756618618965, "epoch": 1.7006568658116985, "grad_norm": 9.437564849853516, "learning_rate": 1.5382856295619622e-05, "loss": 0.6237570762634277, "mean_token_accuracy": 0.7745466388761997, "num_tokens": 1987487.0, "step": 680 }, { "entropy": 1.9909536838531494, "epoch": 1.7256803253049733, "grad_norm": 2.8845739364624023, "learning_rate": 1.5245681408608946e-05, "loss": 0.7010098934173584, "mean_token_accuracy": 0.7885188397020102, "num_tokens": 2018878.0, "step": 690 }, { "entropy": 2.076315422356129, "epoch": 1.7507037847982483, "grad_norm": 6.985860824584961, "learning_rate": 1.5107130932563151e-05, "loss": 0.6411010265350342, "mean_token_accuracy": 0.8081331729888916, "num_tokens": 2042835.0, "step": 700 }, { "epoch": 1.7507037847982483, "eval_entropy": 2.0568552212789655, "eval_loss": 0.8686603307723999, "eval_mean_token_accuracy": 0.797932347166352, "eval_num_tokens": 2042835.0, "eval_runtime": 25.9268, "eval_samples_per_second": 19.748, "eval_steps_per_second": 9.874, "step": 700 }, { "entropy": 2.1089097008109094, "epoch": 1.7757272442915233, "grad_norm": 5.072978496551514, "learning_rate": 1.4967241199943332e-05, "loss": 0.7032583236694336, "mean_token_accuracy": 0.8087547916918993, "num_tokens": 2066598.0, "step": 710 }, { "entropy": 2.0160682946443558, "epoch": 1.8007507037847983, "grad_norm": 3.0282862186431885, "learning_rate": 1.4826048894407396e-05, "loss": 0.7899590492248535, "mean_token_accuracy": 0.7788747299462557, "num_tokens": 2100878.0, "step": 720 }, { "entropy": 2.0212891355156897, "epoch": 1.8257741632780733, "grad_norm": 12.743298530578613, "learning_rate": 1.4683591041190433e-05, "loss": 0.6348478317260742, "mean_token_accuracy": 0.7777544744312763, "num_tokens": 2143751.0, "step": 730 }, { "entropy": 2.032179595530033, "epoch": 1.8507976227713483, "grad_norm": 2.7245559692382812, "learning_rate": 1.4539904997395468e-05, "loss": 1.0501715660095214, "mean_token_accuracy": 0.741650390997529, "num_tokens": 2176229.0, "step": 740 }, { "entropy": 2.078695350885391, "epoch": 1.875821082264623, "grad_norm": 3.2556543350219727, "learning_rate": 1.4395028442197231e-05, "loss": 0.7042528629302979, "mean_token_accuracy": 0.8024522136896849, "num_tokens": 2201679.0, "step": 750 }, { "entropy": 2.094234761595726, "epoch": 1.900844541757898, "grad_norm": 4.391955375671387, "learning_rate": 1.424899936696143e-05, "loss": 0.5421716690063476, "mean_token_accuracy": 0.8527256866917015, "num_tokens": 2230744.0, "step": 760 }, { "entropy": 2.0378435380756854, "epoch": 1.9258680012511729, "grad_norm": 3.724647045135498, "learning_rate": 1.4101856065282174e-05, "loss": 0.6828950405120849, "mean_token_accuracy": 0.7861546307802201, "num_tokens": 2258618.0, "step": 770 }, { "entropy": 2.0710975214838983, "epoch": 1.9508914607444479, "grad_norm": 4.124916076660156, "learning_rate": 1.3953637122940147e-05, "loss": 0.559223222732544, "mean_token_accuracy": 0.8332096721976996, "num_tokens": 2285444.0, "step": 780 }, { "entropy": 1.9646637082099914, "epoch": 1.9759149202377229, "grad_norm": 8.622406959533691, "learning_rate": 1.380438140778416e-05, "loss": 0.5959304332733154, "mean_token_accuracy": 0.7976583287119865, "num_tokens": 2315763.0, "step": 790 }, { "entropy": 1.9505030063839701, "epoch": 2.0, "grad_norm": 5.460987567901611, "learning_rate": 1.365412805953872e-05, "loss": 0.6341047763824463, "mean_token_accuracy": 0.8356712561149102, "num_tokens": 2345140.0, "step": 800 }, { "epoch": 2.0, "eval_entropy": 1.9633747367188334, "eval_loss": 0.8708455562591553, "eval_mean_token_accuracy": 0.7948170631425455, "eval_num_tokens": 2345140.0, "eval_runtime": 25.9388, "eval_samples_per_second": 19.739, "eval_steps_per_second": 9.869, "step": 800 }, { "entropy": 1.9318091489374638, "epoch": 2.025023459493275, "grad_norm": 2.679042339324951, "learning_rate": 1.3502916479540327e-05, "loss": 0.5399890899658203, "mean_token_accuracy": 0.8556223087012768, "num_tokens": 2374724.0, "step": 810 }, { "entropy": 1.9705582335591316, "epoch": 2.05004691898655, "grad_norm": 10.042631149291992, "learning_rate": 1.3350786320405145e-05, "loss": 0.7448606967926026, "mean_token_accuracy": 0.8062794495373964, "num_tokens": 2399771.0, "step": 820 }, { "entropy": 1.9878843411803246, "epoch": 2.075070378479825, "grad_norm": 3.2470123767852783, "learning_rate": 1.31977774756308e-05, "loss": 0.667686939239502, "mean_token_accuracy": 0.8486393585801124, "num_tokens": 2428832.0, "step": 830 }, { "entropy": 1.9709952771663666, "epoch": 2.1000938379731, "grad_norm": 8.831825256347656, "learning_rate": 1.3043930069134998e-05, "loss": 0.5341938018798829, "mean_token_accuracy": 0.8500256590545178, "num_tokens": 2454191.0, "step": 840 }, { "entropy": 1.8401574552059174, "epoch": 2.1251172974663746, "grad_norm": 10.572880744934082, "learning_rate": 1.2889284444733722e-05, "loss": 0.4976643085479736, "mean_token_accuracy": 0.8540774881839752, "num_tokens": 2489898.0, "step": 850 }, { "entropy": 1.8839198268949986, "epoch": 2.1501407569596496, "grad_norm": 10.400339126586914, "learning_rate": 1.273388115556177e-05, "loss": 0.6440447330474853, "mean_token_accuracy": 0.8276639927178622, "num_tokens": 2520146.0, "step": 860 }, { "entropy": 1.905003009736538, "epoch": 2.1751642164529246, "grad_norm": 2.6133148670196533, "learning_rate": 1.2577760953438382e-05, "loss": 0.6256015300750732, "mean_token_accuracy": 0.8756786741316318, "num_tokens": 2552346.0, "step": 870 }, { "entropy": 1.8848004803061484, "epoch": 2.2001876759461996, "grad_norm": 7.504598140716553, "learning_rate": 1.2420964778180815e-05, "loss": 0.49793548583984376, "mean_token_accuracy": 0.8614703625440597, "num_tokens": 2583354.0, "step": 880 }, { "entropy": 1.8399935081601142, "epoch": 2.2252111354394746, "grad_norm": 7.4189910888671875, "learning_rate": 1.2263533746868552e-05, "loss": 0.48105669021606445, "mean_token_accuracy": 0.8231813054531812, "num_tokens": 2610641.0, "step": 890 }, { "entropy": 1.9149093806743622, "epoch": 2.2502345949327496, "grad_norm": 11.7152681350708, "learning_rate": 1.2105509143061072e-05, "loss": 0.6658933162689209, "mean_token_accuracy": 0.8178440190851688, "num_tokens": 2645185.0, "step": 900 }, { "epoch": 2.2502345949327496, "eval_entropy": 1.9004527027718723, "eval_loss": 0.9131948947906494, "eval_mean_token_accuracy": 0.7987493762047961, "eval_num_tokens": 2645185.0, "eval_runtime": 25.9604, "eval_samples_per_second": 19.722, "eval_steps_per_second": 9.861, "step": 900 }, { "entropy": 1.9318789586424827, "epoch": 2.2752580544260246, "grad_norm": 3.531498432159424, "learning_rate": 1.194693240597196e-05, "loss": 0.4017048358917236, "mean_token_accuracy": 0.8562013734132051, "num_tokens": 2671042.0, "step": 910 }, { "entropy": 1.9958116054534911, "epoch": 2.300281513919299, "grad_norm": 7.350344181060791, "learning_rate": 1.1787845119602184e-05, "loss": 0.5403977394104004, "mean_token_accuracy": 0.8568222790956497, "num_tokens": 2697233.0, "step": 920 }, { "entropy": 1.8747393786907196, "epoch": 2.325304973412574, "grad_norm": 4.025563716888428, "learning_rate": 1.1628289001835405e-05, "loss": 0.5700911045074463, "mean_token_accuracy": 0.8201610699295998, "num_tokens": 2726631.0, "step": 930 }, { "entropy": 1.930973158031702, "epoch": 2.350328432905849, "grad_norm": 6.795775413513184, "learning_rate": 1.1468305893498204e-05, "loss": 0.49820170402526853, "mean_token_accuracy": 0.8798530824482441, "num_tokens": 2755100.0, "step": 940 }, { "entropy": 1.897780492901802, "epoch": 2.375351892399124, "grad_norm": 8.053464889526367, "learning_rate": 1.1307937747388034e-05, "loss": 0.6488244533538818, "mean_token_accuracy": 0.8430951170623302, "num_tokens": 2783867.0, "step": 950 }, { "entropy": 1.9086807489395141, "epoch": 2.400375351892399, "grad_norm": 4.615073204040527, "learning_rate": 1.114722661727182e-05, "loss": 0.4686459541320801, "mean_token_accuracy": 0.8605645731091499, "num_tokens": 2811469.0, "step": 960 }, { "entropy": 1.912995307147503, "epoch": 2.425398811385674, "grad_norm": 10.602889060974121, "learning_rate": 1.0986214646858115e-05, "loss": 0.5210050106048584, "mean_token_accuracy": 0.851820009201765, "num_tokens": 2839283.0, "step": 970 }, { "entropy": 1.908480130136013, "epoch": 2.450422270878949, "grad_norm": 9.512876510620117, "learning_rate": 1.0824944058745623e-05, "loss": 0.4891656875610352, "mean_token_accuracy": 0.8372205581516028, "num_tokens": 2867350.0, "step": 980 }, { "entropy": 1.904689647257328, "epoch": 2.475445730372224, "grad_norm": 7.750706195831299, "learning_rate": 1.0663457143351044e-05, "loss": 0.4556453704833984, "mean_token_accuracy": 0.8883680656552315, "num_tokens": 2899406.0, "step": 990 }, { "entropy": 1.8602308124303817, "epoch": 2.500469189865499, "grad_norm": 11.667961120605469, "learning_rate": 1.0501796247819176e-05, "loss": 0.5987341403961182, "mean_token_accuracy": 0.843272651731968, "num_tokens": 2925940.0, "step": 1000 }, { "epoch": 2.500469189865499, "eval_entropy": 1.8962222184054554, "eval_loss": 0.9162221550941467, "eval_mean_token_accuracy": 0.7972135632298887, "eval_num_tokens": 2925940.0, "eval_runtime": 25.9027, "eval_samples_per_second": 19.766, "eval_steps_per_second": 9.883, "step": 1000 }, { "entropy": 1.9050709769129752, "epoch": 2.5254926493587737, "grad_norm": 5.943221569061279, "learning_rate": 1.0340003764918078e-05, "loss": 0.6280072212219239, "mean_token_accuracy": 0.8301476046442986, "num_tokens": 2950373.0, "step": 1010 }, { "entropy": 1.883858135342598, "epoch": 2.5505161088520487, "grad_norm": 9.269403457641602, "learning_rate": 1.0178122121922324e-05, "loss": 0.7295412063598633, "mean_token_accuracy": 0.8335961733013392, "num_tokens": 2979264.0, "step": 1020 }, { "entropy": 1.8532995566725732, "epoch": 2.5755395683453237, "grad_norm": 8.404088020324707, "learning_rate": 1.001619376948718e-05, "loss": 0.42704405784606936, "mean_token_accuracy": 0.88788600564003, "num_tokens": 3006306.0, "step": 1030 }, { "entropy": 1.8335328698158264, "epoch": 2.6005630278385987, "grad_norm": 6.614358901977539, "learning_rate": 9.854261170516648e-06, "loss": 0.5608331680297851, "mean_token_accuracy": 0.8688548248261213, "num_tokens": 3035157.0, "step": 1040 }, { "entropy": 1.9003934860229492, "epoch": 2.6255864873318737, "grad_norm": 17.050065994262695, "learning_rate": 9.692366789028308e-06, "loss": 0.5226325511932373, "mean_token_accuracy": 0.860271492600441, "num_tokens": 3059665.0, "step": 1050 }, { "entropy": 1.8391575522720813, "epoch": 2.6506099468251487, "grad_norm": 6.052427768707275, "learning_rate": 9.530553079017872e-06, "loss": 0.5446209907531738, "mean_token_accuracy": 0.8525044105947017, "num_tokens": 3088789.0, "step": 1060 }, { "entropy": 1.8850280776619912, "epoch": 2.6756334063184237, "grad_norm": 10.781261444091797, "learning_rate": 9.368862473326355e-06, "loss": 0.431397533416748, "mean_token_accuracy": 0.8595373194664717, "num_tokens": 3116869.0, "step": 1070 }, { "entropy": 1.816288386285305, "epoch": 2.7006568658116983, "grad_norm": 5.157824993133545, "learning_rate": 9.207337372512797e-06, "loss": 0.48740544319152834, "mean_token_accuracy": 0.8544141486287117, "num_tokens": 3143639.0, "step": 1080 }, { "entropy": 1.8121181055903435, "epoch": 2.7256803253049733, "grad_norm": 6.038757801055908, "learning_rate": 9.046020133735455e-06, "loss": 0.5854983806610108, "mean_token_accuracy": 0.8188087772578001, "num_tokens": 3169971.0, "step": 1090 }, { "entropy": 1.8628299854695798, "epoch": 2.7507037847982483, "grad_norm": 4.648480415344238, "learning_rate": 8.88495305964436e-06, "loss": 0.435821008682251, "mean_token_accuracy": 0.8851393271237612, "num_tokens": 3198712.0, "step": 1100 }, { "epoch": 2.7507037847982483, "eval_entropy": 1.8285482935607433, "eval_loss": 0.9385225176811218, "eval_mean_token_accuracy": 0.795126418932341, "eval_num_tokens": 3198712.0, "eval_runtime": 25.9475, "eval_samples_per_second": 19.732, "eval_steps_per_second": 9.866, "step": 1100 }, { "entropy": 1.8046948611736298, "epoch": 2.7757272442915233, "grad_norm": 12.604764938354492, "learning_rate": 8.724178387288202e-06, "loss": 0.45838608741760256, "mean_token_accuracy": 0.8950696140527725, "num_tokens": 3243283.0, "step": 1110 }, { "entropy": 1.8318631589412688, "epoch": 2.8007507037847983, "grad_norm": 4.84168815612793, "learning_rate": 8.563738277038376e-06, "loss": 0.5975977897644043, "mean_token_accuracy": 0.8312116377055645, "num_tokens": 3274397.0, "step": 1120 }, { "entropy": 1.772474942356348, "epoch": 2.8257741632780733, "grad_norm": 3.9503281116485596, "learning_rate": 8.40367480153316e-06, "loss": 0.5480531692504883, "mean_token_accuracy": 0.8570948846638202, "num_tokens": 3302787.0, "step": 1130 }, { "entropy": 1.837375022470951, "epoch": 2.8507976227713483, "grad_norm": 7.392024993896484, "learning_rate": 8.244029934644916e-06, "loss": 0.4703363418579102, "mean_token_accuracy": 0.8493639241904021, "num_tokens": 3331250.0, "step": 1140 }, { "entropy": 1.7589601434767246, "epoch": 2.875821082264623, "grad_norm": 9.2200927734375, "learning_rate": 8.084845540473127e-06, "loss": 0.5588334083557129, "mean_token_accuracy": 0.864871158450842, "num_tokens": 3365799.0, "step": 1150 }, { "entropy": 1.7889262288808823, "epoch": 2.9008445417578983, "grad_norm": 6.014739513397217, "learning_rate": 7.9261633623663e-06, "loss": 0.4394832611083984, "mean_token_accuracy": 0.9105578908696771, "num_tokens": 3397907.0, "step": 1160 }, { "entropy": 1.793429036438465, "epoch": 2.925868001251173, "grad_norm": 9.53693675994873, "learning_rate": 7.768025011975481e-06, "loss": 0.5310945987701416, "mean_token_accuracy": 0.8654899284243583, "num_tokens": 3428029.0, "step": 1170 }, { "entropy": 1.791808745265007, "epoch": 2.950891460744448, "grad_norm": 8.377955436706543, "learning_rate": 7.610471958342326e-06, "loss": 0.4895362377166748, "mean_token_accuracy": 0.8723569042980671, "num_tokens": 3462452.0, "step": 1180 }, { "entropy": 1.8754499897360801, "epoch": 2.975914920237723, "grad_norm": 5.267559051513672, "learning_rate": 7.4535455170245476e-06, "loss": 0.521859884262085, "mean_token_accuracy": 0.8782990558072925, "num_tokens": 3490604.0, "step": 1190 }, { "entropy": 1.8749133162684255, "epoch": 3.0, "grad_norm": 18.580663681030273, "learning_rate": 7.297286839261659e-06, "loss": 0.532118558883667, "mean_token_accuracy": 0.8816164097228607, "num_tokens": 3517710.0, "step": 1200 }, { "epoch": 3.0, "eval_entropy": 1.8413750138133764, "eval_loss": 0.9358716607093811, "eval_mean_token_accuracy": 0.798108211136423, "eval_num_tokens": 3517710.0, "eval_runtime": 25.9137, "eval_samples_per_second": 19.758, "eval_steps_per_second": 9.879, "step": 1200 }, { "entropy": 1.8603122062981128, "epoch": 3.025023459493275, "grad_norm": 2.6409912109375, "learning_rate": 7.1417369011837355e-06, "loss": 0.30042328834533694, "mean_token_accuracy": 0.925163534283638, "num_tokens": 3542903.0, "step": 1210 }, { "entropy": 1.8052339434623719, "epoch": 3.05004691898655, "grad_norm": 6.589583873748779, "learning_rate": 6.986936493066165e-06, "loss": 0.5724757194519043, "mean_token_accuracy": 0.8914085768163205, "num_tokens": 3574728.0, "step": 1220 }, { "entropy": 1.828494493663311, "epoch": 3.075070378479825, "grad_norm": 11.607176780700684, "learning_rate": 6.8329262086330864e-06, "loss": 0.3562421083450317, "mean_token_accuracy": 0.9141828007996082, "num_tokens": 3598841.0, "step": 1230 }, { "entropy": 1.663271901011467, "epoch": 3.1000938379731, "grad_norm": 4.842362880706787, "learning_rate": 6.6797464344124045e-06, "loss": 0.3805184602737427, "mean_token_accuracy": 0.9177524700760842, "num_tokens": 3627929.0, "step": 1240 }, { "entropy": 1.6602753311395646, "epoch": 3.1251172974663746, "grad_norm": 8.315300941467285, "learning_rate": 6.527437339145097e-06, "loss": 0.37636704444885255, "mean_token_accuracy": 0.9170884318649769, "num_tokens": 3656057.0, "step": 1250 }, { "entropy": 1.6945518404245377, "epoch": 3.1501407569596496, "grad_norm": 7.678070068359375, "learning_rate": 6.376038863251706e-06, "loss": 0.2868889570236206, "mean_token_accuracy": 0.8948385491967201, "num_tokens": 3682522.0, "step": 1260 }, { "entropy": 1.679956039786339, "epoch": 3.1751642164529246, "grad_norm": 5.331015586853027, "learning_rate": 6.225590708358596e-06, "loss": 0.40270466804504396, "mean_token_accuracy": 0.8960530921816826, "num_tokens": 3710068.0, "step": 1270 }, { "entropy": 1.8229594185948372, "epoch": 3.2001876759461996, "grad_norm": 5.083068370819092, "learning_rate": 6.076132326886934e-06, "loss": 0.30500695705413816, "mean_token_accuracy": 0.9170692026615143, "num_tokens": 3736077.0, "step": 1280 }, { "entropy": 1.712825458496809, "epoch": 3.2252111354394746, "grad_norm": 6.959349155426025, "learning_rate": 5.927702911706961e-06, "loss": 0.4808387279510498, "mean_token_accuracy": 0.9020599089562893, "num_tokens": 3767499.0, "step": 1290 }, { "entropy": 1.7548687763512134, "epoch": 3.2502345949327496, "grad_norm": 3.395026683807373, "learning_rate": 5.780341385860333e-06, "loss": 0.44341039657592773, "mean_token_accuracy": 0.8619493830949068, "num_tokens": 3796930.0, "step": 1300 }, { "epoch": 3.2502345949327496, "eval_entropy": 1.7076403074897826, "eval_loss": 1.0389351844787598, "eval_mean_token_accuracy": 0.797438750974834, "eval_num_tokens": 3796930.0, "eval_runtime": 25.945, "eval_samples_per_second": 19.734, "eval_steps_per_second": 9.867, "step": 1300 }, { "entropy": 1.7037555642426014, "epoch": 3.2752580544260246, "grad_norm": 7.0480732917785645, "learning_rate": 5.63408639235324e-06, "loss": 0.4138573169708252, "mean_token_accuracy": 0.8853528399020434, "num_tokens": 3826118.0, "step": 1310 }, { "entropy": 1.7090844802558423, "epoch": 3.300281513919299, "grad_norm": 9.457049369812012, "learning_rate": 5.488976284022953e-06, "loss": 0.38932549953460693, "mean_token_accuracy": 0.9061979863792657, "num_tokens": 3855132.0, "step": 1320 }, { "entropy": 1.669133372604847, "epoch": 3.325304973412574, "grad_norm": 5.351191997528076, "learning_rate": 5.3450491134804416e-06, "loss": 0.28699569702148436, "mean_token_accuracy": 0.9288982767611742, "num_tokens": 3888805.0, "step": 1330 }, { "entropy": 1.6410670965909957, "epoch": 3.350328432905849, "grad_norm": 15.305230140686035, "learning_rate": 5.202342623131731e-06, "loss": 0.27166821956634524, "mean_token_accuracy": 0.9167630560696125, "num_tokens": 3918738.0, "step": 1340 }, { "entropy": 1.629030243307352, "epoch": 3.375351892399124, "grad_norm": 2.671210289001465, "learning_rate": 5.060894235280637e-06, "loss": 0.4683689117431641, "mean_token_accuracy": 0.9109336912631989, "num_tokens": 3952972.0, "step": 1350 }, { "entropy": 1.6706741809844972, "epoch": 3.400375351892399, "grad_norm": 3.6248090267181396, "learning_rate": 4.9207410423153925e-06, "loss": 0.37804474830627444, "mean_token_accuracy": 0.905234795063734, "num_tokens": 3979693.0, "step": 1360 }, { "entropy": 1.6092427238821982, "epoch": 3.425398811385674, "grad_norm": 6.463558673858643, "learning_rate": 4.781919796981818e-06, "loss": 0.30335488319396975, "mean_token_accuracy": 0.9086175173521042, "num_tokens": 4008543.0, "step": 1370 }, { "entropy": 1.680896159261465, "epoch": 3.450422270878949, "grad_norm": 5.749578475952148, "learning_rate": 4.6444669027455615e-06, "loss": 0.3422755479812622, "mean_token_accuracy": 0.9261092841625214, "num_tokens": 4046924.0, "step": 1380 }, { "entropy": 1.6322893902659417, "epoch": 3.475445730372224, "grad_norm": 4.440096378326416, "learning_rate": 4.508418404245903e-06, "loss": 0.39899749755859376, "mean_token_accuracy": 0.8951961848884821, "num_tokens": 4078782.0, "step": 1390 }, { "entropy": 1.6997947439551353, "epoch": 3.500469189865499, "grad_norm": 6.773232936859131, "learning_rate": 4.373809977843676e-06, "loss": 0.3821007490158081, "mean_token_accuracy": 0.9003218419849872, "num_tokens": 4104561.0, "step": 1400 }, { "epoch": 3.500469189865499, "eval_entropy": 1.6739916959777474, "eval_loss": 1.0589442253112793, "eval_mean_token_accuracy": 0.7994736307300627, "eval_num_tokens": 4104561.0, "eval_runtime": 25.9285, "eval_samples_per_second": 19.747, "eval_steps_per_second": 9.873, "step": 1400 }, { "entropy": 1.6732337854802608, "epoch": 3.5254926493587737, "grad_norm": 4.1359333992004395, "learning_rate": 4.240676922265774e-06, "loss": 0.46932153701782225, "mean_token_accuracy": 0.8881252314895391, "num_tokens": 4131335.0, "step": 1410 }, { "entropy": 1.7210186302661896, "epoch": 3.5505161088520487, "grad_norm": 5.5349907875061035, "learning_rate": 4.1090541493486555e-06, "loss": 0.17852014303207397, "mean_token_accuracy": 0.933498764038086, "num_tokens": 4163748.0, "step": 1420 }, { "entropy": 1.6368225537240506, "epoch": 3.5755395683453237, "grad_norm": 11.67458724975586, "learning_rate": 3.978976174883329e-06, "loss": 0.3674903869628906, "mean_token_accuracy": 0.9103257186710835, "num_tokens": 4190535.0, "step": 1430 }, { "entropy": 1.6206367388367653, "epoch": 3.6005630278385987, "grad_norm": 6.8839850425720215, "learning_rate": 3.8504771095641905e-06, "loss": 0.37578022480010986, "mean_token_accuracy": 0.9048499703407288, "num_tokens": 4220983.0, "step": 1440 }, { "entropy": 1.630747129023075, "epoch": 3.6255864873318737, "grad_norm": 6.57778263092041, "learning_rate": 3.7235906500440576e-06, "loss": 0.3801173448562622, "mean_token_accuracy": 0.8957030765712262, "num_tokens": 4251936.0, "step": 1450 }, { "entropy": 1.6560029186308385, "epoch": 3.6506099468251487, "grad_norm": 8.60834789276123, "learning_rate": 3.5983500700978425e-06, "loss": 0.2505526542663574, "mean_token_accuracy": 0.9320986948907375, "num_tokens": 4286899.0, "step": 1460 }, { "entropy": 1.6910177335143088, "epoch": 3.6756334063184237, "grad_norm": 1.4734828472137451, "learning_rate": 3.4747882118970565e-06, "loss": 0.33303678035736084, "mean_token_accuracy": 0.9101938150823117, "num_tokens": 4309725.0, "step": 1470 }, { "entropy": 1.6117219008505344, "epoch": 3.7006568658116983, "grad_norm": 15.304268836975098, "learning_rate": 3.35293747739753e-06, "loss": 0.30569963455200194, "mean_token_accuracy": 0.9167403355240822, "num_tokens": 4338763.0, "step": 1480 }, { "entropy": 1.6450724273920059, "epoch": 3.7256803253049733, "grad_norm": 8.834023475646973, "learning_rate": 3.2328298198425556e-06, "loss": 0.3241915225982666, "mean_token_accuracy": 0.9080854427069426, "num_tokens": 4369516.0, "step": 1490 }, { "entropy": 1.618889082968235, "epoch": 3.7507037847982483, "grad_norm": 31.705739974975586, "learning_rate": 3.1144967353837196e-06, "loss": 0.35978450775146487, "mean_token_accuracy": 0.8960370164364576, "num_tokens": 4408106.0, "step": 1500 }, { "epoch": 3.7507037847982483, "eval_entropy": 1.6368464617989957, "eval_loss": 1.0974652767181396, "eval_mean_token_accuracy": 0.799555316218175, "eval_num_tokens": 4408106.0, "eval_runtime": 25.9318, "eval_samples_per_second": 19.744, "eval_steps_per_second": 9.872, "step": 1500 }, { "entropy": 1.6849488005042077, "epoch": 3.7757272442915233, "grad_norm": 18.618749618530273, "learning_rate": 2.997969254821548e-06, "loss": 0.434901762008667, "mean_token_accuracy": 0.9058444269001484, "num_tokens": 4433839.0, "step": 1510 }, { "entropy": 1.612816944718361, "epoch": 3.8007507037847983, "grad_norm": 5.040231704711914, "learning_rate": 2.883277935468254e-06, "loss": 0.3056370496749878, "mean_token_accuracy": 0.9123396039009094, "num_tokens": 4461853.0, "step": 1520 }, { "entropy": 1.6486859299242496, "epoch": 3.8257741632780733, "grad_norm": 5.422091960906982, "learning_rate": 2.770452853134593e-06, "loss": 0.33029043674468994, "mean_token_accuracy": 0.8901268295943737, "num_tokens": 4486391.0, "step": 1530 }, { "entropy": 1.6774092495441437, "epoch": 3.8507976227713483, "grad_norm": 11.819265365600586, "learning_rate": 2.6595235942430044e-06, "loss": 0.33706488609313967, "mean_token_accuracy": 0.9124666847288608, "num_tokens": 4515681.0, "step": 1540 }, { "entropy": 1.6730435192584991, "epoch": 3.875821082264623, "grad_norm": 24.53825569152832, "learning_rate": 2.5505192480690865e-06, "loss": 0.37698824405670167, "mean_token_accuracy": 0.8898964198306203, "num_tokens": 4547834.0, "step": 1550 }, { "entropy": 1.6430152557790279, "epoch": 3.9008445417578983, "grad_norm": 6.23613166809082, "learning_rate": 2.4434683991134476e-06, "loss": 0.46486830711364746, "mean_token_accuracy": 0.9119167998433113, "num_tokens": 4578695.0, "step": 1560 }, { "entropy": 1.6305080100893974, "epoch": 3.925868001251173, "grad_norm": 4.144184589385986, "learning_rate": 2.3383991196058918e-06, "loss": 0.24737279415130614, "mean_token_accuracy": 0.9135037533938885, "num_tokens": 4608040.0, "step": 1570 }, { "entropy": 1.6553510926663875, "epoch": 3.950891460744448, "grad_norm": 16.744169235229492, "learning_rate": 2.23533896214399e-06, "loss": 0.5578951358795166, "mean_token_accuracy": 0.864304494485259, "num_tokens": 4637749.0, "step": 1580 }, { "entropy": 1.6376118659973145, "epoch": 3.975914920237723, "grad_norm": 10.743280410766602, "learning_rate": 2.134314952467873e-06, "loss": 0.4247287273406982, "mean_token_accuracy": 0.9346457831561565, "num_tokens": 4667740.0, "step": 1590 }, { "entropy": 1.651349587873979, "epoch": 4.0, "grad_norm": 12.263242721557617, "learning_rate": 2.0353535823732053e-06, "loss": 0.31544477939605714, "mean_token_accuracy": 0.9375330341326726, "num_tokens": 4690280.0, "step": 1600 }, { "epoch": 4.0, "eval_entropy": 1.6643165587447584, "eval_loss": 1.0606985092163086, "eval_mean_token_accuracy": 0.8001990197226405, "eval_num_tokens": 4690280.0, "eval_runtime": 26.1584, "eval_samples_per_second": 19.573, "eval_steps_per_second": 9.787, "step": 1600 }, { "entropy": 1.7042509287595748, "epoch": 4.025023459493275, "grad_norm": 7.552699089050293, "learning_rate": 1.9384808027641666e-06, "loss": 0.36326165199279786, "mean_token_accuracy": 0.9143396973609924, "num_tokens": 4717742.0, "step": 1610 }, { "entropy": 1.6511176168918609, "epoch": 4.05004691898655, "grad_norm": 3.7373321056365967, "learning_rate": 1.8437220168482839e-06, "loss": 0.30488340854644774, "mean_token_accuracy": 0.9318673349916935, "num_tokens": 4743049.0, "step": 1620 }, { "entropy": 1.6843073666095734, "epoch": 4.075070378479825, "grad_norm": 13.701045989990234, "learning_rate": 1.751102073474873e-06, "loss": 0.32843029499053955, "mean_token_accuracy": 0.916875434666872, "num_tokens": 4768941.0, "step": 1630 }, { "entropy": 1.61899134516716, "epoch": 4.1000938379731, "grad_norm": 7.880395412445068, "learning_rate": 1.660645260618864e-06, "loss": 0.27330944538116453, "mean_token_accuracy": 0.9596377186477184, "num_tokens": 4795326.0, "step": 1640 }, { "entropy": 1.6763562865555286, "epoch": 4.125117297466375, "grad_norm": 12.53801441192627, "learning_rate": 1.5723752990116948e-06, "loss": 0.4172815322875977, "mean_token_accuracy": 0.941043658182025, "num_tokens": 4822845.0, "step": 1650 }, { "entropy": 1.6501429066061974, "epoch": 4.15014075695965, "grad_norm": 6.694977283477783, "learning_rate": 1.4863153359209693e-06, "loss": 0.2577322006225586, "mean_token_accuracy": 0.9485772825777531, "num_tokens": 4851779.0, "step": 1660 }, { "entropy": 1.5867636039853097, "epoch": 4.175164216452925, "grad_norm": 5.9964399337768555, "learning_rate": 1.402487939080479e-06, "loss": 0.1994820237159729, "mean_token_accuracy": 0.948756018280983, "num_tokens": 4883534.0, "step": 1670 }, { "entropy": 1.5995635233819485, "epoch": 4.2001876759462, "grad_norm": 11.450733184814453, "learning_rate": 1.3209150907722124e-06, "loss": 0.33185758590698244, "mean_token_accuracy": 0.9658055681735277, "num_tokens": 4911765.0, "step": 1680 }, { "entropy": 1.5724807053804397, "epoch": 4.225211135439475, "grad_norm": 5.831696510314941, "learning_rate": 1.2416181820618745e-06, "loss": 0.21310560703277587, "mean_token_accuracy": 0.9530413594096899, "num_tokens": 4936187.0, "step": 1690 }, { "entropy": 1.578537244349718, "epoch": 4.250234594932749, "grad_norm": 12.092418670654297, "learning_rate": 1.1646180071894608e-06, "loss": 0.2562295913696289, "mean_token_accuracy": 0.908930304646492, "num_tokens": 4963304.0, "step": 1700 }, { "epoch": 4.250234594932749, "eval_entropy": 1.5953729255124927, "eval_loss": 1.187790870666504, "eval_mean_token_accuracy": 0.794221238233149, "eval_num_tokens": 4963304.0, "eval_runtime": 25.9026, "eval_samples_per_second": 19.766, "eval_steps_per_second": 9.883, "step": 1700 }, { "entropy": 1.5515916496515274, "epoch": 4.275258054426025, "grad_norm": 13.895337104797363, "learning_rate": 1.0899347581163222e-06, "loss": 0.22003817558288574, "mean_token_accuracy": 0.966774944961071, "num_tokens": 4992177.0, "step": 1710 }, { "entropy": 1.5425172574818133, "epoch": 4.300281513919299, "grad_norm": 4.921901226043701, "learning_rate": 1.0175880192301713e-06, "loss": 0.21641993522644043, "mean_token_accuracy": 0.9237028583884239, "num_tokens": 5021461.0, "step": 1720 }, { "entropy": 1.5836301006376743, "epoch": 4.325304973412575, "grad_norm": 8.30781078338623, "learning_rate": 9.475967622094207e-07, "loss": 0.3954111337661743, "mean_token_accuracy": 0.9054292887449265, "num_tokens": 5051436.0, "step": 1730 }, { "entropy": 1.6088024839758872, "epoch": 4.350328432905849, "grad_norm": 10.216887474060059, "learning_rate": 8.799793410481871e-07, "loss": 0.3576494693756104, "mean_token_accuracy": 0.8998017378151417, "num_tokens": 5079060.0, "step": 1740 }, { "entropy": 1.5869264729321002, "epoch": 4.375351892399125, "grad_norm": 20.037282943725586, "learning_rate": 8.147534872432761e-07, "loss": 0.2675585985183716, "mean_token_accuracy": 0.9245493724942208, "num_tokens": 5113633.0, "step": 1750 }, { "entropy": 1.5867690846323967, "epoch": 4.400375351892399, "grad_norm": 7.718106746673584, "learning_rate": 7.519363051443996e-07, "loss": 0.25213844776153566, "mean_token_accuracy": 0.9569812349975109, "num_tokens": 5146820.0, "step": 1760 }, { "entropy": 1.5456994101405144, "epoch": 4.425398811385674, "grad_norm": 6.885993003845215, "learning_rate": 6.915442674688633e-07, "loss": 0.269787335395813, "mean_token_accuracy": 0.9462529934942723, "num_tokens": 5180110.0, "step": 1770 }, { "entropy": 1.5553630605340003, "epoch": 4.450422270878949, "grad_norm": 6.1017866134643555, "learning_rate": 6.335932109818754e-07, "loss": 0.19417457580566405, "mean_token_accuracy": 0.9338026508688927, "num_tokens": 5209095.0, "step": 1780 }, { "entropy": 1.558656318485737, "epoch": 4.475445730372224, "grad_norm": 10.72335433959961, "learning_rate": 5.780983323436374e-07, "loss": 0.19903587102890014, "mean_token_accuracy": 0.9535072252154351, "num_tokens": 5247292.0, "step": 1790 }, { "entropy": 1.617298749089241, "epoch": 4.500469189865499, "grad_norm": 22.27625274658203, "learning_rate": 5.250741841242735e-07, "loss": 0.34345569610595705, "mean_token_accuracy": 0.9497337996959686, "num_tokens": 5275299.0, "step": 1800 }, { "epoch": 4.500469189865499, "eval_entropy": 1.5757260229438543, "eval_loss": 1.227073311805725, "eval_mean_token_accuracy": 0.7966270901961252, "eval_num_tokens": 5275299.0, "eval_runtime": 25.953, "eval_samples_per_second": 19.728, "eval_steps_per_second": 9.864, "step": 1800 }, { "entropy": 1.5715236023068428, "epoch": 4.525492649358774, "grad_norm": 9.199799537658691, "learning_rate": 4.745346709876786e-07, "loss": 0.5030104160308838, "mean_token_accuracy": 0.9179459355771542, "num_tokens": 5304526.0, "step": 1810 }, { "entropy": 1.5571273937821388, "epoch": 4.550516108852049, "grad_norm": 6.441845417022705, "learning_rate": 4.26493046045261e-07, "loss": 0.2179567813873291, "mean_token_accuracy": 0.9635193780064583, "num_tokens": 5332333.0, "step": 1820 }, { "entropy": 1.509796992689371, "epoch": 4.575539568345324, "grad_norm": 6.927135944366455, "learning_rate": 3.8096190738053815e-07, "loss": 0.29149680137634276, "mean_token_accuracy": 0.9451167620718479, "num_tokens": 5358537.0, "step": 1830 }, { "entropy": 1.5602086365222931, "epoch": 4.600563027838598, "grad_norm": 9.667234420776367, "learning_rate": 3.379531947455128e-07, "loss": 0.3328777551651001, "mean_token_accuracy": 0.9238914847373962, "num_tokens": 5386183.0, "step": 1840 }, { "entropy": 1.6346124820411205, "epoch": 4.625586487331874, "grad_norm": 5.629216194152832, "learning_rate": 2.974781864296783e-07, "loss": 0.32338910102844237, "mean_token_accuracy": 0.9202292047441005, "num_tokens": 5413879.0, "step": 1850 }, { "entropy": 1.5475087754428387, "epoch": 4.650609946825148, "grad_norm": 8.701130867004395, "learning_rate": 2.5954749630248355e-07, "loss": 0.21816930770874024, "mean_token_accuracy": 0.9383991964161396, "num_tokens": 5441818.0, "step": 1860 }, { "entropy": 1.5626422986388206, "epoch": 4.675633406318424, "grad_norm": 7.034233570098877, "learning_rate": 2.24171071030026e-07, "loss": 0.2586040496826172, "mean_token_accuracy": 0.9426060438156127, "num_tokens": 5470335.0, "step": 1870 }, { "entropy": 1.5956231564283372, "epoch": 4.700656865811698, "grad_norm": 5.224846839904785, "learning_rate": 1.9135818746671587e-07, "loss": 0.28032145500183103, "mean_token_accuracy": 0.9485361717641354, "num_tokens": 5502253.0, "step": 1880 }, { "entropy": 1.6720745049417018, "epoch": 4.725680325304974, "grad_norm": 8.012434959411621, "learning_rate": 1.6111745022257873e-07, "loss": 0.25192699432373045, "mean_token_accuracy": 0.9423566825687886, "num_tokens": 5526629.0, "step": 1890 }, { "entropy": 1.5510374516248704, "epoch": 4.750703784798248, "grad_norm": 23.2949161529541, "learning_rate": 1.3345678940684615e-07, "loss": 0.3542658805847168, "mean_token_accuracy": 0.9370437204837799, "num_tokens": 5555020.0, "step": 1900 }, { "epoch": 4.750703784798248, "eval_entropy": 1.5751400967128575, "eval_loss": 1.221580982208252, "eval_mean_token_accuracy": 0.7978551599662751, "eval_num_tokens": 5555020.0, "eval_runtime": 25.9514, "eval_samples_per_second": 19.729, "eval_steps_per_second": 9.865, "step": 1900 }, { "entropy": 1.5750564984977244, "epoch": 4.775727244291524, "grad_norm": 6.1790971755981445, "learning_rate": 1.0838345854842447e-07, "loss": 0.29637510776519777, "mean_token_accuracy": 0.9434729963541031, "num_tokens": 5585024.0, "step": 1910 }, { "entropy": 1.5804114930331707, "epoch": 4.800750703784798, "grad_norm": 8.236255645751953, "learning_rate": 8.590403269377656e-08, "loss": 0.261922812461853, "mean_token_accuracy": 0.944847946614027, "num_tokens": 5612458.0, "step": 1920 }, { "entropy": 1.5763019055128098, "epoch": 4.825774163278073, "grad_norm": 11.511734008789062, "learning_rate": 6.602440668273758e-08, "loss": 0.18001055717468262, "mean_token_accuracy": 0.9452268406748772, "num_tokens": 5641935.0, "step": 1930 }, { "entropy": 1.5883605182170868, "epoch": 4.850797622771348, "grad_norm": 21.67671012878418, "learning_rate": 4.874979360268928e-08, "loss": 0.3848047494888306, "mean_token_accuracy": 0.914675597846508, "num_tokens": 5672477.0, "step": 1940 }, { "entropy": 1.5379426710307598, "epoch": 4.875821082264623, "grad_norm": 11.815309524536133, "learning_rate": 3.408472342152136e-08, "loss": 0.1684113025665283, "mean_token_accuracy": 0.9466052651405334, "num_tokens": 5703555.0, "step": 1950 }, { "entropy": 1.581675297766924, "epoch": 4.900844541757898, "grad_norm": 6.978868007659912, "learning_rate": 2.2033041799723877e-08, "loss": 0.190066134929657, "mean_token_accuracy": 0.9330016218125821, "num_tokens": 5733510.0, "step": 1960 }, { "entropy": 1.5166504696011542, "epoch": 4.925868001251173, "grad_norm": 5.1840009689331055, "learning_rate": 1.2597909081931702e-08, "loss": 0.3155841588973999, "mean_token_accuracy": 0.931412010639906, "num_tokens": 5762332.0, "step": 1970 }, { "entropy": 1.5898376658558846, "epoch": 4.950891460744448, "grad_norm": 5.622488498687744, "learning_rate": 5.781799468177473e-09, "loss": 0.3049920558929443, "mean_token_accuracy": 0.910175010561943, "num_tokens": 5792982.0, "step": 1980 }, { "entropy": 1.5722950287163258, "epoch": 4.975914920237723, "grad_norm": 9.232101440429688, "learning_rate": 1.5865003650761268e-09, "loss": 0.17625534534454346, "mean_token_accuracy": 0.9326556093990803, "num_tokens": 5828000.0, "step": 1990 }, { "entropy": 1.6074273145044005, "epoch": 5.0, "grad_norm": 2.723400354385376, "learning_rate": 1.311191710651194e-11, "loss": 0.2906452417373657, "mean_token_accuracy": 0.9403998186061908, "num_tokens": 5862850.0, "step": 2000 }, { "epoch": 5.0, "eval_entropy": 1.5754532138817012, "eval_loss": 1.2249126434326172, "eval_mean_token_accuracy": 0.7969964473741129, "eval_num_tokens": 5862850.0, "eval_runtime": 25.9681, "eval_samples_per_second": 19.716, "eval_steps_per_second": 9.858, "step": 2000 }, { "epoch": 5.0, "step": 2000, "total_flos": 2.649974039752704e+17, "train_loss": 0.5825153150558472, "train_runtime": 3830.0191, "train_samples_per_second": 4.174, "train_steps_per_second": 0.522 } ], "logging_steps": 10, "max_steps": 2000, "num_input_tokens_seen": 0, "num_train_epochs": 5, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 2.649974039752704e+17, "train_batch_size": 1, "trial_name": null, "trial_params": null }