| { |
| "best_global_step": 700, |
| "best_metric": 0.8686603307723999, |
| "best_model_checkpoint": "/data6/jongbinwon/evolving_MoE/checkpoints/sft_llama31_8b_sni_ours-c_13393_ffn_only_5ep/checkpoint-700", |
| "epoch": 5.0, |
| "eval_steps": 100, |
| "global_step": 2000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.723603368550539, |
| "epoch": 0.025023459493274947, |
| "grad_norm": 8.617183685302734, |
| "learning_rate": 3e-06, |
| "loss": 1.8632482528686523, |
| "mean_token_accuracy": 0.45814704075455664, |
| "num_tokens": 26675.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.6786307826638223, |
| "epoch": 0.05004691898654989, |
| "grad_norm": 5.2456488609313965, |
| "learning_rate": 6.333333333333333e-06, |
| "loss": 1.7982595443725586, |
| "mean_token_accuracy": 0.48966174516826866, |
| "num_tokens": 51872.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 1.7768761590123177, |
| "epoch": 0.07507037847982484, |
| "grad_norm": 4.166147708892822, |
| "learning_rate": 9.666666666666667e-06, |
| "loss": 2.042780876159668, |
| "mean_token_accuracy": 0.5317133395001292, |
| "num_tokens": 85745.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 1.9048189118504524, |
| "epoch": 0.10009383797309979, |
| "grad_norm": 4.267604351043701, |
| "learning_rate": 1.3000000000000001e-05, |
| "loss": 1.585220718383789, |
| "mean_token_accuracy": 0.6397202219814062, |
| "num_tokens": 118442.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 2.0537888914346696, |
| "epoch": 0.12511729746637473, |
| "grad_norm": 9.502376556396484, |
| "learning_rate": 1.6333333333333335e-05, |
| "loss": 1.1469281196594239, |
| "mean_token_accuracy": 0.6897181114181876, |
| "num_tokens": 147201.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 2.2640592947602274, |
| "epoch": 0.15014075695964968, |
| "grad_norm": 2.122598171234131, |
| "learning_rate": 1.9666666666666666e-05, |
| "loss": 1.0125106811523437, |
| "mean_token_accuracy": 0.7641670696437359, |
| "num_tokens": 170569.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 2.1425670742988587, |
| "epoch": 0.17516421645292463, |
| "grad_norm": 1.710357666015625, |
| "learning_rate": 1.999893795328188e-05, |
| "loss": 1.1022185325622558, |
| "mean_token_accuracy": 0.7504967793822288, |
| "num_tokens": 198185.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 2.247272165119648, |
| "epoch": 0.20018767594619957, |
| "grad_norm": 2.6029555797576904, |
| "learning_rate": 1.9995266970296856e-05, |
| "loss": 1.1327926635742187, |
| "mean_token_accuracy": 0.7273309765383601, |
| "num_tokens": 221480.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 2.2861083179712294, |
| "epoch": 0.22521113543947452, |
| "grad_norm": 2.214897632598877, |
| "learning_rate": 1.9988974901779482e-05, |
| "loss": 1.0936859130859375, |
| "mean_token_accuracy": 0.744602588750422, |
| "num_tokens": 244772.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 2.250556546449661, |
| "epoch": 0.25023459493274947, |
| "grad_norm": 2.8186545372009277, |
| "learning_rate": 1.9980063397715685e-05, |
| "loss": 0.89981050491333, |
| "mean_token_accuracy": 0.714475267007947, |
| "num_tokens": 273805.0, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.25023459493274947, |
| "eval_entropy": 2.2342376695014536, |
| "eval_loss": 0.9610199332237244, |
| "eval_mean_token_accuracy": 0.7704979736590758, |
| "eval_num_tokens": 273805.0, |
| "eval_runtime": 25.919, |
| "eval_samples_per_second": 19.754, |
| "eval_steps_per_second": 9.877, |
| "step": 100 |
| }, |
| { |
| "entropy": 2.279540453851223, |
| "epoch": 0.2752580544260244, |
| "grad_norm": 5.038643836975098, |
| "learning_rate": 1.9968534794992947e-05, |
| "loss": 0.9877372741699219, |
| "mean_token_accuracy": 0.7308646870777011, |
| "num_tokens": 302872.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 2.119027265906334, |
| "epoch": 0.30028151391929936, |
| "grad_norm": 4.737979412078857, |
| "learning_rate": 1.995439211678754e-05, |
| "loss": 0.853663444519043, |
| "mean_token_accuracy": 0.76122229360044, |
| "num_tokens": 330737.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 2.1482249490916727, |
| "epoch": 0.3253049734125743, |
| "grad_norm": 2.8202335834503174, |
| "learning_rate": 1.9937639071771704e-05, |
| "loss": 0.889527416229248, |
| "mean_token_accuracy": 0.6904077736660839, |
| "num_tokens": 362657.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 2.179778201878071, |
| "epoch": 0.35032843290584925, |
| "grad_norm": 4.295614719390869, |
| "learning_rate": 1.9918280053141144e-05, |
| "loss": 1.098190689086914, |
| "mean_token_accuracy": 0.6943824682384729, |
| "num_tokens": 390971.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 2.1700742825865746, |
| "epoch": 0.3753518923991242, |
| "grad_norm": 2.5075595378875732, |
| "learning_rate": 1.9896320137462984e-05, |
| "loss": 1.0178719520568849, |
| "mean_token_accuracy": 0.7038579940795898, |
| "num_tokens": 417595.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 2.135371221601963, |
| "epoch": 0.40037535189239915, |
| "grad_norm": 3.29361629486084, |
| "learning_rate": 1.987176508334451e-05, |
| "loss": 0.8292133331298828, |
| "mean_token_accuracy": 0.756744677759707, |
| "num_tokens": 442918.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 2.1454797983169556, |
| "epoch": 0.4253988113856741, |
| "grad_norm": 2.2850234508514404, |
| "learning_rate": 1.98446213299231e-05, |
| "loss": 0.7698661804199218, |
| "mean_token_accuracy": 0.7885082781314849, |
| "num_tokens": 477921.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 2.17876655459404, |
| "epoch": 0.45042227087894904, |
| "grad_norm": 1.6584603786468506, |
| "learning_rate": 1.9814895995177653e-05, |
| "loss": 0.9373016357421875, |
| "mean_token_accuracy": 0.7417640507221221, |
| "num_tokens": 504910.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 2.213928133249283, |
| "epoch": 0.475445730372224, |
| "grad_norm": 3.5426175594329834, |
| "learning_rate": 1.9782596874062028e-05, |
| "loss": 0.9217703819274903, |
| "mean_token_accuracy": 0.7309052364900708, |
| "num_tokens": 531010.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 2.079026885330677, |
| "epoch": 0.5004691898654989, |
| "grad_norm": 3.355250358581543, |
| "learning_rate": 1.9747732436460955e-05, |
| "loss": 0.808747673034668, |
| "mean_token_accuracy": 0.7717655746266245, |
| "num_tokens": 561639.0, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.5004691898654989, |
| "eval_entropy": 2.1456044730730355, |
| "eval_loss": 0.9099652767181396, |
| "eval_mean_token_accuracy": 0.7726191087858751, |
| "eval_num_tokens": 561639.0, |
| "eval_runtime": 25.922, |
| "eval_samples_per_second": 19.752, |
| "eval_steps_per_second": 9.876, |
| "step": 200 |
| }, |
| { |
| "entropy": 2.1017219856381417, |
| "epoch": 0.5254926493587738, |
| "grad_norm": 2.787653684616089, |
| "learning_rate": 1.9710311824968942e-05, |
| "loss": 0.8461710929870605, |
| "mean_token_accuracy": 0.7469903081655502, |
| "num_tokens": 589942.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 2.134711952507496, |
| "epoch": 0.5505161088520488, |
| "grad_norm": 3.67427921295166, |
| "learning_rate": 1.9670344852492814e-05, |
| "loss": 0.7912054538726807, |
| "mean_token_accuracy": 0.7155324574559927, |
| "num_tokens": 617432.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 2.1465815491974354, |
| "epoch": 0.5755395683453237, |
| "grad_norm": 5.175668239593506, |
| "learning_rate": 1.9627841999678422e-05, |
| "loss": 0.9203764915466308, |
| "mean_token_accuracy": 0.7518326181918382, |
| "num_tokens": 651978.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 2.257516998052597, |
| "epoch": 0.6005630278385987, |
| "grad_norm": 8.140710830688477, |
| "learning_rate": 1.9582814412162288e-05, |
| "loss": 0.9473254203796386, |
| "mean_token_accuracy": 0.7131743170320988, |
| "num_tokens": 687677.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 2.1520414032042026, |
| "epoch": 0.6255864873318736, |
| "grad_norm": 8.945752143859863, |
| "learning_rate": 1.9535273897648857e-05, |
| "loss": 0.9689723968505859, |
| "mean_token_accuracy": 0.7218846149742604, |
| "num_tokens": 717302.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 2.2178710743784906, |
| "epoch": 0.6506099468251486, |
| "grad_norm": 2.2079780101776123, |
| "learning_rate": 1.9485232922814117e-05, |
| "loss": 0.6868520736694336, |
| "mean_token_accuracy": 0.750518599525094, |
| "num_tokens": 749747.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 2.233424980938435, |
| "epoch": 0.6756334063184235, |
| "grad_norm": 2.361921548843384, |
| "learning_rate": 1.9432704610036448e-05, |
| "loss": 0.9331287384033203, |
| "mean_token_accuracy": 0.7324752386659383, |
| "num_tokens": 780201.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 2.2572396978735925, |
| "epoch": 0.7006568658116985, |
| "grad_norm": 8.503108024597168, |
| "learning_rate": 1.9377702733955493e-05, |
| "loss": 0.9354991912841797, |
| "mean_token_accuracy": 0.7744514495134354, |
| "num_tokens": 809852.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 2.2340517044067383, |
| "epoch": 0.7256803253049734, |
| "grad_norm": 2.9905447959899902, |
| "learning_rate": 1.9320241717860007e-05, |
| "loss": 0.821927833557129, |
| "mean_token_accuracy": 0.7892053715884686, |
| "num_tokens": 839080.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 2.1411201044917108, |
| "epoch": 0.7507037847982484, |
| "grad_norm": 4.045820236206055, |
| "learning_rate": 1.926033662990558e-05, |
| "loss": 1.0363512992858888, |
| "mean_token_accuracy": 0.695093622803688, |
| "num_tokens": 872158.0, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.7507037847982484, |
| "eval_entropy": 2.1794460150413215, |
| "eval_loss": 0.8942313194274902, |
| "eval_mean_token_accuracy": 0.7791136712767184, |
| "eval_num_tokens": 872158.0, |
| "eval_runtime": 25.9051, |
| "eval_samples_per_second": 19.764, |
| "eval_steps_per_second": 9.882, |
| "step": 300 |
| }, |
| { |
| "entropy": 2.2423775270581245, |
| "epoch": 0.7757272442915233, |
| "grad_norm": 2.8176357746124268, |
| "learning_rate": 1.9198003179163308e-05, |
| "loss": 0.9462035179138184, |
| "mean_token_accuracy": 0.7392122287303209, |
| "num_tokens": 900576.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 2.2096558839082716, |
| "epoch": 0.8007507037847983, |
| "grad_norm": 6.080548286437988, |
| "learning_rate": 1.9133257711500318e-05, |
| "loss": 0.833648681640625, |
| "mean_token_accuracy": 0.7921354983001947, |
| "num_tokens": 941726.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 2.186213733255863, |
| "epoch": 0.8257741632780732, |
| "grad_norm": 2.5898144245147705, |
| "learning_rate": 1.9066117205293393e-05, |
| "loss": 0.7522900581359864, |
| "mean_token_accuracy": 0.7382632024586201, |
| "num_tokens": 970413.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 2.2108209684491156, |
| "epoch": 0.8507976227713482, |
| "grad_norm": 4.91647481918335, |
| "learning_rate": 1.8996599266976658e-05, |
| "loss": 0.7787142753601074, |
| "mean_token_accuracy": 0.7524209380149841, |
| "num_tokens": 993790.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 2.1969844341278075, |
| "epoch": 0.8758210822646231, |
| "grad_norm": 6.000455379486084, |
| "learning_rate": 1.892472212642459e-05, |
| "loss": 0.8661828994750976, |
| "mean_token_accuracy": 0.7468080155551433, |
| "num_tokens": 1026126.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 2.1586176477372647, |
| "epoch": 0.9008445417578981, |
| "grad_norm": 3.4196536540985107, |
| "learning_rate": 1.885050463217159e-05, |
| "loss": 0.8391610145568847, |
| "mean_token_accuracy": 0.7550970822572708, |
| "num_tokens": 1055053.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 2.106422890722752, |
| "epoch": 0.925868001251173, |
| "grad_norm": 2.27886962890625, |
| "learning_rate": 1.8773966246469238e-05, |
| "loss": 0.7569664478302002, |
| "mean_token_accuracy": 0.7327868092805148, |
| "num_tokens": 1086499.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 2.150851938128471, |
| "epoch": 0.950891460744448, |
| "grad_norm": 2.702540159225464, |
| "learning_rate": 1.8695127040182678e-05, |
| "loss": 1.048074722290039, |
| "mean_token_accuracy": 0.7364495120942592, |
| "num_tokens": 1118377.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 2.207886445522308, |
| "epoch": 0.9759149202377229, |
| "grad_norm": 6.179843902587891, |
| "learning_rate": 1.8614007687527374e-05, |
| "loss": 0.9187170982360839, |
| "mean_token_accuracy": 0.7189946949481965, |
| "num_tokens": 1146754.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 2.1371198025616733, |
| "epoch": 1.0, |
| "grad_norm": 5.99985933303833, |
| "learning_rate": 1.8530629460647658e-05, |
| "loss": 0.6151233673095703, |
| "mean_token_accuracy": 0.7796741042818341, |
| "num_tokens": 1172570.0, |
| "step": 400 |
| }, |
| { |
| "epoch": 1.0, |
| "eval_entropy": 2.1289648609235883, |
| "eval_loss": 0.8714665174484253, |
| "eval_mean_token_accuracy": 0.7819707952439785, |
| "eval_num_tokens": 1172570.0, |
| "eval_runtime": 25.9103, |
| "eval_samples_per_second": 19.76, |
| "eval_steps_per_second": 9.88, |
| "step": 400 |
| }, |
| { |
| "entropy": 2.098095750808716, |
| "epoch": 1.025023459493275, |
| "grad_norm": 2.6623897552490234, |
| "learning_rate": 1.8445014224038485e-05, |
| "loss": 0.6235037803649902, |
| "mean_token_accuracy": 0.823400478810072, |
| "num_tokens": 1201801.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 2.1191157147288324, |
| "epoch": 1.05004691898655, |
| "grad_norm": 4.902485370635986, |
| "learning_rate": 1.835718442881183e-05, |
| "loss": 0.8132280349731446, |
| "mean_token_accuracy": 0.7433615382760763, |
| "num_tokens": 1229848.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 2.057385340332985, |
| "epoch": 1.0750703784798248, |
| "grad_norm": 10.126677513122559, |
| "learning_rate": 1.8267163106809288e-05, |
| "loss": 0.8311259269714355, |
| "mean_token_accuracy": 0.7856612842530012, |
| "num_tokens": 1256935.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 2.050567016005516, |
| "epoch": 1.1000938379730998, |
| "grad_norm": 3.812150478363037, |
| "learning_rate": 1.817497386456238e-05, |
| "loss": 0.7683423042297364, |
| "mean_token_accuracy": 0.7796268314123154, |
| "num_tokens": 1286284.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 2.1574359863996504, |
| "epoch": 1.1251172974663748, |
| "grad_norm": 3.055018186569214, |
| "learning_rate": 1.808064087710212e-05, |
| "loss": 0.6059474468231201, |
| "mean_token_accuracy": 0.8015186853706837, |
| "num_tokens": 1312395.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 2.0954012736678123, |
| "epoch": 1.1501407569596496, |
| "grad_norm": 1.8690502643585205, |
| "learning_rate": 1.7984188881619563e-05, |
| "loss": 0.7081769943237305, |
| "mean_token_accuracy": 0.8020102433860302, |
| "num_tokens": 1342341.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 2.080510801076889, |
| "epoch": 1.1751642164529246, |
| "grad_norm": 5.163702964782715, |
| "learning_rate": 1.788564317097889e-05, |
| "loss": 0.6229144096374511, |
| "mean_token_accuracy": 0.8259935293346643, |
| "num_tokens": 1368026.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 2.0373916894197466, |
| "epoch": 1.2001876759461996, |
| "grad_norm": 3.3629536628723145, |
| "learning_rate": 1.7785029587084793e-05, |
| "loss": 0.7988658905029297, |
| "mean_token_accuracy": 0.836330172047019, |
| "num_tokens": 1408554.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 2.0612950578331946, |
| "epoch": 1.2252111354394746, |
| "grad_norm": 2.9767253398895264, |
| "learning_rate": 1.768237451410589e-05, |
| "loss": 0.8266899108886718, |
| "mean_token_accuracy": 0.8041692972183228, |
| "num_tokens": 1433615.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 2.1810658618807794, |
| "epoch": 1.2502345949327496, |
| "grad_norm": 8.24252700805664, |
| "learning_rate": 1.7577704871555924e-05, |
| "loss": 0.75057373046875, |
| "mean_token_accuracy": 0.7435132082551718, |
| "num_tokens": 1461697.0, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.2502345949327496, |
| "eval_entropy": 2.0794698572717607, |
| "eval_loss": 0.8950290679931641, |
| "eval_mean_token_accuracy": 0.7889105979120359, |
| "eval_num_tokens": 1461697.0, |
| "eval_runtime": 25.9129, |
| "eval_samples_per_second": 19.758, |
| "eval_steps_per_second": 9.879, |
| "step": 500 |
| }, |
| { |
| "entropy": 1.9847739323973657, |
| "epoch": 1.2752580544260244, |
| "grad_norm": 3.5981249809265137, |
| "learning_rate": 1.74710481072346e-05, |
| "loss": 0.6473482131958008, |
| "mean_token_accuracy": 0.8046811826527118, |
| "num_tokens": 1492119.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 2.144796669483185, |
| "epoch": 1.3002815139192994, |
| "grad_norm": 10.027125358581543, |
| "learning_rate": 1.7362432190029862e-05, |
| "loss": 1.0633076667785644, |
| "mean_token_accuracy": 0.7895992513746023, |
| "num_tokens": 1518884.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 2.1153031289577484, |
| "epoch": 1.3253049734125744, |
| "grad_norm": 5.87807035446167, |
| "learning_rate": 1.7251885602583547e-05, |
| "loss": 0.7672069549560547, |
| "mean_token_accuracy": 0.8115054946392775, |
| "num_tokens": 1547318.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 2.1572294265031813, |
| "epoch": 1.3503284329058491, |
| "grad_norm": 8.224700927734375, |
| "learning_rate": 1.7139437333822303e-05, |
| "loss": 0.7436501502990722, |
| "mean_token_accuracy": 0.8101725693792104, |
| "num_tokens": 1576089.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 2.138009563088417, |
| "epoch": 1.3753518923991241, |
| "grad_norm": 2.2141480445861816, |
| "learning_rate": 1.7025116871355737e-05, |
| "loss": 0.654974365234375, |
| "mean_token_accuracy": 0.7730139032006264, |
| "num_tokens": 1606732.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 2.139923092722893, |
| "epoch": 1.4003753518923991, |
| "grad_norm": 5.900820732116699, |
| "learning_rate": 1.6908954193743816e-05, |
| "loss": 0.7841741561889648, |
| "mean_token_accuracy": 0.7526014145463705, |
| "num_tokens": 1639646.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 2.1373246192932127, |
| "epoch": 1.4253988113856741, |
| "grad_norm": 6.760275840759277, |
| "learning_rate": 1.6790979762635497e-05, |
| "loss": 0.7275867462158203, |
| "mean_token_accuracy": 0.7795548062771559, |
| "num_tokens": 1668659.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 2.187282508611679, |
| "epoch": 1.4504222708789491, |
| "grad_norm": 4.651549339294434, |
| "learning_rate": 1.6671224514780692e-05, |
| "loss": 0.9761672019958496, |
| "mean_token_accuracy": 0.7901028756052255, |
| "num_tokens": 1695170.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 2.097830060124397, |
| "epoch": 1.475445730372224, |
| "grad_norm": 3.098156690597534, |
| "learning_rate": 1.654971985391764e-05, |
| "loss": 0.7796518802642822, |
| "mean_token_accuracy": 0.8037835378199816, |
| "num_tokens": 1721326.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 2.190341380238533, |
| "epoch": 1.500469189865499, |
| "grad_norm": 2.757059097290039, |
| "learning_rate": 1.6426497642537826e-05, |
| "loss": 0.648505973815918, |
| "mean_token_accuracy": 0.824278862029314, |
| "num_tokens": 1748835.0, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.500469189865499, |
| "eval_entropy": 2.1228315108455718, |
| "eval_loss": 0.8891530632972717, |
| "eval_mean_token_accuracy": 0.7763976352289319, |
| "eval_num_tokens": 1748835.0, |
| "eval_runtime": 25.9131, |
| "eval_samples_per_second": 19.758, |
| "eval_steps_per_second": 9.879, |
| "step": 600 |
| }, |
| { |
| "entropy": 2.1038011401891707, |
| "epoch": 1.5254926493587737, |
| "grad_norm": 4.028401851654053, |
| "learning_rate": 1.6301590193530585e-05, |
| "loss": 0.7076716423034668, |
| "mean_token_accuracy": 0.792342029325664, |
| "num_tokens": 1781732.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 2.050699570029974, |
| "epoch": 1.5505161088520487, |
| "grad_norm": 2.7474255561828613, |
| "learning_rate": 1.6175030261709615e-05, |
| "loss": 0.8747171401977539, |
| "mean_token_accuracy": 0.7811730474233627, |
| "num_tokens": 1809560.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 2.0647204160690307, |
| "epoch": 1.5755395683453237, |
| "grad_norm": 9.905837059020996, |
| "learning_rate": 1.6046851035223594e-05, |
| "loss": 0.7337132930755615, |
| "mean_token_accuracy": 0.7674034627154469, |
| "num_tokens": 1838965.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 2.054245483875275, |
| "epoch": 1.6005630278385987, |
| "grad_norm": 3.5519328117370605, |
| "learning_rate": 1.591708612685316e-05, |
| "loss": 0.6100962638854981, |
| "mean_token_accuracy": 0.833642303198576, |
| "num_tokens": 1871841.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 2.149427868425846, |
| "epoch": 1.6255864873318737, |
| "grad_norm": 3.8471930027008057, |
| "learning_rate": 1.5785769565196543e-05, |
| "loss": 0.8112316131591797, |
| "mean_token_accuracy": 0.7824445836246013, |
| "num_tokens": 1900058.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 2.034525628387928, |
| "epoch": 1.6506099468251487, |
| "grad_norm": 2.7920093536376953, |
| "learning_rate": 1.565293578574615e-05, |
| "loss": 0.5858467102050782, |
| "mean_token_accuracy": 0.7768620416522026, |
| "num_tokens": 1931656.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 2.1494649052619934, |
| "epoch": 1.6756334063184235, |
| "grad_norm": 3.6777379512786865, |
| "learning_rate": 1.5518619621858474e-05, |
| "loss": 0.6075996398925781, |
| "mean_token_accuracy": 0.8004748497158289, |
| "num_tokens": 1960003.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 2.056756618618965, |
| "epoch": 1.7006568658116985, |
| "grad_norm": 9.437564849853516, |
| "learning_rate": 1.5382856295619622e-05, |
| "loss": 0.6237570762634277, |
| "mean_token_accuracy": 0.7745466388761997, |
| "num_tokens": 1987487.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 1.9909536838531494, |
| "epoch": 1.7256803253049733, |
| "grad_norm": 2.8845739364624023, |
| "learning_rate": 1.5245681408608946e-05, |
| "loss": 0.7010098934173584, |
| "mean_token_accuracy": 0.7885188397020102, |
| "num_tokens": 2018878.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 2.076315422356129, |
| "epoch": 1.7507037847982483, |
| "grad_norm": 6.985860824584961, |
| "learning_rate": 1.5107130932563151e-05, |
| "loss": 0.6411010265350342, |
| "mean_token_accuracy": 0.8081331729888916, |
| "num_tokens": 2042835.0, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.7507037847982483, |
| "eval_entropy": 2.0568552212789655, |
| "eval_loss": 0.8686603307723999, |
| "eval_mean_token_accuracy": 0.797932347166352, |
| "eval_num_tokens": 2042835.0, |
| "eval_runtime": 25.9268, |
| "eval_samples_per_second": 19.748, |
| "eval_steps_per_second": 9.874, |
| "step": 700 |
| }, |
| { |
| "entropy": 2.1089097008109094, |
| "epoch": 1.7757272442915233, |
| "grad_norm": 5.072978496551514, |
| "learning_rate": 1.4967241199943332e-05, |
| "loss": 0.7032583236694336, |
| "mean_token_accuracy": 0.8087547916918993, |
| "num_tokens": 2066598.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 2.0160682946443558, |
| "epoch": 1.8007507037847983, |
| "grad_norm": 3.0282862186431885, |
| "learning_rate": 1.4826048894407396e-05, |
| "loss": 0.7899590492248535, |
| "mean_token_accuracy": 0.7788747299462557, |
| "num_tokens": 2100878.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 2.0212891355156897, |
| "epoch": 1.8257741632780733, |
| "grad_norm": 12.743298530578613, |
| "learning_rate": 1.4683591041190433e-05, |
| "loss": 0.6348478317260742, |
| "mean_token_accuracy": 0.7777544744312763, |
| "num_tokens": 2143751.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 2.032179595530033, |
| "epoch": 1.8507976227713483, |
| "grad_norm": 2.7245559692382812, |
| "learning_rate": 1.4539904997395468e-05, |
| "loss": 1.0501715660095214, |
| "mean_token_accuracy": 0.741650390997529, |
| "num_tokens": 2176229.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 2.078695350885391, |
| "epoch": 1.875821082264623, |
| "grad_norm": 3.2556543350219727, |
| "learning_rate": 1.4395028442197231e-05, |
| "loss": 0.7042528629302979, |
| "mean_token_accuracy": 0.8024522136896849, |
| "num_tokens": 2201679.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 2.094234761595726, |
| "epoch": 1.900844541757898, |
| "grad_norm": 4.391955375671387, |
| "learning_rate": 1.424899936696143e-05, |
| "loss": 0.5421716690063476, |
| "mean_token_accuracy": 0.8527256866917015, |
| "num_tokens": 2230744.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 2.0378435380756854, |
| "epoch": 1.9258680012511729, |
| "grad_norm": 3.724647045135498, |
| "learning_rate": 1.4101856065282174e-05, |
| "loss": 0.6828950405120849, |
| "mean_token_accuracy": 0.7861546307802201, |
| "num_tokens": 2258618.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 2.0710975214838983, |
| "epoch": 1.9508914607444479, |
| "grad_norm": 4.124916076660156, |
| "learning_rate": 1.3953637122940147e-05, |
| "loss": 0.559223222732544, |
| "mean_token_accuracy": 0.8332096721976996, |
| "num_tokens": 2285444.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 1.9646637082099914, |
| "epoch": 1.9759149202377229, |
| "grad_norm": 8.622406959533691, |
| "learning_rate": 1.380438140778416e-05, |
| "loss": 0.5959304332733154, |
| "mean_token_accuracy": 0.7976583287119865, |
| "num_tokens": 2315763.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 1.9505030063839701, |
| "epoch": 2.0, |
| "grad_norm": 5.460987567901611, |
| "learning_rate": 1.365412805953872e-05, |
| "loss": 0.6341047763824463, |
| "mean_token_accuracy": 0.8356712561149102, |
| "num_tokens": 2345140.0, |
| "step": 800 |
| }, |
| { |
| "epoch": 2.0, |
| "eval_entropy": 1.9633747367188334, |
| "eval_loss": 0.8708455562591553, |
| "eval_mean_token_accuracy": 0.7948170631425455, |
| "eval_num_tokens": 2345140.0, |
| "eval_runtime": 25.9388, |
| "eval_samples_per_second": 19.739, |
| "eval_steps_per_second": 9.869, |
| "step": 800 |
| }, |
| { |
| "entropy": 1.9318091489374638, |
| "epoch": 2.025023459493275, |
| "grad_norm": 2.679042339324951, |
| "learning_rate": 1.3502916479540327e-05, |
| "loss": 0.5399890899658203, |
| "mean_token_accuracy": 0.8556223087012768, |
| "num_tokens": 2374724.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 1.9705582335591316, |
| "epoch": 2.05004691898655, |
| "grad_norm": 10.042631149291992, |
| "learning_rate": 1.3350786320405145e-05, |
| "loss": 0.7448606967926026, |
| "mean_token_accuracy": 0.8062794495373964, |
| "num_tokens": 2399771.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 1.9878843411803246, |
| "epoch": 2.075070378479825, |
| "grad_norm": 3.2470123767852783, |
| "learning_rate": 1.31977774756308e-05, |
| "loss": 0.667686939239502, |
| "mean_token_accuracy": 0.8486393585801124, |
| "num_tokens": 2428832.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 1.9709952771663666, |
| "epoch": 2.1000938379731, |
| "grad_norm": 8.831825256347656, |
| "learning_rate": 1.3043930069134998e-05, |
| "loss": 0.5341938018798829, |
| "mean_token_accuracy": 0.8500256590545178, |
| "num_tokens": 2454191.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 1.8401574552059174, |
| "epoch": 2.1251172974663746, |
| "grad_norm": 10.572880744934082, |
| "learning_rate": 1.2889284444733722e-05, |
| "loss": 0.4976643085479736, |
| "mean_token_accuracy": 0.8540774881839752, |
| "num_tokens": 2489898.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 1.8839198268949986, |
| "epoch": 2.1501407569596496, |
| "grad_norm": 10.400339126586914, |
| "learning_rate": 1.273388115556177e-05, |
| "loss": 0.6440447330474853, |
| "mean_token_accuracy": 0.8276639927178622, |
| "num_tokens": 2520146.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 1.905003009736538, |
| "epoch": 2.1751642164529246, |
| "grad_norm": 2.6133148670196533, |
| "learning_rate": 1.2577760953438382e-05, |
| "loss": 0.6256015300750732, |
| "mean_token_accuracy": 0.8756786741316318, |
| "num_tokens": 2552346.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 1.8848004803061484, |
| "epoch": 2.2001876759461996, |
| "grad_norm": 7.504598140716553, |
| "learning_rate": 1.2420964778180815e-05, |
| "loss": 0.49793548583984376, |
| "mean_token_accuracy": 0.8614703625440597, |
| "num_tokens": 2583354.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 1.8399935081601142, |
| "epoch": 2.2252111354394746, |
| "grad_norm": 7.4189910888671875, |
| "learning_rate": 1.2263533746868552e-05, |
| "loss": 0.48105669021606445, |
| "mean_token_accuracy": 0.8231813054531812, |
| "num_tokens": 2610641.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 1.9149093806743622, |
| "epoch": 2.2502345949327496, |
| "grad_norm": 11.7152681350708, |
| "learning_rate": 1.2105509143061072e-05, |
| "loss": 0.6658933162689209, |
| "mean_token_accuracy": 0.8178440190851688, |
| "num_tokens": 2645185.0, |
| "step": 900 |
| }, |
| { |
| "epoch": 2.2502345949327496, |
| "eval_entropy": 1.9004527027718723, |
| "eval_loss": 0.9131948947906494, |
| "eval_mean_token_accuracy": 0.7987493762047961, |
| "eval_num_tokens": 2645185.0, |
| "eval_runtime": 25.9604, |
| "eval_samples_per_second": 19.722, |
| "eval_steps_per_second": 9.861, |
| "step": 900 |
| }, |
| { |
| "entropy": 1.9318789586424827, |
| "epoch": 2.2752580544260246, |
| "grad_norm": 3.531498432159424, |
| "learning_rate": 1.194693240597196e-05, |
| "loss": 0.4017048358917236, |
| "mean_token_accuracy": 0.8562013734132051, |
| "num_tokens": 2671042.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 1.9958116054534911, |
| "epoch": 2.300281513919299, |
| "grad_norm": 7.350344181060791, |
| "learning_rate": 1.1787845119602184e-05, |
| "loss": 0.5403977394104004, |
| "mean_token_accuracy": 0.8568222790956497, |
| "num_tokens": 2697233.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 1.8747393786907196, |
| "epoch": 2.325304973412574, |
| "grad_norm": 4.025563716888428, |
| "learning_rate": 1.1628289001835405e-05, |
| "loss": 0.5700911045074463, |
| "mean_token_accuracy": 0.8201610699295998, |
| "num_tokens": 2726631.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 1.930973158031702, |
| "epoch": 2.350328432905849, |
| "grad_norm": 6.795775413513184, |
| "learning_rate": 1.1468305893498204e-05, |
| "loss": 0.49820170402526853, |
| "mean_token_accuracy": 0.8798530824482441, |
| "num_tokens": 2755100.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 1.897780492901802, |
| "epoch": 2.375351892399124, |
| "grad_norm": 8.053464889526367, |
| "learning_rate": 1.1307937747388034e-05, |
| "loss": 0.6488244533538818, |
| "mean_token_accuracy": 0.8430951170623302, |
| "num_tokens": 2783867.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 1.9086807489395141, |
| "epoch": 2.400375351892399, |
| "grad_norm": 4.615073204040527, |
| "learning_rate": 1.114722661727182e-05, |
| "loss": 0.4686459541320801, |
| "mean_token_accuracy": 0.8605645731091499, |
| "num_tokens": 2811469.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 1.912995307147503, |
| "epoch": 2.425398811385674, |
| "grad_norm": 10.602889060974121, |
| "learning_rate": 1.0986214646858115e-05, |
| "loss": 0.5210050106048584, |
| "mean_token_accuracy": 0.851820009201765, |
| "num_tokens": 2839283.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 1.908480130136013, |
| "epoch": 2.450422270878949, |
| "grad_norm": 9.512876510620117, |
| "learning_rate": 1.0824944058745623e-05, |
| "loss": 0.4891656875610352, |
| "mean_token_accuracy": 0.8372205581516028, |
| "num_tokens": 2867350.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 1.904689647257328, |
| "epoch": 2.475445730372224, |
| "grad_norm": 7.750706195831299, |
| "learning_rate": 1.0663457143351044e-05, |
| "loss": 0.4556453704833984, |
| "mean_token_accuracy": 0.8883680656552315, |
| "num_tokens": 2899406.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 1.8602308124303817, |
| "epoch": 2.500469189865499, |
| "grad_norm": 11.667961120605469, |
| "learning_rate": 1.0501796247819176e-05, |
| "loss": 0.5987341403961182, |
| "mean_token_accuracy": 0.843272651731968, |
| "num_tokens": 2925940.0, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.500469189865499, |
| "eval_entropy": 1.8962222184054554, |
| "eval_loss": 0.9162221550941467, |
| "eval_mean_token_accuracy": 0.7972135632298887, |
| "eval_num_tokens": 2925940.0, |
| "eval_runtime": 25.9027, |
| "eval_samples_per_second": 19.766, |
| "eval_steps_per_second": 9.883, |
| "step": 1000 |
| }, |
| { |
| "entropy": 1.9050709769129752, |
| "epoch": 2.5254926493587737, |
| "grad_norm": 5.943221569061279, |
| "learning_rate": 1.0340003764918078e-05, |
| "loss": 0.6280072212219239, |
| "mean_token_accuracy": 0.8301476046442986, |
| "num_tokens": 2950373.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 1.883858135342598, |
| "epoch": 2.5505161088520487, |
| "grad_norm": 9.269403457641602, |
| "learning_rate": 1.0178122121922324e-05, |
| "loss": 0.7295412063598633, |
| "mean_token_accuracy": 0.8335961733013392, |
| "num_tokens": 2979264.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 1.8532995566725732, |
| "epoch": 2.5755395683453237, |
| "grad_norm": 8.404088020324707, |
| "learning_rate": 1.001619376948718e-05, |
| "loss": 0.42704405784606936, |
| "mean_token_accuracy": 0.88788600564003, |
| "num_tokens": 3006306.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 1.8335328698158264, |
| "epoch": 2.6005630278385987, |
| "grad_norm": 6.614358901977539, |
| "learning_rate": 9.854261170516648e-06, |
| "loss": 0.5608331680297851, |
| "mean_token_accuracy": 0.8688548248261213, |
| "num_tokens": 3035157.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 1.9003934860229492, |
| "epoch": 2.6255864873318737, |
| "grad_norm": 17.050065994262695, |
| "learning_rate": 9.692366789028308e-06, |
| "loss": 0.5226325511932373, |
| "mean_token_accuracy": 0.860271492600441, |
| "num_tokens": 3059665.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 1.8391575522720813, |
| "epoch": 2.6506099468251487, |
| "grad_norm": 6.052427768707275, |
| "learning_rate": 9.530553079017872e-06, |
| "loss": 0.5446209907531738, |
| "mean_token_accuracy": 0.8525044105947017, |
| "num_tokens": 3088789.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 1.8850280776619912, |
| "epoch": 2.6756334063184237, |
| "grad_norm": 10.781261444091797, |
| "learning_rate": 9.368862473326355e-06, |
| "loss": 0.431397533416748, |
| "mean_token_accuracy": 0.8595373194664717, |
| "num_tokens": 3116869.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 1.816288386285305, |
| "epoch": 2.7006568658116983, |
| "grad_norm": 5.157824993133545, |
| "learning_rate": 9.207337372512797e-06, |
| "loss": 0.48740544319152834, |
| "mean_token_accuracy": 0.8544141486287117, |
| "num_tokens": 3143639.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 1.8121181055903435, |
| "epoch": 2.7256803253049733, |
| "grad_norm": 6.038757801055908, |
| "learning_rate": 9.046020133735455e-06, |
| "loss": 0.5854983806610108, |
| "mean_token_accuracy": 0.8188087772578001, |
| "num_tokens": 3169971.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 1.8628299854695798, |
| "epoch": 2.7507037847982483, |
| "grad_norm": 4.648480415344238, |
| "learning_rate": 8.88495305964436e-06, |
| "loss": 0.435821008682251, |
| "mean_token_accuracy": 0.8851393271237612, |
| "num_tokens": 3198712.0, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.7507037847982483, |
| "eval_entropy": 1.8285482935607433, |
| "eval_loss": 0.9385225176811218, |
| "eval_mean_token_accuracy": 0.795126418932341, |
| "eval_num_tokens": 3198712.0, |
| "eval_runtime": 25.9475, |
| "eval_samples_per_second": 19.732, |
| "eval_steps_per_second": 9.866, |
| "step": 1100 |
| }, |
| { |
| "entropy": 1.8046948611736298, |
| "epoch": 2.7757272442915233, |
| "grad_norm": 12.604764938354492, |
| "learning_rate": 8.724178387288202e-06, |
| "loss": 0.45838608741760256, |
| "mean_token_accuracy": 0.8950696140527725, |
| "num_tokens": 3243283.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 1.8318631589412688, |
| "epoch": 2.8007507037847983, |
| "grad_norm": 4.84168815612793, |
| "learning_rate": 8.563738277038376e-06, |
| "loss": 0.5975977897644043, |
| "mean_token_accuracy": 0.8312116377055645, |
| "num_tokens": 3274397.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 1.772474942356348, |
| "epoch": 2.8257741632780733, |
| "grad_norm": 3.9503281116485596, |
| "learning_rate": 8.40367480153316e-06, |
| "loss": 0.5480531692504883, |
| "mean_token_accuracy": 0.8570948846638202, |
| "num_tokens": 3302787.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 1.837375022470951, |
| "epoch": 2.8507976227713483, |
| "grad_norm": 7.392024993896484, |
| "learning_rate": 8.244029934644916e-06, |
| "loss": 0.4703363418579102, |
| "mean_token_accuracy": 0.8493639241904021, |
| "num_tokens": 3331250.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 1.7589601434767246, |
| "epoch": 2.875821082264623, |
| "grad_norm": 9.2200927734375, |
| "learning_rate": 8.084845540473127e-06, |
| "loss": 0.5588334083557129, |
| "mean_token_accuracy": 0.864871158450842, |
| "num_tokens": 3365799.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 1.7889262288808823, |
| "epoch": 2.9008445417578983, |
| "grad_norm": 6.014739513397217, |
| "learning_rate": 7.9261633623663e-06, |
| "loss": 0.4394832611083984, |
| "mean_token_accuracy": 0.9105578908696771, |
| "num_tokens": 3397907.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 1.793429036438465, |
| "epoch": 2.925868001251173, |
| "grad_norm": 9.53693675994873, |
| "learning_rate": 7.768025011975481e-06, |
| "loss": 0.5310945987701416, |
| "mean_token_accuracy": 0.8654899284243583, |
| "num_tokens": 3428029.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 1.791808745265007, |
| "epoch": 2.950891460744448, |
| "grad_norm": 8.377955436706543, |
| "learning_rate": 7.610471958342326e-06, |
| "loss": 0.4895362377166748, |
| "mean_token_accuracy": 0.8723569042980671, |
| "num_tokens": 3462452.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 1.8754499897360801, |
| "epoch": 2.975914920237723, |
| "grad_norm": 5.267559051513672, |
| "learning_rate": 7.4535455170245476e-06, |
| "loss": 0.521859884262085, |
| "mean_token_accuracy": 0.8782990558072925, |
| "num_tokens": 3490604.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 1.8749133162684255, |
| "epoch": 3.0, |
| "grad_norm": 18.580663681030273, |
| "learning_rate": 7.297286839261659e-06, |
| "loss": 0.532118558883667, |
| "mean_token_accuracy": 0.8816164097228607, |
| "num_tokens": 3517710.0, |
| "step": 1200 |
| }, |
| { |
| "epoch": 3.0, |
| "eval_entropy": 1.8413750138133764, |
| "eval_loss": 0.9358716607093811, |
| "eval_mean_token_accuracy": 0.798108211136423, |
| "eval_num_tokens": 3517710.0, |
| "eval_runtime": 25.9137, |
| "eval_samples_per_second": 19.758, |
| "eval_steps_per_second": 9.879, |
| "step": 1200 |
| }, |
| { |
| "entropy": 1.8603122062981128, |
| "epoch": 3.025023459493275, |
| "grad_norm": 2.6409912109375, |
| "learning_rate": 7.1417369011837355e-06, |
| "loss": 0.30042328834533694, |
| "mean_token_accuracy": 0.925163534283638, |
| "num_tokens": 3542903.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 1.8052339434623719, |
| "epoch": 3.05004691898655, |
| "grad_norm": 6.589583873748779, |
| "learning_rate": 6.986936493066165e-06, |
| "loss": 0.5724757194519043, |
| "mean_token_accuracy": 0.8914085768163205, |
| "num_tokens": 3574728.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 1.828494493663311, |
| "epoch": 3.075070378479825, |
| "grad_norm": 11.607176780700684, |
| "learning_rate": 6.8329262086330864e-06, |
| "loss": 0.3562421083450317, |
| "mean_token_accuracy": 0.9141828007996082, |
| "num_tokens": 3598841.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 1.663271901011467, |
| "epoch": 3.1000938379731, |
| "grad_norm": 4.842362880706787, |
| "learning_rate": 6.6797464344124045e-06, |
| "loss": 0.3805184602737427, |
| "mean_token_accuracy": 0.9177524700760842, |
| "num_tokens": 3627929.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 1.6602753311395646, |
| "epoch": 3.1251172974663746, |
| "grad_norm": 8.315300941467285, |
| "learning_rate": 6.527437339145097e-06, |
| "loss": 0.37636704444885255, |
| "mean_token_accuracy": 0.9170884318649769, |
| "num_tokens": 3656057.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 1.6945518404245377, |
| "epoch": 3.1501407569596496, |
| "grad_norm": 7.678070068359375, |
| "learning_rate": 6.376038863251706e-06, |
| "loss": 0.2868889570236206, |
| "mean_token_accuracy": 0.8948385491967201, |
| "num_tokens": 3682522.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 1.679956039786339, |
| "epoch": 3.1751642164529246, |
| "grad_norm": 5.331015586853027, |
| "learning_rate": 6.225590708358596e-06, |
| "loss": 0.40270466804504396, |
| "mean_token_accuracy": 0.8960530921816826, |
| "num_tokens": 3710068.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 1.8229594185948372, |
| "epoch": 3.2001876759461996, |
| "grad_norm": 5.083068370819092, |
| "learning_rate": 6.076132326886934e-06, |
| "loss": 0.30500695705413816, |
| "mean_token_accuracy": 0.9170692026615143, |
| "num_tokens": 3736077.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 1.712825458496809, |
| "epoch": 3.2252111354394746, |
| "grad_norm": 6.959349155426025, |
| "learning_rate": 5.927702911706961e-06, |
| "loss": 0.4808387279510498, |
| "mean_token_accuracy": 0.9020599089562893, |
| "num_tokens": 3767499.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 1.7548687763512134, |
| "epoch": 3.2502345949327496, |
| "grad_norm": 3.395026683807373, |
| "learning_rate": 5.780341385860333e-06, |
| "loss": 0.44341039657592773, |
| "mean_token_accuracy": 0.8619493830949068, |
| "num_tokens": 3796930.0, |
| "step": 1300 |
| }, |
| { |
| "epoch": 3.2502345949327496, |
| "eval_entropy": 1.7076403074897826, |
| "eval_loss": 1.0389351844787598, |
| "eval_mean_token_accuracy": 0.797438750974834, |
| "eval_num_tokens": 3796930.0, |
| "eval_runtime": 25.945, |
| "eval_samples_per_second": 19.734, |
| "eval_steps_per_second": 9.867, |
| "step": 1300 |
| }, |
| { |
| "entropy": 1.7037555642426014, |
| "epoch": 3.2752580544260246, |
| "grad_norm": 7.0480732917785645, |
| "learning_rate": 5.63408639235324e-06, |
| "loss": 0.4138573169708252, |
| "mean_token_accuracy": 0.8853528399020434, |
| "num_tokens": 3826118.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 1.7090844802558423, |
| "epoch": 3.300281513919299, |
| "grad_norm": 9.457049369812012, |
| "learning_rate": 5.488976284022953e-06, |
| "loss": 0.38932549953460693, |
| "mean_token_accuracy": 0.9061979863792657, |
| "num_tokens": 3855132.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 1.669133372604847, |
| "epoch": 3.325304973412574, |
| "grad_norm": 5.351191997528076, |
| "learning_rate": 5.3450491134804416e-06, |
| "loss": 0.28699569702148436, |
| "mean_token_accuracy": 0.9288982767611742, |
| "num_tokens": 3888805.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 1.6410670965909957, |
| "epoch": 3.350328432905849, |
| "grad_norm": 15.305230140686035, |
| "learning_rate": 5.202342623131731e-06, |
| "loss": 0.27166821956634524, |
| "mean_token_accuracy": 0.9167630560696125, |
| "num_tokens": 3918738.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 1.629030243307352, |
| "epoch": 3.375351892399124, |
| "grad_norm": 2.671210289001465, |
| "learning_rate": 5.060894235280637e-06, |
| "loss": 0.4683689117431641, |
| "mean_token_accuracy": 0.9109336912631989, |
| "num_tokens": 3952972.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 1.6706741809844972, |
| "epoch": 3.400375351892399, |
| "grad_norm": 3.6248090267181396, |
| "learning_rate": 4.9207410423153925e-06, |
| "loss": 0.37804474830627444, |
| "mean_token_accuracy": 0.905234795063734, |
| "num_tokens": 3979693.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 1.6092427238821982, |
| "epoch": 3.425398811385674, |
| "grad_norm": 6.463558673858643, |
| "learning_rate": 4.781919796981818e-06, |
| "loss": 0.30335488319396975, |
| "mean_token_accuracy": 0.9086175173521042, |
| "num_tokens": 4008543.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 1.680896159261465, |
| "epoch": 3.450422270878949, |
| "grad_norm": 5.749578475952148, |
| "learning_rate": 4.6444669027455615e-06, |
| "loss": 0.3422755479812622, |
| "mean_token_accuracy": 0.9261092841625214, |
| "num_tokens": 4046924.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 1.6322893902659417, |
| "epoch": 3.475445730372224, |
| "grad_norm": 4.440096378326416, |
| "learning_rate": 4.508418404245903e-06, |
| "loss": 0.39899749755859376, |
| "mean_token_accuracy": 0.8951961848884821, |
| "num_tokens": 4078782.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 1.6997947439551353, |
| "epoch": 3.500469189865499, |
| "grad_norm": 6.773232936859131, |
| "learning_rate": 4.373809977843676e-06, |
| "loss": 0.3821007490158081, |
| "mean_token_accuracy": 0.9003218419849872, |
| "num_tokens": 4104561.0, |
| "step": 1400 |
| }, |
| { |
| "epoch": 3.500469189865499, |
| "eval_entropy": 1.6739916959777474, |
| "eval_loss": 1.0589442253112793, |
| "eval_mean_token_accuracy": 0.7994736307300627, |
| "eval_num_tokens": 4104561.0, |
| "eval_runtime": 25.9285, |
| "eval_samples_per_second": 19.747, |
| "eval_steps_per_second": 9.873, |
| "step": 1400 |
| }, |
| { |
| "entropy": 1.6732337854802608, |
| "epoch": 3.5254926493587737, |
| "grad_norm": 4.1359333992004395, |
| "learning_rate": 4.240676922265774e-06, |
| "loss": 0.46932153701782225, |
| "mean_token_accuracy": 0.8881252314895391, |
| "num_tokens": 4131335.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 1.7210186302661896, |
| "epoch": 3.5505161088520487, |
| "grad_norm": 5.5349907875061035, |
| "learning_rate": 4.1090541493486555e-06, |
| "loss": 0.17852014303207397, |
| "mean_token_accuracy": 0.933498764038086, |
| "num_tokens": 4163748.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 1.6368225537240506, |
| "epoch": 3.5755395683453237, |
| "grad_norm": 11.67458724975586, |
| "learning_rate": 3.978976174883329e-06, |
| "loss": 0.3674903869628906, |
| "mean_token_accuracy": 0.9103257186710835, |
| "num_tokens": 4190535.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 1.6206367388367653, |
| "epoch": 3.6005630278385987, |
| "grad_norm": 6.8839850425720215, |
| "learning_rate": 3.8504771095641905e-06, |
| "loss": 0.37578022480010986, |
| "mean_token_accuracy": 0.9048499703407288, |
| "num_tokens": 4220983.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 1.630747129023075, |
| "epoch": 3.6255864873318737, |
| "grad_norm": 6.57778263092041, |
| "learning_rate": 3.7235906500440576e-06, |
| "loss": 0.3801173448562622, |
| "mean_token_accuracy": 0.8957030765712262, |
| "num_tokens": 4251936.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 1.6560029186308385, |
| "epoch": 3.6506099468251487, |
| "grad_norm": 8.60834789276123, |
| "learning_rate": 3.5983500700978425e-06, |
| "loss": 0.2505526542663574, |
| "mean_token_accuracy": 0.9320986948907375, |
| "num_tokens": 4286899.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 1.6910177335143088, |
| "epoch": 3.6756334063184237, |
| "grad_norm": 1.4734828472137451, |
| "learning_rate": 3.4747882118970565e-06, |
| "loss": 0.33303678035736084, |
| "mean_token_accuracy": 0.9101938150823117, |
| "num_tokens": 4309725.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 1.6117219008505344, |
| "epoch": 3.7006568658116983, |
| "grad_norm": 15.304268836975098, |
| "learning_rate": 3.35293747739753e-06, |
| "loss": 0.30569963455200194, |
| "mean_token_accuracy": 0.9167403355240822, |
| "num_tokens": 4338763.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 1.6450724273920059, |
| "epoch": 3.7256803253049733, |
| "grad_norm": 8.834023475646973, |
| "learning_rate": 3.2328298198425556e-06, |
| "loss": 0.3241915225982666, |
| "mean_token_accuracy": 0.9080854427069426, |
| "num_tokens": 4369516.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 1.618889082968235, |
| "epoch": 3.7507037847982483, |
| "grad_norm": 31.705739974975586, |
| "learning_rate": 3.1144967353837196e-06, |
| "loss": 0.35978450775146487, |
| "mean_token_accuracy": 0.8960370164364576, |
| "num_tokens": 4408106.0, |
| "step": 1500 |
| }, |
| { |
| "epoch": 3.7507037847982483, |
| "eval_entropy": 1.6368464617989957, |
| "eval_loss": 1.0974652767181396, |
| "eval_mean_token_accuracy": 0.799555316218175, |
| "eval_num_tokens": 4408106.0, |
| "eval_runtime": 25.9318, |
| "eval_samples_per_second": 19.744, |
| "eval_steps_per_second": 9.872, |
| "step": 1500 |
| }, |
| { |
| "entropy": 1.6849488005042077, |
| "epoch": 3.7757272442915233, |
| "grad_norm": 18.618749618530273, |
| "learning_rate": 2.997969254821548e-06, |
| "loss": 0.434901762008667, |
| "mean_token_accuracy": 0.9058444269001484, |
| "num_tokens": 4433839.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 1.612816944718361, |
| "epoch": 3.8007507037847983, |
| "grad_norm": 5.040231704711914, |
| "learning_rate": 2.883277935468254e-06, |
| "loss": 0.3056370496749878, |
| "mean_token_accuracy": 0.9123396039009094, |
| "num_tokens": 4461853.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 1.6486859299242496, |
| "epoch": 3.8257741632780733, |
| "grad_norm": 5.422091960906982, |
| "learning_rate": 2.770452853134593e-06, |
| "loss": 0.33029043674468994, |
| "mean_token_accuracy": 0.8901268295943737, |
| "num_tokens": 4486391.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 1.6774092495441437, |
| "epoch": 3.8507976227713483, |
| "grad_norm": 11.819265365600586, |
| "learning_rate": 2.6595235942430044e-06, |
| "loss": 0.33706488609313967, |
| "mean_token_accuracy": 0.9124666847288608, |
| "num_tokens": 4515681.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 1.6730435192584991, |
| "epoch": 3.875821082264623, |
| "grad_norm": 24.53825569152832, |
| "learning_rate": 2.5505192480690865e-06, |
| "loss": 0.37698824405670167, |
| "mean_token_accuracy": 0.8898964198306203, |
| "num_tokens": 4547834.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 1.6430152557790279, |
| "epoch": 3.9008445417578983, |
| "grad_norm": 6.23613166809082, |
| "learning_rate": 2.4434683991134476e-06, |
| "loss": 0.46486830711364746, |
| "mean_token_accuracy": 0.9119167998433113, |
| "num_tokens": 4578695.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 1.6305080100893974, |
| "epoch": 3.925868001251173, |
| "grad_norm": 4.144184589385986, |
| "learning_rate": 2.3383991196058918e-06, |
| "loss": 0.24737279415130614, |
| "mean_token_accuracy": 0.9135037533938885, |
| "num_tokens": 4608040.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 1.6553510926663875, |
| "epoch": 3.950891460744448, |
| "grad_norm": 16.744169235229492, |
| "learning_rate": 2.23533896214399e-06, |
| "loss": 0.5578951358795166, |
| "mean_token_accuracy": 0.864304494485259, |
| "num_tokens": 4637749.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 1.6376118659973145, |
| "epoch": 3.975914920237723, |
| "grad_norm": 10.743280410766602, |
| "learning_rate": 2.134314952467873e-06, |
| "loss": 0.4247287273406982, |
| "mean_token_accuracy": 0.9346457831561565, |
| "num_tokens": 4667740.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 1.651349587873979, |
| "epoch": 4.0, |
| "grad_norm": 12.263242721557617, |
| "learning_rate": 2.0353535823732053e-06, |
| "loss": 0.31544477939605714, |
| "mean_token_accuracy": 0.9375330341326726, |
| "num_tokens": 4690280.0, |
| "step": 1600 |
| }, |
| { |
| "epoch": 4.0, |
| "eval_entropy": 1.6643165587447584, |
| "eval_loss": 1.0606985092163086, |
| "eval_mean_token_accuracy": 0.8001990197226405, |
| "eval_num_tokens": 4690280.0, |
| "eval_runtime": 26.1584, |
| "eval_samples_per_second": 19.573, |
| "eval_steps_per_second": 9.787, |
| "step": 1600 |
| }, |
| { |
| "entropy": 1.7042509287595748, |
| "epoch": 4.025023459493275, |
| "grad_norm": 7.552699089050293, |
| "learning_rate": 1.9384808027641666e-06, |
| "loss": 0.36326165199279786, |
| "mean_token_accuracy": 0.9143396973609924, |
| "num_tokens": 4717742.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 1.6511176168918609, |
| "epoch": 4.05004691898655, |
| "grad_norm": 3.7373321056365967, |
| "learning_rate": 1.8437220168482839e-06, |
| "loss": 0.30488340854644774, |
| "mean_token_accuracy": 0.9318673349916935, |
| "num_tokens": 4743049.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 1.6843073666095734, |
| "epoch": 4.075070378479825, |
| "grad_norm": 13.701045989990234, |
| "learning_rate": 1.751102073474873e-06, |
| "loss": 0.32843029499053955, |
| "mean_token_accuracy": 0.916875434666872, |
| "num_tokens": 4768941.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 1.61899134516716, |
| "epoch": 4.1000938379731, |
| "grad_norm": 7.880395412445068, |
| "learning_rate": 1.660645260618864e-06, |
| "loss": 0.27330944538116453, |
| "mean_token_accuracy": 0.9596377186477184, |
| "num_tokens": 4795326.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 1.6763562865555286, |
| "epoch": 4.125117297466375, |
| "grad_norm": 12.53801441192627, |
| "learning_rate": 1.5723752990116948e-06, |
| "loss": 0.4172815322875977, |
| "mean_token_accuracy": 0.941043658182025, |
| "num_tokens": 4822845.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 1.6501429066061974, |
| "epoch": 4.15014075695965, |
| "grad_norm": 6.694977283477783, |
| "learning_rate": 1.4863153359209693e-06, |
| "loss": 0.2577322006225586, |
| "mean_token_accuracy": 0.9485772825777531, |
| "num_tokens": 4851779.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 1.5867636039853097, |
| "epoch": 4.175164216452925, |
| "grad_norm": 5.9964399337768555, |
| "learning_rate": 1.402487939080479e-06, |
| "loss": 0.1994820237159729, |
| "mean_token_accuracy": 0.948756018280983, |
| "num_tokens": 4883534.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 1.5995635233819485, |
| "epoch": 4.2001876759462, |
| "grad_norm": 11.450733184814453, |
| "learning_rate": 1.3209150907722124e-06, |
| "loss": 0.33185758590698244, |
| "mean_token_accuracy": 0.9658055681735277, |
| "num_tokens": 4911765.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 1.5724807053804397, |
| "epoch": 4.225211135439475, |
| "grad_norm": 5.831696510314941, |
| "learning_rate": 1.2416181820618745e-06, |
| "loss": 0.21310560703277587, |
| "mean_token_accuracy": 0.9530413594096899, |
| "num_tokens": 4936187.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 1.578537244349718, |
| "epoch": 4.250234594932749, |
| "grad_norm": 12.092418670654297, |
| "learning_rate": 1.1646180071894608e-06, |
| "loss": 0.2562295913696289, |
| "mean_token_accuracy": 0.908930304646492, |
| "num_tokens": 4963304.0, |
| "step": 1700 |
| }, |
| { |
| "epoch": 4.250234594932749, |
| "eval_entropy": 1.5953729255124927, |
| "eval_loss": 1.187790870666504, |
| "eval_mean_token_accuracy": 0.794221238233149, |
| "eval_num_tokens": 4963304.0, |
| "eval_runtime": 25.9026, |
| "eval_samples_per_second": 19.766, |
| "eval_steps_per_second": 9.883, |
| "step": 1700 |
| }, |
| { |
| "entropy": 1.5515916496515274, |
| "epoch": 4.275258054426025, |
| "grad_norm": 13.895337104797363, |
| "learning_rate": 1.0899347581163222e-06, |
| "loss": 0.22003817558288574, |
| "mean_token_accuracy": 0.966774944961071, |
| "num_tokens": 4992177.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 1.5425172574818133, |
| "epoch": 4.300281513919299, |
| "grad_norm": 4.921901226043701, |
| "learning_rate": 1.0175880192301713e-06, |
| "loss": 0.21641993522644043, |
| "mean_token_accuracy": 0.9237028583884239, |
| "num_tokens": 5021461.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 1.5836301006376743, |
| "epoch": 4.325304973412575, |
| "grad_norm": 8.30781078338623, |
| "learning_rate": 9.475967622094207e-07, |
| "loss": 0.3954111337661743, |
| "mean_token_accuracy": 0.9054292887449265, |
| "num_tokens": 5051436.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 1.6088024839758872, |
| "epoch": 4.350328432905849, |
| "grad_norm": 10.216887474060059, |
| "learning_rate": 8.799793410481871e-07, |
| "loss": 0.3576494693756104, |
| "mean_token_accuracy": 0.8998017378151417, |
| "num_tokens": 5079060.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 1.5869264729321002, |
| "epoch": 4.375351892399125, |
| "grad_norm": 20.037282943725586, |
| "learning_rate": 8.147534872432761e-07, |
| "loss": 0.2675585985183716, |
| "mean_token_accuracy": 0.9245493724942208, |
| "num_tokens": 5113633.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 1.5867690846323967, |
| "epoch": 4.400375351892399, |
| "grad_norm": 7.718106746673584, |
| "learning_rate": 7.519363051443996e-07, |
| "loss": 0.25213844776153566, |
| "mean_token_accuracy": 0.9569812349975109, |
| "num_tokens": 5146820.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 1.5456994101405144, |
| "epoch": 4.425398811385674, |
| "grad_norm": 6.885993003845215, |
| "learning_rate": 6.915442674688633e-07, |
| "loss": 0.269787335395813, |
| "mean_token_accuracy": 0.9462529934942723, |
| "num_tokens": 5180110.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 1.5553630605340003, |
| "epoch": 4.450422270878949, |
| "grad_norm": 6.1017866134643555, |
| "learning_rate": 6.335932109818754e-07, |
| "loss": 0.19417457580566405, |
| "mean_token_accuracy": 0.9338026508688927, |
| "num_tokens": 5209095.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 1.558656318485737, |
| "epoch": 4.475445730372224, |
| "grad_norm": 10.72335433959961, |
| "learning_rate": 5.780983323436374e-07, |
| "loss": 0.19903587102890014, |
| "mean_token_accuracy": 0.9535072252154351, |
| "num_tokens": 5247292.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 1.617298749089241, |
| "epoch": 4.500469189865499, |
| "grad_norm": 22.27625274658203, |
| "learning_rate": 5.250741841242735e-07, |
| "loss": 0.34345569610595705, |
| "mean_token_accuracy": 0.9497337996959686, |
| "num_tokens": 5275299.0, |
| "step": 1800 |
| }, |
| { |
| "epoch": 4.500469189865499, |
| "eval_entropy": 1.5757260229438543, |
| "eval_loss": 1.227073311805725, |
| "eval_mean_token_accuracy": 0.7966270901961252, |
| "eval_num_tokens": 5275299.0, |
| "eval_runtime": 25.953, |
| "eval_samples_per_second": 19.728, |
| "eval_steps_per_second": 9.864, |
| "step": 1800 |
| }, |
| { |
| "entropy": 1.5715236023068428, |
| "epoch": 4.525492649358774, |
| "grad_norm": 9.199799537658691, |
| "learning_rate": 4.745346709876786e-07, |
| "loss": 0.5030104160308838, |
| "mean_token_accuracy": 0.9179459355771542, |
| "num_tokens": 5304526.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 1.5571273937821388, |
| "epoch": 4.550516108852049, |
| "grad_norm": 6.441845417022705, |
| "learning_rate": 4.26493046045261e-07, |
| "loss": 0.2179567813873291, |
| "mean_token_accuracy": 0.9635193780064583, |
| "num_tokens": 5332333.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 1.509796992689371, |
| "epoch": 4.575539568345324, |
| "grad_norm": 6.927135944366455, |
| "learning_rate": 3.8096190738053815e-07, |
| "loss": 0.29149680137634276, |
| "mean_token_accuracy": 0.9451167620718479, |
| "num_tokens": 5358537.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 1.5602086365222931, |
| "epoch": 4.600563027838598, |
| "grad_norm": 9.667234420776367, |
| "learning_rate": 3.379531947455128e-07, |
| "loss": 0.3328777551651001, |
| "mean_token_accuracy": 0.9238914847373962, |
| "num_tokens": 5386183.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 1.6346124820411205, |
| "epoch": 4.625586487331874, |
| "grad_norm": 5.629216194152832, |
| "learning_rate": 2.974781864296783e-07, |
| "loss": 0.32338910102844237, |
| "mean_token_accuracy": 0.9202292047441005, |
| "num_tokens": 5413879.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 1.5475087754428387, |
| "epoch": 4.650609946825148, |
| "grad_norm": 8.701130867004395, |
| "learning_rate": 2.5954749630248355e-07, |
| "loss": 0.21816930770874024, |
| "mean_token_accuracy": 0.9383991964161396, |
| "num_tokens": 5441818.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 1.5626422986388206, |
| "epoch": 4.675633406318424, |
| "grad_norm": 7.034233570098877, |
| "learning_rate": 2.24171071030026e-07, |
| "loss": 0.2586040496826172, |
| "mean_token_accuracy": 0.9426060438156127, |
| "num_tokens": 5470335.0, |
| "step": 1870 |
| }, |
| { |
| "entropy": 1.5956231564283372, |
| "epoch": 4.700656865811698, |
| "grad_norm": 5.224846839904785, |
| "learning_rate": 1.9135818746671587e-07, |
| "loss": 0.28032145500183103, |
| "mean_token_accuracy": 0.9485361717641354, |
| "num_tokens": 5502253.0, |
| "step": 1880 |
| }, |
| { |
| "entropy": 1.6720745049417018, |
| "epoch": 4.725680325304974, |
| "grad_norm": 8.012434959411621, |
| "learning_rate": 1.6111745022257873e-07, |
| "loss": 0.25192699432373045, |
| "mean_token_accuracy": 0.9423566825687886, |
| "num_tokens": 5526629.0, |
| "step": 1890 |
| }, |
| { |
| "entropy": 1.5510374516248704, |
| "epoch": 4.750703784798248, |
| "grad_norm": 23.2949161529541, |
| "learning_rate": 1.3345678940684615e-07, |
| "loss": 0.3542658805847168, |
| "mean_token_accuracy": 0.9370437204837799, |
| "num_tokens": 5555020.0, |
| "step": 1900 |
| }, |
| { |
| "epoch": 4.750703784798248, |
| "eval_entropy": 1.5751400967128575, |
| "eval_loss": 1.221580982208252, |
| "eval_mean_token_accuracy": 0.7978551599662751, |
| "eval_num_tokens": 5555020.0, |
| "eval_runtime": 25.9514, |
| "eval_samples_per_second": 19.729, |
| "eval_steps_per_second": 9.865, |
| "step": 1900 |
| }, |
| { |
| "entropy": 1.5750564984977244, |
| "epoch": 4.775727244291524, |
| "grad_norm": 6.1790971755981445, |
| "learning_rate": 1.0838345854842447e-07, |
| "loss": 0.29637510776519777, |
| "mean_token_accuracy": 0.9434729963541031, |
| "num_tokens": 5585024.0, |
| "step": 1910 |
| }, |
| { |
| "entropy": 1.5804114930331707, |
| "epoch": 4.800750703784798, |
| "grad_norm": 8.236255645751953, |
| "learning_rate": 8.590403269377656e-08, |
| "loss": 0.261922812461853, |
| "mean_token_accuracy": 0.944847946614027, |
| "num_tokens": 5612458.0, |
| "step": 1920 |
| }, |
| { |
| "entropy": 1.5763019055128098, |
| "epoch": 4.825774163278073, |
| "grad_norm": 11.511734008789062, |
| "learning_rate": 6.602440668273758e-08, |
| "loss": 0.18001055717468262, |
| "mean_token_accuracy": 0.9452268406748772, |
| "num_tokens": 5641935.0, |
| "step": 1930 |
| }, |
| { |
| "entropy": 1.5883605182170868, |
| "epoch": 4.850797622771348, |
| "grad_norm": 21.67671012878418, |
| "learning_rate": 4.874979360268928e-08, |
| "loss": 0.3848047494888306, |
| "mean_token_accuracy": 0.914675597846508, |
| "num_tokens": 5672477.0, |
| "step": 1940 |
| }, |
| { |
| "entropy": 1.5379426710307598, |
| "epoch": 4.875821082264623, |
| "grad_norm": 11.815309524536133, |
| "learning_rate": 3.408472342152136e-08, |
| "loss": 0.1684113025665283, |
| "mean_token_accuracy": 0.9466052651405334, |
| "num_tokens": 5703555.0, |
| "step": 1950 |
| }, |
| { |
| "entropy": 1.581675297766924, |
| "epoch": 4.900844541757898, |
| "grad_norm": 6.978868007659912, |
| "learning_rate": 2.2033041799723877e-08, |
| "loss": 0.190066134929657, |
| "mean_token_accuracy": 0.9330016218125821, |
| "num_tokens": 5733510.0, |
| "step": 1960 |
| }, |
| { |
| "entropy": 1.5166504696011542, |
| "epoch": 4.925868001251173, |
| "grad_norm": 5.1840009689331055, |
| "learning_rate": 1.2597909081931702e-08, |
| "loss": 0.3155841588973999, |
| "mean_token_accuracy": 0.931412010639906, |
| "num_tokens": 5762332.0, |
| "step": 1970 |
| }, |
| { |
| "entropy": 1.5898376658558846, |
| "epoch": 4.950891460744448, |
| "grad_norm": 5.622488498687744, |
| "learning_rate": 5.781799468177473e-09, |
| "loss": 0.3049920558929443, |
| "mean_token_accuracy": 0.910175010561943, |
| "num_tokens": 5792982.0, |
| "step": 1980 |
| }, |
| { |
| "entropy": 1.5722950287163258, |
| "epoch": 4.975914920237723, |
| "grad_norm": 9.232101440429688, |
| "learning_rate": 1.5865003650761268e-09, |
| "loss": 0.17625534534454346, |
| "mean_token_accuracy": 0.9326556093990803, |
| "num_tokens": 5828000.0, |
| "step": 1990 |
| }, |
| { |
| "entropy": 1.6074273145044005, |
| "epoch": 5.0, |
| "grad_norm": 2.723400354385376, |
| "learning_rate": 1.311191710651194e-11, |
| "loss": 0.2906452417373657, |
| "mean_token_accuracy": 0.9403998186061908, |
| "num_tokens": 5862850.0, |
| "step": 2000 |
| }, |
| { |
| "epoch": 5.0, |
| "eval_entropy": 1.5754532138817012, |
| "eval_loss": 1.2249126434326172, |
| "eval_mean_token_accuracy": 0.7969964473741129, |
| "eval_num_tokens": 5862850.0, |
| "eval_runtime": 25.9681, |
| "eval_samples_per_second": 19.716, |
| "eval_steps_per_second": 9.858, |
| "step": 2000 |
| }, |
| { |
| "epoch": 5.0, |
| "step": 2000, |
| "total_flos": 2.649974039752704e+17, |
| "train_loss": 0.5825153150558472, |
| "train_runtime": 3830.0191, |
| "train_samples_per_second": 4.174, |
| "train_steps_per_second": 0.522 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 2000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 5, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 2.649974039752704e+17, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
