{ "best_global_step": 300, "best_metric": 1.0401721000671387, "best_model_checkpoint": "/data6/jongbinwon/evolving_MoE/checkpoints/sft_llama31_8b_sni_ours-c_18467_ffn_only_5ep/checkpoint-300", "epoch": 5.0, "eval_steps": 100, "global_step": 2050, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "entropy": 1.8605872005224229, "epoch": 0.024390243902439025, "grad_norm": 20.60421371459961, "learning_rate": 2.903225806451613e-06, "loss": 1.7898401260375976, "mean_token_accuracy": 0.410875541344285, "num_tokens": 27553.0, "step": 10 }, { "entropy": 1.8053070664405824, "epoch": 0.04878048780487805, "grad_norm": 20.3812255859375, "learning_rate": 6.129032258064517e-06, "loss": 1.7648082733154298, "mean_token_accuracy": 0.38130847699940207, "num_tokens": 57610.0, "step": 20 }, { "entropy": 1.8405818074941636, "epoch": 0.07317073170731707, "grad_norm": 6.929360389709473, "learning_rate": 9.35483870967742e-06, "loss": 1.5310235023498535, "mean_token_accuracy": 0.5096342623233795, "num_tokens": 91084.0, "step": 30 }, { "entropy": 2.0280986458063124, "epoch": 0.0975609756097561, "grad_norm": 3.4556729793548584, "learning_rate": 1.2580645161290324e-05, "loss": 0.9672060012817383, "mean_token_accuracy": 0.6548571057617665, "num_tokens": 115647.0, "step": 40 }, { "entropy": 2.0338595777750017, "epoch": 0.12195121951219512, "grad_norm": 2.3577516078948975, "learning_rate": 1.5806451612903226e-05, "loss": 0.6828746318817138, "mean_token_accuracy": 0.7553045634180308, "num_tokens": 145943.0, "step": 50 }, { "entropy": 2.149640719592571, "epoch": 0.14634146341463414, "grad_norm": 5.422067642211914, "learning_rate": 1.903225806451613e-05, "loss": 0.5620139598846435, "mean_token_accuracy": 0.7796026851981879, "num_tokens": 175404.0, "step": 60 }, { "entropy": 2.123129604756832, "epoch": 0.17073170731707318, "grad_norm": 5.797729969024658, "learning_rate": 1.9999388172996495e-05, "loss": 0.6369808197021485, "mean_token_accuracy": 0.7617549113929272, "num_tokens": 205999.0, "step": 70 }, { "entropy": 2.123254044353962, "epoch": 0.1951219512195122, "grad_norm": 3.454728364944458, "learning_rate": 1.9996391649532744e-05, "loss": 0.5668575763702393, "mean_token_accuracy": 0.7390173625200986, "num_tokens": 242686.0, "step": 80 }, { "entropy": 2.1996585920453073, "epoch": 0.21951219512195122, "grad_norm": 1.8380242586135864, "learning_rate": 1.999089880058469e-05, "loss": 0.5168749332427979, "mean_token_accuracy": 0.8007261864840984, "num_tokens": 283486.0, "step": 90 }, { "entropy": 2.1687889844179153, "epoch": 0.24390243902439024, "grad_norm": 3.541299819946289, "learning_rate": 1.9982910997841175e-05, "loss": 0.5402149200439453, "mean_token_accuracy": 0.8154684893786908, "num_tokens": 314067.0, "step": 100 }, { "epoch": 0.24390243902439024, "eval_entropy": 2.0592461647465825, "eval_loss": 1.106112003326416, "eval_mean_token_accuracy": 0.7109428983530961, "eval_num_tokens": 314067.0, "eval_runtime": 25.9133, "eval_samples_per_second": 19.758, "eval_steps_per_second": 9.879, "step": 100 }, { "entropy": 2.108909949660301, "epoch": 0.2682926829268293, "grad_norm": 2.8216497898101807, "learning_rate": 1.9972430236037522e-05, "loss": 0.6249359130859375, "mean_token_accuracy": 0.8037990782409906, "num_tokens": 341387.0, "step": 110 }, { "entropy": 2.207451155781746, "epoch": 0.2926829268292683, "grad_norm": 2.257850408554077, "learning_rate": 1.9959459132457415e-05, "loss": 0.5127166748046875, "mean_token_accuracy": 0.7718870434910059, "num_tokens": 368588.0, "step": 120 }, { "entropy": 2.2370947152376175, "epoch": 0.3170731707317073, "grad_norm": 2.140078544616699, "learning_rate": 1.9944000926279304e-05, "loss": 0.36243553161621095, "mean_token_accuracy": 0.8322107203304767, "num_tokens": 401629.0, "step": 130 }, { "entropy": 2.140230692923069, "epoch": 0.34146341463414637, "grad_norm": 3.153487205505371, "learning_rate": 1.992605947776752e-05, "loss": 0.43494548797607424, "mean_token_accuracy": 0.8130006022751332, "num_tokens": 435002.0, "step": 140 }, { "entropy": 2.2302474692463874, "epoch": 0.36585365853658536, "grad_norm": 3.9841318130493164, "learning_rate": 1.9905639267308244e-05, "loss": 0.48342504501342776, "mean_token_accuracy": 0.8375152945518494, "num_tokens": 461144.0, "step": 150 }, { "entropy": 2.1675660595297814, "epoch": 0.3902439024390244, "grad_norm": 3.773319959640503, "learning_rate": 1.9882745394290705e-05, "loss": 0.49612889289855955, "mean_token_accuracy": 0.8277705665677786, "num_tokens": 492159.0, "step": 160 }, { "entropy": 2.1969783782958983, "epoch": 0.4146341463414634, "grad_norm": 2.6369755268096924, "learning_rate": 1.9857383575833693e-05, "loss": 0.3935831546783447, "mean_token_accuracy": 0.8360724337399006, "num_tokens": 523123.0, "step": 170 }, { "entropy": 2.084891338646412, "epoch": 0.43902439024390244, "grad_norm": 1.4077402353286743, "learning_rate": 1.9829560145357906e-05, "loss": 0.5125601291656494, "mean_token_accuracy": 0.7995197489857674, "num_tokens": 555773.0, "step": 180 }, { "entropy": 2.2110578060150146, "epoch": 0.4634146341463415, "grad_norm": 1.185247540473938, "learning_rate": 1.979928205100434e-05, "loss": 0.49929103851318357, "mean_token_accuracy": 0.8070170473307371, "num_tokens": 584599.0, "step": 190 }, { "entropy": 2.215707804262638, "epoch": 0.4878048780487805, "grad_norm": 2.001645088195801, "learning_rate": 1.9766556853899153e-05, "loss": 0.3552170515060425, "mean_token_accuracy": 0.8613414619117975, "num_tokens": 609287.0, "step": 200 }, { "epoch": 0.4878048780487805, "eval_entropy": 2.079895834904164, "eval_loss": 1.053781509399414, "eval_mean_token_accuracy": 0.7179834527196363, "eval_num_tokens": 609287.0, "eval_runtime": 25.896, "eval_samples_per_second": 19.771, "eval_steps_per_second": 9.886, "step": 200 }, { "entropy": 2.2184708252549172, "epoch": 0.5121951219512195, "grad_norm": 1.9533485174179077, "learning_rate": 1.9731392726265538e-05, "loss": 0.5081103801727295, "mean_token_accuracy": 0.8310256440192461, "num_tokens": 636926.0, "step": 210 }, { "entropy": 2.1484748467803003, "epoch": 0.5365853658536586, "grad_norm": 3.2435407638549805, "learning_rate": 1.9693798449382873e-05, "loss": 0.36309473514556884, "mean_token_accuracy": 0.8431417915970087, "num_tokens": 675494.0, "step": 220 }, { "entropy": 2.099889335036278, "epoch": 0.5609756097560976, "grad_norm": 2.490366220474243, "learning_rate": 1.9653783411393875e-05, "loss": 0.5986989974975586, "mean_token_accuracy": 0.8006603576242923, "num_tokens": 707879.0, "step": 230 }, { "entropy": 2.1755491733551025, "epoch": 0.5853658536585366, "grad_norm": 2.7216265201568604, "learning_rate": 1.9611357604960156e-05, "loss": 0.42953081130981446, "mean_token_accuracy": 0.8588973596692085, "num_tokens": 735253.0, "step": 240 }, { "entropy": 2.1838101163506507, "epoch": 0.6097560975609756, "grad_norm": 5.845120906829834, "learning_rate": 1.956653162476683e-05, "loss": 0.5157174110412598, "mean_token_accuracy": 0.8359234441071749, "num_tokens": 764318.0, "step": 250 }, { "entropy": 2.1837168082594873, "epoch": 0.6341463414634146, "grad_norm": 3.6001269817352295, "learning_rate": 1.9519316664876782e-05, "loss": 0.4741124153137207, "mean_token_accuracy": 0.8372056260704994, "num_tokens": 793972.0, "step": 260 }, { "entropy": 2.2769947752356527, "epoch": 0.6585365853658537, "grad_norm": 5.620599269866943, "learning_rate": 1.9469724515935242e-05, "loss": 0.3968926668167114, "mean_token_accuracy": 0.8357874505221844, "num_tokens": 823116.0, "step": 270 }, { "entropy": 2.1274041652679445, "epoch": 0.6829268292682927, "grad_norm": 6.058096885681152, "learning_rate": 1.9417767562225422e-05, "loss": 0.44854040145874025, "mean_token_accuracy": 0.8963484466075897, "num_tokens": 851276.0, "step": 280 }, { "entropy": 2.123626720905304, "epoch": 0.7073170731707317, "grad_norm": 5.264728546142578, "learning_rate": 1.9363458778575857e-05, "loss": 0.29857568740844725, "mean_token_accuracy": 0.8969821333885193, "num_tokens": 879449.0, "step": 290 }, { "entropy": 2.080774872004986, "epoch": 0.7317073170731707, "grad_norm": 1.193994402885437, "learning_rate": 1.9306811727120305e-05, "loss": 0.3774923801422119, "mean_token_accuracy": 0.8890460595488549, "num_tokens": 912499.0, "step": 300 }, { "epoch": 0.7317073170731707, "eval_entropy": 2.029030416626483, "eval_loss": 1.0401721000671387, "eval_mean_token_accuracy": 0.7339693020912819, "eval_num_tokens": 912499.0, "eval_runtime": 25.8287, "eval_samples_per_second": 19.823, "eval_steps_per_second": 9.911, "step": 300 }, { "entropy": 2.2068835213780402, "epoch": 0.7560975609756098, "grad_norm": 0.858304500579834, "learning_rate": 1.9247840553910974e-05, "loss": 0.4824486255645752, "mean_token_accuracy": 0.8096422411501407, "num_tokens": 941831.0, "step": 310 }, { "entropy": 2.2409408137202265, "epoch": 0.7804878048780488, "grad_norm": 1.7290642261505127, "learning_rate": 1.918655998538593e-05, "loss": 0.3556920766830444, "mean_token_accuracy": 0.8704973738640547, "num_tokens": 970171.0, "step": 320 }, { "entropy": 2.148376648128033, "epoch": 0.8048780487804879, "grad_norm": 3.8836400508880615, "learning_rate": 1.9122985324691576e-05, "loss": 0.39916319847106935, "mean_token_accuracy": 0.8851064193993807, "num_tokens": 1002712.0, "step": 330 }, { "entropy": 2.2421788349747658, "epoch": 0.8292682926829268, "grad_norm": 11.581186294555664, "learning_rate": 1.9057132447861104e-05, "loss": 0.3940950870513916, "mean_token_accuracy": 0.854437343031168, "num_tokens": 1030748.0, "step": 340 }, { "entropy": 2.1712904781103135, "epoch": 0.8536585365853658, "grad_norm": 3.6925837993621826, "learning_rate": 1.8989017799849896e-05, "loss": 0.25206544399261477, "mean_token_accuracy": 0.8935850247740745, "num_tokens": 1057892.0, "step": 350 }, { "entropy": 2.186696508526802, "epoch": 0.8780487804878049, "grad_norm": 2.2644331455230713, "learning_rate": 1.891865839042884e-05, "loss": 0.48485827445983887, "mean_token_accuracy": 0.8456208303570747, "num_tokens": 1085000.0, "step": 360 }, { "entropy": 2.1996654465794565, "epoch": 0.9024390243902439, "grad_norm": 3.6770546436309814, "learning_rate": 1.8846071789936614e-05, "loss": 0.42680583000183103, "mean_token_accuracy": 0.8636934407055378, "num_tokens": 1114894.0, "step": 370 }, { "entropy": 2.2552217707037925, "epoch": 0.926829268292683, "grad_norm": 4.966202735900879, "learning_rate": 1.8771276124891957e-05, "loss": 0.46676030158996584, "mean_token_accuracy": 0.8419601581990719, "num_tokens": 1146273.0, "step": 380 }, { "entropy": 2.1198509678244593, "epoch": 0.9512195121951219, "grad_norm": 1.9863747358322144, "learning_rate": 1.8694290073467095e-05, "loss": 0.33879837989807127, "mean_token_accuracy": 0.8636636339128018, "num_tokens": 1173401.0, "step": 390 }, { "entropy": 2.2081318989396097, "epoch": 0.975609756097561, "grad_norm": 4.736605167388916, "learning_rate": 1.861513286082334e-05, "loss": 0.49434628486633303, "mean_token_accuracy": 0.8567144408822059, "num_tokens": 1200427.0, "step": 400 }, { "epoch": 0.975609756097561, "eval_entropy": 2.0618814867921174, "eval_loss": 1.076478362083435, "eval_mean_token_accuracy": 0.7202240317128599, "eval_num_tokens": 1200427.0, "eval_runtime": 25.8906, "eval_samples_per_second": 19.776, "eval_steps_per_second": 9.888, "step": 400 }, { "entropy": 2.194084380567074, "epoch": 1.0, "grad_norm": 4.52867317199707, "learning_rate": 1.853382425431017e-05, "loss": 0.3407764911651611, "mean_token_accuracy": 0.8748487565666437, "num_tokens": 1235621.0, "step": 410 }, { "entropy": 2.164950945973396, "epoch": 1.024390243902439, "grad_norm": 9.276742935180664, "learning_rate": 1.8450384558528848e-05, "loss": 0.40042719841003416, "mean_token_accuracy": 0.8702093034982681, "num_tokens": 1263848.0, "step": 420 }, { "entropy": 2.1703254342079163, "epoch": 1.048780487804878, "grad_norm": 3.6132359504699707, "learning_rate": 1.8364834610261916e-05, "loss": 0.34138031005859376, "mean_token_accuracy": 0.8627566002309323, "num_tokens": 1293260.0, "step": 430 }, { "entropy": 2.0617744967341425, "epoch": 1.0731707317073171, "grad_norm": 2.0677478313446045, "learning_rate": 1.8277195773269766e-05, "loss": 0.30842764377593995, "mean_token_accuracy": 0.8859433621168137, "num_tokens": 1326028.0, "step": 440 }, { "entropy": 2.0998841643333437, "epoch": 1.0975609756097562, "grad_norm": 2.509185314178467, "learning_rate": 1.818748993295564e-05, "loss": 0.4127968311309814, "mean_token_accuracy": 0.8594737853854895, "num_tokens": 1358618.0, "step": 450 }, { "entropy": 2.0519236624240875, "epoch": 1.1219512195121952, "grad_norm": 3.162795066833496, "learning_rate": 1.8095739490900344e-05, "loss": 0.3142627000808716, "mean_token_accuracy": 0.8876401141285897, "num_tokens": 1387418.0, "step": 460 }, { "entropy": 2.0459494218230247, "epoch": 1.146341463414634, "grad_norm": 7.157248497009277, "learning_rate": 1.8001967359268057e-05, "loss": 0.2823573112487793, "mean_token_accuracy": 0.9053764116019011, "num_tokens": 1415352.0, "step": 470 }, { "entropy": 2.0482651829719543, "epoch": 1.170731707317073, "grad_norm": 3.106424570083618, "learning_rate": 1.7906196955084655e-05, "loss": 0.17932592630386351, "mean_token_accuracy": 0.9373987130820751, "num_tokens": 1443694.0, "step": 480 }, { "entropy": 1.9499709144234658, "epoch": 1.1951219512195121, "grad_norm": 2.2994019985198975, "learning_rate": 1.780845219438994e-05, "loss": 0.38098506927490233, "mean_token_accuracy": 0.8761991690844297, "num_tokens": 1479239.0, "step": 490 }, { "entropy": 1.979680036008358, "epoch": 1.2195121951219512, "grad_norm": 8.579169273376465, "learning_rate": 1.7708757486265272e-05, "loss": 0.3121361494064331, "mean_token_accuracy": 0.8895363163203001, "num_tokens": 1510220.0, "step": 500 }, { "epoch": 1.2195121951219512, "eval_entropy": 1.933678675442934, "eval_loss": 1.1306657791137695, "eval_mean_token_accuracy": 0.7303212948027067, "eval_num_tokens": 1510220.0, "eval_runtime": 25.9532, "eval_samples_per_second": 19.728, "eval_steps_per_second": 9.864, "step": 500 }, { "entropy": 2.1035942107439043, "epoch": 1.2439024390243902, "grad_norm": 3.168657064437866, "learning_rate": 1.7607137726738074e-05, "loss": 0.16653977632522582, "mean_token_accuracy": 0.9589490376412868, "num_tokens": 1537969.0, "step": 510 }, { "entropy": 2.0047017872333526, "epoch": 1.2682926829268293, "grad_norm": 4.511350154876709, "learning_rate": 1.7503618292564698e-05, "loss": 0.3610692024230957, "mean_token_accuracy": 0.8535416685044765, "num_tokens": 1566202.0, "step": 520 }, { "entropy": 2.0103910475969315, "epoch": 1.2926829268292683, "grad_norm": 12.234400749206543, "learning_rate": 1.7398225034893316e-05, "loss": 0.3847961902618408, "mean_token_accuracy": 0.8493602510541678, "num_tokens": 1606282.0, "step": 530 }, { "entropy": 2.028377616405487, "epoch": 1.3170731707317074, "grad_norm": 3.233703374862671, "learning_rate": 1.7290984272808276e-05, "loss": 0.28517172336578367, "mean_token_accuracy": 0.8965774483978748, "num_tokens": 1634067.0, "step": 540 }, { "entropy": 2.0385069623589516, "epoch": 1.3414634146341464, "grad_norm": 1.668707013130188, "learning_rate": 1.718192278675763e-05, "loss": 0.4459042549133301, "mean_token_accuracy": 0.8568475291132926, "num_tokens": 1666984.0, "step": 550 }, { "entropy": 2.0857202276587485, "epoch": 1.3658536585365852, "grad_norm": 3.642622470855713, "learning_rate": 1.7071067811865477e-05, "loss": 0.3298908233642578, "mean_token_accuracy": 0.8802830599248409, "num_tokens": 1696671.0, "step": 560 }, { "entropy": 2.042790485918522, "epoch": 1.3902439024390243, "grad_norm": 3.806304931640625, "learning_rate": 1.695844703113069e-05, "loss": 0.4180549144744873, "mean_token_accuracy": 0.8694050557911396, "num_tokens": 1721385.0, "step": 570 }, { "entropy": 2.039932794868946, "epoch": 1.4146341463414633, "grad_norm": 2.3932442665100098, "learning_rate": 1.6844088568513885e-05, "loss": 0.2970730304718018, "mean_token_accuracy": 0.8963715672492981, "num_tokens": 1744926.0, "step": 580 }, { "entropy": 2.0380129903554915, "epoch": 1.4390243902439024, "grad_norm": 1.5062286853790283, "learning_rate": 1.6728020981914207e-05, "loss": 0.36641407012939453, "mean_token_accuracy": 0.8786697931587696, "num_tokens": 1784717.0, "step": 590 }, { "entropy": 2.06325720846653, "epoch": 1.4634146341463414, "grad_norm": 2.767944812774658, "learning_rate": 1.661027325603779e-05, "loss": 0.22891387939453126, "mean_token_accuracy": 0.9110789362341165, "num_tokens": 1817997.0, "step": 600 }, { "epoch": 1.4634146341463414, "eval_entropy": 1.9500981532037258, "eval_loss": 1.1467669010162354, "eval_mean_token_accuracy": 0.711899112269748, "eval_num_tokens": 1817997.0, "eval_runtime": 25.8393, "eval_samples_per_second": 19.815, "eval_steps_per_second": 9.907, "step": 600 }, { "entropy": 2.0766924560070037, "epoch": 1.4878048780487805, "grad_norm": 2.603064775466919, "learning_rate": 1.6490874795159613e-05, "loss": 0.3146881103515625, "mean_token_accuracy": 0.8997606918215751, "num_tokens": 1845639.0, "step": 610 }, { "entropy": 2.04001639559865, "epoch": 1.5121951219512195, "grad_norm": 4.884147644042969, "learning_rate": 1.6369855415780577e-05, "loss": 0.19958556890487672, "mean_token_accuracy": 0.895722258836031, "num_tokens": 1872147.0, "step": 620 }, { "entropy": 2.007948975265026, "epoch": 1.5365853658536586, "grad_norm": 3.795300245285034, "learning_rate": 1.624724533918166e-05, "loss": 0.1927574872970581, "mean_token_accuracy": 0.9161218356341123, "num_tokens": 1902422.0, "step": 630 }, { "entropy": 2.024307279288769, "epoch": 1.5609756097560976, "grad_norm": 2.5253334045410156, "learning_rate": 1.612307518387699e-05, "loss": 0.29662456512451174, "mean_token_accuracy": 0.8797389216721058, "num_tokens": 1928898.0, "step": 640 }, { "entropy": 2.0686685636639597, "epoch": 1.5853658536585367, "grad_norm": 2.867074966430664, "learning_rate": 1.5997375957967672e-05, "loss": 0.21333327293395996, "mean_token_accuracy": 0.9226264454424381, "num_tokens": 1961105.0, "step": 650 }, { "entropy": 1.9833966985344886, "epoch": 1.6097560975609757, "grad_norm": 8.466132164001465, "learning_rate": 1.5870179051398405e-05, "loss": 0.4845562934875488, "mean_token_accuracy": 0.8716852784156799, "num_tokens": 1991410.0, "step": 660 }, { "entropy": 2.013328048586845, "epoch": 1.6341463414634148, "grad_norm": 1.5435198545455933, "learning_rate": 1.5741516228118677e-05, "loss": 0.3464773654937744, "mean_token_accuracy": 0.8920355424284935, "num_tokens": 2019949.0, "step": 670 }, { "entropy": 2.041088505089283, "epoch": 1.6585365853658538, "grad_norm": 0.872617244720459, "learning_rate": 1.561141961815058e-05, "loss": 0.32896013259887696, "mean_token_accuracy": 0.8908748209476471, "num_tokens": 2053949.0, "step": 680 }, { "entropy": 1.99617058634758, "epoch": 1.6829268292682928, "grad_norm": 6.470183849334717, "learning_rate": 1.5479921709565244e-05, "loss": 0.265201473236084, "mean_token_accuracy": 0.9074951324611902, "num_tokens": 2081016.0, "step": 690 }, { "entropy": 2.0920645907521247, "epoch": 1.7073170731707317, "grad_norm": 2.3583178520202637, "learning_rate": 1.5347055340369806e-05, "loss": 0.22746286392211915, "mean_token_accuracy": 0.9044146835803986, "num_tokens": 2111353.0, "step": 700 }, { "epoch": 1.7073170731707317, "eval_entropy": 1.9395378669723868, "eval_loss": 1.163757562637329, "eval_mean_token_accuracy": 0.7133319691056386, "eval_num_tokens": 2111353.0, "eval_runtime": 25.8458, "eval_samples_per_second": 19.81, "eval_steps_per_second": 9.905, "step": 700 }, { "entropy": 2.0234655663371086, "epoch": 1.7317073170731707, "grad_norm": 3.27419114112854, "learning_rate": 1.521285369030702e-05, "loss": 0.27246096134185793, "mean_token_accuracy": 0.9247618146240711, "num_tokens": 2138965.0, "step": 710 }, { "entropy": 2.0525792986154556, "epoch": 1.7560975609756098, "grad_norm": 6.4840922355651855, "learning_rate": 1.5077350272569531e-05, "loss": 0.3803213834762573, "mean_token_accuracy": 0.877236220613122, "num_tokens": 2173679.0, "step": 720 }, { "entropy": 2.101880183070898, "epoch": 1.7804878048780488, "grad_norm": 2.194840431213379, "learning_rate": 1.4940578925430866e-05, "loss": 0.3040621280670166, "mean_token_accuracy": 0.8798809833824635, "num_tokens": 2213777.0, "step": 730 }, { "entropy": 1.9950981095433236, "epoch": 1.8048780487804879, "grad_norm": 3.7434253692626953, "learning_rate": 1.480257380379525e-05, "loss": 0.28268771171569823, "mean_token_accuracy": 0.8911542497575283, "num_tokens": 2247231.0, "step": 740 }, { "entropy": 2.0733004450798034, "epoch": 1.8292682926829267, "grad_norm": 2.6025116443634033, "learning_rate": 1.4663369370668348e-05, "loss": 0.2559257984161377, "mean_token_accuracy": 0.8971491321921349, "num_tokens": 2276344.0, "step": 750 }, { "entropy": 2.0892585411667826, "epoch": 1.8536585365853657, "grad_norm": 4.131836891174316, "learning_rate": 1.4523000388551089e-05, "loss": 0.3017284393310547, "mean_token_accuracy": 0.8876780085265636, "num_tokens": 2301643.0, "step": 760 }, { "entropy": 2.001885339617729, "epoch": 1.8780487804878048, "grad_norm": 8.75141716003418, "learning_rate": 1.4381501910758662e-05, "loss": 0.2728187322616577, "mean_token_accuracy": 0.9122106492519378, "num_tokens": 2328314.0, "step": 770 }, { "entropy": 2.016417753696442, "epoch": 1.9024390243902438, "grad_norm": 2.980997323989868, "learning_rate": 1.4238909272666919e-05, "loss": 0.30828332901000977, "mean_token_accuracy": 0.8823501909151673, "num_tokens": 2355364.0, "step": 780 }, { "entropy": 2.104606804251671, "epoch": 1.9268292682926829, "grad_norm": 3.4084644317626953, "learning_rate": 1.409525808288833e-05, "loss": 0.24617531299591064, "mean_token_accuracy": 0.9195199601352215, "num_tokens": 2380127.0, "step": 790 }, { "entropy": 1.978138083219528, "epoch": 1.951219512195122, "grad_norm": 6.704203128814697, "learning_rate": 1.3950584214379703e-05, "loss": 0.3253644466400146, "mean_token_accuracy": 0.9069369226694107, "num_tokens": 2411486.0, "step": 800 }, { "epoch": 1.951219512195122, "eval_entropy": 1.9054545871913433, "eval_loss": 1.2401102781295776, "eval_mean_token_accuracy": 0.7084640916436911, "eval_num_tokens": 2411486.0, "eval_runtime": 25.817, "eval_samples_per_second": 19.832, "eval_steps_per_second": 9.916, "step": 800 }, { "entropy": 2.045858007669449, "epoch": 1.975609756097561, "grad_norm": 4.906912803649902, "learning_rate": 1.38049237954839e-05, "loss": 0.2625818014144897, "mean_token_accuracy": 0.9373483300209046, "num_tokens": 2440608.0, "step": 810 }, { "entropy": 2.0264281585812567, "epoch": 2.0, "grad_norm": 4.688982963562012, "learning_rate": 1.365831320090776e-05, "loss": 0.2906618595123291, "mean_token_accuracy": 0.87566629499197, "num_tokens": 2471242.0, "step": 820 }, { "entropy": 1.978080615401268, "epoch": 2.024390243902439, "grad_norm": 3.9429972171783447, "learning_rate": 1.3510789042638525e-05, "loss": 0.14863593578338624, "mean_token_accuracy": 0.9327735617756844, "num_tokens": 2499311.0, "step": 830 }, { "entropy": 1.9506158411502839, "epoch": 2.048780487804878, "grad_norm": 2.100968360900879, "learning_rate": 1.3362388160800992e-05, "loss": 0.28978102207183837, "mean_token_accuracy": 0.9088719516992569, "num_tokens": 2528792.0, "step": 840 }, { "entropy": 1.986858731508255, "epoch": 2.073170731707317, "grad_norm": 3.8541693687438965, "learning_rate": 1.321314761445768e-05, "loss": 0.19675934314727783, "mean_token_accuracy": 0.9515588477253913, "num_tokens": 2558280.0, "step": 850 }, { "entropy": 1.9902780383825303, "epoch": 2.097560975609756, "grad_norm": 4.1620283126831055, "learning_rate": 1.3063104672354384e-05, "loss": 0.20792100429534913, "mean_token_accuracy": 0.9283792935311794, "num_tokens": 2593282.0, "step": 860 }, { "entropy": 1.9834949254989624, "epoch": 2.1219512195121952, "grad_norm": 5.908895015716553, "learning_rate": 1.2912296803613275e-05, "loss": 0.19119216203689576, "mean_token_accuracy": 0.9366185910999775, "num_tokens": 2623994.0, "step": 870 }, { "entropy": 1.897814567387104, "epoch": 2.1463414634146343, "grad_norm": 1.8507294654846191, "learning_rate": 1.2760761668376063e-05, "loss": 0.1396199345588684, "mean_token_accuracy": 0.9462398022413254, "num_tokens": 2649101.0, "step": 880 }, { "entropy": 1.930545824766159, "epoch": 2.1707317073170733, "grad_norm": 4.111316680908203, "learning_rate": 1.2608537108399354e-05, "loss": 0.23715174198150635, "mean_token_accuracy": 0.9175595246255398, "num_tokens": 2673340.0, "step": 890 }, { "entropy": 1.9516058579087256, "epoch": 2.1951219512195124, "grad_norm": 4.849324703216553, "learning_rate": 1.245566113760475e-05, "loss": 0.17189667224884034, "mean_token_accuracy": 0.9540057420730591, "num_tokens": 2707123.0, "step": 900 }, { "epoch": 2.1951219512195124, "eval_entropy": 1.7826519338414073, "eval_loss": 1.453092098236084, "eval_mean_token_accuracy": 0.7111870584194548, "eval_num_tokens": 2707123.0, "eval_runtime": 25.8656, "eval_samples_per_second": 19.795, "eval_steps_per_second": 9.897, "step": 900 }, { "entropy": 1.9171078875660896, "epoch": 2.2195121951219514, "grad_norm": 10.37508773803711, "learning_rate": 1.2302171932585885e-05, "loss": 0.2812458276748657, "mean_token_accuracy": 0.9279143020510674, "num_tokens": 2747070.0, "step": 910 }, { "entropy": 1.8540041401982308, "epoch": 2.2439024390243905, "grad_norm": 4.996469020843506, "learning_rate": 1.2148107823074873e-05, "loss": 0.23256118297576905, "mean_token_accuracy": 0.9310475498437881, "num_tokens": 2774117.0, "step": 920 }, { "entropy": 1.8542732641100883, "epoch": 2.2682926829268295, "grad_norm": 3.8328895568847656, "learning_rate": 1.1993507282370514e-05, "loss": 0.2446988821029663, "mean_token_accuracy": 0.9261137820780277, "num_tokens": 2806162.0, "step": 930 }, { "entropy": 1.8964425906538964, "epoch": 2.292682926829268, "grad_norm": 4.214897155761719, "learning_rate": 1.183840891773062e-05, "loss": 0.18508377075195312, "mean_token_accuracy": 0.9357328426092864, "num_tokens": 2832074.0, "step": 940 }, { "entropy": 1.9193901032209397, "epoch": 2.317073170731707, "grad_norm": 5.607541084289551, "learning_rate": 1.168285146073092e-05, "loss": 0.17059940099716187, "mean_token_accuracy": 0.9317200519144535, "num_tokens": 2859667.0, "step": 950 }, { "entropy": 1.8618369534611703, "epoch": 2.341463414634146, "grad_norm": 0.7818632125854492, "learning_rate": 1.1526873757592918e-05, "loss": 0.1565119981765747, "mean_token_accuracy": 0.9144253842532635, "num_tokens": 2886878.0, "step": 960 }, { "entropy": 1.8702380433678627, "epoch": 2.3658536585365852, "grad_norm": 4.2469282150268555, "learning_rate": 1.1370514759483082e-05, "loss": 0.13556138277053834, "mean_token_accuracy": 0.9443627454340457, "num_tokens": 2920137.0, "step": 970 }, { "entropy": 1.866177822649479, "epoch": 2.3902439024390243, "grad_norm": 10.995609283447266, "learning_rate": 1.1213813512785897e-05, "loss": 0.18037887811660766, "mean_token_accuracy": 0.9162259615957737, "num_tokens": 2947968.0, "step": 980 }, { "entropy": 1.9102648600935936, "epoch": 2.4146341463414633, "grad_norm": 2.3502182960510254, "learning_rate": 1.1056809149353054e-05, "loss": 0.26104583740234377, "mean_token_accuracy": 0.9086524747312069, "num_tokens": 2975976.0, "step": 990 }, { "entropy": 1.8607132963836193, "epoch": 2.4390243902439024, "grad_norm": 3.956981897354126, "learning_rate": 1.0899540876731366e-05, "loss": 0.17562326192855834, "mean_token_accuracy": 0.9512669969350099, "num_tokens": 3014385.0, "step": 1000 }, { "epoch": 2.4390243902439024, "eval_entropy": 1.762216470669955, "eval_loss": 1.3761112689971924, "eval_mean_token_accuracy": 0.7159432659391314, "eval_num_tokens": 3014385.0, "eval_runtime": 25.8636, "eval_samples_per_second": 19.796, "eval_steps_per_second": 9.898, "step": 1000 }, { "entropy": 1.833851473033428, "epoch": 2.4634146341463414, "grad_norm": 14.941336631774902, "learning_rate": 1.074204796837176e-05, "loss": 0.2952086925506592, "mean_token_accuracy": 0.9224237583577632, "num_tokens": 3044093.0, "step": 1010 }, { "entropy": 1.8050521090626717, "epoch": 2.4878048780487805, "grad_norm": 4.791407585144043, "learning_rate": 1.0584369753821781e-05, "loss": 0.29694275856018065, "mean_token_accuracy": 0.9350910216569901, "num_tokens": 3072764.0, "step": 1020 }, { "entropy": 1.8835955306887626, "epoch": 2.5121951219512195, "grad_norm": 4.77954626083374, "learning_rate": 1.0426545608904141e-05, "loss": 0.17291536331176757, "mean_token_accuracy": 0.9146630242466927, "num_tokens": 3102495.0, "step": 1030 }, { "entropy": 1.904179336130619, "epoch": 2.5365853658536586, "grad_norm": 3.219622850418091, "learning_rate": 1.0268614945883664e-05, "loss": 0.2621953010559082, "mean_token_accuracy": 0.9137581184506416, "num_tokens": 3130677.0, "step": 1040 }, { "entropy": 1.8850647330284118, "epoch": 2.5609756097560976, "grad_norm": 6.881985664367676, "learning_rate": 1.011061720362516e-05, "loss": 0.24364147186279297, "mean_token_accuracy": 0.9113392867147923, "num_tokens": 3161512.0, "step": 1050 }, { "entropy": 1.8635155737400055, "epoch": 2.5853658536585367, "grad_norm": 7.630672454833984, "learning_rate": 9.952591837744642e-06, "loss": 0.22055633068084718, "mean_token_accuracy": 0.9235416676849126, "num_tokens": 3194224.0, "step": 1060 }, { "entropy": 1.9115569680929183, "epoch": 2.6097560975609757, "grad_norm": 16.096616744995117, "learning_rate": 9.794578310756374e-06, "loss": 0.18903634548187256, "mean_token_accuracy": 0.9383679322898388, "num_tokens": 3226905.0, "step": 1070 }, { "entropy": 1.8599299892783165, "epoch": 2.6341463414634148, "grad_norm": 3.6872153282165527, "learning_rate": 9.63661608221817e-06, "loss": 0.16812350749969482, "mean_token_accuracy": 0.9036011725664139, "num_tokens": 3255535.0, "step": 1080 }, { "entropy": 1.896452783048153, "epoch": 2.658536585365854, "grad_norm": 18.830297470092773, "learning_rate": 9.478744598877469e-06, "loss": 0.18473185300827027, "mean_token_accuracy": 0.9346686199307441, "num_tokens": 3282903.0, "step": 1090 }, { "entropy": 1.8907853648066522, "epoch": 2.682926829268293, "grad_norm": 4.009158611297607, "learning_rate": 9.321003284820576e-06, "loss": 0.22757289409637452, "mean_token_accuracy": 0.8958239264786243, "num_tokens": 3312828.0, "step": 1100 }, { "epoch": 2.682926829268293, "eval_entropy": 1.7771105198189616, "eval_loss": 1.376572847366333, "eval_mean_token_accuracy": 0.7098285738611594, "eval_num_tokens": 3312828.0, "eval_runtime": 25.8822, "eval_samples_per_second": 19.782, "eval_steps_per_second": 9.891, "step": 1100 }, { "entropy": 1.9168027386069297, "epoch": 2.7073170731707314, "grad_norm": 3.3059804439544678, "learning_rate": 9.163431531627569e-06, "loss": 0.1886500120162964, "mean_token_accuracy": 0.9344054467976093, "num_tokens": 3339687.0, "step": 1110 }, { "entropy": 1.8417317017912864, "epoch": 2.7317073170731705, "grad_norm": 6.027083396911621, "learning_rate": 9.006068688535342e-06, "loss": 0.1749600052833557, "mean_token_accuracy": 0.9155175268650055, "num_tokens": 3375196.0, "step": 1120 }, { "entropy": 1.7573589943349361, "epoch": 2.7560975609756095, "grad_norm": 5.423085689544678, "learning_rate": 8.848954052611206e-06, "loss": 0.17365559339523315, "mean_token_accuracy": 0.9337028414011002, "num_tokens": 3407340.0, "step": 1130 }, { "entropy": 1.833992850780487, "epoch": 2.7804878048780486, "grad_norm": 1.2189613580703735, "learning_rate": 8.692126858939515e-06, "loss": 0.18834598064422609, "mean_token_accuracy": 0.9459356218576431, "num_tokens": 3436803.0, "step": 1140 }, { "entropy": 1.8947727873921394, "epoch": 2.8048780487804876, "grad_norm": 13.552462577819824, "learning_rate": 8.535626270823782e-06, "loss": 0.1771175742149353, "mean_token_accuracy": 0.9330002166330814, "num_tokens": 3474754.0, "step": 1150 }, { "entropy": 1.7967736013233662, "epoch": 2.8292682926829267, "grad_norm": 2.9942822456359863, "learning_rate": 8.379491370006711e-06, "loss": 0.1513003706932068, "mean_token_accuracy": 0.9353059165179729, "num_tokens": 3503522.0, "step": 1160 }, { "entropy": 1.869030448794365, "epoch": 2.8536585365853657, "grad_norm": 2.178328037261963, "learning_rate": 8.223761146910603e-06, "loss": 0.07608035206794739, "mean_token_accuracy": 0.959375, "num_tokens": 3533136.0, "step": 1170 }, { "entropy": 1.8311837516725062, "epoch": 2.8780487804878048, "grad_norm": 6.313469409942627, "learning_rate": 8.068474490900558e-06, "loss": 0.1867022156715393, "mean_token_accuracy": 0.9270378790795804, "num_tokens": 3566043.0, "step": 1180 }, { "entropy": 1.7838971972465516, "epoch": 2.902439024390244, "grad_norm": 11.847040176391602, "learning_rate": 7.913670180572936e-06, "loss": 0.24486143589019777, "mean_token_accuracy": 0.9272108852863312, "num_tokens": 3599095.0, "step": 1190 }, { "entropy": 1.8346887156367302, "epoch": 2.926829268292683, "grad_norm": 2.1996490955352783, "learning_rate": 7.759386874071444e-06, "loss": 0.25213873386383057, "mean_token_accuracy": 0.9063655398786068, "num_tokens": 3625835.0, "step": 1200 }, { "epoch": 2.926829268292683, "eval_entropy": 1.7363364584743977, "eval_loss": 1.410238265991211, "eval_mean_token_accuracy": 0.7108910096576437, "eval_num_tokens": 3625835.0, "eval_runtime": 25.8843, "eval_samples_per_second": 19.78, "eval_steps_per_second": 9.89, "step": 1200 }, { "entropy": 1.7842099949717523, "epoch": 2.951219512195122, "grad_norm": 8.536894798278809, "learning_rate": 7.60566309943333e-06, "loss": 0.12886732816696167, "mean_token_accuracy": 0.9607394687831402, "num_tokens": 3653904.0, "step": 1210 }, { "entropy": 1.8873861357569695, "epoch": 2.975609756097561, "grad_norm": 4.909502983093262, "learning_rate": 7.4525372449680555e-06, "loss": 0.2193126916885376, "mean_token_accuracy": 0.9048924177885056, "num_tokens": 3679817.0, "step": 1220 }, { "entropy": 1.769970566034317, "epoch": 3.0, "grad_norm": 4.552456855773926, "learning_rate": 7.300047549670871e-06, "loss": 0.23077039718627929, "mean_token_accuracy": 0.9360259227454663, "num_tokens": 3706863.0, "step": 1230 }, { "entropy": 1.7791202768683434, "epoch": 3.024390243902439, "grad_norm": 10.748607635498047, "learning_rate": 7.148232093673672e-06, "loss": 0.08667872548103332, "mean_token_accuracy": 0.9532332256436348, "num_tokens": 3733333.0, "step": 1240 }, { "entropy": 1.8118198722600938, "epoch": 3.048780487804878, "grad_norm": 3.7303555011749268, "learning_rate": 6.9971287887355254e-06, "loss": 0.11020035743713379, "mean_token_accuracy": 0.9588119477033615, "num_tokens": 3762382.0, "step": 1250 }, { "entropy": 1.8652134254574775, "epoch": 3.073170731707317, "grad_norm": 2.8951289653778076, "learning_rate": 6.846775368775231e-06, "loss": 0.09475313425064087, "mean_token_accuracy": 0.978449946641922, "num_tokens": 3795183.0, "step": 1260 }, { "entropy": 1.797010785341263, "epoch": 3.097560975609756, "grad_norm": 6.627988815307617, "learning_rate": 6.697209380448333e-06, "loss": 0.15345336198806764, "mean_token_accuracy": 0.9757440477609635, "num_tokens": 3823044.0, "step": 1270 }, { "entropy": 1.7353664010763168, "epoch": 3.1219512195121952, "grad_norm": 1.0590440034866333, "learning_rate": 6.548468173770845e-06, "loss": 0.13489140272140504, "mean_token_accuracy": 0.962930253893137, "num_tokens": 3849197.0, "step": 1280 }, { "entropy": 1.697338229417801, "epoch": 3.1463414634146343, "grad_norm": 3.2843127250671387, "learning_rate": 6.400588892792127e-06, "loss": 0.0719787061214447, "mean_token_accuracy": 0.9517784558236599, "num_tokens": 3878232.0, "step": 1290 }, { "entropy": 1.7221568048000335, "epoch": 3.1707317073170733, "grad_norm": 10.110462188720703, "learning_rate": 6.253608466319157e-06, "loss": 0.1476641058921814, "mean_token_accuracy": 0.9608659982681275, "num_tokens": 3901609.0, "step": 1300 }, { "epoch": 3.1707317073170733, "eval_entropy": 1.617616279516369, "eval_loss": 1.7025296688079834, "eval_mean_token_accuracy": 0.7084888513199985, "eval_num_tokens": 3901609.0, "eval_runtime": 25.8445, "eval_samples_per_second": 19.811, "eval_steps_per_second": 9.905, "step": 1300 }, { "entropy": 1.6494078099727632, "epoch": 3.1951219512195124, "grad_norm": 19.496536254882812, "learning_rate": 6.107563598694577e-06, "loss": 0.12370929718017579, "mean_token_accuracy": 0.9384686142206192, "num_tokens": 3934470.0, "step": 1310 }, { "entropy": 1.7082608208060264, "epoch": 3.2195121951219514, "grad_norm": 8.636344909667969, "learning_rate": 5.962490760630787e-06, "loss": 0.07684900164604187, "mean_token_accuracy": 0.9750022895634174, "num_tokens": 3965484.0, "step": 1320 }, { "entropy": 1.6917474582791328, "epoch": 3.2439024390243905, "grad_norm": 5.012264728546143, "learning_rate": 5.818426180102382e-06, "loss": 0.13596385717391968, "mean_token_accuracy": 0.9754807695746421, "num_tokens": 3995664.0, "step": 1330 }, { "entropy": 1.674880462884903, "epoch": 3.2682926829268295, "grad_norm": 11.996963500976562, "learning_rate": 5.675405833299215e-06, "loss": 0.14785323143005372, "mean_token_accuracy": 0.945119047909975, "num_tokens": 4022795.0, "step": 1340 }, { "entropy": 1.670597495138645, "epoch": 3.292682926829268, "grad_norm": 4.758082389831543, "learning_rate": 5.533465435642305e-06, "loss": 0.15053837299346923, "mean_token_accuracy": 0.9385013237595559, "num_tokens": 4051622.0, "step": 1350 }, { "entropy": 1.7326559245586395, "epoch": 3.317073170731707, "grad_norm": 3.25938081741333, "learning_rate": 5.392640432864913e-06, "loss": 0.1384304404258728, "mean_token_accuracy": 0.9724495135247707, "num_tokens": 4079449.0, "step": 1360 }, { "entropy": 1.7230241522192955, "epoch": 3.341463414634146, "grad_norm": 5.058661460876465, "learning_rate": 5.252965992160914e-06, "loss": 0.18940384387969972, "mean_token_accuracy": 0.9403771705925464, "num_tokens": 4110411.0, "step": 1370 }, { "entropy": 1.7641312688589097, "epoch": 3.3658536585365852, "grad_norm": 1.001191258430481, "learning_rate": 5.114476993402772e-06, "loss": 0.1598334789276123, "mean_token_accuracy": 0.9734844461083412, "num_tokens": 4142316.0, "step": 1380 }, { "entropy": 1.7149900317192077, "epoch": 3.3902439024390243, "grad_norm": 1.689876914024353, "learning_rate": 4.977208020431246e-06, "loss": 0.08270336389541626, "mean_token_accuracy": 0.9644473820924759, "num_tokens": 4175477.0, "step": 1390 }, { "entropy": 1.6951182693243028, "epoch": 3.4146341463414633, "grad_norm": 2.8464765548706055, "learning_rate": 4.841193352419003e-06, "loss": 0.0659868061542511, "mean_token_accuracy": 0.9773275084793568, "num_tokens": 4205106.0, "step": 1400 }, { "epoch": 3.4146341463414633, "eval_entropy": 1.6031765793450177, "eval_loss": 1.806748390197754, "eval_mean_token_accuracy": 0.7046625635703094, "eval_num_tokens": 4205106.0, "eval_runtime": 25.8817, "eval_samples_per_second": 19.782, "eval_steps_per_second": 9.891, "step": 1400 }, { "entropy": 1.5698539711534978, "epoch": 3.4390243902439024, "grad_norm": 1.9268624782562256, "learning_rate": 4.706466955310343e-06, "loss": 0.1374589443206787, "mean_token_accuracy": 0.970430264621973, "num_tokens": 4245146.0, "step": 1410 }, { "entropy": 1.6444569684565067, "epoch": 3.4634146341463414, "grad_norm": 0.09189051389694214, "learning_rate": 4.573062473339135e-06, "loss": 0.10669373273849488, "mean_token_accuracy": 0.9647710219025611, "num_tokens": 4286404.0, "step": 1420 }, { "entropy": 1.6077862866222858, "epoch": 3.4878048780487805, "grad_norm": 1.4636071920394897, "learning_rate": 4.441013220627087e-06, "loss": 0.10974626541137696, "mean_token_accuracy": 0.9651883531361818, "num_tokens": 4315887.0, "step": 1430 }, { "entropy": 1.597128450870514, "epoch": 3.5121951219512195, "grad_norm": 2.6442878246307373, "learning_rate": 4.310352172864468e-06, "loss": 0.07910129427909851, "mean_token_accuracy": 0.9693680226802825, "num_tokens": 4348889.0, "step": 1440 }, { "entropy": 1.6682945892214776, "epoch": 3.5365853658536586, "grad_norm": 0.8266032338142395, "learning_rate": 4.181111959075295e-06, "loss": 0.12142558097839355, "mean_token_accuracy": 0.9667701870203018, "num_tokens": 4375994.0, "step": 1450 }, { "entropy": 1.632444016635418, "epoch": 3.5609756097560976, "grad_norm": 2.069300651550293, "learning_rate": 4.053324853469164e-06, "loss": 0.06663359999656678, "mean_token_accuracy": 0.9756163962185382, "num_tokens": 4407398.0, "step": 1460 }, { "entropy": 1.6774661853909492, "epoch": 3.5853658536585367, "grad_norm": 1.953863501548767, "learning_rate": 3.927022767381634e-06, "loss": 0.141204571723938, "mean_token_accuracy": 0.9584236457943917, "num_tokens": 4433323.0, "step": 1470 }, { "entropy": 1.6051794439554214, "epoch": 3.6097560975609757, "grad_norm": 1.12648606300354, "learning_rate": 3.8022372413052424e-06, "loss": 0.13420920372009276, "mean_token_accuracy": 0.9492732554674148, "num_tokens": 4459482.0, "step": 1480 }, { "entropy": 1.619800090789795, "epoch": 3.6341463414634148, "grad_norm": 5.512538433074951, "learning_rate": 3.678999437013129e-06, "loss": 0.19428257942199706, "mean_token_accuracy": 0.9662267230451107, "num_tokens": 4483124.0, "step": 1490 }, { "entropy": 1.6692272156476975, "epoch": 3.658536585365854, "grad_norm": 2.8572192192077637, "learning_rate": 3.5573401297772404e-06, "loss": 0.07371877431869507, "mean_token_accuracy": 0.9690535716712475, "num_tokens": 4512129.0, "step": 1500 }, { "epoch": 3.658536585365854, "eval_entropy": 1.577660508453846, "eval_loss": 1.8952677249908447, "eval_mean_token_accuracy": 0.7052186403889209, "eval_num_tokens": 4512129.0, "eval_runtime": 25.8489, "eval_samples_per_second": 19.807, "eval_steps_per_second": 9.904, "step": 1500 }, { "entropy": 1.722746029496193, "epoch": 3.682926829268293, "grad_norm": 8.862682342529297, "learning_rate": 3.437289700683042e-06, "loss": 0.10439562797546387, "mean_token_accuracy": 0.9602027926594019, "num_tokens": 4543269.0, "step": 1510 }, { "entropy": 1.6523637086153031, "epoch": 3.7073170731707314, "grad_norm": 13.63913631439209, "learning_rate": 3.318878129042675e-06, "loss": 0.11178983449935913, "mean_token_accuracy": 0.953480490297079, "num_tokens": 4574267.0, "step": 1520 }, { "entropy": 1.6459150895476342, "epoch": 3.7317073170731705, "grad_norm": 3.3351998329162598, "learning_rate": 3.2021349849084425e-06, "loss": 0.13762412071228028, "mean_token_accuracy": 0.9543538652360439, "num_tokens": 4603986.0, "step": 1530 }, { "entropy": 1.5934583507478237, "epoch": 3.7560975609756095, "grad_norm": 11.15025520324707, "learning_rate": 3.0870894216884994e-06, "loss": 0.15420351028442383, "mean_token_accuracy": 0.9434968508780003, "num_tokens": 4631314.0, "step": 1540 }, { "entropy": 1.6086938992142676, "epoch": 3.7804878048780486, "grad_norm": 6.632437705993652, "learning_rate": 2.97377016886658e-06, "loss": 0.1366713285446167, "mean_token_accuracy": 0.9587422519922256, "num_tokens": 4660281.0, "step": 1550 }, { "entropy": 1.6767722338438034, "epoch": 3.8048780487804876, "grad_norm": 26.195016860961914, "learning_rate": 2.8622055248276037e-06, "loss": 0.10660663843154908, "mean_token_accuracy": 0.9429924249649048, "num_tokens": 4687655.0, "step": 1560 }, { "entropy": 1.6780916333198548, "epoch": 3.8292682926829267, "grad_norm": 12.992459297180176, "learning_rate": 2.7524233497909183e-06, "loss": 0.0768752634525299, "mean_token_accuracy": 0.9729667611420154, "num_tokens": 4722107.0, "step": 1570 }, { "entropy": 1.6294035658240318, "epoch": 3.8536585365853657, "grad_norm": 10.002275466918945, "learning_rate": 2.644451058852979e-06, "loss": 0.1883617162704468, "mean_token_accuracy": 0.9308527745306492, "num_tokens": 4761243.0, "step": 1580 }, { "entropy": 1.6836735352873802, "epoch": 3.8780487804878048, "grad_norm": 13.285955429077148, "learning_rate": 2.5383156151411736e-06, "loss": 0.09115610718727112, "mean_token_accuracy": 0.9692550510168075, "num_tokens": 4789879.0, "step": 1590 }, { "entropy": 1.6084473103284835, "epoch": 3.902439024390244, "grad_norm": 15.486495018005371, "learning_rate": 2.4340435230805235e-06, "loss": 0.08411281108856201, "mean_token_accuracy": 0.9599707998335362, "num_tokens": 4819154.0, "step": 1600 }, { "epoch": 3.902439024390244, "eval_entropy": 1.5618506374303252, "eval_loss": 1.8181430101394653, "eval_mean_token_accuracy": 0.7102930387482047, "eval_num_tokens": 4819154.0, "eval_runtime": 25.8912, "eval_samples_per_second": 19.775, "eval_steps_per_second": 9.888, "step": 1600 }, { "entropy": 1.6830802142620087, "epoch": 3.926829268292683, "grad_norm": 5.259362697601318, "learning_rate": 2.331660821774915e-06, "loss": 0.05890817642211914, "mean_token_accuracy": 0.9772120036184788, "num_tokens": 4847586.0, "step": 1610 }, { "entropy": 1.6499988630414009, "epoch": 3.951219512195122, "grad_norm": 16.05006217956543, "learning_rate": 2.231193078504561e-06, "loss": 0.17039716243743896, "mean_token_accuracy": 0.9628652513027192, "num_tokens": 4875353.0, "step": 1620 }, { "entropy": 1.7116807617247105, "epoch": 3.975609756097561, "grad_norm": 3.7669856548309326, "learning_rate": 2.1326653823412703e-06, "loss": 0.08578027486801147, "mean_token_accuracy": 0.9435943216085434, "num_tokens": 4908273.0, "step": 1630 }, { "entropy": 1.5733015567064286, "epoch": 4.0, "grad_norm": 0.10747440159320831, "learning_rate": 2.036102337883129e-06, "loss": 0.09882248640060425, "mean_token_accuracy": 0.9706439398229122, "num_tokens": 4942484.0, "step": 1640 }, { "entropy": 1.650183080136776, "epoch": 4.024390243902439, "grad_norm": 0.07803265005350113, "learning_rate": 1.9415280591101738e-06, "loss": 0.03629283905029297, "mean_token_accuracy": 0.9803571425378322, "num_tokens": 4971861.0, "step": 1650 }, { "entropy": 1.6938461996614933, "epoch": 4.048780487804878, "grad_norm": 1.196933388710022, "learning_rate": 1.8489661633625955e-06, "loss": 0.06059607267379761, "mean_token_accuracy": 0.980208333581686, "num_tokens": 4998814.0, "step": 1660 }, { "entropy": 1.6491495966911316, "epoch": 4.073170731707317, "grad_norm": 6.6899518966674805, "learning_rate": 1.7584397654429475e-06, "loss": 0.13204474449157716, "mean_token_accuracy": 0.9651960790157318, "num_tokens": 5026456.0, "step": 1670 }, { "entropy": 1.6061969593167305, "epoch": 4.097560975609756, "grad_norm": 5.504490375518799, "learning_rate": 1.6699714718438643e-06, "loss": 0.0931332528591156, "mean_token_accuracy": 0.9626355342566967, "num_tokens": 5054456.0, "step": 1680 }, { "entropy": 1.6022393718361854, "epoch": 4.121951219512195, "grad_norm": 1.4340182542800903, "learning_rate": 1.5835833751026853e-06, "loss": 0.0475650817155838, "mean_token_accuracy": 0.969767627120018, "num_tokens": 5081929.0, "step": 1690 }, { "entropy": 1.6138322927057742, "epoch": 4.146341463414634, "grad_norm": 1.5020872354507446, "learning_rate": 1.4992970482844759e-06, "loss": 0.04676715731620788, "mean_token_accuracy": 0.9869006261229515, "num_tokens": 5116262.0, "step": 1700 }, { "epoch": 4.146341463414634, "eval_entropy": 1.5413944267202169, "eval_loss": 1.938948392868042, "eval_mean_token_accuracy": 0.7062437398126349, "eval_num_tokens": 5116262.0, "eval_runtime": 25.8736, "eval_samples_per_second": 19.789, "eval_steps_per_second": 9.894, "step": 1700 }, { "entropy": 1.6139937132596969, "epoch": 4.170731707317073, "grad_norm": 1.462416172027588, "learning_rate": 1.4171335395947272e-06, "loss": 0.13126976490020753, "mean_token_accuracy": 0.982309315353632, "num_tokens": 5149087.0, "step": 1710 }, { "entropy": 1.6237235128879548, "epoch": 4.195121951219512, "grad_norm": 0.335536926984787, "learning_rate": 1.3371133671231473e-06, "loss": 0.11835663318634033, "mean_token_accuracy": 0.978372187167406, "num_tokens": 5181250.0, "step": 1720 }, { "entropy": 1.5813236683607101, "epoch": 4.219512195121951, "grad_norm": 11.065515518188477, "learning_rate": 1.2592565137198298e-06, "loss": 0.10499415397644044, "mean_token_accuracy": 0.954606880992651, "num_tokens": 5214533.0, "step": 1730 }, { "entropy": 1.5916456162929535, "epoch": 4.2439024390243905, "grad_norm": 2.2649731636047363, "learning_rate": 1.1835824220050796e-06, "loss": 0.04095487892627716, "mean_token_accuracy": 0.9981433153152466, "num_tokens": 5243169.0, "step": 1740 }, { "entropy": 1.5513525933027268, "epoch": 4.2682926829268295, "grad_norm": 6.82105827331543, "learning_rate": 1.1101099895141542e-06, "loss": 0.053831398487091064, "mean_token_accuracy": 0.978318965435028, "num_tokens": 5273306.0, "step": 1750 }, { "entropy": 1.5593497730791568, "epoch": 4.2926829268292686, "grad_norm": 4.913758754730225, "learning_rate": 1.0388575639781217e-06, "loss": 0.06152739524841309, "mean_token_accuracy": 0.9857870370149613, "num_tokens": 5304330.0, "step": 1760 }, { "entropy": 1.600521869212389, "epoch": 4.317073170731708, "grad_norm": 6.424625873565674, "learning_rate": 9.698429387420126e-07, "loss": 0.12111327648162842, "mean_token_accuracy": 0.9749501794576645, "num_tokens": 5338400.0, "step": 1770 }, { "entropy": 1.5368691369891168, "epoch": 4.341463414634147, "grad_norm": 0.3439267873764038, "learning_rate": 9.030833483214208e-07, "loss": 0.0153617262840271, "mean_token_accuracy": 0.9935606062412262, "num_tokens": 5366394.0, "step": 1780 }, { "entropy": 1.5737651720643044, "epoch": 4.365853658536586, "grad_norm": 5.005470275878906, "learning_rate": 8.385954640986538e-07, "loss": 0.05194443464279175, "mean_token_accuracy": 0.9777217738330364, "num_tokens": 5393693.0, "step": 1790 }, { "entropy": 1.5871941670775414, "epoch": 4.390243902439025, "grad_norm": 0.5401604175567627, "learning_rate": 7.763953901595111e-07, "loss": 0.07116225957870484, "mean_token_accuracy": 0.9889907136559486, "num_tokens": 5421124.0, "step": 1800 }, { "epoch": 4.390243902439025, "eval_entropy": 1.4946594985667616, "eval_loss": 2.0964248180389404, "eval_mean_token_accuracy": 0.7045429775025696, "eval_num_tokens": 5421124.0, "eval_runtime": 25.9064, "eval_samples_per_second": 19.763, "eval_steps_per_second": 9.882, "step": 1800 }, { "entropy": 1.5554208427667617, "epoch": 4.414634146341464, "grad_norm": 24.223447799682617, "learning_rate": 7.164986592717327e-07, "loss": 0.10402250289916992, "mean_token_accuracy": 0.9780024513602257, "num_tokens": 5447960.0, "step": 1810 }, { "entropy": 1.546605361253023, "epoch": 4.439024390243903, "grad_norm": 2.0834453105926514, "learning_rate": 6.589202290061015e-07, "loss": 0.05245453119277954, "mean_token_accuracy": 0.9917803034186363, "num_tokens": 5486657.0, "step": 1820 }, { "entropy": 1.5830803610384465, "epoch": 4.463414634146342, "grad_norm": 1.1376031637191772, "learning_rate": 6.036744780012227e-07, "loss": 0.09751057028770446, "mean_token_accuracy": 0.941666667163372, "num_tokens": 5516329.0, "step": 1830 }, { "entropy": 1.5013430513441564, "epoch": 4.487804878048781, "grad_norm": 3.6684820652008057, "learning_rate": 5.507752023728286e-07, "loss": 0.04153382182121277, "mean_token_accuracy": 0.9785602673888206, "num_tokens": 5547317.0, "step": 1840 }, { "entropy": 1.5333821460604669, "epoch": 4.512195121951219, "grad_norm": 0.326924204826355, "learning_rate": 5.002356122685914e-07, "loss": 0.07603456377983094, "mean_token_accuracy": 0.993276233971119, "num_tokens": 5575282.0, "step": 1850 }, { "entropy": 1.518524892628193, "epoch": 4.536585365853659, "grad_norm": 14.980679512023926, "learning_rate": 4.5206832856923845e-07, "loss": 0.07430620789527893, "mean_token_accuracy": 0.9883873172104358, "num_tokens": 5612804.0, "step": 1860 }, { "entropy": 1.5543180502951146, "epoch": 4.560975609756097, "grad_norm": 6.43148946762085, "learning_rate": 4.0628537973684046e-07, "loss": 0.1106789231300354, "mean_token_accuracy": 0.9912989094853402, "num_tokens": 5645237.0, "step": 1870 }, { "entropy": 1.5496280819177628, "epoch": 4.585365853658536, "grad_norm": 1.8432977199554443, "learning_rate": 3.628981988110336e-07, "loss": 0.04811676144599915, "mean_token_accuracy": 0.96875, "num_tokens": 5676121.0, "step": 1880 }, { "entropy": 1.535101333260536, "epoch": 4.609756097560975, "grad_norm": 4.813788890838623, "learning_rate": 3.219176205539243e-07, "loss": 0.0236665740609169, "mean_token_accuracy": 0.9846153847873211, "num_tokens": 5717000.0, "step": 1890 }, { "entropy": 1.550465030223131, "epoch": 4.634146341463414, "grad_norm": 0.5678615570068359, "learning_rate": 2.8335387874440124e-07, "loss": 0.11958084106445313, "mean_token_accuracy": 0.9657242063432931, "num_tokens": 5743465.0, "step": 1900 }, { "epoch": 4.634146341463414, "eval_entropy": 1.4860404375940561, "eval_loss": 2.1459527015686035, "eval_mean_token_accuracy": 0.7026900857454166, "eval_num_tokens": 5743465.0, "eval_runtime": 25.8968, "eval_samples_per_second": 19.771, "eval_steps_per_second": 9.885, "step": 1900 }, { "entropy": 1.5757036216557025, "epoch": 4.658536585365853, "grad_norm": 9.97290325164795, "learning_rate": 2.4721660362253875e-07, "loss": 0.09184517860412597, "mean_token_accuracy": 0.95625, "num_tokens": 5772857.0, "step": 1910 }, { "entropy": 1.5368487030267715, "epoch": 4.682926829268292, "grad_norm": 0.04057607799768448, "learning_rate": 2.1351481948470142e-07, "loss": 0.050920414924621585, "mean_token_accuracy": 0.985744047909975, "num_tokens": 5806885.0, "step": 1920 }, { "entropy": 1.5537334114313126, "epoch": 4.7073170731707314, "grad_norm": 6.230452060699463, "learning_rate": 1.8225694242997583e-07, "loss": 0.037062066793441775, "mean_token_accuracy": 0.9870689652860165, "num_tokens": 5836765.0, "step": 1930 }, { "entropy": 1.55542683750391, "epoch": 4.7317073170731705, "grad_norm": 2.9326839447021484, "learning_rate": 1.5345077825847266e-07, "loss": 0.07759016752243042, "mean_token_accuracy": 0.9804261364042759, "num_tokens": 5863759.0, "step": 1940 }, { "entropy": 1.540198379009962, "epoch": 4.7560975609756095, "grad_norm": 1.1355870962142944, "learning_rate": 1.2710352052205166e-07, "loss": 0.0746928870677948, "mean_token_accuracy": 0.9800629302859306, "num_tokens": 5892661.0, "step": 1950 }, { "entropy": 1.5091066129505635, "epoch": 4.780487804878049, "grad_norm": 0.023747609928250313, "learning_rate": 1.032217487279219e-07, "loss": 0.0806429624557495, "mean_token_accuracy": 0.9624692976474762, "num_tokens": 5919300.0, "step": 1960 }, { "entropy": 1.570461356639862, "epoch": 4.804878048780488, "grad_norm": 18.05229949951172, "learning_rate": 8.181142669559383e-08, "loss": 0.044895905256271365, "mean_token_accuracy": 0.9864312775433064, "num_tokens": 5951012.0, "step": 1970 }, { "entropy": 1.5230771772563458, "epoch": 4.829268292682927, "grad_norm": 25.309349060058594, "learning_rate": 6.287790106757396e-08, "loss": 0.08148726224899291, "mean_token_accuracy": 0.9669642865657806, "num_tokens": 5977981.0, "step": 1980 }, { "entropy": 1.5778639271855355, "epoch": 4.853658536585366, "grad_norm": 0.9248256683349609, "learning_rate": 4.642589997419178e-08, "loss": 0.051714646816253665, "mean_token_accuracy": 0.9888888888061047, "num_tokens": 6007088.0, "step": 1990 }, { "entropy": 1.5951696388423442, "epoch": 4.878048780487805, "grad_norm": 0.006268620491027832, "learning_rate": 3.2459531852873094e-08, "loss": 0.07305824756622314, "mean_token_accuracy": 0.9735119052231311, "num_tokens": 6032377.0, "step": 2000 }, { "epoch": 4.878048780487805, "eval_entropy": 1.486908347113058, "eval_loss": 2.1505281925201416, "eval_mean_token_accuracy": 0.7025353723438457, "eval_num_tokens": 6032377.0, "eval_runtime": 25.8858, "eval_samples_per_second": 19.779, "eval_steps_per_second": 9.89, "step": 2000 }, { "entropy": 1.5873251944780349, "epoch": 4.902439024390244, "grad_norm": 1.0092874765396118, "learning_rate": 2.0982284422177424e-08, "loss": 0.11313344240188598, "mean_token_accuracy": 0.9539617486298084, "num_tokens": 6062030.0, "step": 2010 }, { "entropy": 1.6132668308913707, "epoch": 4.926829268292683, "grad_norm": 2.0686683654785156, "learning_rate": 1.1997023810829122e-08, "loss": 0.08837978839874268, "mean_token_accuracy": 0.9671875, "num_tokens": 6092905.0, "step": 2020 }, { "entropy": 1.5839506693184375, "epoch": 4.951219512195122, "grad_norm": 3.7344248294830322, "learning_rate": 5.505993841982138e-09, "loss": 0.07675654292106629, "mean_token_accuracy": 0.9829166673123837, "num_tokens": 6119924.0, "step": 2030 }, { "entropy": 1.597297092527151, "epoch": 4.975609756097561, "grad_norm": 2.4275758266448975, "learning_rate": 1.5108154728882273e-09, "loss": 0.10278631448745727, "mean_token_accuracy": 0.968489583581686, "num_tokens": 6151903.0, "step": 2040 }, { "entropy": 1.5558482602238655, "epoch": 5.0, "grad_norm": 0.7581784725189209, "learning_rate": 1.2486390105204138e-11, "loss": 0.0749094009399414, "mean_token_accuracy": 0.9579861663281918, "num_tokens": 6178105.0, "step": 2050 }, { "epoch": 5.0, "eval_entropy": 1.4859420859720558, "eval_loss": 2.1463794708251953, "eval_mean_token_accuracy": 0.7037711599841714, "eval_num_tokens": 6178105.0, "eval_runtime": 25.8943, "eval_samples_per_second": 19.773, "eval_steps_per_second": 9.886, "step": 2050 }, { "epoch": 5.0, "step": 2050, "total_flos": 2.7924674629005312e+17, "train_loss": 0.25280428157347007, "train_runtime": 3979.2596, "train_samples_per_second": 4.121, "train_steps_per_second": 0.515 } ], "logging_steps": 10, "max_steps": 2050, "num_input_tokens_seen": 0, "num_train_epochs": 5, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 2.7924674629005312e+17, "train_batch_size": 1, "trial_name": null, "trial_params": null }