Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 471.30it/s]

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:00<00:00,  7.17it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00, 13.96it/s]
{
  "parameters": 505350400,
  "device": "cuda",
  "sequence_length": 2048,
  "train_sequences": 22155,
  "validation_sequences": 300,
  "epochs": 2.0,
  "sequences_per_optimizer_step": 60,
  "tokens_per_optimizer_step": 122880,
  "max_steps": 739,
  "planned_capacity_tokens": 90808320,
  "start_step": 10,
  "initialized_from": null,
  "assistant_only_loss": true,
  "data_source": "huggingface_streaming",
  "precision": "bf16",
  "compile": false,
  "gradient_checkpointing": false
}
Loading a state dict of a shuffle buffer of a dataset without the buffer content.The shuffle buffer will be refilled before starting to yield new examples.
{"step": 20, "max_steps": 739, "train_loss": 2.200546, "learning_rate": 1.8181818181818182e-05, "grad_norm": 0.4163810908794403, "supervised_tokens_in_step": 91838, "tokens_per_second": 27230}
{"step": 30, "max_steps": 739, "train_loss": 2.137885, "learning_rate": 1.9994471472761282e-05, "grad_norm": 0.385495662689209, "supervised_tokens_in_step": 89768, "tokens_per_second": 31070}
{"step": 40, "max_steps": 739, "train_loss": 2.082109, "learning_rate": 1.9972023471167004e-05, "grad_norm": 0.3679565191268921, "supervised_tokens_in_step": 90269, "tokens_per_second": 31056}
{"step": 50, "max_steps": 739, "train_loss": 2.0298, "learning_rate": 1.9932353518414295e-05, "grad_norm": 0.3575701415538788, "supervised_tokens_in_step": 93988, "tokens_per_second": 31047}
{"step": 60, "max_steps": 739, "train_loss": 2.049464, "learning_rate": 1.9875537761650004e-05, "grad_norm": 0.36862215399742126, "supervised_tokens_in_step": 89768, "tokens_per_second": 31032}
{"step": 70, "max_steps": 739, "train_loss": 1.958528, "learning_rate": 1.9801685259682452e-05, "grad_norm": 0.3624249994754791, "supervised_tokens_in_step": 92373, "tokens_per_second": 31034}
{"step": 80, "max_steps": 739, "train_loss": 2.029217, "learning_rate": 1.971093777364121e-05, "grad_norm": 0.3733138144016266, "supervised_tokens_in_step": 90226, "tokens_per_second": 31040}
{"step": 90, "max_steps": 739, "train_loss": 1.929385, "learning_rate": 1.9603469494864215e-05, "grad_norm": 0.3631318211555481, "supervised_tokens_in_step": 89650, "tokens_per_second": 31044}
{"step": 100, "max_steps": 739, "train_loss": 1.983628, "learning_rate": 1.947948671053451e-05, "grad_norm": 0.37106239795684814, "supervised_tokens_in_step": 91416, "tokens_per_second": 31029}
{"step": 100, "validation_loss": 2.081041, "validation_perplexity": 8.0128, "validation_supervised_tokens": 368077}
{"step": 110, "max_steps": 739, "train_loss": 2.091144, "learning_rate": 1.933922740770854e-05, "grad_norm": 0.36857303977012634, "supervised_tokens_in_step": 91081, "tokens_per_second": 27352}
{"step": 120, "max_steps": 739, "train_loss": 2.006558, "learning_rate": 1.918296081649594e-05, "grad_norm": 0.3746139109134674, "supervised_tokens_in_step": 89317, "tokens_per_second": 31022}
{"step": 130, "max_steps": 739, "train_loss": 1.994367, "learning_rate": 1.9010986893267776e-05, "grad_norm": 0.35678595304489136, "supervised_tokens_in_step": 93730, "tokens_per_second": 31038}
{"step": 140, "max_steps": 739, "train_loss": 1.969047, "learning_rate": 1.8823635744885215e-05, "grad_norm": 0.36139315366744995, "supervised_tokens_in_step": 93137, "tokens_per_second": 31034}
{"step": 150, "max_steps": 739, "train_loss": 1.977563, "learning_rate": 1.8621266995053796e-05, "grad_norm": 0.35925647616386414, "supervised_tokens_in_step": 92158, "tokens_per_second": 31033}
{"step": 160, "max_steps": 739, "train_loss": 1.958745, "learning_rate": 1.8404269094019652e-05, "grad_norm": 0.35704389214515686, "supervised_tokens_in_step": 93347, "tokens_per_second": 31039}
{"step": 170, "max_steps": 739, "train_loss": 1.955936, "learning_rate": 1.8173058572932637e-05, "grad_norm": 0.35927313566207886, "supervised_tokens_in_step": 90266, "tokens_per_second": 31035}
{"step": 180, "max_steps": 739, "train_loss": 1.970204, "learning_rate": 1.7928079244307776e-05, "grad_norm": 0.3543277382850647, "supervised_tokens_in_step": 89477, "tokens_per_second": 31046}
{"step": 190, "max_steps": 739, "train_loss": 1.934904, "learning_rate": 1.7669801350119565e-05, "grad_norm": 0.36543169617652893, "supervised_tokens_in_step": 91818, "tokens_per_second": 31050}
{"step": 200, "max_steps": 739, "train_loss": 2.007836, "learning_rate": 1.7398720659164565e-05, "grad_norm": 0.3641034662723541, "supervised_tokens_in_step": 90828, "tokens_per_second": 31045}
{"step": 200, "validation_loss": 2.037366, "validation_perplexity": 7.6704, "validation_supervised_tokens": 368077}
Loading a state dict of a shuffle buffer of a dataset without the buffer content.The shuffle buffer will be refilled before starting to yield new examples.
{"step": 210, "max_steps": 739, "train_loss": 1.948719, "learning_rate": 1.7115357515424788e-05, "grad_norm": 0.36110395193099976, "supervised_tokens_in_step": 91207, "tokens_per_second": 24060}
{"step": 220, "max_steps": 739, "train_loss": 1.928805, "learning_rate": 1.682025583925858e-05, "grad_norm": 0.36596813797950745, "supervised_tokens_in_step": 91495, "tokens_per_second": 31018}
{"step": 230, "max_steps": 739, "train_loss": 1.951567, "learning_rate": 1.6513982083336294e-05, "grad_norm": 0.3660127520561218, "supervised_tokens_in_step": 91789, "tokens_per_second": 31025}
{"step": 240, "max_steps": 739, "train_loss": 1.906401, "learning_rate": 1.6197124145324775e-05, "grad_norm": 0.35649800300598145, "supervised_tokens_in_step": 92093, "tokens_per_second": 31041}
{"step": 250, "max_steps": 739, "train_loss": 1.893618, "learning_rate": 1.5870290239407834e-05, "grad_norm": 0.38765037059783936, "supervised_tokens_in_step": 89919, "tokens_per_second": 31042}
Saved checkpoint: runs/kawk500m-ko-instruct-v1-a100/checkpoint-00000250
{"step": 260, "max_steps": 739, "train_loss": 1.838687, "learning_rate": 1.5534107728808798e-05, "grad_norm": 0.3714355230331421, "supervised_tokens_in_step": 91009, "tokens_per_second": 25208}
{"step": 270, "max_steps": 739, "train_loss": 1.857017, "learning_rate": 1.5189221921556159e-05, "grad_norm": 0.3687601387500763, "supervised_tokens_in_step": 91754, "tokens_per_second": 31019}
{"step": 280, "max_steps": 739, "train_loss": 1.880019, "learning_rate": 1.4836294831803904e-05, "grad_norm": 0.3878323435783386, "supervised_tokens_in_step": 90070, "tokens_per_second": 31027}
{"step": 290, "max_steps": 739, "train_loss": 1.874783, "learning_rate": 1.4476003909084096e-05, "grad_norm": 0.3904125690460205, "supervised_tokens_in_step": 89451, "tokens_per_second": 31038}
{"step": 300, "max_steps": 739, "train_loss": 1.90464, "learning_rate": 1.4109040737931015e-05, "grad_norm": 0.37880393862724304, "supervised_tokens_in_step": 89735, "tokens_per_second": 31041}
{"step": 300, "validation_loss": 2.017481, "validation_perplexity": 7.5194, "validation_supervised_tokens": 368077}
{"step": 310, "max_steps": 739, "train_loss": 1.890699, "learning_rate": 1.3736109710372933e-05, "grad_norm": 0.3769417703151703, "supervised_tokens_in_step": 92323, "tokens_per_second": 27371}
{"step": 320, "max_steps": 739, "train_loss": 1.836743, "learning_rate": 1.3357926673839667e-05, "grad_norm": 0.3678024411201477, "supervised_tokens_in_step": 90536, "tokens_per_second": 31032}
{"step": 330, "max_steps": 739, "train_loss": 1.931257, "learning_rate": 1.2975217557081295e-05, "grad_norm": 0.38104498386383057, "supervised_tokens_in_step": 91111, "tokens_per_second": 31053}
{"step": 340, "max_steps": 739, "train_loss": 1.824471, "learning_rate": 1.2588716976735622e-05, "grad_norm": 0.36527207493782043, "supervised_tokens_in_step": 92740, "tokens_per_second": 31048}
{"step": 350, "max_steps": 739, "train_loss": 1.902425, "learning_rate": 1.2199166827219094e-05, "grad_norm": 0.3782408535480499, "supervised_tokens_in_step": 89779, "tokens_per_second": 31043}
{"step": 360, "max_steps": 739, "train_loss": 1.895964, "learning_rate": 1.1807314856647862e-05, "grad_norm": 0.38417351245880127, "supervised_tokens_in_step": 87455, "tokens_per_second": 31047}
{"step": 370, "max_steps": 739, "train_loss": 1.938641, "learning_rate": 1.1413913231522637e-05, "grad_norm": 0.37378209829330444, "supervised_tokens_in_step": 88765, "tokens_per_second": 31032}
{"step": 380, "max_steps": 739, "train_loss": 1.86831, "learning_rate": 1.1019717092932303e-05, "grad_norm": 0.37865233421325684, "supervised_tokens_in_step": 89250, "tokens_per_second": 31047}
{"step": 390, "max_steps": 739, "train_loss": 1.862856, "learning_rate": 1.0625483107047814e-05, "grad_norm": 0.37694498896598816, "supervised_tokens_in_step": 88867, "tokens_per_second": 31049}
{"step": 400, "max_steps": 739, "train_loss": 1.990085, "learning_rate": 1.0231968012688647e-05, "grad_norm": 0.3922199606895447, "supervised_tokens_in_step": 90160, "tokens_per_second": 29131}
{"step": 400, "validation_loss": 2.002938, "validation_perplexity": 7.4108, "validation_supervised_tokens": 368077}
{"step": 410, "max_steps": 739, "train_loss": 1.96719, "learning_rate": 9.839927168749815e-06, "grad_norm": 0.3786531388759613, "supervised_tokens_in_step": 93584, "tokens_per_second": 27341}
{"step": 420, "max_steps": 739, "train_loss": 2.051971, "learning_rate": 9.450113104277645e-06, "grad_norm": 0.3788132667541504, "supervised_tokens_in_step": 93003, "tokens_per_second": 31014}
{"step": 430, "max_steps": 739, "train_loss": 2.049608, "learning_rate": 9.063274073977529e-06, "grad_norm": 0.39005762338638306, "supervised_tokens_in_step": 93502, "tokens_per_second": 31032}
{"step": 440, "max_steps": 739, "train_loss": 1.944249, "learning_rate": 8.68015262192634e-06, "grad_norm": 0.3816198408603668, "supervised_tokens_in_step": 90205, "tokens_per_second": 30638}
{"step": 450, "max_steps": 739, "train_loss": 1.924675, "learning_rate": 8.301484156246458e-06, "grad_norm": 0.37720900774002075, "supervised_tokens_in_step": 92207, "tokens_per_second": 31020}
{"step": 460, "max_steps": 739, "train_loss": 2.006875, "learning_rate": 7.927995537477446e-06, "grad_norm": 0.3737892508506775, "supervised_tokens_in_step": 91867, "tokens_per_second": 31029}
{"step": 470, "max_steps": 739, "train_loss": 1.949734, "learning_rate": 7.560403683354902e-06, "grad_norm": 0.3696380853652954, "supervised_tokens_in_step": 92146, "tokens_per_second": 31021}
{"step": 480, "max_steps": 739, "train_loss": 1.981081, "learning_rate": 7.199414192674712e-06, "grad_norm": 0.37900790572166443, "supervised_tokens_in_step": 93959, "tokens_per_second": 31036}
{"step": 490, "max_steps": 739, "train_loss": 1.972043, "learning_rate": 6.8457199908841985e-06, "grad_norm": 0.36629000306129456, "supervised_tokens_in_step": 95639, "tokens_per_second": 31022}
{"step": 500, "max_steps": 739, "train_loss": 1.950306, "learning_rate": 6.500000000000003e-06, "grad_norm": 0.35963720083236694, "supervised_tokens_in_step": 94424, "tokens_per_second": 30649}
{"step": 500, "validation_loss": 1.986996, "validation_perplexity": 7.2936, "validation_supervised_tokens": 368077}
Saved checkpoint: runs/kawk500m-ko-instruct-v1-a100/checkpoint-00000500
{"step": 510, "max_steps": 739, "train_loss": 1.94603, "learning_rate": 6.162917835405712e-06, "grad_norm": 0.374806672334671, "supervised_tokens_in_step": 91776, "tokens_per_second": 23102}
{"step": 520, "max_steps": 739, "train_loss": 1.950595, "learning_rate": 5.835120532030926e-06, "grad_norm": 0.3778649866580963, "supervised_tokens_in_step": 90132, "tokens_per_second": 31028}
{"step": 530, "max_steps": 739, "train_loss": 1.96701, "learning_rate": 5.517237302356732e-06, "grad_norm": 0.3780269920825958, "supervised_tokens_in_step": 91961, "tokens_per_second": 31012}
{"step": 540, "max_steps": 739, "train_loss": 1.978193, "learning_rate": 5.2098783286316545e-06, "grad_norm": 0.3767697513103485, "supervised_tokens_in_step": 93106, "tokens_per_second": 30995}
{"step": 550, "max_steps": 739, "train_loss": 1.950969, "learning_rate": 4.9136335916165115e-06, "grad_norm": 0.3750850558280945, "supervised_tokens_in_step": 93075, "tokens_per_second": 31031}
{"step": 560, "max_steps": 739, "train_loss": 1.927163, "learning_rate": 4.629071738106343e-06, "grad_norm": 0.37781381607055664, "supervised_tokens_in_step": 90968, "tokens_per_second": 30755}
{"step": 570, "max_steps": 739, "train_loss": 1.941846, "learning_rate": 4.356738989403217e-06, "grad_norm": 0.3899576663970947, "supervised_tokens_in_step": 86836, "tokens_per_second": 30752}
{"step": 580, "max_steps": 739, "train_loss": 1.919732, "learning_rate": 4.097158092835219e-06, "grad_norm": 0.3869037926197052, "supervised_tokens_in_step": 91489, "tokens_per_second": 31037}
{"step": 590, "max_steps": 739, "train_loss": 1.872858, "learning_rate": 3.850827318334068e-06, "grad_norm": 0.36998265981674194, "supervised_tokens_in_step": 93877, "tokens_per_second": 31037}
{"step": 600, "max_steps": 739, "train_loss": 1.882916, "learning_rate": 3.618219501997533e-06, "grad_norm": 0.38547229766845703, "supervised_tokens_in_step": 88594, "tokens_per_second": 31027}
{"step": 600, "validation_loss": 1.981575, "validation_perplexity": 7.2542, "validation_supervised_tokens": 368077}
{"step": 610, "max_steps": 739, "train_loss": 1.909304, "learning_rate": 3.3997811384725533e-06, "grad_norm": 0.3757702112197876, "supervised_tokens_in_step": 92555, "tokens_per_second": 27343}
{"step": 620, "max_steps": 739, "train_loss": 1.864604, "learning_rate": 3.1959315239011684e-06, "grad_norm": 0.38048025965690613, "supervised_tokens_in_step": 90828, "tokens_per_second": 31038}
{"step": 630, "max_steps": 739, "train_loss": 1.8555, "learning_rate": 3.0070619510744687e-06, "grad_norm": 0.380898654460907, "supervised_tokens_in_step": 86254, "tokens_per_second": 31035}
{"step": 640, "max_steps": 739, "train_loss": 1.876318, "learning_rate": 2.8335349583394663e-06, "grad_norm": 0.36896955966949463, "supervised_tokens_in_step": 91306, "tokens_per_second": 31041}
{"step": 650, "max_steps": 739, "train_loss": 1.927439, "learning_rate": 2.6756836337005715e-06, "grad_norm": 0.37641212344169617, "supervised_tokens_in_step": 92129, "tokens_per_second": 31021}
{"step": 660, "max_steps": 739, "train_loss": 1.83481, "learning_rate": 2.5338109754515294e-06, "grad_norm": 0.37352287769317627, "supervised_tokens_in_step": 86178, "tokens_per_second": 31038}
{"step": 670, "max_steps": 739, "train_loss": 1.851432, "learning_rate": 2.408189310565064e-06, "grad_norm": 0.36761924624443054, "supervised_tokens_in_step": 91539, "tokens_per_second": 31029}
{"step": 680, "max_steps": 739, "train_loss": 1.942716, "learning_rate": 2.2990597719566276e-06, "grad_norm": 0.3929079473018646, "supervised_tokens_in_step": 90785, "tokens_per_second": 31030}
{"step": 690, "max_steps": 739, "train_loss": 1.876109, "learning_rate": 2.206631835625685e-06, "grad_norm": 0.3768686354160309, "supervised_tokens_in_step": 89262, "tokens_per_second": 31014}
{"step": 700, "max_steps": 739, "train_loss": 1.836297, "learning_rate": 2.1310829185629837e-06, "grad_norm": 0.37402093410491943, "supervised_tokens_in_step": 92439, "tokens_per_second": 31017}
{"step": 700, "validation_loss": 1.978615, "validation_perplexity": 7.2327, "validation_supervised_tokens": 368077}
{"step": 710, "max_steps": 739, "train_loss": 1.837864, "learning_rate": 2.0725580381956298e-06, "grad_norm": 0.36420106887817383, "supervised_tokens_in_step": 92903, "tokens_per_second": 27349}
{"step": 720, "max_steps": 739, "train_loss": 1.85419, "learning_rate": 2.0311695340236796e-06, "grad_norm": 0.37577784061431885, "supervised_tokens_in_step": 88511, "tokens_per_second": 31007}
{"step": 730, "max_steps": 739, "train_loss": 1.892897, "learning_rate": 2.0069968519825643e-06, "grad_norm": 0.3862324655056, "supervised_tokens_in_step": 88540, "tokens_per_second": 31036}
Final checkpoint: runs/kawk500m-ko-instruct-v1-a100/checkpoint-00000739