| { |
| "config": { |
| "name": "C5", |
| "pos_encoding": "sinusoidal", |
| "attention": "mha", |
| "norm": "layernorm", |
| "tokenization": "blt", |
| "d_model": 256, |
| "n_heads": 8, |
| "n_kv_heads": 2, |
| "n_enc_layers": 4, |
| "n_dec_layers": 4, |
| "d_ff": 1024, |
| "dropout": 0.1, |
| "src_vocab_size": 4096, |
| "tgt_vocab_size": 8192, |
| "max_src_len": 768, |
| "max_tgt_len": 384, |
| "src_patch_size": 16, |
| "tgt_patch_size": 4, |
| "local_layers": 2, |
| "d_local": 64, |
| "local_heads": 4, |
| "blt_max_src_bytes": 10240, |
| "blt_max_tgt_bytes": 1288, |
| "batch_size": 16, |
| "accum_steps": 2, |
| "lr": 0.0003, |
| "warmup_steps": 500, |
| "epochs": 30, |
| "label_smoothing": 0.1, |
| "grad_clip": 1.0, |
| "seed": 42, |
| "amp": true, |
| "max_raw_cipher_chars": 10240, |
| "val_frac": 0.1, |
| "test_frac": 0.1 |
| }, |
| "n_params": 7712004, |
| "train_seconds": 1090.1626682281494, |
| "best_val_loss": 2.959963167818104, |
| "peak_mem_gib": 5.163149356842041, |
| "test": { |
| "loss": 2.9563522463934033, |
| "ppl": 19.22770574389416, |
| "bit_accuracy": 0.0912937977810586, |
| "sequence_accuracy": 0.0, |
| "levenshtein": 416.4621848739496, |
| "levenshtein_norm": 0.7325332789903187, |
| "greedy_decode_sec": 129.7128336429596 |
| }, |
| "history": [ |
| { |
| "epoch": 1, |
| "train_loss": 4.8856480285743284, |
| "val_loss": 4.366412441294718, |
| "val_ppl": 78.76056609140832, |
| "epoch_sec": 36.37962794303894, |
| "tokens_per_sec": 57370.04796387304, |
| "lr": 7.139999999999999e-05, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 2, |
| "train_loss": 4.017434895385889, |
| "val_loss": 3.66580420896976, |
| "val_ppl": 39.08755809133063, |
| "epoch_sec": 35.767035722732544, |
| "tokens_per_sec": 58307.45986798462, |
| "lr": 0.00014279999999999997, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 3, |
| "train_loss": 3.5286779530186765, |
| "val_loss": 3.4381912673805632, |
| "val_ppl": 31.130600283306546, |
| "epoch_sec": 36.28211236000061, |
| "tokens_per_sec": 57474.63045450877, |
| "lr": 0.00021419999999999998, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 4, |
| "train_loss": 3.414794660852919, |
| "val_loss": 3.3655481051912495, |
| "val_ppl": 28.949360246005433, |
| "epoch_sec": 35.8679313659668, |
| "tokens_per_sec": 58184.78848713909, |
| "lr": 0.00028559999999999995, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 5, |
| "train_loss": 3.33122580577287, |
| "val_loss": 3.2656278741588136, |
| "val_ppl": 26.19655396750886, |
| "epoch_sec": 35.89056324958801, |
| "tokens_per_sec": 58146.147929956365, |
| "lr": 0.00027500954910846334, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 6, |
| "train_loss": 3.258343439180606, |
| "val_loss": 3.216050086578376, |
| "val_ppl": 24.929456258769516, |
| "epoch_sec": 36.45792627334595, |
| "tokens_per_sec": 57213.621651458954, |
| "lr": 0.0002510482226267169, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 7, |
| "train_loss": 3.2190938078369262, |
| "val_loss": 3.183741140569163, |
| "val_ppl": 24.136884328162783, |
| "epoch_sec": 36.145684480667114, |
| "tokens_per_sec": 57726.365677651425, |
| "lr": 0.00023242549051998872, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 8, |
| "train_loss": 3.1909731308092413, |
| "val_loss": 3.1610012521179307, |
| "val_ppl": 23.594207856936794, |
| "epoch_sec": 36.0872757434845, |
| "tokens_per_sec": 57822.62465120392, |
| "lr": 0.00021741413836966813, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 9, |
| "train_loss": 3.170286599244441, |
| "val_loss": 3.142047068762836, |
| "val_ppl": 23.151210504185514, |
| "epoch_sec": 36.17484188079834, |
| "tokens_per_sec": 57643.79031347906, |
| "lr": 0.0002049800154226969, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 10, |
| "train_loss": 3.151909489605079, |
| "val_loss": 3.1233615767220058, |
| "val_ppl": 22.722635283605857, |
| "epoch_sec": 35.889060258865356, |
| "tokens_per_sec": 58143.09388289265, |
| "lr": 0.0001944611170656493, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 11, |
| "train_loss": 3.1358774769276097, |
| "val_loss": 3.1098914752281215, |
| "val_ppl": 22.418611294046038, |
| "epoch_sec": 35.825498819351196, |
| "tokens_per_sec": 58283.34758236911, |
| "lr": 0.0001854114001850042, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 12, |
| "train_loss": 3.1213864852483115, |
| "val_loss": 3.095396877660626, |
| "val_ppl": 22.09600620883744, |
| "epoch_sec": 36.12416625022888, |
| "tokens_per_sec": 57756.76552775195, |
| "lr": 0.00017751790062418157, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 13, |
| "train_loss": 3.107604542756909, |
| "val_loss": 3.0846324056481307, |
| "val_ppl": 21.859429962020776, |
| "epoch_sec": 35.75457000732422, |
| "tokens_per_sec": 58351.42191816658, |
| "lr": 0.00017055368216583073, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 14, |
| "train_loss": 3.095432237961438, |
| "val_loss": 3.071521989232364, |
| "val_ppl": 21.5747141819647, |
| "epoch_sec": 35.99679756164551, |
| "tokens_per_sec": 57972.045886200955, |
| "lr": 0.00016434964046729364, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 15, |
| "train_loss": 3.0841269372763924, |
| "val_loss": 3.061921389576639, |
| "val_ppl": 21.36857510115458, |
| "epoch_sec": 35.79954791069031, |
| "tokens_per_sec": 58247.57913709059, |
| "lr": 0.00015877683720748892, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 16, |
| "train_loss": 3.0723313667054386, |
| "val_loss": 3.0498964309454477, |
| "val_ppl": 21.113157639525703, |
| "epoch_sec": 36.16572141647339, |
| "tokens_per_sec": 57720.43023726741, |
| "lr": 0.00015373501156702268, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 17, |
| "train_loss": 3.062301852304857, |
| "val_loss": 3.039605276343999, |
| "val_ppl": 20.89699306942831, |
| "epoch_sec": 35.97733950614929, |
| "tokens_per_sec": 57989.64094172124, |
| "lr": 0.0001491448684815029, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 18, |
| "train_loss": 3.0529227420823193, |
| "val_loss": 3.0307262499070586, |
| "val_ppl": 20.712269412218593, |
| "epoch_sec": 36.08746528625488, |
| "tokens_per_sec": 57822.099264887176, |
| "lr": 0.0001449427589131121, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 19, |
| "train_loss": 3.0429874217637956, |
| "val_loss": 3.0226253830864067, |
| "val_ppl": 20.545159856151887, |
| "epoch_sec": 35.95555114746094, |
| "tokens_per_sec": 58046.64185066689, |
| "lr": 0.00014107692200099125, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 20, |
| "train_loss": 3.034629042195703, |
| "val_loss": 3.0152378949947223, |
| "val_ppl": 20.39394198022706, |
| "epoch_sec": 35.92344522476196, |
| "tokens_per_sec": 58055.56752564562, |
| "lr": 0.00013750477455423167, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 21, |
| "train_loss": 3.026096388460377, |
| "val_loss": 3.0081024792558453, |
| "val_ppl": 20.24894066320124, |
| "epoch_sec": 35.79409074783325, |
| "tokens_per_sec": 58242.68633297236, |
| "lr": 0.0001341909195182463, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 22, |
| "train_loss": 3.0178336800439802, |
| "val_loss": 3.001203268693511, |
| "val_ppl": 20.109719767271528, |
| "epoch_sec": 36.0793514251709, |
| "tokens_per_sec": 57889.17808935106, |
| "lr": 0.00013110565838010918, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 23, |
| "train_loss": 3.01066998057014, |
| "val_loss": 2.996551458374599, |
| "val_ppl": 20.016390408882653, |
| "epoch_sec": 35.93640065193176, |
| "tokens_per_sec": 58042.012059097986, |
| "lr": 0.00012822386342782138, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 24, |
| "train_loss": 3.0036904106618825, |
| "val_loss": 2.9893544447212204, |
| "val_ppl": 19.872849326964413, |
| "epoch_sec": 35.878557443618774, |
| "tokens_per_sec": 58113.23388005483, |
| "lr": 0.00012552411131335845, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 25, |
| "train_loss": 2.996913657717417, |
| "val_loss": 2.9835222242595556, |
| "val_ppl": 19.757283817829542, |
| "epoch_sec": 36.10230779647827, |
| "tokens_per_sec": 57774.20135461437, |
| "lr": 0.00012298800925361817, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 26, |
| "train_loss": 2.989658152692368, |
| "val_loss": 2.978006158703192, |
| "val_ppl": 19.64860137040021, |
| "epoch_sec": 36.05625867843628, |
| "tokens_per_sec": 57875.91604028568, |
| "lr": 0.00012059966521579404, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 27, |
| "train_loss": 2.9836042881902016, |
| "val_loss": 2.9727318855725997, |
| "val_ppl": 19.54524209227385, |
| "epoch_sec": 35.98175501823425, |
| "tokens_per_sec": 57978.494905065236, |
| "lr": 0.00011834526708278771, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 28, |
| "train_loss": 2.977233608978246, |
| "val_loss": 2.968758717068618, |
| "val_ppl": 19.46773961913767, |
| "epoch_sec": 35.78663873672485, |
| "tokens_per_sec": 58274.31895300869, |
| "lr": 0.00011621274525999436, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 29, |
| "train_loss": 2.9718500992439765, |
| "val_loss": 2.9640991903384575, |
| "val_ppl": 19.37724017204039, |
| "epoch_sec": 35.99818825721741, |
| "tokens_per_sec": 57916.80362085975, |
| "lr": 0.00011419149985036504, |
| "peak_mem_gib": 5.163149356842041 |
| }, |
| { |
| "epoch": 30, |
| "train_loss": 2.966067832870334, |
| "val_loss": 2.959963167818104, |
| "val_ppl": 19.297260982186582, |
| "epoch_sec": 35.59194898605347, |
| "tokens_per_sec": 58641.99796470411, |
| "lr": 0.00011227217828476796, |
| "peak_mem_gib": 5.163149356842041 |
| } |
| ] |
| } |
