| { |
| "config": { |
| "name": "C3", |
| "pos_encoding": "sinusoidal", |
| "attention": "gqa", |
| "norm": "layernorm", |
| "tokenization": "bpe", |
| "d_model": 256, |
| "n_heads": 8, |
| "n_kv_heads": 2, |
| "n_enc_layers": 4, |
| "n_dec_layers": 4, |
| "d_ff": 1024, |
| "dropout": 0.1, |
| "src_vocab_size": 4096, |
| "tgt_vocab_size": 8192, |
| "max_src_len": 768, |
| "max_tgt_len": 384, |
| "src_patch_size": 16, |
| "tgt_patch_size": 4, |
| "local_layers": 2, |
| "d_local": 64, |
| "local_heads": 4, |
| "blt_max_src_bytes": 10240, |
| "blt_max_tgt_bytes": 1288, |
| "batch_size": 16, |
| "accum_steps": 2, |
| "lr": 0.0003, |
| "warmup_steps": 500, |
| "epochs": 30, |
| "label_smoothing": 0.1, |
| "grad_clip": 1.0, |
| "seed": 42, |
| "amp": true, |
| "max_raw_cipher_chars": 10240, |
| "val_frac": 0.1, |
| "test_frac": 0.1 |
| }, |
| "n_params": 11432960, |
| "train_seconds": 702.2708430290222, |
| "best_val_loss": 6.065617772560229, |
| "peak_mem_gib": 4.573119163513184, |
| "test": { |
| "loss": 6.064282278245915, |
| "ppl": 430.213793040676, |
| "bit_accuracy": 0.07168867569390866, |
| "sequence_accuracy": 0.0, |
| "levenshtein": 413.24789915966386, |
| "levenshtein_norm": 0.7356565054539704, |
| "bleu": 1.0587898468593648, |
| "rouge1": 0.20878252073717582, |
| "rouge2": 0.03246129621877159, |
| "rougeL": 0.16201144756220592, |
| "greedy_decode_sec": 73.27188539505005 |
| }, |
| "history": [ |
| { |
| "epoch": 1, |
| "train_loss": 8.666249432952137, |
| "val_loss": 8.198083422331626, |
| "val_ppl": 3633.978826108288, |
| "epoch_sec": 32.991130113601685, |
| "tokens_per_sec": 13894.704377253467, |
| "lr": 7.139999999999999e-05, |
| "peak_mem_gib": 4.478940963745117 |
| }, |
| { |
| "epoch": 2, |
| "train_loss": 7.824309250238496, |
| "val_loss": 7.624079632915133, |
| "val_ppl": 2046.8957023857647, |
| "epoch_sec": 22.461591958999634, |
| "tokens_per_sec": 20409.550704901016, |
| "lr": 0.00014279999999999997, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 3, |
| "train_loss": 7.520816068801131, |
| "val_loss": 7.373348980090785, |
| "val_ppl": 1592.9596501089438, |
| "epoch_sec": 22.598292350769043, |
| "tokens_per_sec": 20291.93148235356, |
| "lr": 0.00021419999999999998, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 4, |
| "train_loss": 7.237775218739889, |
| "val_loss": 7.090926617298936, |
| "val_ppl": 1201.0201712273515, |
| "epoch_sec": 22.431107759475708, |
| "tokens_per_sec": 20455.43202413489, |
| "lr": 0.00028559999999999995, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 5, |
| "train_loss": 6.97803251973895, |
| "val_loss": 6.89697231811092, |
| "val_ppl": 989.2749668536017, |
| "epoch_sec": 22.68420672416687, |
| "tokens_per_sec": 20217.193643867373, |
| "lr": 0.00027500954910846334, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 6, |
| "train_loss": 6.788357346591319, |
| "val_loss": 6.765579688010909, |
| "val_ppl": 867.4689234013784, |
| "epoch_sec": 22.676483869552612, |
| "tokens_per_sec": 20222.403201393994, |
| "lr": 0.0002510482226267169, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 7, |
| "train_loss": 6.629533332610492, |
| "val_loss": 6.6624268559486115, |
| "val_ppl": 782.4475222503605, |
| "epoch_sec": 22.77009153366089, |
| "tokens_per_sec": 20137.556290546832, |
| "lr": 0.00023242549051998872, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 8, |
| "train_loss": 6.489869470300078, |
| "val_loss": 6.564996251037285, |
| "val_ppl": 709.8092351435845, |
| "epoch_sec": 22.95131206512451, |
| "tokens_per_sec": 19972.583645732117, |
| "lr": 0.00021741413836966813, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 9, |
| "train_loss": 6.364245431624562, |
| "val_loss": 6.490266808232799, |
| "val_ppl": 658.6990861111321, |
| "epoch_sec": 22.69159436225891, |
| "tokens_per_sec": 20209.289513949738, |
| "lr": 0.0002049800154226969, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 10, |
| "train_loss": 6.251341307085133, |
| "val_loss": 6.424389596816817, |
| "val_ppl": 616.7042643838491, |
| "epoch_sec": 22.461724281311035, |
| "tokens_per_sec": 20398.52302795949, |
| "lr": 0.0001944611170656493, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 11, |
| "train_loss": 6.14681397262172, |
| "val_loss": 6.369069526059093, |
| "val_ppl": 583.5146310564346, |
| "epoch_sec": 22.919347763061523, |
| "tokens_per_sec": 20006.372115833285, |
| "lr": 0.0001854114001850042, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 12, |
| "train_loss": 6.05125893580923, |
| "val_loss": 6.314809204496219, |
| "val_ppl": 552.6965985050921, |
| "epoch_sec": 22.541085720062256, |
| "tokens_per_sec": 20345.160197489076, |
| "lr": 0.00017751790062418157, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 13, |
| "train_loss": 5.961064552582484, |
| "val_loss": 6.267394850353683, |
| "val_ppl": 527.102407003704, |
| "epoch_sec": 22.53730034828186, |
| "tokens_per_sec": 20342.986645024328, |
| "lr": 0.00017055368216583073, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 14, |
| "train_loss": 5.876982218490247, |
| "val_loss": 6.225303183811826, |
| "val_ppl": 505.3762411271679, |
| "epoch_sec": 22.694039821624756, |
| "tokens_per_sec": 20208.213416590657, |
| "lr": 0.00016434964046729364, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 15, |
| "train_loss": 5.799299590924299, |
| "val_loss": 6.20042628951472, |
| "val_ppl": 492.95913962108074, |
| "epoch_sec": 22.612531185150146, |
| "tokens_per_sec": 20279.4192408299, |
| "lr": 0.00015877683720748892, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 16, |
| "train_loss": 5.727273020222414, |
| "val_loss": 6.173640811681388, |
| "val_ppl": 479.9302648002502, |
| "epoch_sec": 22.526395559310913, |
| "tokens_per_sec": 20345.420943767702, |
| "lr": 0.00015373501156702268, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 17, |
| "train_loss": 5.658098405940642, |
| "val_loss": 6.14677290176261, |
| "val_ppl": 467.20722775604986, |
| "epoch_sec": 22.482492685317993, |
| "tokens_per_sec": 20394.713629749574, |
| "lr": 0.0001491448684815029, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 18, |
| "train_loss": 5.595762052809414, |
| "val_loss": 6.132504606326968, |
| "val_ppl": 460.58830963229815, |
| "epoch_sec": 22.90033769607544, |
| "tokens_per_sec": 20013.198324081615, |
| "lr": 0.0001449427589131121, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 19, |
| "train_loss": 5.536297566259913, |
| "val_loss": 6.1130555950769105, |
| "val_ppl": 451.7168724015767, |
| "epoch_sec": 22.869896173477173, |
| "tokens_per_sec": 20053.129953946434, |
| "lr": 0.00014107692200099125, |
| "peak_mem_gib": 4.564123153686523 |
| }, |
| { |
| "epoch": 20, |
| "train_loss": 5.479562937387878, |
| "val_loss": 6.1117970505583346, |
| "val_ppl": 451.1487242027606, |
| "epoch_sec": 22.574064016342163, |
| "tokens_per_sec": 20301.927010937092, |
| "lr": 0.00013750477455423167, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 21, |
| "train_loss": 5.425869344832037, |
| "val_loss": 6.091444310471403, |
| "val_ppl": 442.05942134388613, |
| "epoch_sec": 22.867355823516846, |
| "tokens_per_sec": 20052.690111570486, |
| "lr": 0.0001341909195182463, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 22, |
| "train_loss": 5.374141672669874, |
| "val_loss": 6.0855736070080155, |
| "val_ppl": 439.47182450666025, |
| "epoch_sec": 22.70615267753601, |
| "tokens_per_sec": 20185.453982851344, |
| "lr": 0.00013110565838010918, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 23, |
| "train_loss": 5.325132388283005, |
| "val_loss": 6.079210498257359, |
| "val_ppl": 436.68429557472865, |
| "epoch_sec": 22.85144066810608, |
| "tokens_per_sec": 20078.51525266775, |
| "lr": 0.00012822386342782138, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 24, |
| "train_loss": 5.2781339674359105, |
| "val_loss": 6.072077576588776, |
| "val_ppl": 433.5805432710534, |
| "epoch_sec": 22.73417901992798, |
| "tokens_per_sec": 20180.055747685117, |
| "lr": 0.00012552411131335845, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 25, |
| "train_loss": 5.232884620040095, |
| "val_loss": 6.065617772560229, |
| "val_ppl": 430.7887249389105, |
| "epoch_sec": 22.5778706073761, |
| "tokens_per_sec": 20307.229497994016, |
| "lr": 0.00012298800925361817, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 26, |
| "train_loss": 5.189752116410751, |
| "val_loss": 6.080037675181538, |
| "val_ppl": 437.04566018280764, |
| "epoch_sec": 22.7062349319458, |
| "tokens_per_sec": 20199.25370166582, |
| "lr": 0.00012059966521579404, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 27, |
| "train_loss": 5.147430500576669, |
| "val_loss": 6.071555076483253, |
| "val_ppl": 433.35405656625767, |
| "epoch_sec": 22.61909556388855, |
| "tokens_per_sec": 20267.83072316564, |
| "lr": 0.00011834526708278771, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 28, |
| "train_loss": 5.10784804674173, |
| "val_loss": 6.075399348631383, |
| "val_ppl": 435.02319374938554, |
| "epoch_sec": 22.16178035736084, |
| "tokens_per_sec": 20693.238205823134, |
| "lr": 0.00011621274525999436, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 29, |
| "train_loss": 5.066572172150591, |
| "val_loss": 6.075840941522121, |
| "val_ppl": 435.21533932098805, |
| "epoch_sec": 22.596670389175415, |
| "tokens_per_sec": 20287.19240953306, |
| "lr": 0.00011419149985036504, |
| "peak_mem_gib": 4.573119163513184 |
| }, |
| { |
| "epoch": 30, |
| "train_loss": 5.028765135503165, |
| "val_loss": 6.087151586179226, |
| "val_ppl": 440.16584932640916, |
| "epoch_sec": 22.642820835113525, |
| "tokens_per_sec": 20260.770658423127, |
| "lr": 0.00011227217828476796, |
| "peak_mem_gib": 4.573119163513184 |
| } |
| ] |
| } |
