{
  "config": {
    "name": "C5",
    "pos_encoding": "sinusoidal",
    "attention": "mha",
    "norm": "layernorm",
    "tokenization": "blt",
    "d_model": 256,
    "n_heads": 8,
    "n_kv_heads": 2,
    "n_enc_layers": 4,
    "n_dec_layers": 4,
    "d_ff": 1024,
    "dropout": 0.1,
    "src_vocab_size": 4096,
    "tgt_vocab_size": 8192,
    "max_src_len": 768,
    "max_tgt_len": 384,
    "src_patch_size": 16,
    "tgt_patch_size": 4,
    "local_layers": 2,
    "d_local": 64,
    "local_heads": 4,
    "blt_max_src_bytes": 10240,
    "blt_max_tgt_bytes": 1288,
    "batch_size": 16,
    "accum_steps": 2,
    "lr": 0.0003,
    "warmup_steps": 500,
    "epochs": 30,
    "label_smoothing": 0.1,
    "grad_clip": 1.0,
    "seed": 42,
    "amp": true,
    "max_raw_cipher_chars": 10240,
    "val_frac": 0.1,
    "test_frac": 0.1
  },
  "n_params": 7712004,
  "train_seconds": 1090.1626682281494,
  "best_val_loss": 2.959963167818104,
  "peak_mem_gib": 5.163149356842041,
  "test": {
    "loss": 2.9563522463934033,
    "ppl": 19.22770574389416,
    "bit_accuracy": 0.0912937977810586,
    "sequence_accuracy": 0.0,
    "levenshtein": 416.4621848739496,
    "levenshtein_norm": 0.7325332789903187,
    "greedy_decode_sec": 129.7128336429596
  },
  "history": [
    {
      "epoch": 1,
      "train_loss": 4.8856480285743284,
      "val_loss": 4.366412441294718,
      "val_ppl": 78.76056609140832,
      "epoch_sec": 36.37962794303894,
      "tokens_per_sec": 57370.04796387304,
      "lr": 7.139999999999999e-05,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 2,
      "train_loss": 4.017434895385889,
      "val_loss": 3.66580420896976,
      "val_ppl": 39.08755809133063,
      "epoch_sec": 35.767035722732544,
      "tokens_per_sec": 58307.45986798462,
      "lr": 0.00014279999999999997,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 3,
      "train_loss": 3.5286779530186765,
      "val_loss": 3.4381912673805632,
      "val_ppl": 31.130600283306546,
      "epoch_sec": 36.28211236000061,
      "tokens_per_sec": 57474.63045450877,
      "lr": 0.00021419999999999998,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 4,
      "train_loss": 3.414794660852919,
      "val_loss": 3.3655481051912495,
      "val_ppl": 28.949360246005433,
      "epoch_sec": 35.8679313659668,
      "tokens_per_sec": 58184.78848713909,
      "lr": 0.00028559999999999995,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 5,
      "train_loss": 3.33122580577287,
      "val_loss": 3.2656278741588136,
      "val_ppl": 26.19655396750886,
      "epoch_sec": 35.89056324958801,
      "tokens_per_sec": 58146.147929956365,
      "lr": 0.00027500954910846334,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 6,
      "train_loss": 3.258343439180606,
      "val_loss": 3.216050086578376,
      "val_ppl": 24.929456258769516,
      "epoch_sec": 36.45792627334595,
      "tokens_per_sec": 57213.621651458954,
      "lr": 0.0002510482226267169,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 7,
      "train_loss": 3.2190938078369262,
      "val_loss": 3.183741140569163,
      "val_ppl": 24.136884328162783,
      "epoch_sec": 36.145684480667114,
      "tokens_per_sec": 57726.365677651425,
      "lr": 0.00023242549051998872,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 8,
      "train_loss": 3.1909731308092413,
      "val_loss": 3.1610012521179307,
      "val_ppl": 23.594207856936794,
      "epoch_sec": 36.0872757434845,
      "tokens_per_sec": 57822.62465120392,
      "lr": 0.00021741413836966813,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 9,
      "train_loss": 3.170286599244441,
      "val_loss": 3.142047068762836,
      "val_ppl": 23.151210504185514,
      "epoch_sec": 36.17484188079834,
      "tokens_per_sec": 57643.79031347906,
      "lr": 0.0002049800154226969,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 10,
      "train_loss": 3.151909489605079,
      "val_loss": 3.1233615767220058,
      "val_ppl": 22.722635283605857,
      "epoch_sec": 35.889060258865356,
      "tokens_per_sec": 58143.09388289265,
      "lr": 0.0001944611170656493,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 11,
      "train_loss": 3.1358774769276097,
      "val_loss": 3.1098914752281215,
      "val_ppl": 22.418611294046038,
      "epoch_sec": 35.825498819351196,
      "tokens_per_sec": 58283.34758236911,
      "lr": 0.0001854114001850042,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 12,
      "train_loss": 3.1213864852483115,
      "val_loss": 3.095396877660626,
      "val_ppl": 22.09600620883744,
      "epoch_sec": 36.12416625022888,
      "tokens_per_sec": 57756.76552775195,
      "lr": 0.00017751790062418157,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 13,
      "train_loss": 3.107604542756909,
      "val_loss": 3.0846324056481307,
      "val_ppl": 21.859429962020776,
      "epoch_sec": 35.75457000732422,
      "tokens_per_sec": 58351.42191816658,
      "lr": 0.00017055368216583073,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 14,
      "train_loss": 3.095432237961438,
      "val_loss": 3.071521989232364,
      "val_ppl": 21.5747141819647,
      "epoch_sec": 35.99679756164551,
      "tokens_per_sec": 57972.045886200955,
      "lr": 0.00016434964046729364,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 15,
      "train_loss": 3.0841269372763924,
      "val_loss": 3.061921389576639,
      "val_ppl": 21.36857510115458,
      "epoch_sec": 35.79954791069031,
      "tokens_per_sec": 58247.57913709059,
      "lr": 0.00015877683720748892,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 16,
      "train_loss": 3.0723313667054386,
      "val_loss": 3.0498964309454477,
      "val_ppl": 21.113157639525703,
      "epoch_sec": 36.16572141647339,
      "tokens_per_sec": 57720.43023726741,
      "lr": 0.00015373501156702268,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 17,
      "train_loss": 3.062301852304857,
      "val_loss": 3.039605276343999,
      "val_ppl": 20.89699306942831,
      "epoch_sec": 35.97733950614929,
      "tokens_per_sec": 57989.64094172124,
      "lr": 0.0001491448684815029,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 18,
      "train_loss": 3.0529227420823193,
      "val_loss": 3.0307262499070586,
      "val_ppl": 20.712269412218593,
      "epoch_sec": 36.08746528625488,
      "tokens_per_sec": 57822.099264887176,
      "lr": 0.0001449427589131121,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 19,
      "train_loss": 3.0429874217637956,
      "val_loss": 3.0226253830864067,
      "val_ppl": 20.545159856151887,
      "epoch_sec": 35.95555114746094,
      "tokens_per_sec": 58046.64185066689,
      "lr": 0.00014107692200099125,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 20,
      "train_loss": 3.034629042195703,
      "val_loss": 3.0152378949947223,
      "val_ppl": 20.39394198022706,
      "epoch_sec": 35.92344522476196,
      "tokens_per_sec": 58055.56752564562,
      "lr": 0.00013750477455423167,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 21,
      "train_loss": 3.026096388460377,
      "val_loss": 3.0081024792558453,
      "val_ppl": 20.24894066320124,
      "epoch_sec": 35.79409074783325,
      "tokens_per_sec": 58242.68633297236,
      "lr": 0.0001341909195182463,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 22,
      "train_loss": 3.0178336800439802,
      "val_loss": 3.001203268693511,
      "val_ppl": 20.109719767271528,
      "epoch_sec": 36.0793514251709,
      "tokens_per_sec": 57889.17808935106,
      "lr": 0.00013110565838010918,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 23,
      "train_loss": 3.01066998057014,
      "val_loss": 2.996551458374599,
      "val_ppl": 20.016390408882653,
      "epoch_sec": 35.93640065193176,
      "tokens_per_sec": 58042.012059097986,
      "lr": 0.00012822386342782138,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 24,
      "train_loss": 3.0036904106618825,
      "val_loss": 2.9893544447212204,
      "val_ppl": 19.872849326964413,
      "epoch_sec": 35.878557443618774,
      "tokens_per_sec": 58113.23388005483,
      "lr": 0.00012552411131335845,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 25,
      "train_loss": 2.996913657717417,
      "val_loss": 2.9835222242595556,
      "val_ppl": 19.757283817829542,
      "epoch_sec": 36.10230779647827,
      "tokens_per_sec": 57774.20135461437,
      "lr": 0.00012298800925361817,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 26,
      "train_loss": 2.989658152692368,
      "val_loss": 2.978006158703192,
      "val_ppl": 19.64860137040021,
      "epoch_sec": 36.05625867843628,
      "tokens_per_sec": 57875.91604028568,
      "lr": 0.00012059966521579404,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 27,
      "train_loss": 2.9836042881902016,
      "val_loss": 2.9727318855725997,
      "val_ppl": 19.54524209227385,
      "epoch_sec": 35.98175501823425,
      "tokens_per_sec": 57978.494905065236,
      "lr": 0.00011834526708278771,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 28,
      "train_loss": 2.977233608978246,
      "val_loss": 2.968758717068618,
      "val_ppl": 19.46773961913767,
      "epoch_sec": 35.78663873672485,
      "tokens_per_sec": 58274.31895300869,
      "lr": 0.00011621274525999436,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 29,
      "train_loss": 2.9718500992439765,
      "val_loss": 2.9640991903384575,
      "val_ppl": 19.37724017204039,
      "epoch_sec": 35.99818825721741,
      "tokens_per_sec": 57916.80362085975,
      "lr": 0.00011419149985036504,
      "peak_mem_gib": 5.163149356842041
    },
    {
      "epoch": 30,
      "train_loss": 2.966067832870334,
      "val_loss": 2.959963167818104,
      "val_ppl": 19.297260982186582,
      "epoch_sec": 35.59194898605347,
      "tokens_per_sec": 58641.99796470411,
      "lr": 0.00011227217828476796,
      "peak_mem_gib": 5.163149356842041
    }
  ]
}