{ "written": "2026-09-06T18:40:33Z", "config": { "model": { "n_layer": 12, "d_model": 1280, "n_head": 20 }, "train": { "seed": 1337, "data_seed": 1337, "batch": 64, "grad_accum": 1, "iters": 76000, "lr": 0.0006, "min_lr": 6e-05, "warmup": 1000, "decay_frac": 0.1, "cycle": 76000, "weight_decay": 0.01, "grad_clip": 1.0, "beta1": 0.9, "beta2": 0.95, "log_every": 20, "eval_every": 1000, "eval_urls": 2000, "final_eval_urls": 100000, "ckpt_every": 1000, "snapshot_every": 4000, "compile": true, "attn": "auto", "qat_iters": 7600, "qat_lr": 6e-05, "grad_hook": "bf16", "loss": "fused", "bucket_mb": 25, "bucket_view": false, "powersgd_rank": 32, "powersgd_start": 10, "awq": 0, "awq_batches": 16, "qat_bits": 4, "qat_learn_scale": false, "qat_grad_accum": 2, "qat_compile": true, "fp8": false, "optimizer": "adamw", "muon_lr": 0.02, "muon_momentum": 0.95, "muon_wd": 0.0, "compile_mode": "max-autotune-no-cudagraphs", "peak_flops": 200000000000000.0, "dtype": "bfloat16" }, "data": { "bins": "/root/data/linklet/tokens/tokens-s0000[0-9].bin", "tokenizer": "/root/data/linklet/tokens/tokenizer", "eval_shard": 512 }, "run": { "out": "/root/models/linklet", "name": "prod-target-base", "init": "", "resume": true, "force_config": false }, "wandb": { "project": "linklet", "group": "production", "job_type": "production", "tags": [ "target", "baseline" ], "mode": "online" } }, "model_config": { "n_layer": 12, "d_model": 1280, "n_head": 20, "n_kv_head": 20, "d_attn": 1280, "d_mlp": 3392, "vocab": 8192, "block": 512, "struct_pos": false, "head_dim": 64, "mlp": "swiglu", "host_rows": 0, "host_dim": 64, "prefix_rows": 0, "prefix_dim": 64 }, "model": { "n_params": 246087680, "artifact_bytes": 130862112, "artifact_mib": 124.79983520507812, "exportable": true }, "data": { "bins": [ { "path": "/root/data/linklet/tokens/tokens-s00000.bin", "bytes": 2596447828, "tokens": 1298223914, "urls": 36578304, "source_parquet": "part-00000.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00001.bin", "bytes": 2596484504, "tokens": 1298242252, "urls": 36582969, "source_parquet": "part-00001.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00002.bin", "bytes": 2595421618, "tokens": 1297710809, "urls": 36567201, "source_parquet": "part-00002.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00003.bin", "bytes": 2597225440, "tokens": 1298612720, "urls": 36586549, "source_parquet": "part-00003.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00004.bin", "bytes": 2595658946, "tokens": 1297829473, "urls": 36571079, "source_parquet": "part-00004.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00005.bin", "bytes": 2596718326, "tokens": 1298359163, "urls": 36581527, "source_parquet": "part-00005.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00006.bin", "bytes": 2596866352, "tokens": 1298433176, "urls": 36583575, "source_parquet": "part-00006.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00007.bin", "bytes": 2595681408, "tokens": 1297840704, "urls": 36572621, "source_parquet": "part-00007.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00008.bin", "bytes": 2597495968, "tokens": 1298747984, "urls": 36585947, "source_parquet": "part-00008.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" }, { "path": "/root/data/linklet/tokens/tokens-s00009.bin", "bytes": 2595850662, "tokens": 1297925331, "urls": 36569751, "source_parquet": "part-00009.parquet", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374" } ], "total_tokens": 12981925526, "total_urls": 365779523, "tokenizer": "/root/data/linklet/tokens/tokenizer", "tokenizer_sha256": "6e4f2b4fa9de153b27adaee016dc3f2586ac886423f0153858b89e13f31be374", "eval_shard": 512, "eval_source": "ks46/urls-sampled part-00512.parquet, even stride" }, "schedule": { "tokens_per_iter": 262144, "iters": 76000, "qat_iters": 7600, "total_tokens_planned": 21915238400, "snapshot_every_iters": 4000, "snapshot_every_tokens": 1048576000 }, "code": { "git": "unknown", "argv": [ "/root/work/linklet/training/train.py", "--config", "training/configs/target-base.toml", "run.name=\"prod-target-base\"", "data.bins=\"/root/data/linklet/tokens/tokens-s000[0-3][0-9].bin\"", "train.awq=0", "train.weight_decay=0.01", "train.compile_mode=\"max-autotune-no-cudagraphs\"", "data.bins=\"/root/data/linklet/tokens/tokens-s0000[0-9].bin\"", "train.iters=76000", "train.cycle=76000", "train.warmup=1000", "train.qat_iters=7600", "train.snapshot_every=4000", "train.eval_every=1000", "train.final_eval_urls=100000", "train.ckpt_every=1000", "wandb.group=\"production\"", "wandb.job_type=\"production\"" ], "cwd": "/root/work/linklet" }, "node": { "hostname": "dbafddc1fff5", "platform": "Linux-5.15.0-160-generic-x86_64-with-glibc2.39", "cpu_count": 96, "gpus": [ "NVIDIA GeForce RTX 5090", "NVIDIA GeForce RTX 5090", "NVIDIA GeForce RTX 5090", "NVIDIA GeForce RTX 5090", "NVIDIA GeForce RTX 5090", "NVIDIA GeForce RTX 5090", "NVIDIA GeForce RTX 5090", "NVIDIA GeForce RTX 5090" ], "world_size": 8, "device": "cuda:0", "lk_job": "20260906-175341-prod-tail-49e8" }, "software": { "python": "3.14.7", "torch": "2.14.0+cu130", "torchao": "0.18.0", "cuda": "13.0", "nccl": "2.30.7" }, "env": { "NCCL_VERSION": "2.25.1-1", "PYTORCH_BACKEND": "cu128", "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", "PYTORCH_VERSION": "2.11.0", "TORCHINDUCTOR_CACHE_DIR": "/root/models/linklet/inductor-cache", "OMP_NUM_THREADS": "1" }, "wandb": { "id": "402vdj7x", "url": "https://wandb.ai/ks99-sc/linklet/runs/402vdj7x", "project": "linklet", "group": "production" }, "finished": "2026-09-06T18:40:33Z", "final": { "final/bits_per_char": 1.263393708877395, "final/bits_per_char_skipped": 74, "final/bits_per_char_kernel_sim": 1.2637269047434312, "final/bits_per_char_kernel_sim_skipped": 74 }, "iterations_completed": 83600, "tokens_seen": 21915238400, "snapshots": [ "ckpt-i004000.pt", "ckpt-i020000.pt", "ckpt-i040000.pt", "ckpt-i060000.pt", "ckpt-i072000.pt", "ckpt-i080000.pt" ], "qat_applied": true, "qat_scales_learned": false }