| { |
| "run_name": "F_inject_add_s1_4090", |
| "params": { |
| "total": 9441152, |
| "embedding": 3145728, |
| "non_embedding": 6295424 |
| }, |
| "tokens_trained": 99999744, |
| "wall_clock_s": 3477.89067530632, |
| "val_loss_by_loops": { |
| "1": 7.405364990234375, |
| "2": 6.412084007263184, |
| "4": 5.2127032995224, |
| "8": 4.134036874771118, |
| "16": 3.7898231625556944, |
| "24": 3.8697949051856995, |
| "32": 3.9913308143615724, |
| "48": 4.1892451524734495, |
| "64": 4.327530193328857, |
| "96": 4.507761836051941, |
| "128": 4.6194603681564335 |
| }, |
| "val_ppl_by_loops": { |
| "1": 1644.7850570163394, |
| "2": 609.1618568928884, |
| "4": 183.5896858870107, |
| "8": 62.429434742451754, |
| "16": 44.248574779205214, |
| "24": 47.9325543543014, |
| "32": 54.12687427505751, |
| "48": 65.97297262494666, |
| "64": 75.75695030099443, |
| "96": 90.7185481353149, |
| "128": 101.43927749287803 |
| }, |
| "val_bpb_by_loops": { |
| "1": 2.839405015475947, |
| "2": 2.4585558596889587, |
| "4": 1.9986828349946704, |
| "8": 1.585094732247102, |
| "16": 1.4531144527944435, |
| "24": 1.4837776500061182, |
| "32": 1.5303776042999957, |
| "48": 1.6062629880738588, |
| "64": 1.6592849848407885, |
| "96": 1.7283903741052271, |
| "128": 1.7712184281845387 |
| }, |
| "best_val_loss": 3.7898231625556944, |
| "config": { |
| "model": { |
| "vocab_size": 8192, |
| "d_model": 384, |
| "n_heads": 6, |
| "n_kv_heads": 2, |
| "head_dim": 64, |
| "d_ff": 1024, |
| "max_seq_len": 512, |
| "rope_theta": 10000.0, |
| "rms_eps": 1e-06, |
| "tie_embeddings": true, |
| "n_prelude": 1, |
| "n_recurrent": 2, |
| "n_coda": 1, |
| "n_loops": 16, |
| "max_loops": 256, |
| "state_init": "prelude", |
| "state_init_std": 0.4, |
| "input_injection": "add", |
| "state_norm": "none", |
| "update_rule": "residual", |
| "update_gate_init": 3.0, |
| "depth_cond": "none", |
| "depth_cond_input": "progress", |
| "depth_cond_dim": 64, |
| "loop_noise": 0.0, |
| "noise_schedule": "linear", |
| "halting": "none", |
| "halt_prior": 0.1, |
| "halt_kl_weight": 0.01, |
| "init_std": 0.02, |
| "depth_scaled_init": true |
| }, |
| "train": { |
| "data_dir": "/root/looped/data", |
| "out_dir": "/root/looped/runs", |
| "run_name": "F_inject_add_s1_4090", |
| "notes": "", |
| "max_tokens": 100000000, |
| "seq_len": 512, |
| "micro_batch": 16, |
| "grad_accum": 1, |
| "lr": 0.0015, |
| "min_lr_ratio": 0.1, |
| "warmup_frac": 0.02, |
| "weight_decay": 0.1, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "grad_clip": 1.0, |
| "seed": 1, |
| "loop_schedule": "fixed", |
| "loop_mean": 16, |
| "loop_min": 1, |
| "loop_max": 16, |
| "backprop_loops": 0, |
| "grad_checkpoint": false, |
| "aux_loss_weight": 0.0, |
| "aux_readouts": 1, |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "compile": true, |
| "eval_every": 500, |
| "eval_batches": 20, |
| "log_every": 25, |
| "eval_loops": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 24, |
| 32, |
| 48, |
| 64, |
| 96, |
| 128 |
| ], |
| "save_checkpoint": true |
| }, |
| "params": { |
| "total": 9441152, |
| "embedding": 3145728, |
| "non_embedding": 6295424 |
| }, |
| "tokens_per_step": 8192, |
| "total_steps": 12207, |
| "corpus": { |
| "tokenizer_vocab_size": 8192, |
| "train_tokens": 110197899, |
| "val_tokens": 1674551, |
| "train_docs": 133732, |
| "val_docs": 2098, |
| "train_bytes": 414483859, |
| "val_bytes": 6300747, |
| "train_bytes_per_token": 3.7612682524918193, |
| "val_bytes_per_token": 3.7626486144644145, |
| "shard": "sample/10BT/000_00000.parquet", |
| "val_every": 500 |
| }, |
| "torch": "2.9.0+cu129", |
| "gpu": "NVIDIA GeForce RTX 4090" |
| } |
| } |
