{
  "run_name": "F_inject_add_s1_4090",
  "params": {
    "total": 9441152,
    "embedding": 3145728,
    "non_embedding": 6295424
  },
  "tokens_trained": 99999744,
  "wall_clock_s": 3477.89067530632,
  "val_loss_by_loops": {
    "1": 7.405364990234375,
    "2": 6.412084007263184,
    "4": 5.2127032995224,
    "8": 4.134036874771118,
    "16": 3.7898231625556944,
    "24": 3.8697949051856995,
    "32": 3.9913308143615724,
    "48": 4.1892451524734495,
    "64": 4.327530193328857,
    "96": 4.507761836051941,
    "128": 4.6194603681564335
  },
  "val_ppl_by_loops": {
    "1": 1644.7850570163394,
    "2": 609.1618568928884,
    "4": 183.5896858870107,
    "8": 62.429434742451754,
    "16": 44.248574779205214,
    "24": 47.9325543543014,
    "32": 54.12687427505751,
    "48": 65.97297262494666,
    "64": 75.75695030099443,
    "96": 90.7185481353149,
    "128": 101.43927749287803
  },
  "val_bpb_by_loops": {
    "1": 2.839405015475947,
    "2": 2.4585558596889587,
    "4": 1.9986828349946704,
    "8": 1.585094732247102,
    "16": 1.4531144527944435,
    "24": 1.4837776500061182,
    "32": 1.5303776042999957,
    "48": 1.6062629880738588,
    "64": 1.6592849848407885,
    "96": 1.7283903741052271,
    "128": 1.7712184281845387
  },
  "best_val_loss": 3.7898231625556944,
  "config": {
    "model": {
      "vocab_size": 8192,
      "d_model": 384,
      "n_heads": 6,
      "n_kv_heads": 2,
      "head_dim": 64,
      "d_ff": 1024,
      "max_seq_len": 512,
      "rope_theta": 10000.0,
      "rms_eps": 1e-06,
      "tie_embeddings": true,
      "n_prelude": 1,
      "n_recurrent": 2,
      "n_coda": 1,
      "n_loops": 16,
      "max_loops": 256,
      "state_init": "prelude",
      "state_init_std": 0.4,
      "input_injection": "add",
      "state_norm": "none",
      "update_rule": "residual",
      "update_gate_init": 3.0,
      "depth_cond": "none",
      "depth_cond_input": "progress",
      "depth_cond_dim": 64,
      "loop_noise": 0.0,
      "noise_schedule": "linear",
      "halting": "none",
      "halt_prior": 0.1,
      "halt_kl_weight": 0.01,
      "init_std": 0.02,
      "depth_scaled_init": true
    },
    "train": {
      "data_dir": "/root/looped/data",
      "out_dir": "/root/looped/runs",
      "run_name": "F_inject_add_s1_4090",
      "notes": "",
      "max_tokens": 100000000,
      "seq_len": 512,
      "micro_batch": 16,
      "grad_accum": 1,
      "lr": 0.0015,
      "min_lr_ratio": 0.1,
      "warmup_frac": 0.02,
      "weight_decay": 0.1,
      "beta1": 0.9,
      "beta2": 0.95,
      "grad_clip": 1.0,
      "seed": 1,
      "loop_schedule": "fixed",
      "loop_mean": 16,
      "loop_min": 1,
      "loop_max": 16,
      "backprop_loops": 0,
      "grad_checkpoint": false,
      "aux_loss_weight": 0.0,
      "aux_readouts": 1,
      "device": "cuda",
      "dtype": "bfloat16",
      "compile": true,
      "eval_every": 500,
      "eval_batches": 20,
      "log_every": 25,
      "eval_loops": [
        1,
        2,
        4,
        8,
        16,
        24,
        32,
        48,
        64,
        96,
        128
      ],
      "save_checkpoint": true
    },
    "params": {
      "total": 9441152,
      "embedding": 3145728,
      "non_embedding": 6295424
    },
    "tokens_per_step": 8192,
    "total_steps": 12207,
    "corpus": {
      "tokenizer_vocab_size": 8192,
      "train_tokens": 110197899,
      "val_tokens": 1674551,
      "train_docs": 133732,
      "val_docs": 2098,
      "train_bytes": 414483859,
      "val_bytes": 6300747,
      "train_bytes_per_token": 3.7612682524918193,
      "val_bytes_per_token": 3.7626486144644145,
      "shard": "sample/10BT/000_00000.parquet",
      "val_every": 500
    },
    "torch": "2.9.0+cu129",
    "gpu": "NVIDIA GeForce RTX 4090"
  }
}