{
  "active_params": 222926080,
  "amp_dtype": "torch.bfloat16",
  "arch": "looped",
  "base_d_ff": 704,
  "base_d_model": 256,
  "batch_size": 16,
  "beta1": 0.9,
  "beta2": 0.999,
  "context_length": 1024,
  "d_ff": 1728,
  "d_ff_run": 1728,
  "d_model": 640,
  "d_model_run": 640,
  "effective_depth": 32,
  "eps": 1e-08,
  "estimated_flops_active": 1000003621607178240,
  "eval_batches": 50,
  "flop_budget": 1e+18,
  "init_seed": 42,
  "looped": true,
  "lr": 0.005,
  "moe": false,
  "mup": true,
  "num_heads": 10,
  "num_heads_run": 10,
  "num_layers_in_stack": 16,
  "num_params": 143627520,
  "num_stacks": 2,
  "rope_theta": 10000.0,
  "steps": 45632,
  "steps_run": 45632,
  "train_path": "data/fw_train_tokens.npy",
  "val_path": "data/fw_valid_tokens.npy",
  "vocab_size": 50257,
  "wandb_project": "looped-scaling-32L-isoflop",
  "warmup": 4563,
  "warmup_frac": 0.1,
  "weight_decay": 0.0001,
  "weight_tied": false,
  "width_ratio": 2.5
}