{
  "model_type": "small_lm",
  "vocab_size": 402,
  "d_model": 256,
  "n_heads": 4,
  "n_kv_heads": 2,
  "n_layers": 6,
  "intermediate_size": 1024,
  "max_seq_len": 128,
  "norm_type": "rms",
  "norm_eps": 1e-05,
  "rope_theta": 10000.0,
  "rope_type": "neox",
  "activation": "gelu",
  "n_experts": 0,
  "n_loops": 1,
  "loop_mode": "per_layer"
}