{
  "architectures": ["SmallLM"],
  "model_type": "small_lm",
  "vocab_size": 24189,
  "d_model": 512,
  "n_heads": 8,
  "n_kv_heads": 4,
  "n_layers": 8,
  "intermediate_size": 2048,
  "max_seq_len": 128,
  "norm_type": "rms",
  "rope_type": "neox",
  "n_experts": 0,
  "n_loops": 1,
  "loop_mode": "per_layer",
  "activation": "silu",
  "dropout": 0.0,
  "torch_dtype": "float16",
  "transformers_version": "4.40.0"
}