config.json

{
    "n_loops":  1,
    "n_kv_heads":  4,
    "n_layers":  8,
    "loop_mode":  "per_layer",
    "n_experts":  0,
    "intermediate_size":  2048,
    "vocab_size":  402,
    "n_heads":  8,
    "rope_type":  "neox",
    "norm_type":  "rms",
    "max_seq_len":  128,
    "d_model":  512
}