{
  "architectures": [
    "LimiteForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_limite.LimiteConfig",
    "AutoModel": "modeling_limite.LimiteModel",
    "AutoModelForCausalLM": "modeling_limite.LimiteForCausalLM"
  },
  "attention_softmax_scale": 0.1,
  "attn_gate_applied": "per_head_before_o_proj",
  "attn_gate_channels": 128,
  "attn_gate_scale": 2.0,
  "bos_token_id": 151643,
  "checkpoint_step": -1,
  "checkpoint_world_size": 48,
  "eos_token_id": 151643,
  "final_softcap": 0.0,
  "global_every": 4,
  "global_layers": [
    3,
    7,
    11,
    15,
    19,
    23,
    27,
    31,
    35,
    39,
    43,
    47
  ],
  "global_nope": true,
  "global_window": -1,
  "head_dim": 128,
  "hidden_size": 1280,
  "intermediate_size": 3328,
  "lm_head_precision_mode": "oracle_exact",
  "max_position_embeddings": 131072,
  "mlp_formula": "(silu(gate_proj(h)) * up_proj(h)) @ down_proj, no clamp on either factor",
  "mlp_ratio": 4,
  "mlp_type": "swiglu",
  "model_type": "limite",
  "mudd": true,
  "mudd_accumulation": "ordered_left_to_right",
  "mudd_at": [
    24,
    47
  ],
  "mudd_hist_convention": "hist[0] = rms_norm(embedding); hist[j] = output of layer j-1",
  "mudd_inter": 32,
  "mudd_layers": [
    24,
    47
  ],
  "mudd_mlp": true,
  "mudd_r_site": "resid",
  "mudd_tap_idx": {
    "24": [
      0,
      12,
      24
    ],
    "47": [
      0,
      23,
      47
    ]
  },
  "mudd_taps": 3,
  "num_attention_heads": 10,
  "num_hidden_layers": 48,
  "num_key_value_heads": 2,
  "pad_token_id": 151643,
  "padded_vocab_size": 151680,
  "pos_mode": "rope",
  "qk_norm": "rms_pre_rope",
  "rms_norm_eps_mode": "torch_finfo_default",
  "rms_norm_has_weight": false,
  "rope_base_global": 1024.0,
  "rope_base_local": 1024.0,
  "rope_cos_sin_dtype": "bfloat16",
  "rope_frac": 0.5,
  "rope_n_pairs": 32,
  "rope_per_layer": false,
  "rope_style": "interleaved_pairs_odd_lane_sign_flip",
  "sliding_window": 1024,
  "sliding_window_convention": "k >= q - sliding_window, inclusive of the query token (span = sliding_window + 1 keys)",
  "softcap_logits": {
    "a": 23.0,
    "b": 5.0,
    "c": 7.5,
    "kind": "sigmoid"
  },
  "source_format": "guido-export-v1",
  "source_metadata_assumptions": [],
  "tie_word_embeddings": true,
  "tokenizer_vocab_size": 151667,
  "torch_dtype": "bfloat16",
  "ve_applied_before_qk_norm": true,
  "ve_dim": 128,
  "ve_gate_channels": 12,
  "ve_gate_scale": 2.0,
  "ve_head_slice": "first_num_key_value_heads",
  "ve_layers": [
    1,
    4,
    7,
    10,
    13,
    16,
    19,
    22,
    25,
    28,
    31,
    34,
    37,
    40,
    43,
    46
  ],
  "ve_stored_heads": 2,
  "vocab_size": 151680,
  "xsa": true,
  "xsa_layers": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23,
    24,
    25,
    26,
    27,
    28,
    29,
    30,
    31,
    32,
    33,
    34,
    35,
    36,
    37,
    38,
    39,
    40,
    41,
    42,
    43,
    44,
    45,
    46,
    47
  ],
  "xsa_normalize_eps": 0.0001
}