{
  "architectures": [
    "Fuse2ForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "fuse2_model.Fuse2Config",
    "AutoModelForCausalLM": "fuse2_model.Fuse2ForCausalLM"
  },
  "model_type": "fuse2",
  "hidden_size": 2560,
  "intermediate_size": 9728,
  "num_hidden_layers": 36,
  "num_attention_heads": 32,
  "num_key_value_heads": 8,
  "head_dim": 128,
  "hidden_act": "silu",
  "max_position_embeddings": 40960,
  "rms_norm_eps": 1e-06,
  "rope_theta": 1000000.0,
  "use_sliding_window": false,
  "max_window_layers": 36,
  "tie_word_embeddings": true,
  "vocab_size": 151936,
  "bos_token_id": 151643,
  "eos_token_id": 151643,
  "pad_token_id": 151643,
  "attention_bias": false,
  "attention_dropout": 0.0,
  "expert_hidden_size": 4096,
  "expert_intermediate_size": 2048,
  "experts_per_layer": {
    "0": [
      0,
      1,
      2,
      3,
      4,
      5
    ],
    "1": [
      0,
      1,
      2,
      3
    ],
    "2": [
      0,
      1,
      2,
      3,
      4
    ],
    "3": [
      0,
      1,
      2,
      3
    ],
    "4": [
      0,
      1,
      2,
      3
    ],
    "5": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "6": [
      0,
      1,
      2,
      3
    ],
    "7": [
      0,
      1,
      2,
      3,
      4,
      5,
      6
    ],
    "8": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "9": [
      0
    ],
    "10": [
      0,
      1,
      2,
      3,
      4
    ],
    "11": [
      0,
      1,
      2,
      3,
      4,
      5,
      6
    ],
    "12": [
      0,
      1,
      2,
      3
    ],
    "13": [
      0,
      1,
      2,
      3,
      4,
      5,
      6
    ],
    "14": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "15": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8
    ],
    "16": [
      0,
      1,
      2,
      3,
      4,
      5
    ],
    "17": [
      0,
      1,
      2,
      3,
      4,
      5,
      6
    ],
    "18": [
      0,
      1,
      2,
      3,
      4,
      5,
      6
    ],
    "19": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "20": [
      0,
      1,
      2,
      3
    ],
    "21": [
      0,
      1,
      2
    ],
    "22": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9
    ],
    "23": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "24": [
      0,
      1,
      2,
      3,
      4
    ],
    "25": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "26": [
      0,
      1
    ],
    "27": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "28": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12
    ],
    "29": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20
    ],
    "30": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10
    ],
    "31": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17
    ],
    "32": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17
    ],
    "33": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "34": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20,
      21,
      22,
      23,
      24
    ],
    "35": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17
    ]
  },
  "num_augmented_layers": 36,
  "top_k_experts": 2,
  "bridge_rank": 7,
  "coding_enabled": true,
  "router_init_scale": -2.0,
  "load_balance_coef": 0.01,
  "use_cache": false,
  "torch_dtype": "bfloat16",
  "quantization_config": {
    "quant_method": "bitsandbytes",
    "load_in_4bit": true,
    "load_in_8bit": false,
    "bnb_4bit_quant_type": "nf4",
    "bnb_4bit_compute_dtype": "bfloat16",
    "bnb_4bit_use_double_quant": false,
    "llm_int8_enable_fp32_cpu_offload": false,
    "llm_int8_has_fp16_weight": false
  },
  "transformers_version": "5.12.1"
}