{
  "transformers_version": "5.5.0",
  "architectures": [
    "OutlierMoEForCausalLM"
  ],
  "output_hidden_states": false,
  "return_dict": true,
  "dtype": "float32",
  "chunk_size_feed_forward": 0,
  "is_encoder_decoder": false,
  "id2label": {
    "0": "LABEL_0",
    "1": "LABEL_1"
  },
  "label2id": {
    "LABEL_0": 0,
    "LABEL_1": 1
  },
  "problem_type": null,
  "vocab_size": 152064,
  "hidden_size": 3584,
  "intermediate_size": 18944,
  "num_hidden_layers": 28,
  "num_attention_heads": 28,
  "num_key_value_heads": 4,
  "hidden_act": "silu",
  "max_position_embeddings": 32768,
  "initializer_range": 0.02,
  "rms_norm_eps": 1e-06,
  "use_cache": true,
  "tie_word_embeddings": false,
  "rope_parameters": {
    "rope_theta": 1000000.0,
    "rope_type": "default"
  },
  "use_sliding_window": false,
  "sliding_window": null,
  "max_window_layers": 28,
  "layer_types": [
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention",
    "full_attention"
  ],
  "attention_dropout": 0.0,
  "pad_token_id": null,
  "bos_token_id": 151643,
  "eos_token_id": 151645,
  "_name_or_path": "/dev/shm/qwen7b",
  "model_type": "qwen2",
  "output_attentions": false,
  "n_experts": 8,
  "top_k": 2,
  "base_model": "Qwen/Qwen2.5-7B-Instruct",
  "v3_2_pipeline": "zero_delta_cakld_long",
  "v3_2_checkpoint": "outlier-10b-v3.2",
  "v3_2_moe_layers": [
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20
  ],
  "auto_map": {
    "AutoModelForCausalLM": "modeling_outlier_moe.OutlierMoEForCausalLM"
  },
  "moe_layer_indices": [
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20
  ],
  "experts_per_layer": 8,
  "alpha_values": {
    "7": [
      0.01969238929450512,
      0.01805144175887108,
      0.01898551732301712,
      0.020216668024659157,
      0.019787311553955078,
      0.019402215257287025,
      0.0193850789219141,
      0.02014368399977684
    ],
    "8": [
      0.02600640431046486,
      0.022020943462848663,
      0.02276753820478916,
      0.026547007262706757,
      0.027177805081009865,
      0.02274380810558796,
      0.026304863393306732,
      0.02764948271214962
    ],
    "9": [
      0.013534344732761383,
      0.0145828016102314,
      0.013892530463635921,
      0.013967453502118587,
      0.013249906711280346,
      0.01342932227998972,
      0.014618050307035446,
      0.01448456197977066
    ],
    "10": [
      0.022231444716453552,
      0.019972696900367737,
      0.024050315842032433,
      0.021881794556975365,
      0.021667910739779472,
      0.020687896758317947,
      0.021906213834881783,
      0.021762344986200333
    ],
    "11": [
      0.02834983915090561,
      0.028404127806425095,
      0.028010638430714607,
      0.029820352792739868,
      0.029144475236535072,
      0.031495511531829834,
      0.025821425020694733,
      0.02732662297785282
    ],
    "12": [
      0.030554262921214104,
      0.030468732118606567,
      0.030987773090600967,
      0.028986576944589615,
      0.03083137981593609,
      0.02847979962825775,
      0.029302263632416725,
      0.02996886521577835
    ],
    "13": [
      0.03826427087187767,
      0.03209064528346062,
      0.034369178116321564,
      0.030629923567175865,
      0.033462297171354294,
      0.033646147698163986,
      0.031612180173397064,
      0.03360999748110771
    ],
    "14": [
      0.03800996392965317,
      0.035479817539453506,
      0.035590216517448425,
      0.03698544576764107,
      0.036537885665893555,
      0.03591139242053032,
      0.0347776859998703,
      0.035771314054727554
    ],
    "15": [
      0.037486180663108826,
      0.03183320537209511,
      0.033597372472286224,
      0.034959577023983,
      0.033746231347322464,
      0.03387705609202385,
      0.036250147968530655,
      0.03742583096027374
    ],
    "16": [
      0.03308767080307007,
      0.03283284604549408,
      0.03292208909988403,
      0.032567355781793594,
      0.0328839085996151,
      0.03387352451682091,
      0.032794974744319916,
      0.031062856316566467
    ],
    "17": [
      0.03392287716269493,
      0.030720185488462448,
      0.03290826454758644,
      0.03276655077934265,
      0.031194286420941353,
      0.0328129306435585,
      0.03219807893037796,
      0.03303227201104164
    ],
    "18": [
      0.032916367053985596,
      0.03964288905262947,
      0.039214037358760834,
      0.03236524388194084,
      0.03989103063941002,
      0.03882138058543205,
      0.032691407948732376,
      0.03731442242860794
    ],
    "19": [
      0.02582397684454918,
      0.026262322440743446,
      0.026067685335874557,
      0.025103550404310226,
      0.026913922280073166,
      0.02546408213675022,
      0.025271881371736526,
      0.026326699182391167
    ],
    "20": [
      0.037034712731838226,
      0.02768402174115181,
      0.026117460802197456,
      0.03588447719812393,
      0.03711237385869026,
      0.025986790657043457,
      0.026828493922948837,
      0.036583513021469116
    ]
  }
}