File size: 4,033 Bytes
d136afb 9909f20 d136afb 9909f20 d136afb 9909f20 d136afb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | {
"model_type": "bailing_moe_v3_vl",
"architectures": [
"BailingMoeV3VLForConditionalGeneration"
],
"auto_map": {
"AutoConfig": "configuration_bailing_moe_v3_vl.BailingMoeV3VLConfig",
"AutoModelForCausalLM": "modeling_bailing_moe_v3_vl.BailingMoeV3VLForConditionalGeneration"
},
"torch_dtype": "bfloat16",
"transformers_version": "4.56.2",
"tie_word_embeddings": false,
"norm_query_embeds": false,
"image_token_id": 157157,
"video_token_id": 156909,
"vision_start_token_id": 157158,
"vision_end_token_id": 157159,
"mrope_section": [8, 12, 12],
"text_config": {
"image_patch_token": 157157,
"video_patch_token": 156909,
"image_start_token": 157158,
"video_start_token": 157160,
"num_hidden_layers": 42,
"hidden_size": 2560,
"intermediate_size": 6144,
"eos_token_id": 156895,
"pad_token_id": 156892,
"first_k_dense_replace": 2,
"max_position_embeddings": 131072,
"moe_intermediate_size": 768,
"num_experts_per_tok": 8,
"num_attention_heads": 32,
"q_lora_rank": null,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"num_experts": 512,
"num_key_value_heads": 32,
"rope_theta": 6000000,
"rms_norm_eps": 1e-06,
"head_dim": 128,
"vocab_size": 157184,
"partial_rotary_factor": 0.5,
"router_dtype": "fp32",
"moe_router_enable_expert_bias": true,
"routed_scaling_factor": 2.5,
"n_group": 8,
"topk_group": 4,
"use_qk_norm": true,
"score_function": "sigmoid",
"moe_shared_expert_intermediate_size": 768,
"layer_group_size": 6,
"num_kv_heads_for_linear_attn": 0,
"group_norm_size": 1,
"linear_silu": true,
"rotary_dim": 64,
"use_mla_nope": false,
"short_conv_kernel_size": 4,
"use_nGPT": false,
"scale_router_input": false,
"value_norm": false,
"up_proj_norm": false,
"gated_attention_proj_granularity_type": "head_wise",
"mtp_use_kda": false,
"no_kda_lora": true,
"use_kda_lora": false,
"kda_safe_gate": true,
"kda_lower_bound": -5.0,
"norm_topk_prob": true,
"expert_swiglu_limit_list": [
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0,
4, 4, 4, 4, 4, 4, 4
],
"share_expert_swiglu_limit_list": [
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0,
5, 5, 5, 5, 5, 5, 7, 7
]
},
"vision_config": {
"model_type": "qwen3_moe_vit",
"depth": 27,
"hidden_size": 1152,
"hidden_act": "gelu_pytorch_tanh",
"intermediate_size": 4304,
"num_heads": 16,
"in_channels": 3,
"patch_size": 16,
"spatial_merge_size": 2,
"temporal_patch_size": 2,
"out_hidden_size": 4096,
"num_position_embeddings": 2304,
"initializer_range": 0.02,
"disable_merger_proj": true
},
"quantization_config": {
"config_groups": {
"group_0": {
"input_activations": null,
"output_activations": null,
"targets": [
"Linear"
],
"weights": {
"actorder": null,
"block_structure": null,
"dynamic": false,
"group_size": 32,
"num_bits": 4,
"observer": "minmax",
"observer_kwargs": {},
"strategy": "group",
"symmetric": true,
"type": "int"
}
}
},
"format": "pack-quantized",
"global_compression_ratio": null,
"ignore": [
"re:.*visual.*",
"re:.*linear_proj.*",
"re:.*lm_head.*",
"re:.*self_attn.*",
"re:.*attention.*",
"re:.*(mlp|shared_experts)\\.(gate|up|gate_up|down|eh)_proj.*",
"re:.*layers\\.\\d+\\.(gate|up|gate_up|down|eh)_proj.*"
],
"kv_cache_scheme": null,
"quant_method": "compressed-tensors",
"quantization_status": "compressed",
"sparsity_config": {},
"transform_config": {}
}
}
|