# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/lladamoe/modular_lladamoe.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_lladamoe.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # coding=utf-8 # Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved. # # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX # and OPT implementations in this library. It has been modified from its # original forms to accommodate minor architectural differences compared # to GPT-NeoX and OPT used by the Meta AI team that trained the model. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from transformers.configuration_utils import PretrainedConfig, layer_type_validation from transformers.configuration_utils import PretrainedConfig from transformers.modeling_rope_utils import rope_config_validation ALLOWED_LAYER_TYPES = ( "full_attention", "sliding_attention", "chunked_attention", "linear_attention", # used in minimax ) def layer_type_validation(layer_types, num_hidden_layers=None): """Check that `layer_types` is correctly defined.""" if not all(layer_type in ALLOWED_LAYER_TYPES for layer_type in layer_types): raise ValueError(f"The `layer_types` entries must be in {ALLOWED_LAYER_TYPES}") if num_hidden_layers is not None and num_hidden_layers != len(layer_types): raise ValueError( f"`num_hidden_layers` ({num_hidden_layers}) must be equal to the number of layer types " f"({len(layer_types)})" ) class LLaDAMoEVisionConfig(PretrainedConfig): # model_type = "lladamoe" model_type = "lladavision" base_config_key = "vision_config" def __init__( self, depth=32, hidden_size=3584, hidden_act="silu", intermediate_size=3420, num_heads=16, in_channels=3, patch_size=14, spatial_merge_size=2, temporal_patch_size=2, tokens_per_second=4, window_size=112, out_hidden_size=3584, fullatt_block_indexes=[7, 15, 23, 31], initializer_range=0.02, **kwargs, ): super().__init__(**kwargs) self.depth = depth self.hidden_size = hidden_size self.hidden_act = hidden_act self.intermediate_size = intermediate_size self.num_heads = num_heads self.in_channels = in_channels self.patch_size = patch_size self.spatial_merge_size = spatial_merge_size self.temporal_patch_size = temporal_patch_size self.tokens_per_second = tokens_per_second self.window_size = window_size self.fullatt_block_indexes = fullatt_block_indexes self.out_hidden_size = out_hidden_size self.initializer_range = initializer_range class LLaDA2MoeConfig(PretrainedConfig): model_type = "llada2_vl" def __init__( self, vocab_size=30592, hidden_size=1024, intermediate_size=None, num_hidden_layers=24, num_attention_heads=16, num_key_value_heads=0, hidden_act="silu", use_qkv_bias=False, # llada2 only use_qk_norm=False, use_bias=True, # llada2 only rms_norm_eps=1e-05, norm_head=False, # llada2 only tie_word_embeddings=False, # PretrainedConfig key, here change default value. embedding_dropout=0.1, attention_dropout=0.1, output_dropout=0.1, initializer_range=0.02, max_position_embeddings=16384, rope_theta=10000.0, use_cache=True, use_sliding_window=False, sliding_window=4096, max_window_layers=28, rope_scaling=None, pad_token_id=126081, num_experts=16, num_shared_experts=0, num_experts_per_tok=2, n_group=8, topk_group=4, routed_scaling_factor=2.5, moe_intermediate_size=None, first_k_dense_replace=0, head_dim=None, output_router_logits=False, partial_rotary_factor=0.5, **kwargs, ): self.num_hidden_layers = num_hidden_layers self.vocab_size = vocab_size self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.num_attention_heads = num_attention_heads self.num_key_value_heads = num_key_value_heads self.hidden_act = hidden_act self.use_qkv_bias = use_qkv_bias self.use_bias = use_bias self.norm_head = norm_head self.rms_norm_eps = rms_norm_eps self.embedding_dropout = embedding_dropout self.attention_dropout = attention_dropout self.output_dropout = output_dropout self.initializer_range = initializer_range self.max_position_embeddings = max_position_embeddings self.rope_theta = rope_theta self.use_cache = use_cache self.use_sliding_window = use_sliding_window self.sliding_window = sliding_window self.max_window_layers = max_window_layers self.head_dim = head_dim or self.hidden_size // self.num_attention_heads self.rope_scaling = rope_scaling # MoE configs self.num_experts = num_experts self.num_shared_experts = num_shared_experts self.num_experts_per_tok = num_experts_per_tok self.n_group = n_group self.topk_group = topk_group self.moe_intermediate_size = moe_intermediate_size self.first_k_dense_replace = first_k_dense_replace self.output_router_logits = output_router_logits self.routed_scaling_factor = routed_scaling_factor self.partial_rotary_factor = partial_rotary_factor super().__init__(pad_token_id=pad_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs) class LLaDA2VLMoEConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`LLaDAMoEModel`]. It is used to instantiate a LLaDAMoE model according to the specified arguments, defining the model architecture. Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `LLaDAMoETextConfig`): The config object or dictionary of the text backbone. vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `LLaDAMoEVisionConfig`): The config object or dictionary of the vision backbone. image_token_id (`int`, *optional*, defaults to 151655): The image token index to encode the image prompt. video_token_id (`int`, *optional*, defaults to 151656): The video token index to encode the image prompt. vision_start_token_id (`int`, *optional*, defaults to 151652): The token index to denote start of vision input. vision_end_token_id (`int`, *optional*, defaults to 151653): The token index to denote end of vision input. ```python >>> from transformers import LLaDAMoEForConditionalGeneration, LLaDAMoEConfig >>> # Initializing a LLaDAMoE style configuration >>> configuration = LLaDAMoEConfig() >>> # Initializing a model from the LLaDAMoE style configuration >>> model = LLaDAMoEForConditionalGeneration(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```""" # model_type = "lladamoe" model_type = "llada2vl" sub_configs = {"vision_config": LLaDAMoEVisionConfig, "text_config": LLaDA2MoeConfig} keys_to_ignore_at_inference = ["past_key_values"] def __init__( self, text_config=None, vision_config=None, image_token_id=151655, video_token_id=151656, vision_start_token_id=151652, vision_end_token_id=151653, **kwargs, ): self.is_composition = True # We need to init super() here so that it does not reset values # that are in text config to the BaseClass defaults. The Base # config has many text related defaults and not all defaults are same as for `LLaDAMoETextConfig` super().__init__(**kwargs) if isinstance(vision_config, dict): self.vision_config = self.sub_configs["vision_config"](**vision_config) elif vision_config is None: self.vision_config = self.sub_configs["vision_config"]() if isinstance(text_config, dict): self.text_config = self.sub_configs["text_config"](**text_config) elif text_config is None: # For BC use all kwargs to init `TextConfig` self.text_config = self.sub_configs["text_config"](**kwargs) self.image_token_id = image_token_id self.video_token_id = video_token_id self.vision_start_token_id = vision_start_token_id self.vision_end_token_id = vision_end_token_id # Attention implementation to use. It sets it recursively on sub-configs so we call it again in the end self._attn_implementation = kwargs.pop("attn_implementation", None) def __setattr__(self, key, value): if ( (text_config := super().__getattribute__("__dict__").get("text_config")) is not None and key not in ["dtype", "_attn_implementation_internal"] and key in text_config.__dict__ ): setattr(text_config, key, value) else: super().__setattr__(key, value) def __getattribute__(self, key): if "text_config" in super().__getattribute__("__dict__") and key not in [ "dtype", "_attn_implementation_internal", ]: text_config = super().__getattribute__("text_config") if key in text_config.__dict__: return getattr(text_config, key) return super().__getattribute__(key) __all__ = ["LLaDA2VLMoEConfig", "LLaDA2MoeConfig", "LLaDAMoEVisionConfig"]