configuration_bailingmm.py

#!/usr/bin/env python3
# coding=utf-8
# Copyright (c) Ant Group. All rights reserved.

from typing import Dict

from transformers.configuration_utils import PretrainedConfig


class ModelConfig(PretrainedConfig):
    model_type = "bailing_mm"

    def __init__(self, type: str, args: Dict, freeze: bool, half: bool = False, megatron_args: Dict = None, **kwargs):
        self.type = type
        self.args = args
        self.freeze = freeze
        self.half = half
        self.megatron_args = megatron_args

        super().__init__(**kwargs)


class BailingMMConfig(PretrainedConfig):
    model_type = "bailing_mm"

    def __init__(
        self,
        image_size=224,
        num_query_token=32,
        num_query_token_video=64,
        num_query_token_audio=32,
        num_decoder_image_token=1024,
        num_decoder_audio_token=512,
        max_txt_len=32,
        mlp_depth=1,
        add_position_emb=False,
        loss_weight=False,
        tune_word_embeddings=False,
        llm_config: ModelConfig = None,
        pool_after_mlp=False,
        norm_query_embeds=True,
        norm_ds_embeds=True,
        use_second_last_layer_feature=False,
        vit_train_last_layer=None,
        copy_ori_embedding=False,
        use_multi_patch=False,
        multi_patch_pooling=False,
        audio_vocab_size=4099,
        add_audio_token=False,
        vision_config: ModelConfig = None,
        audio_config: ModelConfig = None,
        loss_func: str = None,
        use_llm_3drope: str = None,
        offload_threshold_value: int = 150 * 1024 * 1024,
        offload_activation_dataset: list = None,
        vision_compression_type: str = None,
        vision_compression_keep_ratio: float = 0.25,
        vision_compression_window_size: int = 2,
        vision_compression_pack_mode: str = "boost",
        vision_compression_position: str = "after_bridge",
        **kwargs,
    ):
        self.audio_config = self._parse_model_config(audio_config)
        self.vision_config = self._parse_model_config(vision_config)
        self.llm_config = self._parse_model_config(llm_config)
        self.image_size = image_size
        self.num_query_token = num_query_token
        self.num_query_token_video = num_query_token_video
        self.num_query_token_audio = num_query_token_audio
        self.num_decoder_image_token = num_decoder_image_token
        self.num_decoder_audio_token = num_decoder_audio_token
        self.mlp_depth = mlp_depth
        self.add_position_emb = add_position_emb
        self.tune_word_embeddings = tune_word_embeddings
        self.vit_train_last_layer = vit_train_last_layer
        self.use_second_last_layer_feature = use_second_last_layer_feature
        self.copy_ori_embedding = copy_ori_embedding
        self.use_multi_patch = use_multi_patch
        self.multi_patch_pooling = multi_patch_pooling
        self.max_txt_len = max_txt_len
        self.loss_weight = loss_weight
        self.pool_after_mlp = pool_after_mlp
        self.norm_query_embeds = norm_query_embeds
        self.norm_ds_embeds = norm_ds_embeds
        self.audio_vocab_size = audio_vocab_size
        self.add_audio_token = add_audio_token
        self.loss_func = loss_func
        self.use_llm_3drope = use_llm_3drope
        self.offload_threshold_value = offload_threshold_value
        self.vision_compression_type = vision_compression_type
        self.vision_compression_keep_ratio = vision_compression_keep_ratio
        self.vision_compression_window_size = vision_compression_window_size
        if vision_compression_pack_mode not in {"boost", "preserve"}:
            raise ValueError(
                f"Invalid vision_compression_pack_mode: {vision_compression_pack_mode!r}. "
                "Expected one of ['boost', 'preserve']."
            )
        if vision_compression_pack_mode == "preserve" and vision_compression_type is None:
            # preserve 模式只是 VisionZip 的 packing 节奏开关；缺少 vision_compression_type
            # 时既不会真正压缩，也无 cache rewrite 写入 pack_length，但 PostProcessor.concat
            # 仍会切到 dynamic padding 分支截短 packed-sample，悄无声息地偏离 baseline。
            raise ValueError(
                "vision_compression_pack_mode='preserve' requires vision_compression_type "
                "to be set (e.g. 'visionzip'). preserve 是 VisionZip 的 packing 节奏开关，"
                "不能单独使用 —— 请设置 vision_compression_type，或将 pack_mode 改回 'boost'。"
            )
        self.vision_compression_pack_mode = vision_compression_pack_mode
        if vision_compression_position not in {"after_bridge", "before_bridge"}:
            raise ValueError(
                f"Invalid vision_compression_position: {vision_compression_position!r}. "
                "Expected one of ['after_bridge', 'before_bridge']."
            )
        if vision_compression_position == "before_bridge" and vision_compression_type is None:
            # before_bridge 仅在启用 VisionZip 时有意义：它决定 compressor 是在 linear_proj
            # (vision→LLM 维度的 bridge) 之前还是之后调用，缺少 vision_compression_type 时
            # 整条压缩路径都不存在，该开关无效果。
            raise ValueError(
                "vision_compression_position='before_bridge' requires vision_compression_type "
                "to be set (e.g. 'visionzip')."
            )
        self.vision_compression_position = vision_compression_position
        if offload_activation_dataset is None:
            offload_activation_dataset = []
        self.offload_activation_dataset = offload_activation_dataset
        super().__init__(**kwargs)

    def _parse_model_config(self, ori_config):
        config = ori_config
        if isinstance(ori_config, dict):
            config = ModelConfig(**ori_config)
        return config