File size: 5,743 Bytes
b368c02 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | # --------------------------------------------------------
# InternVL
# Copyright (c) 2023 OpenGVLab
# Licensed under The MIT License [see LICENSE for details]
# --------------------------------------------------------
import copy
from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
class MoonViTConfig(PretrainedConfig):
model_type = "moonvit"
def __init__(
self,
patch_size: int = 14,
init_pos_emb_height: int = 64,
init_pos_emb_width: int = 64,
num_attention_heads: int = 16,
num_hidden_layers: int = 27,
hidden_size: int = 1152,
intermediate_size: int = 4304,
merge_kernel_size: tuple[int, int] = (2, 2),
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
# Positional embedding config
self.init_pos_emb_height = init_pos_emb_height
self.init_pos_emb_width = init_pos_emb_width
# Transformer config
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
# Patch merger config
self.merge_kernel_size = merge_kernel_size
class LocateAnythingConfig(PretrainedConfig):
model_type = 'locateanything'
is_composition = True
sub_configs = {"vision_config": MoonViTConfig, "text_config": Qwen2Config}
def __init__(
self,
vision_config=None,
text_config=None,
use_backbone_lora=0,
backbone_lora_last_layers=0,
use_llm_lora=0,
downsample_ratio=0.5,
template=None,
loss_version='v1',
mlp_checkpoint=False,
image_token_index=151667,
box_start_token_id=151668,
box_end_token_id=151669,
coord_start_token_id=151677,
coord_end_token_id=152677,
ref_start_token_id=151672,
ref_end_token_id=151673,
none_token_id=4064,
**kwargs):
super().__init__(**kwargs)
if vision_config is None:
vision_config = {'model_type': 'moonvit'}
logger.info('vision_config is None. Initializing the MoonViTConfig with default values.')
if text_config is None:
text_config = {'architectures': ['Qwen2ForCausalLM']}
logger.info('text_config is None. Initializing the Qwen2Config config with default values.')
if vision_config['model_type'] == 'moonvit':
self.vision_config = MoonViTConfig(**vision_config)
else:
raise ValueError('Unsupported model_type: {}. Only moonvit is supported.'.format(vision_config['model_type']))
if text_config['architectures'][0] == 'Qwen2ForCausalLM':
self.text_config = Qwen2Config(**text_config)
elif text_config['architectures'][0] == 'Qwen3ForCausalLM':
self.text_config = Qwen3Config(**text_config)
else:
raise ValueError('Unsupported architecture: {}. Only Qwen2ForCausalLM and Qwen3ForCausalLM are supported.'.format(text_config['architectures'][0]))
self.use_backbone_lora = use_backbone_lora
self.backbone_lora_last_layers = backbone_lora_last_layers
self.use_llm_lora = use_llm_lora
self.mlp_checkpoint = mlp_checkpoint
self.downsample_ratio = downsample_ratio
self.template = template
self.loss_version = loss_version
self.tie_word_embeddings = self.text_config.tie_word_embeddings
self.image_token_index = image_token_index
self.box_start_token_id = box_start_token_id
self.box_end_token_id = box_end_token_id
self.coord_start_token_id = coord_start_token_id
self.coord_end_token_id = coord_end_token_id
self.ref_start_token_id = ref_start_token_id
self.ref_end_token_id = ref_end_token_id
self.none_token_id = none_token_id
def to_dict(self):
"""
Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
Returns:
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
"""
output = copy.deepcopy(self.__dict__)
output['vision_config'] = self.vision_config.to_dict()
output['text_config'] = self.text_config.to_dict()
output['model_type'] = self.__class__.model_type
output['use_backbone_lora'] = self.use_backbone_lora
output['backbone_lora_last_layers'] = self.backbone_lora_last_layers
output['use_llm_lora'] = self.use_llm_lora
output['downsample_ratio'] = self.downsample_ratio
output['template'] = self.template
output['image_token_index'] = self.image_token_index
output['box_start_token_id'] = self.box_start_token_id
output['box_end_token_id'] = self.box_end_token_id
output['coord_start_token_id'] = self.coord_start_token_id
output['coord_end_token_id'] = self.coord_end_token_id
output['ref_start_token_id'] = self.ref_start_token_id
output['ref_end_token_id'] = self.ref_end_token_id
output['none_token_id'] = self.none_token_id
output['_attn_implementation'] = self._attn_implementation
if hasattr(self, '_attn_implementation_autoset'):
output['_attn_implementation_autoset'] = self._attn_implementation_autoset
return output
|