Tim Becker commited on
Upload 19 files
Browse files- .gitattributes +1 -0
- Inference_script.py +233 -0
- LICENSE +10 -0
- NOTICE +5 -0
- README.md +339 -8
- added_tokens.json +28 -0
- chat_template.json +3 -0
- config.json +72 -0
- configuration_ovis2_5.py +94 -0
- generation_config.json +15 -0
- merges.txt +0 -0
- model-00001-of-00004.safetensors +3 -0
- model-00004-of-00004.safetensors +3 -0
- model.safetensors.index.json +449 -0
- modeling_ovis2_5.py +999 -0
- preprocessor_config.json +24 -0
- special_tokens_map.json +31 -0
- tokenizer.json +3 -0
- tokenizer_config.json +240 -0
- vocab.json +0 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
Inference_script.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import glob
|
| 4 |
+
import gc
|
| 5 |
+
import torch
|
| 6 |
+
import requests
|
| 7 |
+
from PIL import Image
|
| 8 |
+
from safetensors.torch import load_file
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
|
| 11 |
+
# Set allocator hint before large torch usage
|
| 12 |
+
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
| 13 |
+
|
| 14 |
+
# Import model components
|
| 15 |
+
from sinq.patch_model import AutoSINQHFModel
|
| 16 |
+
from transformers import AutoTokenizer
|
| 17 |
+
|
| 18 |
+
# Configuration
|
| 19 |
+
BASE_MODEL_DIR = "Ovis2.5-9B-sinq-4bit-experimental" #change to your model path
|
| 20 |
+
TARGET_IMAGE_SIZE = (1024, 1024) #Change if needed
|
| 21 |
+
DEFAULT_MAX_NEW_TOKENS = 512 #increase for longer responses
|
| 22 |
+
DEFAULT_THINKING = False #set True to enable "thinking" mode (not recommended for grounding tasks)
|
| 23 |
+
|
| 24 |
+
# Add model repo to path
|
| 25 |
+
if BASE_MODEL_DIR not in sys.path:
|
| 26 |
+
sys.path.insert(0, BASE_MODEL_DIR)
|
| 27 |
+
|
| 28 |
+
# Import model-specific modules
|
| 29 |
+
try:
|
| 30 |
+
import configuration_ovis2_5
|
| 31 |
+
import modeling_ovis2_5
|
| 32 |
+
except ImportError as e:
|
| 33 |
+
print(f"CRITICAL: Could not import model files from {BASE_MODEL_DIR}")
|
| 34 |
+
print(f"Please ensure 'configuration_ovis2_5.py' and 'modeling_ovis2_5.py' exist in that directory.")
|
| 35 |
+
sys.exit(1)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class PrecomputedVisualTokenizer(torch.nn.Module):
|
| 39 |
+
"""Tokenizer that returns precomputed visual tokens."""
|
| 40 |
+
def __init__(self, visual_tokens):
|
| 41 |
+
super().__init__()
|
| 42 |
+
if visual_tokens is not None:
|
| 43 |
+
self.register_buffer("_visual_tokens", visual_tokens, persistent=False)
|
| 44 |
+
else:
|
| 45 |
+
self.register_buffer("_visual_tokens", torch.zeros(1, 1, dtype=torch.bfloat16), persistent=False)
|
| 46 |
+
|
| 47 |
+
def forward(self, pixel_values, grid_thws):
|
| 48 |
+
return self._visual_tokens
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def load_model_components():
|
| 52 |
+
"""Load all model components and return the configured model."""
|
| 53 |
+
# Load tokenizer
|
| 54 |
+
print(f"Loading tokenizer from {BASE_MODEL_DIR}...")
|
| 55 |
+
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_DIR, trust_remote_code=True)
|
| 56 |
+
|
| 57 |
+
# Load quantized LLM
|
| 58 |
+
quantized_llm_dir = f"{BASE_MODEL_DIR}/quantized_llm"
|
| 59 |
+
print(f"Loading quantized LLM from {quantized_llm_dir} (on cuda:0)...")
|
| 60 |
+
qmodel_llm = AutoSINQHFModel.from_quantized(
|
| 61 |
+
quantized_llm_dir,
|
| 62 |
+
device="cuda:0",
|
| 63 |
+
compute_dtype=torch.bfloat16,
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
# Create full model
|
| 67 |
+
print("Instantiating full Ovis2.5 architecture...")
|
| 68 |
+
config = configuration_ovis2_5.Ovis2_5_Config.from_pretrained(BASE_MODEL_DIR)
|
| 69 |
+
config.name_or_path = BASE_MODEL_DIR
|
| 70 |
+
model = modeling_ovis2_5.Ovis2_5(config)
|
| 71 |
+
model.eval()
|
| 72 |
+
|
| 73 |
+
# Replace LLM with quantized version
|
| 74 |
+
model.llm = qmodel_llm
|
| 75 |
+
|
| 76 |
+
# Load non-LLM weights
|
| 77 |
+
load_non_llm_weights(model)
|
| 78 |
+
|
| 79 |
+
# Attach tokenizer
|
| 80 |
+
model.text_tokenizer = tokenizer
|
| 81 |
+
|
| 82 |
+
print("Model loaded successfully (LLM on cuda:0, rest on CPU).")
|
| 83 |
+
return model
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def load_non_llm_weights(model):
|
| 87 |
+
"""Load weights for non-LLM components."""
|
| 88 |
+
# Load base model weights (excluding LLM)
|
| 89 |
+
safetensors_files = sorted(glob.glob(os.path.join(BASE_MODEL_DIR, "*.safetensors")))
|
| 90 |
+
state_dict = {}
|
| 91 |
+
|
| 92 |
+
for shard_path in safetensors_files:
|
| 93 |
+
shard_dict = load_file(shard_path, device="cpu")
|
| 94 |
+
shard_dict = {k: v for k, v in shard_dict.items() if not k.startswith("llm.")}
|
| 95 |
+
state_dict.update(shard_dict)
|
| 96 |
+
|
| 97 |
+
# Load vision encoder weights
|
| 98 |
+
vision_shards = sorted(glob.glob(os.path.join(BASE_MODEL_DIR, "*.safetensors")))
|
| 99 |
+
if vision_shards:
|
| 100 |
+
vision_state_dict = {}
|
| 101 |
+
for shard_path in vision_shards:
|
| 102 |
+
shard_dict = load_file(shard_path, device="cpu")
|
| 103 |
+
vision_state_dict.update(shard_dict)
|
| 104 |
+
state_dict.update(vision_state_dict)
|
| 105 |
+
|
| 106 |
+
# Load state dict
|
| 107 |
+
model.load_state_dict(state_dict, strict=False)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def process_image(model, img_url):
|
| 111 |
+
"""Download, resize, and process the image to get visual tokens."""
|
| 112 |
+
print("Downloading and resizing input image...")
|
| 113 |
+
img = Image.open(requests.get(img_url, stream=True).raw).convert("RGB")
|
| 114 |
+
img = img.resize(TARGET_IMAGE_SIZE, resample=Image.LANCZOS)
|
| 115 |
+
|
| 116 |
+
# Create message with image
|
| 117 |
+
messages = [{
|
| 118 |
+
"role": "user",
|
| 119 |
+
"content": [
|
| 120 |
+
{"type": "image", "image": img},
|
| 121 |
+
{"type": "text", "text": "Calculate the sum of the numbers in the middle box in figure (c)."},
|
| 122 |
+
],
|
| 123 |
+
}]
|
| 124 |
+
|
| 125 |
+
# Preprocess inputs
|
| 126 |
+
print("Preprocessing inputs...")
|
| 127 |
+
input_ids, pixel_values, grid_thws = model.preprocess_inputs(
|
| 128 |
+
messages=messages,
|
| 129 |
+
add_generation_prompt=True,
|
| 130 |
+
enable_thinking=DEFAULT_THINKING
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
# Determine visual device
|
| 134 |
+
visual_device = "cuda:1" if torch.cuda.is_available() and torch.cuda.device_count() > 1 else "cuda:0" # use cpu if no second gpu and not enough vram
|
| 135 |
+
print("if you get OOM, and have only one GPU, change visual_device to 'cpu'")
|
| 136 |
+
print(f"Using {visual_device} for visual processing")
|
| 137 |
+
|
| 138 |
+
# Precompute visual tokens
|
| 139 |
+
precomputed_visual_tokens = None
|
| 140 |
+
if pixel_values is not None:
|
| 141 |
+
print("Precomputing visual tokens...")
|
| 142 |
+
original_device = next(model.visual_tokenizer.parameters()).device
|
| 143 |
+
model.visual_tokenizer = model.visual_tokenizer.to(visual_device)
|
| 144 |
+
|
| 145 |
+
with torch.no_grad(), torch.inference_mode():
|
| 146 |
+
vt = model.visual_tokenizer(
|
| 147 |
+
pixel_values.to(visual_device),
|
| 148 |
+
grid_thws.to(visual_device) if grid_thws is not None else None
|
| 149 |
+
)
|
| 150 |
+
precomputed_visual_tokens = vt.cpu().to(dtype=torch.bfloat16)
|
| 151 |
+
|
| 152 |
+
model.visual_tokenizer = model.visual_tokenizer.to(original_device)
|
| 153 |
+
del pixel_values, grid_thws, vt
|
| 154 |
+
gc.collect()
|
| 155 |
+
torch.cuda.empty_cache()
|
| 156 |
+
|
| 157 |
+
# Replace visual tokenizer with precomputed tokens
|
| 158 |
+
model.visual_tokenizer = PrecomputedVisualTokenizer(precomputed_visual_tokens)
|
| 159 |
+
|
| 160 |
+
return input_ids
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def prepare_for_generation(model, input_ids):
|
| 164 |
+
"""Prepare model and inputs for generation."""
|
| 165 |
+
device = "cuda:0"
|
| 166 |
+
|
| 167 |
+
# Move non-LLM components to device
|
| 168 |
+
print(f"Moving non-LLM components to {device}...")
|
| 169 |
+
for name, module in model.named_children():
|
| 170 |
+
if name != 'llm':
|
| 171 |
+
module.to(device, dtype=torch.bfloat16)
|
| 172 |
+
|
| 173 |
+
# Move top-level buffers
|
| 174 |
+
if hasattr(model, 'indicator_token_indices'):
|
| 175 |
+
model.indicator_token_indices = model.indicator_token_indices.to(device)
|
| 176 |
+
if hasattr(model, 'visual_token_placeholder'):
|
| 177 |
+
model.visual_token_placeholder = model.visual_token_placeholder.to(device)
|
| 178 |
+
|
| 179 |
+
# Move input_ids to device
|
| 180 |
+
input_ids = input_ids.to(device) if input_ids is not None else None
|
| 181 |
+
|
| 182 |
+
# Create placeholder tensors
|
| 183 |
+
pixel_values = torch.zeros(1, 3, 8, 8, device=device, dtype=torch.bfloat16)
|
| 184 |
+
grid_thws = torch.zeros(1, device=device, dtype=torch.int32)
|
| 185 |
+
|
| 186 |
+
# Clean up memory
|
| 187 |
+
gc.collect()
|
| 188 |
+
torch.cuda.empty_cache()
|
| 189 |
+
|
| 190 |
+
return input_ids, pixel_values, grid_thws
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def generate_response(model, input_ids, pixel_values, grid_thws):
|
| 194 |
+
"""Generate response from the model."""
|
| 195 |
+
print("\nGenerating response...")
|
| 196 |
+
with torch.no_grad(), torch.inference_mode():
|
| 197 |
+
outputs = model.generate(
|
| 198 |
+
inputs=input_ids,
|
| 199 |
+
pixel_values=pixel_values,
|
| 200 |
+
grid_thws=grid_thws,
|
| 201 |
+
enable_thinking=DEFAULT_THINKING,
|
| 202 |
+
enable_thinking_budget=False,
|
| 203 |
+
max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
|
| 204 |
+
thinking_budget=2048,
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
# Decode and print response
|
| 208 |
+
print("\n--- Raw Generated Token IDs ---")
|
| 209 |
+
print(outputs[0])
|
| 210 |
+
|
| 211 |
+
response = model.text_tokenizer.decode(outputs[0], skip_special_tokens=True)
|
| 212 |
+
print("\n--- Response ---")
|
| 213 |
+
print(response)
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def main():
|
| 217 |
+
"""Main execution function."""
|
| 218 |
+
# Load model
|
| 219 |
+
model = load_model_components()
|
| 220 |
+
|
| 221 |
+
# Process image and get input_ids
|
| 222 |
+
img_url = "https://cdn-uploads.huggingface.co/production/uploads/658a8a837959448ef5500ce5/TIlymOb86R6_Mez3bpmcB.png"
|
| 223 |
+
input_ids = process_image(model, img_url)
|
| 224 |
+
|
| 225 |
+
# Prepare for generation
|
| 226 |
+
input_ids, pixel_values, grid_thws = prepare_for_generation(model, input_ids)
|
| 227 |
+
|
| 228 |
+
# Generate response
|
| 229 |
+
generate_response(model, input_ids, pixel_values, grid_thws)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
if __name__ == "__main__":
|
| 233 |
+
main()
|
LICENSE
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Copyright (C) 2025 AIDC-AI
|
| 2 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 3 |
+
you may not use this file except in compliance with the License.
|
| 4 |
+
You may obtain a copy of the License at
|
| 5 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 6 |
+
Unless required by applicable law or agreed to in writing, software
|
| 7 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 8 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 9 |
+
See the License for the specific language governing permissions and
|
| 10 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Copyright (C) 2025 AIDC-AI
|
| 2 |
+
Licensed under the Apache 2.0 (the "License").
|
| 3 |
+
The model was trained based on the following models:
|
| 4 |
+
1. Qwen3-8B (https://huggingface.co/Qwen/Qwen3-8B), license: Apache License 2.0 (https://huggingface.co/Qwen/Qwen3-8B/blob/main/LICENSE).
|
| 5 |
+
2. Siglip2 (https://huggingface.co/google/siglip2-so400m-patch16-512), license: Apache License 2.0 (https://choosealicense.com/licenses/apache-2.0/).
|
README.md
CHANGED
|
@@ -1,8 +1,339 @@
|
|
| 1 |
-
---
|
| 2 |
-
license: apache-2.0
|
| 3 |
-
|
| 4 |
-
- AIDC-AI/
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
datasets:
|
| 4 |
+
- AIDC-AI/Ovis-dataset
|
| 5 |
+
library_name: transformers
|
| 6 |
+
tags:
|
| 7 |
+
- MLLM
|
| 8 |
+
pipeline_tag: image-text-to-text
|
| 9 |
+
language:
|
| 10 |
+
- en
|
| 11 |
+
- zh
|
| 12 |
+
---
|
| 13 |
+
# Ovis2.5-9B
|
| 14 |
+
<div align="center">
|
| 15 |
+
<img src=https://cdn-uploads.huggingface.co/production/uploads/637aebed7ce76c3b834cea37/3IK823BZ8w-mz_QfeYkDn.png width="30%"/>
|
| 16 |
+
</div>
|
| 17 |
+
|
| 18 |
+
<p align="center">
|
| 19 |
+
<a href="https://arxiv.org/abs/2508.11737"><img src="https://img.shields.io/badge/📖_Technical_Report-Ovis2.5-b31b1b.svg" alt="technical report"></a>
|
| 20 |
+
<a href="https://github.com/AIDC-AI/Ovis"><img src="https://img.shields.io/badge/GitHub-AIDC--AI/Ovis-blue?style=flat&logo=github" alt="code"></a>
|
| 21 |
+
<a href="https://huggingface.co/spaces/AIDC-AI/Ovis2.5-9B"><img src="https://img.shields.io/badge/🎨_HF_Spaces-AIDC--AI/Ovis2.5--9B-lightblack" alt="demo"></a>
|
| 22 |
+
<a href="https://huggingface.co/collections/AIDC-AI/ovis25-689ec1474633b2aab8809335"><img src="https://img.shields.io/badge/🤗_Models-AIDC--AI/Ovis2.5-yellow" alt="models"></a>
|
| 23 |
+
</p>
|
| 24 |
+
|
| 25 |
+
## Introduction
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
We are pleased to announce the release of **Ovis2.5**, the successor to Ovis2, designed for native-resolution visual perception and enhanced multimodal reasoning.
|
| 29 |
+
It integrates a native-resolution vision transformer (NaViT) that processes images at their original, variable resolutions, eliminating the need for fixed-resolution tiling and preserving both fine details and global layout—crucial for visually dense content such as charts and diagrams.
|
| 30 |
+
To strengthen reasoning, Ovis2.5 is trained not only on linear chain-of-thought (CoT) but also on reflective reasoning, including self-checking and revision.
|
| 31 |
+
This advanced capability is available at inference as an optional *thinking mode*, enabling users to trade latency for higher accuracy on complex inputs.
|
| 32 |
+
|
| 33 |
+
Building on these advances, **Ovis2.5-9B** achieves an average score of 78.3 on the OpenCompass multimodal evaluation suite (SOTA among open-source MLLMs under 40B parameters), while the lightweight **Ovis2.5-2B** scores 73.9, continuing the “small model, big performance” philosophy for resource-constrained scenarios.
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
<div align="center">
|
| 37 |
+
<img src="https://cdn-uploads.huggingface.co/production/uploads/637aebed7ce76c3b834cea37/kh-1dhZRAduP-P4SkIhXr.png" width="100%" />
|
| 38 |
+
</div>
|
| 39 |
+
|
| 40 |
+
**Key Features**
|
| 41 |
+
* **Native-Resolution Perception** — NaViT vision encoder preserves fine details and global structure without lossy tiling.
|
| 42 |
+
* **Deep-Reasoning Capability** — Optional *thinking mode* for self-checking and revision beyond linear CoT. *Thinking budget* supported.
|
| 43 |
+
* **Chart & Document OCR** — State-of-the-art at its scale for complex chart analysis, document understanding (including tables and forms), and OCR.
|
| 44 |
+
* **Broad Task Coverage** — Demonstrates leading performance on image reasoning, video understanding, and grounding benchmarks, showcasing strong general multimodal capability.
|
| 45 |
+
|
| 46 |
+
<div align="center">
|
| 47 |
+
<img src="https://cdn-uploads.huggingface.co/production/uploads/637aebed7ce76c3b834cea37/4kw2RRUhXDiMZdU7wGOfP.png" width="100%" />
|
| 48 |
+
</div>
|
| 49 |
+
|
| 50 |
+
## Quick Inference
|
| 51 |
+
|
| 52 |
+
Below is a simple example demonstrating how to run Ovis2.5 with a single image input. For accelerated inference with **vLLM**, refer to [GitHub](https://github.com/AIDC-AI/Ovis).
|
| 53 |
+
|
| 54 |
+
First, install the required dependencies:
|
| 55 |
+
|
| 56 |
+
```bash
|
| 57 |
+
pip install torch==2.4.0 transformers==4.51.3 numpy==1.25.0 pillow==10.3.0 moviepy==1.0.3
|
| 58 |
+
pip install flash-attn==2.7.0.post2 --no-build-isolation
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Then, run the following code.
|
| 62 |
+
|
| 63 |
+
```python
|
| 64 |
+
import torch
|
| 65 |
+
import requests
|
| 66 |
+
from PIL import Image
|
| 67 |
+
from transformers import AutoModelForCausalLM
|
| 68 |
+
|
| 69 |
+
MODEL_PATH = "AIDC-AI/Ovis2.5-9B"
|
| 70 |
+
|
| 71 |
+
# Thinking mode & budget
|
| 72 |
+
enable_thinking = True
|
| 73 |
+
enable_thinking_budget = True # Only effective if enable_thinking is True.
|
| 74 |
+
|
| 75 |
+
# Total tokens for thinking + answer. Ensure: max_new_tokens > thinking_budget + 25
|
| 76 |
+
max_new_tokens = 3072
|
| 77 |
+
thinking_budget = 2048
|
| 78 |
+
|
| 79 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 80 |
+
MODEL_PATH,
|
| 81 |
+
torch_dtype=torch.bfloat16,
|
| 82 |
+
trust_remote_code=True
|
| 83 |
+
).cuda()
|
| 84 |
+
|
| 85 |
+
messages = [{
|
| 86 |
+
"role": "user",
|
| 87 |
+
"content": [
|
| 88 |
+
{"type": "image", "image": Image.open(requests.get("https://cdn-uploads.huggingface.co/production/uploads/658a8a837959448ef5500ce5/TIlymOb86R6_Mez3bpmcB.png", stream=True).raw)},
|
| 89 |
+
{"type": "text", "text": "Calculate the sum of the numbers in the middle box in figure (c)."},
|
| 90 |
+
],
|
| 91 |
+
}]
|
| 92 |
+
|
| 93 |
+
input_ids, pixel_values, grid_thws = model.preprocess_inputs(
|
| 94 |
+
messages=messages,
|
| 95 |
+
add_generation_prompt=True,
|
| 96 |
+
enable_thinking=enable_thinking
|
| 97 |
+
)
|
| 98 |
+
input_ids = input_ids.cuda()
|
| 99 |
+
pixel_values = pixel_values.cuda() if pixel_values is not None else None
|
| 100 |
+
grid_thws = grid_thws.cuda() if grid_thws is not None else None
|
| 101 |
+
|
| 102 |
+
outputs = model.generate(
|
| 103 |
+
inputs=input_ids,
|
| 104 |
+
pixel_values=pixel_values,
|
| 105 |
+
grid_thws=grid_thws,
|
| 106 |
+
enable_thinking=enable_thinking,
|
| 107 |
+
enable_thinking_budget=enable_thinking_budget,
|
| 108 |
+
max_new_tokens=max_new_tokens,
|
| 109 |
+
thinking_budget=thinking_budget,
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
response = model.text_tokenizer.decode(outputs[0], skip_special_tokens=True)
|
| 113 |
+
print(response)
|
| 114 |
+
```
|
| 115 |
+
|
| 116 |
+
The thinking and thinking budget logic can be applied in the same way for multi-image, video and pure text scenarios.
|
| 117 |
+
|
| 118 |
+
**Note (answer extraction for CoT/Thinking):**
|
| 119 |
+
To make evaluation and usage easier, we recommend appending a fixed suffix to prompts when using chain-of-thought (CoT) or thinking mode. This ensures the model clearly outputs a final answer that can be extracted programmatically:
|
| 120 |
+
|
| 121 |
+
```
|
| 122 |
+
End your response with 'Final answer: '.
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
For example:
|
| 126 |
+
|
| 127 |
+
```
|
| 128 |
+
Calculate the sum of the numbers in the middle box in figure (c).
|
| 129 |
+
End your response with 'Final answer: '.
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
**Tip:** The sections below include an optional streaming helper (compatible with two-phase thinking/budget runs) and extra inference modes: multi-image, video, and text-only.
|
| 134 |
+
|
| 135 |
+
<details>
|
| 136 |
+
<summary>Optional: Streaming (Advanced)</summary>
|
| 137 |
+
|
| 138 |
+
To support thinking budget, we modified the implementation of the Ovis `generate` method and the default `TextIteratorStreamer` is now incompatible. If you need to stream model output, be sure to use the helper class below.
|
| 139 |
+
|
| 140 |
+
```python
|
| 141 |
+
# --- Budget-aware streamer helper ---
|
| 142 |
+
from transformers import TextIteratorStreamer
|
| 143 |
+
|
| 144 |
+
class BudgetAwareTextStreamer(TextIteratorStreamer):
|
| 145 |
+
"""A streamer compatible with Ovis two-phase generation.
|
| 146 |
+
|
| 147 |
+
Call .manual_end() after generation to flush any remaining text.
|
| 148 |
+
"""
|
| 149 |
+
def manual_end(self):
|
| 150 |
+
if len(self.token_cache) > 0:
|
| 151 |
+
text = self.tokenizer.decode(self.token_cache, **self.decode_kwargs)
|
| 152 |
+
printable_text = text[self.print_len:]
|
| 153 |
+
self.token_cache = []
|
| 154 |
+
self.print_len = 0
|
| 155 |
+
else:
|
| 156 |
+
printable_text = ""
|
| 157 |
+
self.next_tokens_are_prompt = True
|
| 158 |
+
self.on_finalized_text(printable_text, stream_end=True)
|
| 159 |
+
|
| 160 |
+
# Disable base class's end hook; we'll finalize via manual_end()
|
| 161 |
+
def end(self):
|
| 162 |
+
pass
|
| 163 |
+
```
|
| 164 |
+
|
| 165 |
+
Example usage:
|
| 166 |
+
|
| 167 |
+
```python
|
| 168 |
+
streamer = BudgetAwareTextStreamer(
|
| 169 |
+
model.text_tokenizer,
|
| 170 |
+
skip_prompt=True,
|
| 171 |
+
skip_special_tokens=True
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
outputs = model.generate(
|
| 175 |
+
inputs=input_ids,
|
| 176 |
+
pixel_values=pixel_values,
|
| 177 |
+
grid_thws=grid_thws,
|
| 178 |
+
enable_thinking=enable_thinking,
|
| 179 |
+
enable_thinking_budget=enable_thinking_budget,
|
| 180 |
+
max_new_tokens=max_new_tokens,
|
| 181 |
+
thinking_budget=thinking_budget,
|
| 182 |
+
streamer=streamer
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
```
|
| 186 |
+
|
| 187 |
+
</details>
|
| 188 |
+
|
| 189 |
+
<details>
|
| 190 |
+
<summary>Example: Multi-image</summary>
|
| 191 |
+
Demonstrates how to run inference with multiple images and a related question.
|
| 192 |
+
|
| 193 |
+
```python
|
| 194 |
+
# Multi-image inference
|
| 195 |
+
multi_image_files = [
|
| 196 |
+
"/path/to/image_1.jpg",
|
| 197 |
+
"/path/to/image_2.jpg",
|
| 198 |
+
"/path/to/image_3.jpg",
|
| 199 |
+
]
|
| 200 |
+
|
| 201 |
+
content = [{"type": "image", "image": Image.open(p).convert("RGB")} for p in multi_image_files]
|
| 202 |
+
content.append({"type": "text", "text": "Describe the images."})
|
| 203 |
+
messages = [{"role": "user", "content": content}]
|
| 204 |
+
|
| 205 |
+
input_ids, pixel_values, grid_thws = model.preprocess_inputs(messages=messages, add_generation_prompt=True, max_pixels=896*896)
|
| 206 |
+
input_ids = input_ids.cuda()
|
| 207 |
+
pixel_values = pixel_values.cuda().to(model.dtype) if pixel_values is not None else None
|
| 208 |
+
grid_thws = grid_thws.cuda() if grid_thws is not None else None
|
| 209 |
+
|
| 210 |
+
with torch.no_grad():
|
| 211 |
+
outputs = model.generate(inputs=input_ids, pixel_values=pixel_values, grid_thws=grid_thws,
|
| 212 |
+
max_new_tokens=1024, do_sample=True,
|
| 213 |
+
eos_token_id=model.text_tokenizer.eos_token_id,
|
| 214 |
+
pad_token_id=model.text_tokenizer.pad_token_id)
|
| 215 |
+
print(model.text_tokenizer.decode(outputs[0], skip_special_tokens=True))
|
| 216 |
+
```
|
| 217 |
+
|
| 218 |
+
</details>
|
| 219 |
+
|
| 220 |
+
<details>
|
| 221 |
+
<summary>Example: Video</summary>
|
| 222 |
+
Demonstrates how to run inference on a video by sampling multiple frames and asking the model to describe the content.
|
| 223 |
+
|
| 224 |
+
```python
|
| 225 |
+
# Video inference
|
| 226 |
+
from moviepy.editor import VideoFileClip # pip install moviepy==1.0.3
|
| 227 |
+
|
| 228 |
+
video_file = "/path/to/video_1.mp4"
|
| 229 |
+
num_frames = 8
|
| 230 |
+
|
| 231 |
+
with VideoFileClip(video_file) as clip:
|
| 232 |
+
total_frames = int(clip.fps * clip.duration)
|
| 233 |
+
indices = [int(i * total_frames / num_frames) for i in range(num_frames)]
|
| 234 |
+
frames = [Image.fromarray(clip.get_frame(t)) for t in (idx / clip.fps for idx in indices)]
|
| 235 |
+
|
| 236 |
+
messages = [{"role": "user", "content": [
|
| 237 |
+
{"type": "video", "video": frames},
|
| 238 |
+
{"type": "text", "text": "Describe this video in detail."},
|
| 239 |
+
]}]
|
| 240 |
+
|
| 241 |
+
input_ids, pixel_values, grid_thws = model.preprocess_inputs(messages=messages, add_generation_prompt=True, max_pixels=896*896)
|
| 242 |
+
input_ids = input_ids.cuda()
|
| 243 |
+
pixel_values = pixel_values.cuda().to(model.dtype) if pixel_values is not None else None
|
| 244 |
+
grid_thws = grid_thws.cuda() if grid_thws is not None else None
|
| 245 |
+
|
| 246 |
+
with torch.no_grad():
|
| 247 |
+
outputs = model.generate(inputs=input_ids, pixel_values=pixel_values, grid_thws=grid_thws,
|
| 248 |
+
max_new_tokens=1024, do_sample=True,
|
| 249 |
+
eos_token_id=model.text_tokenizer.eos_token_id,
|
| 250 |
+
pad_token_id=model.text_tokenizer.pad_token_id)
|
| 251 |
+
print(model.text_tokenizer.decode(outputs[0], skip_special_tokens=True))
|
| 252 |
+
```
|
| 253 |
+
|
| 254 |
+
</details>
|
| 255 |
+
|
| 256 |
+
<details>
|
| 257 |
+
<summary>Example: Text-only</summary>
|
| 258 |
+
Demonstrates how to run inference using only text input without any images or videos.
|
| 259 |
+
|
| 260 |
+
```python
|
| 261 |
+
# Text-only inference
|
| 262 |
+
messages = [{"role": "user", "content": "Hi, please introduce Yellow Mountain."}]
|
| 263 |
+
|
| 264 |
+
input_ids, _, _ = model.preprocess_inputs(messages=messages, add_generation_prompt=True)
|
| 265 |
+
input_ids = input_ids.cuda()
|
| 266 |
+
|
| 267 |
+
with torch.no_grad():
|
| 268 |
+
outputs = model.generate(inputs=input_ids, max_new_tokens=1024, do_sample=True,
|
| 269 |
+
eos_token_id=model.text_tokenizer.eos_token_id,
|
| 270 |
+
pad_token_id=model.text_tokenizer.pad_token_id)
|
| 271 |
+
print(model.text_tokenizer.decode(outputs[0], skip_special_tokens=True))
|
| 272 |
+
```
|
| 273 |
+
|
| 274 |
+
</details>
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
To enable grounding, end your prompt with `Please provide the bounding box coordinates.` (for boxes) or `Please provide the point coordinates.` (for points). To target a specific object, wrap its description in `<ref>` tags, e.g.:
|
| 278 |
+
|
| 279 |
+
```text
|
| 280 |
+
Find the <ref>red apple</ref> in the image. Please provide the bounding box coordinates.
|
| 281 |
+
```
|
| 282 |
+
|
| 283 |
+
Coordinates are normalized to `[0,1)` with the origin `(0,0)` at the top-left corner of the image.
|
| 284 |
+
|
| 285 |
+
* Point: `<point>(x,y)</point>`
|
| 286 |
+
* Bounding box: `<box>(x1,y1),(x2,y2)</box>` where `(x1,y1)` is top-left, `(x2,y2)` is bottom-right.
|
| 287 |
+
* Multiple results can be listed in square brackets: `[<box>(...)</box>,<box>(...)</box> ]`
|
| 288 |
+
|
| 289 |
+
Example:
|
| 290 |
+
|
| 291 |
+
```text
|
| 292 |
+
The image features a serene scene with <ref>three birds</ref>[
|
| 293 |
+
<box>(0.401,0.526),(0.430,0.557)</box>,
|
| 294 |
+
<box>(0.489,0.494),(0.516,0.526)</box>,
|
| 295 |
+
<box>(0.296,0.529),(0.324,0.576)</box>
|
| 296 |
+
] flying in formation against a clear blue sky.
|
| 297 |
+
```
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
## Model Zoo
|
| 302 |
+
|
| 303 |
+
| Ovis MLLMs | ViT | LLM | Model Weights | Demo |
|
| 304 |
+
|:-----------|:-----------------------:|:---------------------:|:-------------------------------------------------------:|:--------------------------------------------------------:|
|
| 305 |
+
| Ovis2.5-2B | siglip2-so400m-patch16-512 | Qwen3-1.7B | [Huggingface](https://huggingface.co/AIDC-AI/Ovis2.5-2B) | [Space](https://huggingface.co/spaces/AIDC-AI/Ovis2.5-2B) |
|
| 306 |
+
| Ovis2.5-9B | siglip2-so400m-patch16-512 | Qwen3-8B | [Huggingface](https://huggingface.co/AIDC-AI/Ovis2.5-9B) | [Space](https://huggingface.co/spaces/AIDC-AI/Ovis2.5-9B) |
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
## Performance
|
| 310 |
+
We evaluate Ovis2.5 using [VLMEvalKit](https://github.com/open-compass/VLMEvalKit), as employed in the OpenCompass multimodal and reasoning evaluation suite.
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
## Citation
|
| 318 |
+
If you find Ovis useful, please consider citing the paper
|
| 319 |
+
```bibtex
|
| 320 |
+
@article{lu2025ovis25technicalreport,
|
| 321 |
+
title={Ovis2.5 Technical Report},
|
| 322 |
+
author={Shiyin Lu and Yang Li and Yu Xia and Yuwei Hu and Shanshan Zhao and Yanqing Ma and Zhichao Wei and Yinglun Li and Lunhao Duan and Jianshan Zhao and Yuxuan Han and Haijun Li and Wanying Chen and Junke Tang and Chengkun Hou and Zhixing Du and Tianli Zhou and Wenjie Zhang and Huping Ding and Jiahe Li and Wen Li and Gui Hu and Yiliang Gu and Siran Yang and Jiamang Wang and Hailong Sun and Yibo Wang and Hui Sun and Jinlong Huang and Yuping He and Shengze Shi and Weihong Zhang and Guodong Zheng and Junpeng Jiang and Sensen Gao and Yi-Feng Wu and Sijia Chen and Yuhui Chen and Qing-Guo Chen and Zhao Xu and Weihua Luo and Kaifu Zhang},
|
| 323 |
+
year={2025},
|
| 324 |
+
journal={arXiv:2508.11737}
|
| 325 |
+
}
|
| 326 |
+
|
| 327 |
+
@article{lu2024ovis,
|
| 328 |
+
title={Ovis: Structural Embedding Alignment for Multimodal Large Language Model},
|
| 329 |
+
author={Shiyin Lu and Yang Li and Qing-Guo Chen and Zhao Xu and Weihua Luo and Kaifu Zhang and Han-Jia Ye},
|
| 330 |
+
year={2024},
|
| 331 |
+
journal={arXiv:2405.20797}
|
| 332 |
+
}
|
| 333 |
+
```
|
| 334 |
+
|
| 335 |
+
## License
|
| 336 |
+
This project is licensed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0.txt) (SPDX-License-Identifier: Apache-2.0).
|
| 337 |
+
|
| 338 |
+
## Disclaimer
|
| 339 |
+
We used compliance-checking algorithms during the training process, to ensure the compliance of the trained model to the best of our ability. Due to the complexity of the data and the diversity of language model usage scenarios, we cannot guarantee that the model is completely free of copyright issues or improper content. If you believe anything infringes on your rights or generates improper content, please contact us, and we will promptly address the matter.
|
added_tokens.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"</think>": 151668,
|
| 3 |
+
"</tool_call>": 151658,
|
| 4 |
+
"</tool_response>": 151666,
|
| 5 |
+
"<think>": 151667,
|
| 6 |
+
"<tool_call>": 151657,
|
| 7 |
+
"<tool_response>": 151665,
|
| 8 |
+
"<|box_end|>": 151649,
|
| 9 |
+
"<|box_start|>": 151648,
|
| 10 |
+
"<|endoftext|>": 151643,
|
| 11 |
+
"<|file_sep|>": 151664,
|
| 12 |
+
"<|fim_middle|>": 151660,
|
| 13 |
+
"<|fim_pad|>": 151662,
|
| 14 |
+
"<|fim_prefix|>": 151659,
|
| 15 |
+
"<|fim_suffix|>": 151661,
|
| 16 |
+
"<|im_end|>": 151645,
|
| 17 |
+
"<|im_start|>": 151644,
|
| 18 |
+
"<|image_pad|>": 151655,
|
| 19 |
+
"<|object_ref_end|>": 151647,
|
| 20 |
+
"<|object_ref_start|>": 151646,
|
| 21 |
+
"<|quad_end|>": 151651,
|
| 22 |
+
"<|quad_start|>": 151650,
|
| 23 |
+
"<|repo_name|>": 151663,
|
| 24 |
+
"<|video_pad|>": 151656,
|
| 25 |
+
"<|vision_end|>": 151653,
|
| 26 |
+
"<|vision_pad|>": 151654,
|
| 27 |
+
"<|vision_start|>": 151652
|
| 28 |
+
}
|
chat_template.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"chat_template": "{%- for message in messages %}{{- '<|im_start|>' + message.role + '\n'}}{%- if message.role == 'system' or message.role == 'user' %}{%- if message.content is string %}{{- message.content | replace('<image>', '') | replace('<video>', '') }}{%- else %}{%- for item in message.content %}{%- if item.type == 'text' and 'text' in item %}{{- item.text | replace('<image>', '') | replace('<video>', '') }}{%- elif item.type == 'image' %}{{- '<image>'}}{%- elif item.type == 'video' %}{{- '<video>'}}{%- else %}{{- raise_exception('Invalid content type. Supported types for system and user are text, image, video.')}}{%- endif %}{%- if not loop.last %}{{- '\n'}}{%- endif %}{%- endfor %}{%- endif %}{%- elif message.role == 'assistant' %}{%- set content = '' %}{%- if message.content is string %}{%- set content = message.content | replace('<image>', '') | replace('<video>', '') %}{%- else %}{%- for item in message.content %}{%- if item.type == 'text' and 'text' in item %}{%- set content = content ~ (item.text | replace('<image>', '') | replace('<video>', '')) %}{%- else %}{{- raise_exception('Invalid content type. Supported type for assistant is text.')}}{%- endif %}{%- endfor %}{%- endif %}{%- set content = content.split('</think>')[-1].lstrip('\n') %}{{- content }}{%- else %}{{- raise_exception('Invalid role. Supported roles are system, user, assistant.')}}{%- endif %}{{- '<|im_end|>\n'}}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- if enable_thinking is defined and enable_thinking is false %}{{- '<think>\n\n</think>\n\n' }}{%- endif %}{%- endif %}"
|
| 3 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Ovis2_5"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_ovis2_5.Ovis2_5_Config",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_ovis2_5.Ovis2_5"
|
| 8 |
+
},
|
| 9 |
+
"conversation_formatter_class": "Qwen3ConversationFormatter",
|
| 10 |
+
"hidden_size": 4096,
|
| 11 |
+
"vocab_size": 151936,
|
| 12 |
+
"num_attention_heads": 32,
|
| 13 |
+
"max_position_embeddings": 40960,
|
| 14 |
+
"llm_config": {
|
| 15 |
+
"_attn_implementation_autoset": true,
|
| 16 |
+
"_name_or_path": "Qwen/Qwen3-8B",
|
| 17 |
+
"architectures": [
|
| 18 |
+
"Qwen3ForCausalLM"
|
| 19 |
+
],
|
| 20 |
+
"attention_bias": false,
|
| 21 |
+
"attention_dropout": 0.0,
|
| 22 |
+
"bos_token_id": 151643,
|
| 23 |
+
"eos_token_id": 151645,
|
| 24 |
+
"head_dim": 128,
|
| 25 |
+
"hidden_act": "silu",
|
| 26 |
+
"hidden_size": 4096,
|
| 27 |
+
"initializer_range": 0.02,
|
| 28 |
+
"intermediate_size": 12288,
|
| 29 |
+
"max_position_embeddings": 40960,
|
| 30 |
+
"max_window_layers": 36,
|
| 31 |
+
"model_type": "qwen3",
|
| 32 |
+
"num_attention_heads": 32,
|
| 33 |
+
"num_hidden_layers": 36,
|
| 34 |
+
"num_key_value_heads": 8,
|
| 35 |
+
"rms_norm_eps": 1e-06,
|
| 36 |
+
"rope_scaling": null,
|
| 37 |
+
"rope_theta": 1000000,
|
| 38 |
+
"sliding_window": null,
|
| 39 |
+
"torch_dtype": "float32",
|
| 40 |
+
"use_cache": true,
|
| 41 |
+
"use_sliding_window": false,
|
| 42 |
+
"vocab_size": 151936
|
| 43 |
+
},
|
| 44 |
+
"model_type": "ovis2_5",
|
| 45 |
+
"torch_dtype": "bfloat16",
|
| 46 |
+
"transformers_version": "4.51.3",
|
| 47 |
+
"use_cache": true,
|
| 48 |
+
"visual_vocab_size": 65536,
|
| 49 |
+
"vit_config": {
|
| 50 |
+
"_attn_implementation_autoset": true,
|
| 51 |
+
"_name_or_path": "google/siglip2-so400m-patch16-512",
|
| 52 |
+
"attention_dropout": 0.0,
|
| 53 |
+
"fullatt_block_indexes": null,
|
| 54 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 55 |
+
"hidden_size": 1152,
|
| 56 |
+
"hidden_stride": 2,
|
| 57 |
+
"image_size": 512,
|
| 58 |
+
"intermediate_size": 4304,
|
| 59 |
+
"layer_norm_eps": 1e-06,
|
| 60 |
+
"model_type": "siglip2_navit",
|
| 61 |
+
"num_attention_heads": 16,
|
| 62 |
+
"num_channels": 3,
|
| 63 |
+
"num_hidden_layers": 27,
|
| 64 |
+
"num_patches": -1,
|
| 65 |
+
"patch_size": 16,
|
| 66 |
+
"preserve_original_pe": true,
|
| 67 |
+
"temporal_patch_size": 1,
|
| 68 |
+
"torch_dtype": "float32",
|
| 69 |
+
"use_rope": true,
|
| 70 |
+
"window_size": 112
|
| 71 |
+
}
|
| 72 |
+
}
|
configuration_ovis2_5.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Optional, List, Union
|
| 2 |
+
|
| 3 |
+
from transformers import Qwen3Config
|
| 4 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 5 |
+
|
| 6 |
+
__all__ = ["Siglip2NavitConfig", "Ovis2_5_Config"]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class Siglip2NavitConfig(PretrainedConfig):
|
| 10 |
+
"""This is the configuration class to store the configuration of an [`AIMv2Model`].
|
| 11 |
+
Instantiating a configuration with the defaults will yield a similar configuration
|
| 12 |
+
to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224).
|
| 13 |
+
Args:
|
| 14 |
+
hidden_size: Dimension of the hidden representations.
|
| 15 |
+
intermediate_size: Dimension of the SwiGLU representations.
|
| 16 |
+
num_hidden_layers: Number of hidden layers in the Transformer.
|
| 17 |
+
num_attention_heads: Number of attention heads for each attention layer
|
| 18 |
+
in the Transformer.
|
| 19 |
+
num_channels: Number of input channels.
|
| 20 |
+
image_size: Image size.
|
| 21 |
+
patch_size: Patch size.
|
| 22 |
+
rms_norm_eps: Epsilon value used for the RMS normalization layer.
|
| 23 |
+
attention_dropout: Dropout ratio for attention probabilities.
|
| 24 |
+
projection_dropout: Dropout ratio for the projection layer after the attention.
|
| 25 |
+
qkv_bias: Whether to add a bias to the queries, keys and values.
|
| 26 |
+
use_bias: Whether to add a bias in the feed-forward and projection layers.
|
| 27 |
+
kwargs: Keyword arguments for the [`PretrainedConfig`].
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
model_type: str = "siglip2_navit"
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
hidden_size: int = 1024,
|
| 35 |
+
intermediate_size: int = 4096,
|
| 36 |
+
num_hidden_layers: int = 24,
|
| 37 |
+
num_attention_heads: int = 16,
|
| 38 |
+
num_channels: int = 3,
|
| 39 |
+
num_patches: int = -1,
|
| 40 |
+
image_size: int = 512,
|
| 41 |
+
patch_size: int = 16,
|
| 42 |
+
hidden_act: str="gelu_pytorch_tanh",
|
| 43 |
+
layer_norm_eps: float = 1e-6,
|
| 44 |
+
attention_dropout: float = 0.0,
|
| 45 |
+
hidden_stride: int = 2,
|
| 46 |
+
window_size: int = 112,
|
| 47 |
+
fullatt_block_indexes: Optional[list] = None,
|
| 48 |
+
temporal_patch_size: int = 1,
|
| 49 |
+
preserve_original_pe: bool = True,
|
| 50 |
+
use_rope: bool = True,
|
| 51 |
+
**kwargs: Any,
|
| 52 |
+
):
|
| 53 |
+
super().__init__(**kwargs)
|
| 54 |
+
self.hidden_size = hidden_size
|
| 55 |
+
self.intermediate_size = intermediate_size
|
| 56 |
+
self.num_hidden_layers = num_hidden_layers
|
| 57 |
+
self.num_attention_heads = num_attention_heads
|
| 58 |
+
self.num_channels = num_channels
|
| 59 |
+
self.num_patches = num_patches
|
| 60 |
+
self.patch_size = patch_size
|
| 61 |
+
self.image_size = image_size
|
| 62 |
+
self.hidden_act = hidden_act
|
| 63 |
+
self.attention_dropout = attention_dropout
|
| 64 |
+
self.layer_norm_eps = layer_norm_eps
|
| 65 |
+
self.hidden_stride = hidden_stride
|
| 66 |
+
self.window_size = window_size
|
| 67 |
+
self.fullatt_block_indexes = fullatt_block_indexes
|
| 68 |
+
self.temporal_patch_size = temporal_patch_size
|
| 69 |
+
self.preserve_original_pe = preserve_original_pe
|
| 70 |
+
self.use_rope = use_rope
|
| 71 |
+
|
| 72 |
+
class Ovis2_5_Config(PretrainedConfig):
|
| 73 |
+
model_type = "ovis2_5"
|
| 74 |
+
sub_configs = dict(llm_config=Qwen3Config, vit_config=Siglip2NavitConfig)
|
| 75 |
+
|
| 76 |
+
def __init__(self,
|
| 77 |
+
llm_config: Optional[Union[Qwen3Config, dict]] = None,
|
| 78 |
+
vit_config: Optional[Union[Siglip2NavitConfig, dict]] = None,
|
| 79 |
+
visual_vocab_size=65536,
|
| 80 |
+
hidden_size=None,
|
| 81 |
+
**kwargs
|
| 82 |
+
):
|
| 83 |
+
super().__init__(**kwargs)
|
| 84 |
+
if isinstance(llm_config, dict):
|
| 85 |
+
llm_config = Qwen3Config(**llm_config)
|
| 86 |
+
self.llm_config = llm_config
|
| 87 |
+
if isinstance(vit_config, dict):
|
| 88 |
+
vit_config = Siglip2NavitConfig(**vit_config)
|
| 89 |
+
self.vit_config = vit_config
|
| 90 |
+
self.visual_vocab_size = visual_vocab_size
|
| 91 |
+
self.hidden_size = hidden_size
|
| 92 |
+
if kwargs.get('attn_implementation'):
|
| 93 |
+
self.llm_config._attn_implementation = kwargs['attn_implementation']
|
| 94 |
+
self.vit_config._attn_implementation = kwargs['attn_implementation']
|
generation_config.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 151643,
|
| 3 |
+
"do_sample": true,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
151645,
|
| 6 |
+
151643
|
| 7 |
+
],
|
| 8 |
+
"multimodal_max_length": 8192,
|
| 9 |
+
"pad_token_id": 151643,
|
| 10 |
+
"repetition_penalty": 1.05,
|
| 11 |
+
"temperature": 0.6,
|
| 12 |
+
"top_k": 20,
|
| 13 |
+
"top_p": 0.95,
|
| 14 |
+
"transformers_version": "4.51.3"
|
| 15 |
+
}
|
merges.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
model-00001-of-00004.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1067142367960639cd8eaef67a19cf0db453c43d8c46b9793fc8fe60b5d9c16d
|
| 3 |
+
size 1244659816
|
model-00004-of-00004.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d41ecd35649386fa479903074fbd344840bac48fa93925b637fa725c9385f415
|
| 3 |
+
size 1968208712
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,449 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_size": 18349615568
|
| 4 |
+
},
|
| 5 |
+
"weight_map": {
|
| 6 |
+
"llm.lm_head.weight": "model-00001-of-00004.safetensors",
|
| 7 |
+
"vte.weight": "model-00004-of-00004.safetensors",
|
| 8 |
+
"visual_tokenizer.head.0.weight": "model-00004-of-00004.safetensors",
|
| 9 |
+
"visual_tokenizer.head.1.bias": "model-00004-of-00004.safetensors",
|
| 10 |
+
"visual_tokenizer.head.1.weight": "model-00004-of-00004.safetensors",
|
| 11 |
+
"visual_tokenizer.vit.vision_model.embeddings.patch_embedding.bias": "model-00004-of-00004.safetensors",
|
| 12 |
+
"visual_tokenizer.vit.vision_model.embeddings.patch_embedding.weight": "model-00004-of-00004.safetensors",
|
| 13 |
+
"visual_tokenizer.vit.vision_model.embeddings.position_embedding.weight": "model-00004-of-00004.safetensors",
|
| 14 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 15 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 16 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 17 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 18 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 19 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 20 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 21 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 22 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 23 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 24 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 25 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 26 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 27 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 28 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 29 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 30 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 31 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 32 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 33 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 34 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 35 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 36 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 37 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 38 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 39 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 40 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 41 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 42 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 43 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 44 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 45 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 46 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 47 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 48 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 49 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 50 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 51 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 52 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 53 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 54 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 55 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 56 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 57 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 58 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 59 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 60 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 61 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 62 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 63 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 64 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 65 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 66 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 67 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 68 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 69 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 70 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 71 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 72 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 73 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 74 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 75 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 76 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 77 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 78 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 79 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 80 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 81 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 82 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 83 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 84 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 85 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 86 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 87 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 88 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 89 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 90 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 91 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 92 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 93 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 94 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 95 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 96 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 97 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 98 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 99 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 100 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 101 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 102 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 103 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 104 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 105 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 106 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 107 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 108 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 109 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 110 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 111 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 112 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 113 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 114 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 115 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 116 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 117 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 118 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 119 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 120 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 121 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 122 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 123 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 124 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 125 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 126 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 127 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 128 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 129 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 130 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 131 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 132 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 133 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 134 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 135 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 136 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 137 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 138 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 139 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 140 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 141 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 142 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 143 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 144 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 145 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 146 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 147 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 148 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 149 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 150 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 151 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 152 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 153 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 154 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 155 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 156 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 157 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 158 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 159 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 160 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 161 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 162 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 163 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 164 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 165 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 166 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 167 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 168 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 169 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 170 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 171 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 172 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 173 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 174 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 175 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 176 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 177 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 178 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 179 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 180 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 181 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 182 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 183 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 184 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 185 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 186 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 187 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 188 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 189 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 190 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 191 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 192 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 193 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 194 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 195 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 196 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 197 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 198 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 199 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 200 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 201 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 202 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 203 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 204 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 205 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 206 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 207 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 208 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 209 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 210 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 211 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 212 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 213 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 214 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 215 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 216 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 217 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 218 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 219 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 220 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 221 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 222 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 223 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 224 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 225 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 226 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 227 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 228 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 229 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 230 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 231 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 232 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 233 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 234 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 235 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 236 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 237 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 238 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 239 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 240 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 241 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 242 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 243 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 244 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 245 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 246 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 247 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 248 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 249 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 250 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 251 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 252 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 253 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 254 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 255 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 256 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 257 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 258 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 259 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 260 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 261 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 262 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 263 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 264 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 265 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 266 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 267 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 268 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 269 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 270 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 271 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 272 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 273 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 274 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 275 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 276 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 277 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 278 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 279 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 280 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 281 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 282 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 283 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 284 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 285 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 286 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 287 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 288 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 289 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 290 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 291 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 292 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 293 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 294 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 295 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 296 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 297 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 298 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 299 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 300 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 301 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 302 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 303 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 304 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 305 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 306 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 307 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 308 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 309 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 310 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 311 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 312 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 313 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 314 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 315 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 316 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 317 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 318 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 319 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 320 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 321 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 322 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 323 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 324 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 325 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 326 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 327 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 328 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 329 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 330 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 331 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 332 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 333 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 334 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 335 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 336 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 337 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 338 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 339 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 340 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 341 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 342 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 343 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 344 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 345 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 346 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 347 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 348 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 349 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 350 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 351 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 352 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 353 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 354 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 355 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 356 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 357 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 358 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 359 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 360 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 361 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 362 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 363 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 364 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 365 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 366 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 367 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 368 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 369 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 370 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 371 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 372 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 373 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 374 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 375 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 376 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 377 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 378 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 379 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 380 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 381 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 382 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 383 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 384 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 385 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 386 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 387 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 388 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 389 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 390 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 391 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 392 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 393 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 394 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 395 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 396 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 397 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 398 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 399 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 400 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 401 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 402 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 403 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 404 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 405 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 406 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 407 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 408 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 409 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 410 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 411 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 412 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 413 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 414 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 415 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 416 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 417 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 418 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 419 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 420 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 421 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 422 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 423 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 424 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 425 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 426 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 427 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 428 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 429 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 430 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.layer_norm1.bias": "model-00004-of-00004.safetensors",
|
| 431 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.layer_norm1.weight": "model-00004-of-00004.safetensors",
|
| 432 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.layer_norm2.bias": "model-00004-of-00004.safetensors",
|
| 433 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.layer_norm2.weight": "model-00004-of-00004.safetensors",
|
| 434 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00004-of-00004.safetensors",
|
| 435 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00004-of-00004.safetensors",
|
| 436 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00004-of-00004.safetensors",
|
| 437 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00004-of-00004.safetensors",
|
| 438 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
| 439 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
| 440 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
|
| 441 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
|
| 442 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
| 443 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
| 444 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
| 445 |
+
"visual_tokenizer.vit.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
| 446 |
+
"visual_tokenizer.vit.vision_model.post_layernorm.bias": "model-00004-of-00004.safetensors",
|
| 447 |
+
"visual_tokenizer.vit.vision_model.post_layernorm.weight": "model-00004-of-00004.safetensors"
|
| 448 |
+
}
|
| 449 |
+
}
|
modeling_ovis2_5.py
ADDED
|
@@ -0,0 +1,999 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
from typing import Dict, List, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import PIL.Image
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
from torch import Tensor, nn
|
| 8 |
+
from torch.nn import functional as F
|
| 9 |
+
from transformers import (
|
| 10 |
+
AutoConfig,
|
| 11 |
+
AutoImageProcessor,
|
| 12 |
+
AutoModel,
|
| 13 |
+
AutoModelForCausalLM,
|
| 14 |
+
AutoTokenizer,
|
| 15 |
+
)
|
| 16 |
+
from transformers.activations import ACT2FN
|
| 17 |
+
from transformers.generation.utils import GenerateOutput
|
| 18 |
+
from transformers.modeling_outputs import BaseModelOutputWithNoAttention
|
| 19 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 20 |
+
from transformers.utils import is_flash_attn_2_available
|
| 21 |
+
|
| 22 |
+
from configuration_ovis2_5 import Siglip2NavitConfig, Ovis2_5_Config
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
if is_flash_attn_2_available():
|
| 26 |
+
from flash_attn import flash_attn_varlen_func
|
| 27 |
+
from flash_attn.layers.rotary import apply_rotary_emb
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
IMAGE_PLACEHOLDER = "<image>"
|
| 31 |
+
IMAGE_PLACEHOLDER_ID = -200
|
| 32 |
+
VIDEO_PLACEHOLDER = "<video>"
|
| 33 |
+
VIDEO_PLACEHOLDER_ID = -201
|
| 34 |
+
|
| 35 |
+
VISUAL_ATOM_ID = -300
|
| 36 |
+
INDICATOR_IDS = [-301, -302, -303, -304]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# copied from qwen2.5-vl
|
| 40 |
+
class VisionRotaryEmbedding(nn.Module):
|
| 41 |
+
def __init__(self, dim: int, theta: float = 10000.0) -> None:
|
| 42 |
+
super().__init__()
|
| 43 |
+
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
|
| 44 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 45 |
+
|
| 46 |
+
def forward(self, seqlen: int) -> torch.Tensor:
|
| 47 |
+
seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype)
|
| 48 |
+
freqs = torch.outer(seq, self.inv_freq)
|
| 49 |
+
return freqs
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class Siglip2VisionEmbeddings(nn.Module):
|
| 53 |
+
def __init__(self, config: Siglip2NavitConfig):
|
| 54 |
+
super().__init__()
|
| 55 |
+
self.config = config
|
| 56 |
+
self.embed_dim = config.hidden_size
|
| 57 |
+
self.patch_size = config.patch_size
|
| 58 |
+
self.image_size = config.image_size
|
| 59 |
+
self.num_patches = config.num_patches
|
| 60 |
+
self.preserve_original_pe = config.preserve_original_pe
|
| 61 |
+
self.hidden_stride = config.hidden_stride
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
# siglip2 naflex
|
| 65 |
+
if self.num_patches > 0:
|
| 66 |
+
self.patch_embedding = nn.Linear(
|
| 67 |
+
in_features=config.num_channels * self.patch_size * self.patch_size,
|
| 68 |
+
out_features=self.embed_dim,
|
| 69 |
+
)
|
| 70 |
+
if self.preserve_original_pe:
|
| 71 |
+
self.position_embedding_size = int(self.num_patches**0.5)
|
| 72 |
+
self.position_embedding = nn.Embedding(self.num_patches, self.embed_dim)
|
| 73 |
+
|
| 74 |
+
else:
|
| 75 |
+
self.patch_embedding = nn.Conv2d(
|
| 76 |
+
in_channels=config.num_channels,
|
| 77 |
+
out_channels=self.embed_dim,
|
| 78 |
+
kernel_size=self.patch_size,
|
| 79 |
+
stride=self.patch_size,
|
| 80 |
+
padding="valid",
|
| 81 |
+
)
|
| 82 |
+
if self.preserve_original_pe:
|
| 83 |
+
self.num_patches = (self.image_size // self.patch_size) ** 2
|
| 84 |
+
self.position_embedding_size = self.image_size // self.patch_size
|
| 85 |
+
self.position_embedding = nn.Embedding(self.num_patches, self.embed_dim)
|
| 86 |
+
|
| 87 |
+
@staticmethod
|
| 88 |
+
def resize_positional_embeddings(
|
| 89 |
+
positional_embeddings: torch.Tensor,
|
| 90 |
+
spatial_shapes: torch.LongTensor,
|
| 91 |
+
max_length: int,
|
| 92 |
+
) -> torch.Tensor:
|
| 93 |
+
"""
|
| 94 |
+
Resize positional embeddings to image-specific size and pad to a fixed size.
|
| 95 |
+
Args:
|
| 96 |
+
positional_embeddings (`torch.Tensor`):
|
| 97 |
+
Position embeddings of shape (height, width, embed_dim)
|
| 98 |
+
spatial_shapes (`torch.LongTensor`):
|
| 99 |
+
Spatial shapes of shape (batch_size, 2) to resize the positional embeddings to
|
| 100 |
+
max_length (`int`):
|
| 101 |
+
Maximum length of the positional embeddings to pad resized positional embeddings to
|
| 102 |
+
Returns:
|
| 103 |
+
`torch.Tensor`: Embeddings of shape (batch_size, max_length, embed_dim)
|
| 104 |
+
"""
|
| 105 |
+
batch_size = spatial_shapes.shape[0]
|
| 106 |
+
embed_dim = positional_embeddings.shape[-1]
|
| 107 |
+
source_dtype = positional_embeddings.dtype
|
| 108 |
+
|
| 109 |
+
resulted_positional_embeddings = torch.empty(
|
| 110 |
+
(batch_size, max_length, embed_dim),
|
| 111 |
+
device=positional_embeddings.device,
|
| 112 |
+
dtype=source_dtype,
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
# (height, width, embed_dim) -> (1, embed_dim, height, width) for interpolation
|
| 116 |
+
positional_embeddings = positional_embeddings.permute(2, 0, 1).unsqueeze(0)
|
| 117 |
+
|
| 118 |
+
# Upcast to float32 on CPU because antialias is not supported for bfloat16/float16 on CPU
|
| 119 |
+
if positional_embeddings.device.type == "cpu":
|
| 120 |
+
positional_embeddings = positional_embeddings.to(torch.float32)
|
| 121 |
+
|
| 122 |
+
for i in range(batch_size):
|
| 123 |
+
# (1, dim, height, width) -> (1, dim, target_height, target_width)
|
| 124 |
+
height, width = spatial_shapes[i]
|
| 125 |
+
resized_embeddings = F.interpolate(
|
| 126 |
+
positional_embeddings,
|
| 127 |
+
size=(height, width),
|
| 128 |
+
mode="bilinear",
|
| 129 |
+
align_corners=False,
|
| 130 |
+
antialias=True,
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
# (1, dim, target_height, target_width) -> (target_height * target_width, dim)
|
| 134 |
+
resized_embeddings = resized_embeddings.reshape(embed_dim, height * width).transpose(0, 1)
|
| 135 |
+
|
| 136 |
+
# Cast to original dtype
|
| 137 |
+
resized_embeddings = resized_embeddings.to(source_dtype)
|
| 138 |
+
|
| 139 |
+
resulted_positional_embeddings[i, : height * width] = resized_embeddings
|
| 140 |
+
resulted_positional_embeddings[i, height * width :] = resized_embeddings[0]
|
| 141 |
+
|
| 142 |
+
return resulted_positional_embeddings
|
| 143 |
+
|
| 144 |
+
def forward(self, pixel_values: torch.FloatTensor,
|
| 145 |
+
grid_thws: Optional[torch.LongTensor] = None) -> torch.Tensor:
|
| 146 |
+
"""
|
| 147 |
+
Args:
|
| 148 |
+
pixel_values (`torch.FloatTensor`):
|
| 149 |
+
Pixel values of shape (num_patches, num_channels * temporal_patch_size * patch_size * patch_size)
|
| 150 |
+
grid_thws: (`torch.LongTensor`):
|
| 151 |
+
grid shape (num_patches, 3)
|
| 152 |
+
"""
|
| 153 |
+
|
| 154 |
+
# Apply patch embeddings to already patchified pixel values
|
| 155 |
+
target_dtype = self.patch_embedding.weight.dtype
|
| 156 |
+
if isinstance(self.patch_embedding, nn.Linear):
|
| 157 |
+
patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype))
|
| 158 |
+
elif isinstance(self.patch_embedding, nn.Conv2d):
|
| 159 |
+
pixel_values = pixel_values.view(-1, self.config.num_channels * self.config.temporal_patch_size, self.patch_size,
|
| 160 |
+
self.patch_size)
|
| 161 |
+
patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype))
|
| 162 |
+
patch_embeds = patch_embeds.reshape(-1, self.embed_dim)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
if self.preserve_original_pe:
|
| 166 |
+
assert grid_thws is not None
|
| 167 |
+
pos_embed_new = torch.zeros_like(patch_embeds)
|
| 168 |
+
ori_h = ori_w = self.position_embedding_size
|
| 169 |
+
positional_embeddings = self.position_embedding.weight.reshape(
|
| 170 |
+
self.position_embedding_size, self.position_embedding_size, -1
|
| 171 |
+
).unsqueeze(0).permute(0,3,1,2)
|
| 172 |
+
# pos_embed = self.pos_embed.reshape(1, ori_h, ori_w, -1).permute(0, 3, 1, 2)
|
| 173 |
+
cnt = 0
|
| 174 |
+
for t, h, w in grid_thws:
|
| 175 |
+
thw = t * h * w
|
| 176 |
+
pe = F.interpolate(positional_embeddings, size=(h, w), mode='bicubic', align_corners=False)
|
| 177 |
+
pe = pe.permute(0, 2, 3, 1).reshape(1, h * w, -1)
|
| 178 |
+
pe = pe[0].repeat(t, 1)
|
| 179 |
+
pe = pe.reshape(t, h // self.hidden_stride, self.hidden_stride, w // self.hidden_stride,
|
| 180 |
+
self.hidden_stride, -1)
|
| 181 |
+
pe = pe.permute(0, 1, 3, 2, 4, 5).reshape(thw, -1)
|
| 182 |
+
pos_embed_new[cnt:cnt + thw] = pe
|
| 183 |
+
cnt += thw
|
| 184 |
+
patch_embeds = patch_embeds + pos_embed_new
|
| 185 |
+
|
| 186 |
+
return patch_embeds
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
# copied from qwen2.5-vl
|
| 190 |
+
def apply_rotary_pos_emb_flashatt(
|
| 191 |
+
q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
|
| 192 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 193 |
+
cos = cos.chunk(2, dim=-1)[0].contiguous()
|
| 194 |
+
sin = sin.chunk(2, dim=-1)[0].contiguous()
|
| 195 |
+
q_embed = apply_rotary_emb(q.float(), cos.float(), sin.float()).type_as(q)
|
| 196 |
+
k_embed = apply_rotary_emb(k.float(), cos.float(), sin.float()).type_as(k)
|
| 197 |
+
return q_embed, k_embed
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
# Copied from transformers.models.llama.modeling_llama.rotate_half
|
| 201 |
+
def rotate_half(x):
|
| 202 |
+
"""Rotates half the hidden dims of the input."""
|
| 203 |
+
x1 = x[..., : x.shape[-1] // 2]
|
| 204 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
| 205 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def apply_rotary_pos_emb_vision(
|
| 209 |
+
q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
|
| 210 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 211 |
+
orig_q_dtype = q.dtype
|
| 212 |
+
orig_k_dtype = k.dtype
|
| 213 |
+
q, k = q.float(), k.float()
|
| 214 |
+
cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()
|
| 215 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
| 216 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
| 217 |
+
q_embed = q_embed.to(orig_q_dtype)
|
| 218 |
+
k_embed = k_embed.to(orig_k_dtype)
|
| 219 |
+
return q_embed, k_embed
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
class Siglip2Attention(nn.Module):
|
| 223 |
+
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
| 224 |
+
|
| 225 |
+
def __init__(self, config):
|
| 226 |
+
super().__init__()
|
| 227 |
+
self.config = config
|
| 228 |
+
self.embed_dim = config.hidden_size
|
| 229 |
+
self.num_heads = config.num_attention_heads
|
| 230 |
+
self.head_dim = self.embed_dim // self.num_heads
|
| 231 |
+
if self.head_dim * self.num_heads != self.embed_dim:
|
| 232 |
+
raise ValueError(
|
| 233 |
+
f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
|
| 234 |
+
f" {self.num_heads})."
|
| 235 |
+
)
|
| 236 |
+
self.scale = self.head_dim**-0.5
|
| 237 |
+
self.dropout = config.attention_dropout
|
| 238 |
+
self.is_causal = False
|
| 239 |
+
|
| 240 |
+
self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 241 |
+
self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 242 |
+
self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 243 |
+
self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 244 |
+
|
| 245 |
+
self.use_rope = config.use_rope
|
| 246 |
+
|
| 247 |
+
def forward(
|
| 248 |
+
self,
|
| 249 |
+
hidden_states: torch.Tensor,
|
| 250 |
+
cu_seqlens: torch.Tensor,
|
| 251 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
| 252 |
+
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
| 253 |
+
"""Input shape: Batch x Time x Channel"""
|
| 254 |
+
|
| 255 |
+
seq_length, embed_dim = hidden_states.shape
|
| 256 |
+
|
| 257 |
+
queries = self.q_proj(hidden_states)
|
| 258 |
+
keys = self.k_proj(hidden_states)
|
| 259 |
+
values = self.v_proj(hidden_states)
|
| 260 |
+
|
| 261 |
+
queries = queries.view(seq_length, self.num_heads, self.head_dim)
|
| 262 |
+
keys = keys.view(seq_length, self.num_heads, self.head_dim)
|
| 263 |
+
values = values.view(seq_length, self.num_heads, self.head_dim)
|
| 264 |
+
|
| 265 |
+
if self.use_rope:
|
| 266 |
+
cos, sin = position_embeddings
|
| 267 |
+
if is_flash_attn_2_available():
|
| 268 |
+
queries, keys = apply_rotary_pos_emb_flashatt(queries.unsqueeze(0), keys.unsqueeze(0), cos, sin)
|
| 269 |
+
else:
|
| 270 |
+
queries, keys = apply_rotary_pos_emb_vision(queries.unsqueeze(0), keys.unsqueeze(0), cos, sin)
|
| 271 |
+
queries = queries.squeeze(0)
|
| 272 |
+
keys = keys.squeeze(0)
|
| 273 |
+
|
| 274 |
+
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
|
| 275 |
+
if is_flash_attn_2_available():
|
| 276 |
+
attn_output = flash_attn_varlen_func(queries, keys, values, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen).reshape(
|
| 277 |
+
seq_length, -1
|
| 278 |
+
)
|
| 279 |
+
else:
|
| 280 |
+
batch_size = cu_seqlens.shape[0] - 1
|
| 281 |
+
outputs = []
|
| 282 |
+
cu = cu_seqlens.tolist()
|
| 283 |
+
for i in range(batch_size):
|
| 284 |
+
start_idx = cu[i]
|
| 285 |
+
end_idx = cu[i + 1]
|
| 286 |
+
# Each sequence is processed independently.
|
| 287 |
+
q_i = queries[start_idx:end_idx].unsqueeze(0)
|
| 288 |
+
k_i = keys[start_idx:end_idx].unsqueeze(0)
|
| 289 |
+
v_i = values[start_idx:end_idx].unsqueeze(0)
|
| 290 |
+
# (1, seq_len, num_heads, head_dim) ->
|
| 291 |
+
# (1, num_heads, seq_len, head_dim)
|
| 292 |
+
q_i, k_i, v_i = [x.transpose(1, 2) for x in (q_i, k_i, v_i)]
|
| 293 |
+
output_i = F.scaled_dot_product_attention(q_i,
|
| 294 |
+
k_i,
|
| 295 |
+
v_i,
|
| 296 |
+
dropout_p=0.0)
|
| 297 |
+
# (1, num_heads, seq_len, head_dim) -> (seq_len, embed_dim)
|
| 298 |
+
output_i = output_i.transpose(1, 2).reshape(-1, self.embed_dim)
|
| 299 |
+
outputs.append(output_i)
|
| 300 |
+
attn_output = torch.cat(outputs, dim=0)
|
| 301 |
+
|
| 302 |
+
attn_output = self.out_proj(attn_output)
|
| 303 |
+
return attn_output
|
| 304 |
+
|
| 305 |
+
class Siglip2MLP(nn.Module):
|
| 306 |
+
def __init__(self, config):
|
| 307 |
+
super().__init__()
|
| 308 |
+
self.config = config
|
| 309 |
+
self.activation_fn = ACT2FN[config.hidden_act]
|
| 310 |
+
self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
|
| 311 |
+
self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
|
| 312 |
+
|
| 313 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 314 |
+
hidden_states = self.fc1(hidden_states)
|
| 315 |
+
hidden_states = self.activation_fn(hidden_states)
|
| 316 |
+
hidden_states = self.fc2(hidden_states)
|
| 317 |
+
return hidden_states
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
class Siglip2EncoderLayer(nn.Module):
|
| 321 |
+
def __init__(self, config: Siglip2NavitConfig):
|
| 322 |
+
super().__init__()
|
| 323 |
+
self.embed_dim = config.hidden_size
|
| 324 |
+
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
| 325 |
+
self.self_attn = Siglip2Attention(config)
|
| 326 |
+
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
| 327 |
+
self.mlp = Siglip2MLP(config)
|
| 328 |
+
|
| 329 |
+
def forward(
|
| 330 |
+
self,
|
| 331 |
+
hidden_states: torch.Tensor,
|
| 332 |
+
cu_seqlens: torch.Tensor,
|
| 333 |
+
position_embeddings: torch.Tensor
|
| 334 |
+
) -> tuple[torch.FloatTensor]:
|
| 335 |
+
"""
|
| 336 |
+
Args:
|
| 337 |
+
hidden_states (`torch.FloatTensor`):
|
| 338 |
+
Input to the layer of shape `(batch, seq_len, embed_dim)`.
|
| 339 |
+
attention_mask (`torch.FloatTensor`):
|
| 340 |
+
Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
|
| 341 |
+
output_attentions (`bool`, *optional*, defaults to `False`):
|
| 342 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 343 |
+
returned tensors for more detail.
|
| 344 |
+
"""
|
| 345 |
+
residual = hidden_states
|
| 346 |
+
|
| 347 |
+
hidden_states = self.layer_norm1(hidden_states)
|
| 348 |
+
hidden_states = self.self_attn(
|
| 349 |
+
hidden_states=hidden_states,
|
| 350 |
+
cu_seqlens=cu_seqlens,
|
| 351 |
+
position_embeddings=position_embeddings
|
| 352 |
+
)
|
| 353 |
+
hidden_states = residual + hidden_states
|
| 354 |
+
|
| 355 |
+
residual = hidden_states
|
| 356 |
+
hidden_states = self.layer_norm2(hidden_states)
|
| 357 |
+
hidden_states = self.mlp(hidden_states)
|
| 358 |
+
hidden_states = residual + hidden_states
|
| 359 |
+
|
| 360 |
+
return hidden_states
|
| 361 |
+
|
| 362 |
+
class Siglip2Encoder(nn.Module):
|
| 363 |
+
"""
|
| 364 |
+
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
|
| 365 |
+
[`Siglip2EncoderLayer`].
|
| 366 |
+
Args:
|
| 367 |
+
config: Siglip2NavitConfig
|
| 368 |
+
"""
|
| 369 |
+
|
| 370 |
+
def __init__(self, config: Siglip2NavitConfig):
|
| 371 |
+
super().__init__()
|
| 372 |
+
self.config = config
|
| 373 |
+
self.layers = nn.ModuleList([Siglip2EncoderLayer(config) for _ in range(config.num_hidden_layers)])
|
| 374 |
+
self.gradient_checkpointing = False
|
| 375 |
+
|
| 376 |
+
self.rotary_pos_emb = VisionRotaryEmbedding(config.hidden_size // config.num_attention_heads // 2)
|
| 377 |
+
self.patch_size = config.patch_size
|
| 378 |
+
self.hidden_stride = config.hidden_stride
|
| 379 |
+
self.window_size = config.window_size
|
| 380 |
+
self.spatial_merge_unit = config.hidden_stride * config.hidden_stride
|
| 381 |
+
self.fullatt_block_indexes = None if config.fullatt_block_indexes is None else [int(i) for i in config.fullatt_block_indexes.split('|')]
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
# copied from qwen2.5_vl
|
| 385 |
+
def rot_pos_emb(self, grid_thw):
|
| 386 |
+
pos_ids = []
|
| 387 |
+
for t, h, w in grid_thw:
|
| 388 |
+
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
| 389 |
+
hpos_ids = hpos_ids.reshape(
|
| 390 |
+
h // self.hidden_stride,
|
| 391 |
+
self.hidden_stride,
|
| 392 |
+
w // self.hidden_stride,
|
| 393 |
+
self.hidden_stride,
|
| 394 |
+
)
|
| 395 |
+
hpos_ids = hpos_ids.permute(0, 2, 1, 3)
|
| 396 |
+
hpos_ids = hpos_ids.flatten()
|
| 397 |
+
|
| 398 |
+
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
| 399 |
+
wpos_ids = wpos_ids.reshape(
|
| 400 |
+
h // self.hidden_stride,
|
| 401 |
+
self.hidden_stride,
|
| 402 |
+
w // self.hidden_stride,
|
| 403 |
+
self.hidden_stride,
|
| 404 |
+
)
|
| 405 |
+
wpos_ids = wpos_ids.permute(0, 2, 1, 3)
|
| 406 |
+
wpos_ids = wpos_ids.flatten()
|
| 407 |
+
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
|
| 408 |
+
pos_ids = torch.cat(pos_ids, dim=0)
|
| 409 |
+
max_grid_size = grid_thw[:, 1:].max()
|
| 410 |
+
rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
|
| 411 |
+
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
|
| 412 |
+
return rotary_pos_emb
|
| 413 |
+
|
| 414 |
+
def get_window_index(self, grid_thw):
|
| 415 |
+
window_index: list = []
|
| 416 |
+
cu_window_seqlens: list = [0]
|
| 417 |
+
window_index_id = 0
|
| 418 |
+
vit_merger_window_size = self.window_size // self.hidden_stride // self.patch_size # patch (after merge) number in each window
|
| 419 |
+
|
| 420 |
+
for grid_t, grid_h, grid_w in grid_thw:
|
| 421 |
+
llm_grid_h, llm_grid_w = (
|
| 422 |
+
grid_h // self.hidden_stride, # number of patch after merge
|
| 423 |
+
grid_w // self.hidden_stride,
|
| 424 |
+
)
|
| 425 |
+
index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w)
|
| 426 |
+
pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
|
| 427 |
+
pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
|
| 428 |
+
num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
|
| 429 |
+
num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
|
| 430 |
+
index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
|
| 431 |
+
index_padded = index_padded.reshape(
|
| 432 |
+
grid_t,
|
| 433 |
+
num_windows_h,
|
| 434 |
+
vit_merger_window_size,
|
| 435 |
+
num_windows_w,
|
| 436 |
+
vit_merger_window_size,
|
| 437 |
+
)
|
| 438 |
+
index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
|
| 439 |
+
grid_t,
|
| 440 |
+
num_windows_h * num_windows_w,
|
| 441 |
+
vit_merger_window_size,
|
| 442 |
+
vit_merger_window_size,
|
| 443 |
+
)
|
| 444 |
+
seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)
|
| 445 |
+
index_padded = index_padded.reshape(-1)
|
| 446 |
+
index_new = index_padded[index_padded != -100]
|
| 447 |
+
window_index.append(index_new + window_index_id)
|
| 448 |
+
cu_seqlens_tmp = seqlens.cumsum(0) * self.spatial_merge_unit + cu_window_seqlens[-1]
|
| 449 |
+
cu_window_seqlens.extend(cu_seqlens_tmp.tolist())
|
| 450 |
+
window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()
|
| 451 |
+
window_index = torch.cat(window_index, dim=0)
|
| 452 |
+
|
| 453 |
+
return window_index, cu_window_seqlens
|
| 454 |
+
|
| 455 |
+
# Ignore copy
|
| 456 |
+
def forward(
|
| 457 |
+
self,
|
| 458 |
+
inputs_embeds,
|
| 459 |
+
grid_thws: torch.Tensor,
|
| 460 |
+
output_hidden_states: bool = False,
|
| 461 |
+
) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]:
|
| 462 |
+
r"""
|
| 463 |
+
Args:
|
| 464 |
+
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
|
| 465 |
+
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
|
| 466 |
+
This is useful if you want more control over how to convert `input_ids` indices into associated vectors
|
| 467 |
+
than the model's internal embedding lookup matrix.
|
| 468 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 469 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 470 |
+
- 1 for tokens that are **not masked**,
|
| 471 |
+
- 0 for tokens that are **masked**.
|
| 472 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 473 |
+
output_attentions (`bool`, *optional*):
|
| 474 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 475 |
+
returned tensors for more detail.
|
| 476 |
+
output_hidden_states (`bool`, *optional*):
|
| 477 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
|
| 478 |
+
for more detail.
|
| 479 |
+
return_dict (`bool`, *optional*):
|
| 480 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 481 |
+
"""
|
| 482 |
+
|
| 483 |
+
rotary_pos_emb = self.rot_pos_emb(grid_thws)
|
| 484 |
+
window_index, cu_window_seqlens = self.get_window_index(grid_thws)
|
| 485 |
+
cu_window_seqlens = torch.tensor(
|
| 486 |
+
cu_window_seqlens,
|
| 487 |
+
device=inputs_embeds.device,
|
| 488 |
+
dtype=grid_thws.dtype if torch.jit.is_tracing() else torch.int32,
|
| 489 |
+
)
|
| 490 |
+
cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
|
| 491 |
+
|
| 492 |
+
seq_len, _ = inputs_embeds.size()
|
| 493 |
+
inputs_embeds = inputs_embeds.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
|
| 494 |
+
inputs_embeds = inputs_embeds[window_index, :, :]
|
| 495 |
+
inputs_embeds = inputs_embeds.reshape(seq_len, -1)
|
| 496 |
+
rotary_pos_emb = rotary_pos_emb.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
|
| 497 |
+
rotary_pos_emb = rotary_pos_emb[window_index, :, :]
|
| 498 |
+
rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)
|
| 499 |
+
emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)
|
| 500 |
+
position_embeddings = (emb.cos(), emb.sin())
|
| 501 |
+
|
| 502 |
+
cu_seqlens = torch.repeat_interleave(grid_thws[:, 1] * grid_thws[:, 2], grid_thws[:, 0]).cumsum(
|
| 503 |
+
dim=0,
|
| 504 |
+
# Select dtype based on the following factors:
|
| 505 |
+
# - FA2 requires that cu_seqlens_q must have dtype int32
|
| 506 |
+
# - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw
|
| 507 |
+
# See https://github.com/huggingface/transformers/pull/34852 for more information
|
| 508 |
+
dtype=grid_thws.dtype if torch.jit.is_tracing() else torch.int32,
|
| 509 |
+
)
|
| 510 |
+
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
|
| 511 |
+
|
| 512 |
+
reverse_indices = torch.argsort(window_index)
|
| 513 |
+
encoder_states = () if output_hidden_states else None
|
| 514 |
+
|
| 515 |
+
hidden_states = inputs_embeds
|
| 516 |
+
for index, block in enumerate(self.layers):
|
| 517 |
+
if self.fullatt_block_indexes is None or index in self.fullatt_block_indexes:
|
| 518 |
+
cu_seqlens_tmp = cu_seqlens
|
| 519 |
+
else:
|
| 520 |
+
cu_seqlens_tmp = cu_window_seqlens
|
| 521 |
+
if self.gradient_checkpointing and self.training:
|
| 522 |
+
hidden_states = self._gradient_checkpointing_func(block.__call__, hidden_states, cu_seqlens_tmp, position_embeddings)
|
| 523 |
+
else:
|
| 524 |
+
hidden_states = block(hidden_states, cu_seqlens_tmp, position_embeddings)
|
| 525 |
+
if output_hidden_states:
|
| 526 |
+
hidden_states_ = hidden_states.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
|
| 527 |
+
encoder_states += (hidden_states_[reverse_indices, :].reshape(seq_len, -1),)
|
| 528 |
+
# tokens = self.post_trunk_norm(tokens)
|
| 529 |
+
hidden_states = hidden_states.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
|
| 530 |
+
hidden_states = hidden_states[reverse_indices, :].reshape(seq_len, -1)
|
| 531 |
+
|
| 532 |
+
return hidden_states, encoder_states
|
| 533 |
+
|
| 534 |
+
class Siglip2VisionTransformer(nn.Module):
|
| 535 |
+
def __init__(self, config: Siglip2NavitConfig):
|
| 536 |
+
super().__init__()
|
| 537 |
+
self.config = config
|
| 538 |
+
embed_dim = config.hidden_size
|
| 539 |
+
|
| 540 |
+
self.embeddings = Siglip2VisionEmbeddings(config)
|
| 541 |
+
self.encoder = Siglip2Encoder(config)
|
| 542 |
+
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
| 543 |
+
self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
|
| 544 |
+
|
| 545 |
+
def forward(
|
| 546 |
+
self,
|
| 547 |
+
pixel_values: torch.FloatTensor,
|
| 548 |
+
grid_thws: torch.LongTensor,
|
| 549 |
+
output_hidden_states: Optional[bool] = True,
|
| 550 |
+
return_dict: Optional[bool] = True,
|
| 551 |
+
) -> Union[
|
| 552 |
+
Tuple[torch.Tensor],
|
| 553 |
+
Tuple[torch.Tensor, Tuple[torch.Tensor, ...]],
|
| 554 |
+
BaseModelOutputWithNoAttention,
|
| 555 |
+
]:
|
| 556 |
+
r"""
|
| 557 |
+
spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
|
| 558 |
+
Tensor containing the spatial dimensions (height, width) of the input images.
|
| 559 |
+
"""
|
| 560 |
+
# output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 561 |
+
# output_hidden_states = (
|
| 562 |
+
# output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 563 |
+
# )
|
| 564 |
+
|
| 565 |
+
hidden_states = self.embeddings(pixel_values, grid_thws)
|
| 566 |
+
|
| 567 |
+
last_hidden_state, hidden_states = self.encoder(hidden_states, grid_thws, output_hidden_states)
|
| 568 |
+
last_hidden_state = self.post_layernorm(last_hidden_state)
|
| 569 |
+
|
| 570 |
+
if not return_dict:
|
| 571 |
+
output = (last_hidden_state,)
|
| 572 |
+
output += (hidden_states,) if output_hidden_states else ()
|
| 573 |
+
return output
|
| 574 |
+
|
| 575 |
+
return BaseModelOutputWithNoAttention(
|
| 576 |
+
last_hidden_state=last_hidden_state,
|
| 577 |
+
hidden_states=hidden_states
|
| 578 |
+
)
|
| 579 |
+
|
| 580 |
+
class Siglip2PreTrainedModel(PreTrainedModel):
|
| 581 |
+
config_class = Siglip2NavitConfig
|
| 582 |
+
base_model_prefix = "siglip2_navit"
|
| 583 |
+
supports_gradient_checkpointing = True
|
| 584 |
+
|
| 585 |
+
_no_split_modules = [
|
| 586 |
+
"Siglip2VisionEmbeddings",
|
| 587 |
+
"Siglip2EncoderLayer",
|
| 588 |
+
]
|
| 589 |
+
_supports_flash_attn_2 = True
|
| 590 |
+
_supports_sdpa = False
|
| 591 |
+
_supports_flex_attn = False
|
| 592 |
+
_supports_attention_backend = True
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
class Siglip2NavitModel(Siglip2PreTrainedModel):
|
| 596 |
+
config_class = Siglip2NavitConfig
|
| 597 |
+
main_input_name = "pixel_values"
|
| 598 |
+
|
| 599 |
+
def __init__(self, config: Siglip2NavitConfig):
|
| 600 |
+
super().__init__(config)
|
| 601 |
+
|
| 602 |
+
self.vision_model = Siglip2VisionTransformer(config)
|
| 603 |
+
|
| 604 |
+
def get_input_embeddings(self) -> nn.Module:
|
| 605 |
+
return self.vision_model.embeddings.patch_embedding
|
| 606 |
+
|
| 607 |
+
def forward(
|
| 608 |
+
self,
|
| 609 |
+
pixel_values: torch.FloatTensor,
|
| 610 |
+
grid_thws: torch.LongTensor,
|
| 611 |
+
output_hidden_states: Optional[bool] = None,
|
| 612 |
+
return_dict: Optional[bool] = None,
|
| 613 |
+
) -> Union[
|
| 614 |
+
Tuple[torch.Tensor],
|
| 615 |
+
Tuple[torch.Tensor, Tuple[torch.Tensor, ...]],
|
| 616 |
+
BaseModelOutputWithNoAttention,
|
| 617 |
+
]:
|
| 618 |
+
|
| 619 |
+
if output_hidden_states is None:
|
| 620 |
+
output_hidden_states = self.config.output_hidden_states
|
| 621 |
+
if return_dict is None:
|
| 622 |
+
return_dict = self.config.use_return_dict
|
| 623 |
+
|
| 624 |
+
return self.vision_model(
|
| 625 |
+
pixel_values=pixel_values,
|
| 626 |
+
grid_thws=grid_thws,
|
| 627 |
+
output_hidden_states=output_hidden_states,
|
| 628 |
+
return_dict=return_dict,
|
| 629 |
+
)
|
| 630 |
+
|
| 631 |
+
class VisualEmbedding(torch.nn.Embedding):
|
| 632 |
+
"""
|
| 633 |
+
A visual embedding layer that can handle both discrete token IDs (long) and continuous
|
| 634 |
+
soft-token probabilities (float).
|
| 635 |
+
"""
|
| 636 |
+
|
| 637 |
+
def forward(self, visual_tokens: Tensor) -> Tensor:
|
| 638 |
+
if visual_tokens.dtype in [torch.int8, torch.int16, torch.int32, torch.int64, torch.long]:
|
| 639 |
+
return super().forward(visual_tokens)
|
| 640 |
+
# Handle soft tokens (probabilities) by matrix multiplication with the embedding weight
|
| 641 |
+
return torch.matmul(visual_tokens, self.weight)
|
| 642 |
+
|
| 643 |
+
|
| 644 |
+
class VisualTokenizer(torch.nn.Module):
|
| 645 |
+
"""
|
| 646 |
+
Tokenizes images or videos into a sequence of continuous visual tokens.
|
| 647 |
+
"""
|
| 648 |
+
|
| 649 |
+
def __init__(self, vit, visual_vocab_size, image_processor_name_or_path, *args, **kwargs):
|
| 650 |
+
super().__init__(*args, **kwargs)
|
| 651 |
+
self.vit = vit
|
| 652 |
+
self.image_processor = AutoImageProcessor.from_pretrained(image_processor_name_or_path, do_center_crop=False)
|
| 653 |
+
head_dim = visual_vocab_size - len(INDICATOR_IDS)
|
| 654 |
+
self.head = torch.nn.Sequential(
|
| 655 |
+
torch.nn.Linear(self.vit.config.hidden_size * self.vit.config.hidden_stride ** 2, head_dim, bias=False),
|
| 656 |
+
torch.nn.LayerNorm(head_dim)
|
| 657 |
+
)
|
| 658 |
+
|
| 659 |
+
def _encode(self, pixel_values, grid_thws):
|
| 660 |
+
output = self.vit(pixel_values, grid_thws, output_hidden_states=True, return_dict=True)
|
| 661 |
+
features = output.hidden_states[-1]
|
| 662 |
+
seq_len, _ = features.shape
|
| 663 |
+
features = features.reshape(seq_len // (self.vit.config.hidden_stride ** 2), -1)
|
| 664 |
+
return features
|
| 665 |
+
|
| 666 |
+
# Adapted from qwen2_vl
|
| 667 |
+
@staticmethod
|
| 668 |
+
def smart_resize(
|
| 669 |
+
height: int, width: int, factor: int = 28, min_pixels: int = 448 * 448, max_pixels: int = 1344 * 1792
|
| 670 |
+
):
|
| 671 |
+
"""Rescales the image so that the following conditions are met:
|
| 672 |
+
1. Both dimensions are divisible by 'factor'.
|
| 673 |
+
2. The total number of pixels is within ['min_pixels', 'max_pixels'].
|
| 674 |
+
3. The aspect ratio is maintained as closely as possible.
|
| 675 |
+
"""
|
| 676 |
+
if height < factor or width < factor:
|
| 677 |
+
if height < width:
|
| 678 |
+
width = round(factor / height * width)
|
| 679 |
+
height = factor
|
| 680 |
+
else:
|
| 681 |
+
height = round(factor / width * height)
|
| 682 |
+
width = factor
|
| 683 |
+
|
| 684 |
+
elif max(height, width) / min(height, width) > 200:
|
| 685 |
+
if height > width:
|
| 686 |
+
height = 200 * width
|
| 687 |
+
else:
|
| 688 |
+
width = 200 * height
|
| 689 |
+
|
| 690 |
+
h_bar = round(height / factor) * factor
|
| 691 |
+
w_bar = round(width / factor) * factor
|
| 692 |
+
if h_bar * w_bar > max_pixels:
|
| 693 |
+
beta = math.sqrt((height * width) / max_pixels)
|
| 694 |
+
h_bar = math.floor(height / beta / factor) * factor
|
| 695 |
+
w_bar = math.floor(width / beta / factor) * factor
|
| 696 |
+
elif h_bar * w_bar < min_pixels:
|
| 697 |
+
beta = math.sqrt(min_pixels / (height * width))
|
| 698 |
+
h_bar = math.ceil(height * beta / factor) * factor
|
| 699 |
+
w_bar = math.ceil(width * beta / factor) * factor
|
| 700 |
+
return h_bar, w_bar
|
| 701 |
+
|
| 702 |
+
def preprocess(
|
| 703 |
+
self,
|
| 704 |
+
image: Optional[PIL.Image.Image] = None,
|
| 705 |
+
video: Optional[List[PIL.Image.Image]] = None,
|
| 706 |
+
min_pixels: Optional[int] = None,
|
| 707 |
+
max_pixels: Optional[int] = None
|
| 708 |
+
):
|
| 709 |
+
patch_size = self.vit.config.patch_size
|
| 710 |
+
temporal_patch_size = self.vit.config.temporal_patch_size
|
| 711 |
+
hidden_stride = self.vit.config.hidden_stride
|
| 712 |
+
assert (image is None) ^ (video is None), "Invalid input: expect either image or video"
|
| 713 |
+
if image is not None:
|
| 714 |
+
images = [image]
|
| 715 |
+
else:
|
| 716 |
+
images = video
|
| 717 |
+
images = [image.convert("RGB") if image.mode != 'RGB' else image for image in images]
|
| 718 |
+
width, height = images[0].size
|
| 719 |
+
processed_images = []
|
| 720 |
+
for image in images:
|
| 721 |
+
resized_height, resized_width = self.smart_resize(
|
| 722 |
+
height,
|
| 723 |
+
width,
|
| 724 |
+
factor=patch_size * hidden_stride,
|
| 725 |
+
min_pixels=min_pixels,
|
| 726 |
+
max_pixels=max_pixels,
|
| 727 |
+
)
|
| 728 |
+
new_size = dict(height=resized_height, width=resized_width)
|
| 729 |
+
new_image = self.image_processor.preprocess(image, size=new_size, return_tensors="np")['pixel_values'][0]
|
| 730 |
+
processed_images.append(new_image)
|
| 731 |
+
|
| 732 |
+
patches = np.array(processed_images)
|
| 733 |
+
if patches.shape[0] % temporal_patch_size != 0:
|
| 734 |
+
repeats = np.repeat(patches[-1][np.newaxis], temporal_patch_size - 1, axis=0)
|
| 735 |
+
patches = np.concatenate([patches, repeats], axis=0)
|
| 736 |
+
channel = patches.shape[1]
|
| 737 |
+
grid_t = patches.shape[0] // temporal_patch_size
|
| 738 |
+
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
| 739 |
+
grid_thw = torch.tensor([[grid_t, grid_h, grid_w]])
|
| 740 |
+
|
| 741 |
+
patches = patches.reshape(
|
| 742 |
+
grid_t, temporal_patch_size, channel,
|
| 743 |
+
grid_h // hidden_stride, hidden_stride, patch_size,
|
| 744 |
+
grid_w // hidden_stride, hidden_stride, patch_size,
|
| 745 |
+
)
|
| 746 |
+
patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8)
|
| 747 |
+
flatten_patches = patches.reshape(
|
| 748 |
+
grid_t * grid_h * grid_w, channel * temporal_patch_size * patch_size * patch_size
|
| 749 |
+
)
|
| 750 |
+
flatten_patches = torch.tensor(flatten_patches)
|
| 751 |
+
|
| 752 |
+
return flatten_patches, grid_thw
|
| 753 |
+
|
| 754 |
+
def forward(
|
| 755 |
+
self, pixel_values, grid_thws
|
| 756 |
+
) -> torch.Tensor: # [BatchSize, ImageShape] -> [BatchSize, #Token, VocabSize]
|
| 757 |
+
features = self._encode(pixel_values, grid_thws)
|
| 758 |
+
logits = self.head(features)
|
| 759 |
+
tokens = torch.softmax(logits, dim=-1, dtype=torch.float32).to(logits.dtype)
|
| 760 |
+
|
| 761 |
+
token_len, _ = tokens.shape
|
| 762 |
+
padding_tensor = torch.zeros(size=(token_len, len(INDICATOR_IDS)),
|
| 763 |
+
dtype=tokens.dtype,
|
| 764 |
+
device=tokens.device,
|
| 765 |
+
layout=tokens.layout,
|
| 766 |
+
requires_grad=False)
|
| 767 |
+
tokens = torch.cat((tokens, padding_tensor), dim=1)
|
| 768 |
+
return tokens
|
| 769 |
+
|
| 770 |
+
|
| 771 |
+
class OvisPreTrainedModel(PreTrainedModel):
|
| 772 |
+
config_class = Ovis2_5_Config
|
| 773 |
+
base_model_prefix = "ovis2_5"
|
| 774 |
+
|
| 775 |
+
|
| 776 |
+
class Ovis2_5(OvisPreTrainedModel):
|
| 777 |
+
_supports_flash_attn_2 = True
|
| 778 |
+
|
| 779 |
+
def __init__(self, config: Ovis2_5_Config, *inputs, **kwargs):
|
| 780 |
+
super().__init__(config, *inputs, **kwargs)
|
| 781 |
+
|
| 782 |
+
self.llm = AutoModelForCausalLM.from_config(self.config.llm_config)
|
| 783 |
+
assert self.config.hidden_size == self.llm.config.hidden_size, "hidden size mismatch"
|
| 784 |
+
self.text_tokenizer = AutoTokenizer.from_pretrained(self.config.name_or_path)
|
| 785 |
+
self.visual_tokenizer = VisualTokenizer(vit=AutoModel.from_config(self.config.vit_config),
|
| 786 |
+
visual_vocab_size=self.config.visual_vocab_size,
|
| 787 |
+
image_processor_name_or_path=self.config.name_or_path)
|
| 788 |
+
|
| 789 |
+
self.vte = VisualEmbedding(self.config.visual_vocab_size, self.config.hidden_size,
|
| 790 |
+
device=self.visual_tokenizer.vit.device, dtype=self.visual_tokenizer.vit.dtype)
|
| 791 |
+
indicator_token_indices = torch.arange(
|
| 792 |
+
self.config.visual_vocab_size - len(INDICATOR_IDS),
|
| 793 |
+
self.config.visual_vocab_size,
|
| 794 |
+
dtype=torch.long
|
| 795 |
+
)
|
| 796 |
+
self.register_buffer("indicator_token_indices", indicator_token_indices, persistent=False)
|
| 797 |
+
|
| 798 |
+
def _merge_modules(modules_list: tuple):
|
| 799 |
+
merged_modules = []
|
| 800 |
+
for modules in modules_list:
|
| 801 |
+
merged_modules.extend(modules if modules else [])
|
| 802 |
+
return merged_modules
|
| 803 |
+
|
| 804 |
+
# Standard model configurations for parallelism and device placement
|
| 805 |
+
self._no_split_modules = _merge_modules(
|
| 806 |
+
(self.llm._no_split_modules, self.visual_tokenizer.vit._no_split_modules))
|
| 807 |
+
self._skip_keys_device_placement = self.llm._skip_keys_device_placement
|
| 808 |
+
self._keep_in_fp32_modules = _merge_modules(
|
| 809 |
+
(self.llm._keep_in_fp32_modules, self.visual_tokenizer.vit._keep_in_fp32_modules))
|
| 810 |
+
self.is_parallelizable = all((self.llm.is_parallelizable, self.visual_tokenizer.vit.is_parallelizable))
|
| 811 |
+
self.supports_gradient_checkpointing = True
|
| 812 |
+
|
| 813 |
+
def tie_weights(self):
|
| 814 |
+
self.llm.tie_weights()
|
| 815 |
+
|
| 816 |
+
def get_wte(self):
|
| 817 |
+
return self.llm.get_input_embeddings()
|
| 818 |
+
|
| 819 |
+
def forward(
|
| 820 |
+
self,
|
| 821 |
+
input_ids: torch.Tensor,
|
| 822 |
+
attention_mask: torch.Tensor,
|
| 823 |
+
pixel_values: Optional[torch.Tensor],
|
| 824 |
+
grid_thws: Optional[torch.Tensor],
|
| 825 |
+
labels: Optional[torch.Tensor] = None,
|
| 826 |
+
**kwargs
|
| 827 |
+
):
|
| 828 |
+
inputs_embeds = self.merge_multimodal(
|
| 829 |
+
input_ids=input_ids,
|
| 830 |
+
pixel_values=pixel_values,
|
| 831 |
+
grid_thws=grid_thws,
|
| 832 |
+
)
|
| 833 |
+
return self.llm(inputs_embeds=inputs_embeds, attention_mask=attention_mask, labels=labels, **kwargs)
|
| 834 |
+
|
| 835 |
+
def merge_multimodal(
|
| 836 |
+
self,
|
| 837 |
+
input_ids: torch.Tensor,
|
| 838 |
+
pixel_values: Optional[torch.Tensor],
|
| 839 |
+
grid_thws: Optional[torch.Tensor],
|
| 840 |
+
):
|
| 841 |
+
placeholder_token_mask = torch.lt(input_ids, 0)
|
| 842 |
+
multimodal_embeds = self.get_wte()(torch.masked_fill(input_ids, placeholder_token_mask, 0))
|
| 843 |
+
|
| 844 |
+
if pixel_values is not None:
|
| 845 |
+
visual_indicator_embeds = self.vte(self.indicator_token_indices).to(
|
| 846 |
+
dtype=multimodal_embeds.dtype, device=multimodal_embeds.device
|
| 847 |
+
)
|
| 848 |
+
visual_tokens = self.visual_tokenizer(pixel_values, grid_thws)
|
| 849 |
+
visual_embeds = self.vte(visual_tokens).to(dtype=multimodal_embeds.dtype, device=multimodal_embeds.device)
|
| 850 |
+
|
| 851 |
+
for i, indicator_id in enumerate(INDICATOR_IDS):
|
| 852 |
+
multimodal_embeds[input_ids == indicator_id] = visual_indicator_embeds[i]
|
| 853 |
+
multimodal_embeds[input_ids == VISUAL_ATOM_ID] = visual_embeds
|
| 854 |
+
|
| 855 |
+
return multimodal_embeds
|
| 856 |
+
|
| 857 |
+
def _merge_inputs(
|
| 858 |
+
self, raw_input_ids, placeholder_id, grid_thws, indicator_begin_id, indicator_end_id
|
| 859 |
+
):
|
| 860 |
+
input_ids = []
|
| 861 |
+
prev_index = 0
|
| 862 |
+
placeholder_indexes = [i for i, v in enumerate(raw_input_ids) if v == placeholder_id]
|
| 863 |
+
for placeholder_index, grid_thw in zip(placeholder_indexes, grid_thws):
|
| 864 |
+
input_ids.extend(raw_input_ids[prev_index:placeholder_index])
|
| 865 |
+
num_image_atoms = grid_thw.prod().item()
|
| 866 |
+
num_image_atoms //= self.visual_tokenizer.vit.config.hidden_stride ** 2
|
| 867 |
+
num_image_atoms //= self.visual_tokenizer.vit.config.temporal_patch_size
|
| 868 |
+
input_ids.extend([indicator_begin_id] + [VISUAL_ATOM_ID] * num_image_atoms + [indicator_end_id])
|
| 869 |
+
prev_index = placeholder_index + 1
|
| 870 |
+
input_ids.extend(raw_input_ids[prev_index:])
|
| 871 |
+
return input_ids
|
| 872 |
+
|
| 873 |
+
def _tokenize_with_visual_placeholder(self, text):
|
| 874 |
+
placeholder = VIDEO_PLACEHOLDER if VIDEO_PLACEHOLDER in text else IMAGE_PLACEHOLDER
|
| 875 |
+
placeholder_id = VIDEO_PLACEHOLDER_ID if VIDEO_PLACEHOLDER in text else IMAGE_PLACEHOLDER_ID
|
| 876 |
+
chunks = [self.text_tokenizer(chunk, add_special_tokens=False).input_ids for chunk in text.split(placeholder)]
|
| 877 |
+
input_ids = chunks[0]
|
| 878 |
+
for chunk in chunks[1:]:
|
| 879 |
+
input_ids.append(placeholder_id)
|
| 880 |
+
input_ids.extend(chunk)
|
| 881 |
+
return input_ids
|
| 882 |
+
|
| 883 |
+
def preprocess_inputs(
|
| 884 |
+
self,
|
| 885 |
+
messages: List[Union[str, Dict]],
|
| 886 |
+
min_pixels=448 * 448,
|
| 887 |
+
max_pixels=1792 * 1792,
|
| 888 |
+
add_generation_prompt=True,
|
| 889 |
+
enable_thinking=False
|
| 890 |
+
):
|
| 891 |
+
text = self.text_tokenizer.apply_chat_template(
|
| 892 |
+
messages,
|
| 893 |
+
tokenize=False,
|
| 894 |
+
add_generation_prompt=add_generation_prompt,
|
| 895 |
+
enable_thinking=enable_thinking
|
| 896 |
+
)
|
| 897 |
+
input_ids = self._tokenize_with_visual_placeholder(text)
|
| 898 |
+
images = []
|
| 899 |
+
videos = []
|
| 900 |
+
for message in messages:
|
| 901 |
+
content = message["content"]
|
| 902 |
+
if isinstance(content, list):
|
| 903 |
+
images.extend([item["image"] for item in content if item.get("image") is not None])
|
| 904 |
+
videos.extend([item["video"] for item in content if item.get("video") is not None])
|
| 905 |
+
if images and videos:
|
| 906 |
+
raise ValueError(
|
| 907 |
+
"Multiple visual input data types detected (both image and video provided). "
|
| 908 |
+
"This model supports only one type of visual input data at a time. "
|
| 909 |
+
"Please provide either image or video, but not both."
|
| 910 |
+
)
|
| 911 |
+
|
| 912 |
+
pixel_values, grid_thws = None, None
|
| 913 |
+
if images:
|
| 914 |
+
pixel_values, grid_thws = zip(
|
| 915 |
+
*(self.visual_tokenizer.preprocess(image=image, min_pixels=min_pixels, max_pixels=max_pixels)
|
| 916 |
+
for image in images)
|
| 917 |
+
)
|
| 918 |
+
input_ids = self._merge_inputs(
|
| 919 |
+
input_ids, IMAGE_PLACEHOLDER_ID, grid_thws, INDICATOR_IDS[0], INDICATOR_IDS[1]
|
| 920 |
+
)
|
| 921 |
+
pixel_values = torch.cat(pixel_values, dim=0)
|
| 922 |
+
grid_thws = torch.cat(grid_thws, dim=0)
|
| 923 |
+
elif videos:
|
| 924 |
+
assert len(videos) == 1, "only support single video"
|
| 925 |
+
pixel_values, grid_thws = self.visual_tokenizer.preprocess(
|
| 926 |
+
video=videos[0], min_pixels=min_pixels, max_pixels=max_pixels
|
| 927 |
+
)
|
| 928 |
+
input_ids = self._merge_inputs(
|
| 929 |
+
input_ids, VIDEO_PLACEHOLDER_ID, grid_thws, INDICATOR_IDS[2], INDICATOR_IDS[3]
|
| 930 |
+
)
|
| 931 |
+
|
| 932 |
+
input_ids = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)
|
| 933 |
+
|
| 934 |
+
return input_ids, pixel_values, grid_thws
|
| 935 |
+
|
| 936 |
+
def generate(
|
| 937 |
+
self,
|
| 938 |
+
inputs: Optional[torch.Tensor] = None,
|
| 939 |
+
**kwargs,
|
| 940 |
+
) -> Union[GenerateOutput, torch.LongTensor]:
|
| 941 |
+
attention_mask = torch.ne(inputs, self.text_tokenizer.pad_token_id).to(device=inputs.device)
|
| 942 |
+
inputs_embeds = self.merge_multimodal(
|
| 943 |
+
input_ids=inputs,
|
| 944 |
+
pixel_values=kwargs.pop('pixel_values', None),
|
| 945 |
+
grid_thws=kwargs.pop('grid_thws', None)
|
| 946 |
+
)
|
| 947 |
+
enable_thinking = kwargs.pop('enable_thinking', False)
|
| 948 |
+
enable_thinking_budget = kwargs.pop('enable_thinking_budget', False)
|
| 949 |
+
thinking_budget = kwargs.pop('thinking_budget', 1024)
|
| 950 |
+
|
| 951 |
+
if enable_thinking and enable_thinking_budget:
|
| 952 |
+
actual_max_new_tokens = kwargs['max_new_tokens']
|
| 953 |
+
kwargs['max_new_tokens'] = thinking_budget
|
| 954 |
+
generated_ids = self.llm.generate(inputs=None, inputs_embeds=inputs_embeds, attention_mask=attention_mask, **kwargs)
|
| 955 |
+
output_ids = generated_ids
|
| 956 |
+
output_ids_list = generated_ids[0]
|
| 957 |
+
|
| 958 |
+
# check if the generation has already finished (151645 is <|im_end|>)
|
| 959 |
+
if 151645 not in output_ids_list:
|
| 960 |
+
# check if the thinking process has finished (151668 is </think>)
|
| 961 |
+
# and prepare the second model input
|
| 962 |
+
if 151668 not in output_ids_list:
|
| 963 |
+
early_stopping_text = "\n\nConsidering the limited time by the user, I have to give the solution based on the thinking directly now.\n</think>\n\n"
|
| 964 |
+
early_stopping_ids = self.text_tokenizer(early_stopping_text, return_tensors="pt", return_attention_mask=False).input_ids.to(inputs.device)
|
| 965 |
+
input_ids_appendent = torch.cat([output_ids, early_stopping_ids], dim=-1)
|
| 966 |
+
kwargs['streamer'].put(early_stopping_ids) if 'streamer' in kwargs else None
|
| 967 |
+
else:
|
| 968 |
+
input_ids_appendent = output_ids
|
| 969 |
+
|
| 970 |
+
|
| 971 |
+
# second generation
|
| 972 |
+
new_inputs = torch.cat([inputs, input_ids_appendent], dim=-1)
|
| 973 |
+
attention_mask = torch.ne(new_inputs, self.text_tokenizer.pad_token_id).to(device=inputs.device)
|
| 974 |
+
inputs_embeds_appendent = self.merge_multimodal(
|
| 975 |
+
input_ids=input_ids_appendent,
|
| 976 |
+
pixel_values=None,
|
| 977 |
+
grid_thws=None
|
| 978 |
+
)
|
| 979 |
+
new_inputs_embeds = torch.cat([inputs_embeds, inputs_embeds_appendent], dim=-2)
|
| 980 |
+
|
| 981 |
+
kwargs['max_new_tokens'] = inputs_embeds.size(-2) + actual_max_new_tokens - new_inputs_embeds.size(-2)
|
| 982 |
+
generated_ids2 = self.llm.generate(inputs=None, inputs_embeds=new_inputs_embeds, attention_mask=attention_mask, **kwargs)
|
| 983 |
+
kwargs['streamer'].manual_end() if 'streamer' in kwargs else None
|
| 984 |
+
return torch.cat([input_ids_appendent, generated_ids2], dim=-1)
|
| 985 |
+
|
| 986 |
+
else:
|
| 987 |
+
kwargs['streamer'].manual_end() if 'streamer' in kwargs else None
|
| 988 |
+
return generated_ids
|
| 989 |
+
|
| 990 |
+
else:
|
| 991 |
+
generated_ids = self.llm.generate(inputs=None, inputs_embeds=inputs_embeds, attention_mask=attention_mask, **kwargs)
|
| 992 |
+
kwargs['streamer'].manual_end() if 'streamer' in kwargs else None
|
| 993 |
+
return generated_ids
|
| 994 |
+
|
| 995 |
+
|
| 996 |
+
AutoConfig.register('siglip2_navit', Siglip2NavitConfig)
|
| 997 |
+
AutoModel.register(Siglip2NavitConfig, Siglip2NavitModel)
|
| 998 |
+
AutoConfig.register("ovis2_5", Ovis2_5_Config)
|
| 999 |
+
AutoModelForCausalLM.register(Ovis2_5_Config, Ovis2_5)
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"do_convert_rgb": null,
|
| 3 |
+
"do_normalize": true,
|
| 4 |
+
"do_rescale": true,
|
| 5 |
+
"do_resize": true,
|
| 6 |
+
"image_mean": [
|
| 7 |
+
0.5,
|
| 8 |
+
0.5,
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
"image_processor_type": "SiglipImageProcessor",
|
| 12 |
+
"image_std": [
|
| 13 |
+
0.5,
|
| 14 |
+
0.5,
|
| 15 |
+
0.5
|
| 16 |
+
],
|
| 17 |
+
"processor_class": "SiglipProcessor",
|
| 18 |
+
"resample": 2,
|
| 19 |
+
"rescale_factor": 0.00392156862745098,
|
| 20 |
+
"size": {
|
| 21 |
+
"height": 512,
|
| 22 |
+
"width": 512
|
| 23 |
+
}
|
| 24 |
+
}
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<|im_start|>",
|
| 4 |
+
"<|im_end|>",
|
| 5 |
+
"<|object_ref_start|>",
|
| 6 |
+
"<|object_ref_end|>",
|
| 7 |
+
"<|box_start|>",
|
| 8 |
+
"<|box_end|>",
|
| 9 |
+
"<|quad_start|>",
|
| 10 |
+
"<|quad_end|>",
|
| 11 |
+
"<|vision_start|>",
|
| 12 |
+
"<|vision_end|>",
|
| 13 |
+
"<|vision_pad|>",
|
| 14 |
+
"<|image_pad|>",
|
| 15 |
+
"<|video_pad|>"
|
| 16 |
+
],
|
| 17 |
+
"eos_token": {
|
| 18 |
+
"content": "<|im_end|>",
|
| 19 |
+
"lstrip": false,
|
| 20 |
+
"normalized": false,
|
| 21 |
+
"rstrip": false,
|
| 22 |
+
"single_word": false
|
| 23 |
+
},
|
| 24 |
+
"pad_token": {
|
| 25 |
+
"content": "<|endoftext|>",
|
| 26 |
+
"lstrip": false,
|
| 27 |
+
"normalized": false,
|
| 28 |
+
"rstrip": false,
|
| 29 |
+
"single_word": false
|
| 30 |
+
}
|
| 31 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
|
| 3 |
+
size 11422654
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_prefix_space": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"151643": {
|
| 6 |
+
"content": "<|endoftext|>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"151644": {
|
| 14 |
+
"content": "<|im_start|>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"151645": {
|
| 22 |
+
"content": "<|im_end|>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"151646": {
|
| 30 |
+
"content": "<|object_ref_start|>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
},
|
| 37 |
+
"151647": {
|
| 38 |
+
"content": "<|object_ref_end|>",
|
| 39 |
+
"lstrip": false,
|
| 40 |
+
"normalized": false,
|
| 41 |
+
"rstrip": false,
|
| 42 |
+
"single_word": false,
|
| 43 |
+
"special": true
|
| 44 |
+
},
|
| 45 |
+
"151648": {
|
| 46 |
+
"content": "<|box_start|>",
|
| 47 |
+
"lstrip": false,
|
| 48 |
+
"normalized": false,
|
| 49 |
+
"rstrip": false,
|
| 50 |
+
"single_word": false,
|
| 51 |
+
"special": true
|
| 52 |
+
},
|
| 53 |
+
"151649": {
|
| 54 |
+
"content": "<|box_end|>",
|
| 55 |
+
"lstrip": false,
|
| 56 |
+
"normalized": false,
|
| 57 |
+
"rstrip": false,
|
| 58 |
+
"single_word": false,
|
| 59 |
+
"special": true
|
| 60 |
+
},
|
| 61 |
+
"151650": {
|
| 62 |
+
"content": "<|quad_start|>",
|
| 63 |
+
"lstrip": false,
|
| 64 |
+
"normalized": false,
|
| 65 |
+
"rstrip": false,
|
| 66 |
+
"single_word": false,
|
| 67 |
+
"special": true
|
| 68 |
+
},
|
| 69 |
+
"151651": {
|
| 70 |
+
"content": "<|quad_end|>",
|
| 71 |
+
"lstrip": false,
|
| 72 |
+
"normalized": false,
|
| 73 |
+
"rstrip": false,
|
| 74 |
+
"single_word": false,
|
| 75 |
+
"special": true
|
| 76 |
+
},
|
| 77 |
+
"151652": {
|
| 78 |
+
"content": "<|vision_start|>",
|
| 79 |
+
"lstrip": false,
|
| 80 |
+
"normalized": false,
|
| 81 |
+
"rstrip": false,
|
| 82 |
+
"single_word": false,
|
| 83 |
+
"special": true
|
| 84 |
+
},
|
| 85 |
+
"151653": {
|
| 86 |
+
"content": "<|vision_end|>",
|
| 87 |
+
"lstrip": false,
|
| 88 |
+
"normalized": false,
|
| 89 |
+
"rstrip": false,
|
| 90 |
+
"single_word": false,
|
| 91 |
+
"special": true
|
| 92 |
+
},
|
| 93 |
+
"151654": {
|
| 94 |
+
"content": "<|vision_pad|>",
|
| 95 |
+
"lstrip": false,
|
| 96 |
+
"normalized": false,
|
| 97 |
+
"rstrip": false,
|
| 98 |
+
"single_word": false,
|
| 99 |
+
"special": true
|
| 100 |
+
},
|
| 101 |
+
"151655": {
|
| 102 |
+
"content": "<|image_pad|>",
|
| 103 |
+
"lstrip": false,
|
| 104 |
+
"normalized": false,
|
| 105 |
+
"rstrip": false,
|
| 106 |
+
"single_word": false,
|
| 107 |
+
"special": true
|
| 108 |
+
},
|
| 109 |
+
"151656": {
|
| 110 |
+
"content": "<|video_pad|>",
|
| 111 |
+
"lstrip": false,
|
| 112 |
+
"normalized": false,
|
| 113 |
+
"rstrip": false,
|
| 114 |
+
"single_word": false,
|
| 115 |
+
"special": true
|
| 116 |
+
},
|
| 117 |
+
"151657": {
|
| 118 |
+
"content": "<tool_call>",
|
| 119 |
+
"lstrip": false,
|
| 120 |
+
"normalized": false,
|
| 121 |
+
"rstrip": false,
|
| 122 |
+
"single_word": false,
|
| 123 |
+
"special": false
|
| 124 |
+
},
|
| 125 |
+
"151658": {
|
| 126 |
+
"content": "</tool_call>",
|
| 127 |
+
"lstrip": false,
|
| 128 |
+
"normalized": false,
|
| 129 |
+
"rstrip": false,
|
| 130 |
+
"single_word": false,
|
| 131 |
+
"special": false
|
| 132 |
+
},
|
| 133 |
+
"151659": {
|
| 134 |
+
"content": "<|fim_prefix|>",
|
| 135 |
+
"lstrip": false,
|
| 136 |
+
"normalized": false,
|
| 137 |
+
"rstrip": false,
|
| 138 |
+
"single_word": false,
|
| 139 |
+
"special": false
|
| 140 |
+
},
|
| 141 |
+
"151660": {
|
| 142 |
+
"content": "<|fim_middle|>",
|
| 143 |
+
"lstrip": false,
|
| 144 |
+
"normalized": false,
|
| 145 |
+
"rstrip": false,
|
| 146 |
+
"single_word": false,
|
| 147 |
+
"special": false
|
| 148 |
+
},
|
| 149 |
+
"151661": {
|
| 150 |
+
"content": "<|fim_suffix|>",
|
| 151 |
+
"lstrip": false,
|
| 152 |
+
"normalized": false,
|
| 153 |
+
"rstrip": false,
|
| 154 |
+
"single_word": false,
|
| 155 |
+
"special": false
|
| 156 |
+
},
|
| 157 |
+
"151662": {
|
| 158 |
+
"content": "<|fim_pad|>",
|
| 159 |
+
"lstrip": false,
|
| 160 |
+
"normalized": false,
|
| 161 |
+
"rstrip": false,
|
| 162 |
+
"single_word": false,
|
| 163 |
+
"special": false
|
| 164 |
+
},
|
| 165 |
+
"151663": {
|
| 166 |
+
"content": "<|repo_name|>",
|
| 167 |
+
"lstrip": false,
|
| 168 |
+
"normalized": false,
|
| 169 |
+
"rstrip": false,
|
| 170 |
+
"single_word": false,
|
| 171 |
+
"special": false
|
| 172 |
+
},
|
| 173 |
+
"151664": {
|
| 174 |
+
"content": "<|file_sep|>",
|
| 175 |
+
"lstrip": false,
|
| 176 |
+
"normalized": false,
|
| 177 |
+
"rstrip": false,
|
| 178 |
+
"single_word": false,
|
| 179 |
+
"special": false
|
| 180 |
+
},
|
| 181 |
+
"151665": {
|
| 182 |
+
"content": "<tool_response>",
|
| 183 |
+
"lstrip": false,
|
| 184 |
+
"normalized": false,
|
| 185 |
+
"rstrip": false,
|
| 186 |
+
"single_word": false,
|
| 187 |
+
"special": false
|
| 188 |
+
},
|
| 189 |
+
"151666": {
|
| 190 |
+
"content": "</tool_response>",
|
| 191 |
+
"lstrip": false,
|
| 192 |
+
"normalized": false,
|
| 193 |
+
"rstrip": false,
|
| 194 |
+
"single_word": false,
|
| 195 |
+
"special": false
|
| 196 |
+
},
|
| 197 |
+
"151667": {
|
| 198 |
+
"content": "<think>",
|
| 199 |
+
"lstrip": false,
|
| 200 |
+
"normalized": false,
|
| 201 |
+
"rstrip": false,
|
| 202 |
+
"single_word": false,
|
| 203 |
+
"special": false
|
| 204 |
+
},
|
| 205 |
+
"151668": {
|
| 206 |
+
"content": "</think>",
|
| 207 |
+
"lstrip": false,
|
| 208 |
+
"normalized": false,
|
| 209 |
+
"rstrip": false,
|
| 210 |
+
"single_word": false,
|
| 211 |
+
"special": false
|
| 212 |
+
}
|
| 213 |
+
},
|
| 214 |
+
"additional_special_tokens": [
|
| 215 |
+
"<|im_start|>",
|
| 216 |
+
"<|im_end|>",
|
| 217 |
+
"<|object_ref_start|>",
|
| 218 |
+
"<|object_ref_end|>",
|
| 219 |
+
"<|box_start|>",
|
| 220 |
+
"<|box_end|>",
|
| 221 |
+
"<|quad_start|>",
|
| 222 |
+
"<|quad_end|>",
|
| 223 |
+
"<|vision_start|>",
|
| 224 |
+
"<|vision_end|>",
|
| 225 |
+
"<|vision_pad|>",
|
| 226 |
+
"<|image_pad|>",
|
| 227 |
+
"<|video_pad|>"
|
| 228 |
+
],
|
| 229 |
+
"bos_token": null,
|
| 230 |
+
"chat_template": "{%- for message in messages %}{{- '<|im_start|>' + message.role + '\n'}}{%- if message.role == 'system' or message.role == 'user' %}{%- if message.content is string %}{{- message.content | replace('<image>', '') | replace('<video>', '') }}{%- else %}{%- for item in message.content %}{%- if item.type == 'text' and 'text' in item %}{{- item.text | replace('<image>', '') | replace('<video>', '') }}{%- elif item.type == 'image' %}{{- '<image>'}}{%- elif item.type == 'video' %}{{- '<video>'}}{%- else %}{{- raise_exception('Invalid content type. Supported types for system and user are text, image, video.')}}{%- endif %}{%- if not loop.last %}{{- '\n'}}{%- endif %}{%- endfor %}{%- endif %}{%- elif message.role == 'assistant' %}{%- set content = '' %}{%- if message.content is string %}{%- set content = message.content | replace('<image>', '') | replace('<video>', '') %}{%- else %}{%- for item in message.content %}{%- if item.type == 'text' and 'text' in item %}{%- set content = content ~ (item.text | replace('<image>', '') | replace('<video>', '')) %}{%- else %}{{- raise_exception('Invalid content type. Supported type for assistant is text.')}}{%- endif %}{%- endfor %}{%- endif %}{%- set content = content.split('</think>')[-1].lstrip('\n') %}{{- content }}{%- else %}{{- raise_exception('Invalid role. Supported roles are system, user, assistant.')}}{%- endif %}{{- '<|im_end|>\n'}}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- if enable_thinking is defined and enable_thinking is false %}{{- '<think>\n\n</think>\n\n' }}{%- endif %}{%- endif %}",
|
| 231 |
+
"clean_up_tokenization_spaces": false,
|
| 232 |
+
"eos_token": "<|im_end|>",
|
| 233 |
+
"errors": "replace",
|
| 234 |
+
"extra_special_tokens": {},
|
| 235 |
+
"model_max_length": 131072,
|
| 236 |
+
"pad_token": "<|endoftext|>",
|
| 237 |
+
"split_special_tokens": false,
|
| 238 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 239 |
+
"unk_token": null
|
| 240 |
+
}
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|