| import torch |
| import torch.nn as nn |
| import torch.nn.functional as F |
|
|
| import PIL.Image |
| from typing import List |
| from friday.util import expand2square, pad_and_stack |
|
|
| from transformers import AutoModel, AutoImageProcessor |
| from friday.util.s2wrapper import forward as multiscale_forward |
|
|
|
|
| class FastVitVisionTower(nn.Module): |
| def __init__(self, pretrained_model_name_or_path, model_params={}, pad_to_square=True, **kwargs): |
| super().__init__() |
|
|
| self.is_loaded = False |
| self.pretrained_model_name_or_path = pretrained_model_name_or_path |
| self.model_params = model_params |
| self.pad_to_square = pad_to_square |
| self.load_model() |
|
|
| @property |
| def output_dim(self): |
| return self.vision_tower.config.embed_dim if self.vision_tower else None |
| |
| def load_model(self): |
| if self.is_loaded: |
| return |
| self.image_processor = AutoImageProcessor.from_pretrained(self.pretrained_model_name_or_path) |
| self.image_processor.crop_size = self.image_processor.size |
| self.vision_tower = AutoModel.from_pretrained( |
| self.pretrained_model_name_or_path, |
| **self.model_params, |
| ) |
| self.vision_tower.requires_grad_(False) |
|
|
| self.is_loaded = True |
| |
| def preprocess_images(self, imgs: List[PIL.Image.Image], pad_and_stack_tensors=True) -> torch.Tensor: |
| img_mean = tuple(int(x * 255) for x in self.image_processor.image_mean) |
| if self.pad_to_square: |
| imgs = [expand2square(img, img_mean) for img in imgs] |
| |
| imgs = [self.image_processor(img, do_resize=True, do_center_crop=False, return_tensors="pt")['pixel_values'][0] for img in imgs] |
| |
|
|
| if pad_and_stack_tensors: |
| imgs = pad_and_stack(imgs, pad_value=0.0) |
| imgs = imgs.to(dtype=torch.float32, device=self.device) |
| |
| return imgs |
|
|
| def forward(self, images): |
| if type(images) is list: |
| image_features = [] |
| for image in images: |
| image_feature = self.vision_tower( |
| image.to(device=self.device, dtype=self.dtype).unsqueeze(0) |
| ) |
| image_features.append(image_feature) |
| else: |
| image_features = self.vision_tower( |
| images.to(device=self.device, dtype=self.dtype), |
| ) |
|
|
| return image_features |
|
|
| @property |
| def dummy_feature(self): |
| return torch.zeros(1, self.embed_dim, device=self.device, dtype=self.dtype) |
|
|
| @property |
| def dtype(self): |
| return self.vision_tower.dtype |
|
|
| @property |
| def device(self): |
| return self.vision_tower.device |
|
|
| @property |
| def config(self): |
| if self.is_loaded: |
| return self.vision_tower.config |
| else: |
| return self.cfg_only |
|
|
| @property |
| def hidden_size(self): |
| return self.config.embed_dim |
|
|
| @property |
| def num_patches(self): |
| return (self.config.image_size // self.config.patch_size) ** 2 |
|
|
|
|
| class FastVitVisionTowerS2(FastVitVisionTower): |
| def __init__(self, pretrained_model_name_or_path, s2_scales, model_params={}, **kwargs): |
| self.s2_scales = list(map(int, s2_scales.split(','))) |
| self.s2_scales.sort() |
| self.s2_split_size = self.s2_scales[0] |
| self.s2_image_size = self.s2_scales[-1] |
|
|
| super().__init__(pretrained_model_name_or_path, model_params) |
|
|
| self.multiscale_forward = multiscale_forward |
| |
| @property |
| def output_dim(self): |
| return (2*self.vision_tower.config.embed_dim) if self.vision_tower else None |
|
|
| def load_model(self): |
| if self.is_loaded: |
| return |
| |
| super().load_model() |
| self.image_processor.size = self.image_processor.crop_size = { |
| "height": self.s2_image_size, |
| "width": self.s2_image_size |
| } |
|
|
| def forward_feature(self, images): |
| image_size = self.vision_tower.config.image_size |
| if images.shape[2] != image_size or images.shape[3] != image_size: |
| images = F.interpolate( |
| images, |
| size=(image_size, image_size), |
| mode="bilinear", |
| align_corners=False, |
| antialias=True |
| ) |
|
|
| return self.vision_tower( |
| images.to(device=self.device, dtype=self.dtype), |
| ) |
|
|
| def forward(self, images): |
| if type(images) is list: |
| image_features = [] |
| for image in images: |
| image_feature = self.multiscale_forward( |
| self.forward_feature, |
| image.unsqueeze(0), |
| img_sizes=self.s2_scales, |
| max_split_size=self.s2_split_size |
| ) |
| image_features.append(image_feature) |
| else: |
| image_features = self.multiscale_forward( |
| self.forward_feature, |
| images, |
| img_sizes=self.s2_scales, |
| max_split_size=self.s2_split_size |
| ) |
|
|
| return image_features |
|
|
| @property |
| def hidden_size(self): |
| return self.config.embed_dim * len(self.s2_scales) |
|
|
