Add golden tests and latest runtime support for Nemotron Parse v1.1

Dockerfile

@@ -0,0 +1,39 @@
+FROM nvcr.io/nvidia/pytorch:26.03-py3
+
+# ---------------------------------------------------------------------------
+# Install uv
+# ---------------------------------------------------------------------------
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /workspace
+
+# ---------------------------------------------------------------------------
+# Virtual environment
+#
+# --system-site-packages makes the venv inherit every package already
+# installed in the NVIDIA base image, including torch, torchvision, and the
+# matching CUDA libraries.  uv will not reinstall those packages (they are
+# declared in [tool.uv] exclude-dependencies), so the base-image builds are
+# left untouched.
+# ---------------------------------------------------------------------------
+RUN uv venv /opt/venv --system-site-packages
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# ---------------------------------------------------------------------------
+# Layer 1 - install dependencies (not the project itself)
+#
+# Only pyproject.toml is present at this point, so Docker re-runs this step
+# only when the dependency list changes, not when source files change.
+# ---------------------------------------------------------------------------
+COPY pyproject.toml .
+RUN uv sync --no-install-project
+
+# ---------------------------------------------------------------------------
+# Layer 2 - copy source and finish
+#
+# With package = false in pyproject.toml there is nothing extra to install;
+# the second sync is a fast no-op that confirms the environment is complete.
+# ---------------------------------------------------------------------------
+COPY . .
+RUN uv sync

README.md

@@ -108,6 +108,8 @@ pip install transformers==4.51.3
 pip install timm==1.0.22
 ```
 
+`transformers==4.51.3` remains the pinned reference environment for this model. The included remote code has also been tested against newer Transformers APIs, including Transformers 5.6.2, while preserving golden output compatibility with the pinned environment.
+
 ### Usage example
 
 ```python
@@ -169,6 +171,8 @@ for bbox in bboxes:
 
 **Update**: Nemotron-Parse-v1.1 is now [available in vllm main](https://github.com/vllm-project/vllm/pull/30864) and can be found in [vllm/vllm-openai:v0.14.1 docker image](https://hub.docker.com/layers/vllm/vllm-openai/v0.14.1/images/sha256-8e67731819426f7df194e5a0dfd6649d3aa3474f80c44f75b1e8711e76f8030a). 
 
+The model has also been validated with vLLM 0.20.1 using both implicit prompts and explicit encoder/decoder prompts.
+
 Note: when running on A100/A10 we recommend running vllm serve with _--attention-backend=TRITON_ATTN_
 
 You will need to install *albumentations* on top, and then follow the VLLM Inference example below:
@@ -376,4 +380,4 @@ Get access to knowledge base articles and support cases or [submit a ticket](htt
       primaryClass={cs.LG},
       url={https://arxiv.org/abs/2511.20478}, 
 }
-```
+```

config.json

@@ -99,8 +99,9 @@
   },
   "decoder_start_token_id": 2,
   "encoder": {
-    "_attn_implementation": "eager",
+    "attn_implementation": "eager",
     "_name_or_path": "nvidia/C-RADIOv2-H",
+    "image_processor_normalizes": true,
     "adaptor_configs": {},
     "adaptor_names": null,
     "add_cross_attention": false,
@@ -382,7 +383,7 @@
     "torchscript": false,
     "transformers_version": "4.51.3",
     "typical_p": 1.0,
-    "use_bfloat16": true,
+    "use_bfloat16": false,
     "version": "radio_v2.5-h",
     "vitdet_window_size": null
   },

docker-compose.yaml

@@ -0,0 +1,33 @@
+services:
+  nemotron-parse:
+    build: .
+    image: nemotron-parse:latest
+    working_dir: /workspace
+
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+
+    volumes:
+      # Project source - live-edits on the host are reflected immediately.
+      - .:/workspace
+      # Shadow the host .venv (wrong Python version) so uv uses /opt/venv inside
+      # the container rather than the host-side virtual environment.
+      - /workspace/.venv
+      # HuggingFace model cache - persists the RADIO encoder weights across
+      # container restarts so they are not re-downloaded on every run.
+      - hf-cache:/root/.cache/huggingface
+
+    environment:
+      HF_HOME: /root/.cache/huggingface
+
+    # Keep the container alive for interactive use (exec, attach, etc.).
+    stdin_open: true
+    tty: true
+
+volumes:
+  hf-cache:

golden_outputs.json

@@ -0,0 +1,133 @@
+{
+  "metadata": {
+    "transformers_version": "4.51.3",
+    "torch_version": "2.11.0+cu130",
+    "device": "cuda:0",
+    "dtype": "torch.bfloat16",
+    "model_path": "/home/oholworthy/Code/NeMo/nemo-retriever-research/models/NVIDIA-Nemotron-Parse-v1.1"
+  },
+  "image_processing": {
+    "shape": [
+      1,
+      3,
+      2048,
+      1648
+    ],
+    "mean": 0.06600654870271683,
+    "std": 0.23970627784729004,
+    "first_20_values": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "encoder_output": {
+    "shape": [
+      1,
+      3201,
+      1024
+    ],
+    "mean": -0.0010558579815551639,
+    "std": 0.9514492154121399,
+    "token0_first16": [
+      0.1474609375,
+      0.84765625,
+      2.09375,
+      -0.002960205078125,
+      0.640625,
+      0.494140625,
+      -0.2373046875,
+      -0.9921875,
+      -0.220703125,
+      0.5078125,
+      0.02734375,
+      0.0615234375,
+      -0.267578125,
+      -1.796875,
+      -0.1962890625,
+      0.17578125
+    ]
+  },
+  "forward_pass": {
+    "logits_shape": [
+      1,
+      1,
+      52352
+    ],
+    "top_k_indices": [
+      0,
+      221,
+      2,
+      276,
+      28,
+      501,
+      702,
+      1004,
+      123,
+      23
+    ],
+    "top_k_values": [
+      54.75,
+      41.0,
+      40.5,
+      40.25,
+      40.25,
+      40.25,
+      40.0,
+      40.0,
+      40.0,
+      40.0
+    ]
+  },
+  "generation": {
+    "max_new_tokens": 50,
+    "token_ids": [
+      2,
+      0,
+      50004,
+      50008,
+      50001,
+      50409,
+      51797,
+      82,
+      2722,
+      113,
+      18121,
+      579,
+      115,
+      113,
+      19321,
+      89,
+      115,
+      221,
+      82,
+      493,
+      113,
+      18121,
+      579,
+      115,
+      50632,
+      51847,
+      52325,
+      2
+    ],
+    "decoded_text": "</s><s><predict_bbox><predict_classes><output_markdown><x_0.3906><y_0.5969>\\begin{tabular}{ccccc}\n\\end{tabular}<x_0.6084><y_0.6359><class_Table></s>"
+  }
+}

hf_nemotron_parse_modeling.py

@@ -23,6 +23,38 @@ from transformers.modeling_attn_mask_utils import (
     _prepare_4d_causal_attention_mask_for_sdpa,
 )
 
+# ---------------------------------------------------------------------------
+# Cache compatibility (transformers 5.x introduced Cache objects;
+# 4.x used plain tuple-of-tuples for past_key_values)
+# ---------------------------------------------------------------------------
+import inspect
+try:
+    from transformers.cache_utils import Cache as _CacheBase
+    def _is_cache_object(obj) -> bool:
+        return isinstance(obj, _CacheBase)
+except ImportError:
+    def _is_cache_object(obj) -> bool:
+        return False
+
+def _past_key_values_length(past_key_values) -> int:
+    """Return the number of already-decoded tokens regardless of cache format."""
+    if past_key_values is None:
+        return 0
+    if _is_cache_object(past_key_values):
+        return past_key_values.get_seq_length()
+    return past_key_values[0][0].shape[2]
+
+# ---------------------------------------------------------------------------
+# MBartDecoderLayer API detection
+#
+# transformers <~4.57: forward() takes `past_key_value` (singular), returns a
+#   tuple (hidden_states, [attentions], [present_key_value])
+# transformers >=~4.57: forward() takes `past_key_values` (plural, Cache).
+#   True 5.x returns a single torch.Tensor (cache updated in-place);
+#   intermediate versions (e.g. 4.57.x) still return a tuple.
+# ---------------------------------------------------------------------------
+_layer_takes_plural_past_kv = 'past_key_values' in inspect.signature(MBartDecoderLayer.forward).parameters
+
 
 class NemotronParseDecoder(MBartPreTrainedModel):
     """
@@ -47,7 +79,11 @@ class NemotronParseDecoder(MBartPreTrainedModel):
         if embed_tokens is not None:
             self.embed_tokens.weight = embed_tokens.weight
 
-        self.layers = nn.ModuleList([MBartDecoderLayer(config) for _ in range(config.decoder_layers)])
+        _layer_supports_idx = 'layer_idx' in inspect.signature(MBartDecoderLayer.__init__).parameters
+        self.layers = nn.ModuleList([
+            MBartDecoderLayer(config, layer_idx=i) if _layer_supports_idx else MBartDecoderLayer(config)
+            for i in range(config.decoder_layers)
+        ])
         self.config = config
 
         self.layernorm_embedding = nn.LayerNorm(config.d_model)
@@ -163,8 +199,8 @@ class NemotronParseDecoder(MBartPreTrainedModel):
         else:
             raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")
 
-        # past_key_values_length
-        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0
+        # past_key_values_length works with tuple-of-tuples (4.x) and Cache objects (5.x).
+        past_key_values_length = _past_key_values_length(past_key_values)
 
         if inputs_embeds is None:
             inputs_embeds = self.embed_tokens(input_ids)
@@ -221,7 +257,22 @@ class NemotronParseDecoder(MBartPreTrainedModel):
         all_hidden_states = () if output_hidden_states else None
         all_self_attns = () if output_attentions else None
         all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
-        next_decoder_cache = () if use_cache else None
+        # In 5.x the Cache object is updated in-place by each layer, so we just
+        # carry the same object through.  In 4.x we collect per-layer tuples.
+        _using_cache_obj = _is_cache_object(past_key_values)
+        next_decoder_cache = past_key_values if (_using_cache_obj and use_cache) else (() if use_cache else None)
+
+        # 5.x: on the first call (past_key_values=None), create an EncoderDecoderCache
+        # so each MBartAttention layer can populate cross-/self-attention KV states
+        # in-place.  This enables proper KV caching during multi-step generation.
+        if _layer_takes_plural_past_kv and use_cache and past_key_values is None:
+            try:
+                from transformers.cache_utils import EncoderDecoderCache, DynamicCache
+                past_key_values = EncoderDecoderCache(DynamicCache(), DynamicCache())
+                _using_cache_obj = True
+                next_decoder_cache = past_key_values
+            except (ImportError, AttributeError, TypeError):
+                pass  # fallback: layers recompute KV each step (correct but slower)
 
         # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired
         for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
@@ -240,45 +291,68 @@ class NemotronParseDecoder(MBartPreTrainedModel):
                 if dropout_probability < self.layerdrop:
                     continue
 
-            past_key_value = past_key_values[idx] if past_key_values is not None else None
-
-            if self.gradient_checkpointing and self.training:
-                layer_outputs = self._gradient_checkpointing_func(
-                    decoder_layer.__call__,
-                    hidden_states,
-                    attention_mask,
-                    encoder_hidden_states,
-                    encoder_attention_mask,
-                    head_mask[idx] if head_mask is not None else None,
-                    cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None,
-                    None,
-                    output_attentions,
-                    use_cache,
-                )
-            else:
+            if _layer_takes_plural_past_kv:
+                # Plural-param API: cache updated in-place, nothing to collect.
                 layer_outputs = decoder_layer(
                     hidden_states,
                     attention_mask=attention_mask,
                     encoder_hidden_states=encoder_hidden_states,
                     encoder_attention_mask=encoder_attention_mask,
-                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
-                    cross_attn_layer_head_mask=(
-                        cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
-                    ),
-                    past_key_value=past_key_value,
-                    output_attentions=output_attentions,
+                    past_key_values=past_key_values if use_cache else None,
                     use_cache=use_cache,
                 )
-            hidden_states = layer_outputs[0]
-
-            if use_cache:
-                next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)
+                # True 5.x returns a single Tensor; intermediate versions
+                # (e.g. 4.57.x) have the renamed parameter but still return
+                # a tuple, so handle both.
+                hidden_states = layer_outputs if isinstance(layer_outputs, torch.Tensor) else layer_outputs[0]
+            else:
+                # Singular-param API: returns a tuple, collect cache per-layer.
+                if past_key_values is None:
+                    past_key_value = None
+                elif _using_cache_obj:
+                    past_key_value = past_key_values  # full Cache object
+                else:
+                    past_key_value = past_key_values[idx]  # per-layer tuple
+
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        decoder_layer.__call__,
+                        hidden_states,
+                        attention_mask,
+                        encoder_hidden_states,
+                        encoder_attention_mask,
+                        head_mask[idx] if head_mask is not None else None,
+                        cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None,
+                        None,
+                        output_attentions,
+                        use_cache,
+                    )
+                else:
+                    layer_outputs = decoder_layer(
+                        hidden_states,
+                        attention_mask=attention_mask,
+                        encoder_hidden_states=encoder_hidden_states,
+                        encoder_attention_mask=encoder_attention_mask,
+                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
+                        cross_attn_layer_head_mask=(
+                            cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
+                        ),
+                        past_key_value=past_key_value,
+                        output_attentions=output_attentions,
+                        use_cache=use_cache,
+                    )
+                hidden_states = layer_outputs[0]
 
-            if output_attentions:
-                all_self_attns += (layer_outputs[1],)
+                if use_cache and not _using_cache_obj:
+                    # 4.x: cache is the last element of layer_outputs.
+                    cache_idx = 3 if output_attentions else 1
+                    if len(layer_outputs) > cache_idx:
+                        next_decoder_cache += (layer_outputs[cache_idx],)
 
-                if encoder_hidden_states is not None:
-                    all_cross_attentions += (layer_outputs[2],)
+                if output_attentions:
+                    all_self_attns += (layer_outputs[1],)
+                    if encoder_hidden_states is not None:
+                        all_cross_attentions += (layer_outputs[2],)
 
         hidden_states = self.layer_norm(hidden_states)
 
@@ -308,6 +382,8 @@ class RadioWithNeck(nn.Module):
     def __init__(self, config):
         super().__init__()
         self.config = config
+        self.image_processor_normalizes = bool(getattr(config, "image_processor_normalizes", False))
+        self._radio_input_conditioner_externalized = False
         
         self.model_encoder = AutoModel.from_config(config, trust_remote_code=True)
         
@@ -319,8 +395,26 @@ class RadioWithNeck(nn.Module):
         self.layer_norm2 = nn.LayerNorm(last_hidden_state, eps=1e-06, elementwise_affine=True)
         self.sum_proj = nn.Linear(3840, last_hidden_state)
         self.layer_norm3 = nn.LayerNorm(last_hidden_state, eps=1e-06, elementwise_affine=True)
+
+    def _externalize_radio_input_conditioner(self):
+        if not self.image_processor_normalizes or self._radio_input_conditioner_externalized:
+            return
+
+        make_external = getattr(self.model_encoder, "make_preprocessor_external", None)
+        if make_external is None:
+            raise ValueError(
+                "image_processor_normalizes=True requires a RADIO encoder with "
+                "make_preprocessor_external()."
+            )
+
+        make_external()
+        self._radio_input_conditioner_externalized = True
         
     def forward(self, pixel_values, output_attentions=False, output_hidden_states=False, return_dict=False, **kwargs):
+        self._externalize_radio_input_conditioner()
+        if self.image_processor_normalizes:
+            dtype = next(self.model_encoder.parameters()).dtype
+            pixel_values = pixel_values.to(dtype=dtype)
         radio_output = self.model_encoder(pixel_values)
         summary, feature = radio_output
         
@@ -533,6 +627,30 @@ class NemotronParseForConditionalGeneration(NemotronParsePreTrainedModel, Genera
             encoder_attentions=encoder_outputs.attentions,
         )
 
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        use_cache=None,
+        encoder_outputs=None,
+        **kwargs,
+    ):
+        if past_key_values is not None:
+            past_length = _past_key_values_length(past_key_values)
+            if input_ids.shape[1] > past_length:
+                input_ids = input_ids[:, past_length:]
+            else:
+                input_ids = input_ids[:, -1:]
+        return {
+            "pixel_values": None,          # encoder_outputs carries the image features
+            "encoder_outputs": encoder_outputs,
+            "past_key_values": past_key_values,
+            "decoder_input_ids": input_ids,
+            "decoder_attention_mask": attention_mask,
+            "use_cache": use_cache,
+        }
+
     def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor):
         return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id)
 

hf_nemotron_parse_processor.py

@@ -7,12 +7,15 @@ import albumentations as A
 import cv2
 import json
 
-from transformers import ProcessorMixin, BaseImageProcessor, ImageProcessingMixin
-from transformers.tokenization_utils_base import BatchEncoding
+from transformers import BatchFeature, ProcessorMixin, BaseImageProcessor, ImageProcessingMixin
 from transformers.image_utils import ChannelDimension, ImageInput, PILImageResampling, infer_channel_dimension_format
 from transformers.utils import TensorType
 
 
+OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
+OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
+
+
 class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
     """
     Image processor for NemotronParse model.
@@ -25,6 +28,9 @@ class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
     def __init__(
         self,
         final_size: tuple = (2048, 1648),
+        do_normalize: bool = False,
+        image_mean: Optional[Union[List[float], tuple]] = None,
+        image_std: Optional[Union[List[float], tuple]] = None,
         **kwargs,
     ):
         clean_kwargs = {}
@@ -45,6 +51,9 @@ class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
                     final_size = (int(size_config['height']), int(size_config['width']))
         
         super().__init__(**clean_kwargs)
+        self.do_normalize = bool(do_normalize)
+        self.image_mean = list(image_mean or OPENAI_CLIP_MEAN)
+        self.image_std = list(image_std or OPENAI_CLIP_STD)
         
         if isinstance(final_size, (list, tuple)) and len(final_size) >= 2:
             self.final_size = (int(final_size[0]), int(final_size[1]))
@@ -74,7 +83,6 @@ class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
         
         self.torch_transform = T.Compose([
             T.ToTensor(),
-            # Note: Normalization is done within RADIO model
         ])
 
     def to_dict(self):
@@ -101,6 +109,10 @@ class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
             final_size = config_dict['final_size']
             if isinstance(final_size, (list, tuple)):
                 config_dict['final_size'] = tuple(int(x) for x in final_size)
+        if 'image_mean' in config_dict:
+            config_dict['image_mean'] = [float(x) for x in config_dict['image_mean']]
+        if 'image_std' in config_dict:
+            config_dict['image_std'] = [float(x) for x in config_dict['image_std']]
         
         try:
             return cls(**config_dict, **kwargs)
@@ -120,13 +132,16 @@ class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
         config = {
             "feature_extractor_type": "NemotronParseImageProcessor",
             "image_processor_type": "NemotronParseImageProcessor", 
-            "processor_class": "NemotronParseImageProcessor",
+            "processor_class": "NemotronParseProcessor",
             "size": {
                 "height": self.final_size[0],
                 "width": self.final_size[1],
                 "longest_edge": self.final_size
             },
             "final_size": self.final_size,
+            "do_normalize": self.do_normalize,
+            "image_mean": self.image_mean,
+            "image_std": self.image_std,
         }
         
         config_path = os.path.join(save_directory, "preprocessor_config.json")
@@ -231,6 +246,11 @@ class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
             pixel_values.append(pixel_values_tensor)
                         
         pixel_values = torch.stack(pixel_values)
+
+        if self.do_normalize:
+            mean = pixel_values.new_tensor(self.image_mean).view(1, -1, 1, 1)
+            std = pixel_values.new_tensor(self.image_std).view(1, -1, 1, 1)
+            pixel_values = (pixel_values - mean) / std
         
         data = {"pixel_values": pixel_values}
         
@@ -285,7 +305,7 @@ class NemotronParseProcessor(ProcessorMixin):
         verbose: bool = True,
         return_tensors: Optional[Union[str, "TensorType"]] = None,
         **kwargs
-    ) -> BatchEncoding:
+    ) -> BatchFeature:
         """
         Main method to prepare for the model one or several text(s) and image(s).
         """
@@ -320,7 +340,7 @@ class NemotronParseProcessor(ProcessorMixin):
             text_inputs = {}
         
         # Combine inputs
-        return BatchEncoding({**image_inputs, **text_inputs})
+        return BatchFeature(data={**image_inputs, **text_inputs})
     
     def decode(self, *args, **kwargs):
         """Decode token ids to strings."""

preprocessor_config.json

@@ -6,7 +6,17 @@
     "AutoImageProcessor": "hf_nemotron_parse_processor.NemotronParseImageProcessor",
     "AutoProcessor": "hf_nemotron_parse_processor.NemotronParseProcessor"
   },
-  "do_normalize": false,
+  "do_normalize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
   "do_rescale": true,
   "rescale_factor": 0.00392156862745098,
   "size": {

pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "nemotron-parse"
+version = "1.1.0"
+description = "NVIDIA Nemotron-Parse document parsing model"
+requires-python = ">=3.10"
+dependencies = [
+    "transformers>=4.51.3",
+    "accelerate==1.12.0",
+    "albumentations==2.0.8",
+    "timm==1.0.22",
+    "einops",
+    "Pillow",
+    "numpy",
+    "opencv-python-headless",
+    "beautifulsoup4",
+    "open-clip-torch>=3.3.0",
+    "pytest>=9.0.3",
+]
+
+[project.optional-dependencies]
+# vLLM serving (install separately in the serving container).
+vllm = ["openai"]
+# Development / testing.
+dev = ["pytest"]
+
+[tool.uv]
+# This repo is a model directory loaded via trust_remote_code, not an
+# installable Python package, so uv should only manage dependencies.
+package = false
+
+# torch and torchvision ship pre-compiled with CUDA support inside the
+# NVIDIA base image (nvcr.io/nvidia/pytorch:*).  uv must not overwrite them.
+exclude-dependencies = ["torch", "torchvision"]

test_golden.py

@@ -0,0 +1,550 @@
+"""
+Golden reference tests for NVIDIA-Nemotron-Parse-v1.1.
+
+Captures reference outputs from the pinned dependency set, then verifies the
+same outputs after dependency changes.
+
+WORKFLOW
+--------
+Step 1 - capture (run once against pinned deps, e.g. transformers==4.51.3):
+
+    python test_golden.py --capture [--model-path /path/to/model]
+
+    This writes golden_outputs.json next to this file.
+
+Step 2 - verify (run against new deps):
+
+    pytest test_golden.py -v
+
+    All tests skip automatically if golden_outputs.json is missing.
+
+TEST LAYERS
+-----------
+1. Image preprocessing   - pixel value stats + first-N raw values (no GPU needed)
+2. Encoder output        - hidden state shape, mean, std, and a fixed-position slice
+3. Decoder forward pass  - top-k logit indices and values at a fixed decoder step
+4. Generation            - exact token ID sequence for 50 greedy-decoded tokens
+"""
+
+import json
+import os
+import sys
+import pytest
+import numpy as np
+import torch
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Paths / constants
+# ---------------------------------------------------------------------------
+MODEL_PATH = str(Path(__file__).parent)
+GOLDEN_FILE = Path(__file__).parent / "golden_outputs.json"
+
+TASK_PROMPT = "</s><s><predict_bbox><predict_classes><output_markdown>"
+MAX_NEW_TOKENS_GOLDEN = 50     # short enough to be fast, long enough to be meaningful
+TOP_K = 10                     # number of top logit predictions to capture
+
+
+# ---------------------------------------------------------------------------
+# Deterministic test image (no external files required)
+# ---------------------------------------------------------------------------
+def make_test_image():
+    """Return a fully deterministic PIL image that loosely resembles a document."""
+    from PIL import Image, ImageDraw
+
+    img = Image.new("RGB", (400, 600), color=(255, 255, 255))
+    draw = ImageDraw.Draw(img)
+
+    # Title bar
+    draw.rectangle([20, 20, 380, 80], fill=(210, 210, 210))
+
+    # Body text area with ruled lines
+    draw.rectangle([20, 100, 380, 480], fill=(245, 245, 245))
+    for y in range(120, 470, 18):
+        draw.line([(40, y), (360, y)], fill=(170, 170, 170), width=1)
+
+    # Table-like grid at the bottom
+    draw.rectangle([20, 500, 380, 580], fill=(200, 220, 200))
+    for x in range(80, 380, 80):
+        draw.line([(x, 500), (x, 580)], fill=(100, 140, 100), width=1)
+    for y in range(520, 580, 20):
+        draw.line([(20, y), (380, y)], fill=(100, 140, 100), width=1)
+
+    return img
+
+
+# ---------------------------------------------------------------------------
+# Golden file helpers
+# ---------------------------------------------------------------------------
+def load_golden():
+    if GOLDEN_FILE.exists():
+        with open(GOLDEN_FILE) as f:
+            return json.load(f)
+    return None
+
+
+def save_golden(data: dict):
+    with open(GOLDEN_FILE, "w") as f:
+        json.dump(data, f, indent=2)
+
+
+def _pixel_values_for_golden(processor, pixel_values: torch.Tensor) -> torch.Tensor:
+    """Return the canonical raw pixel values used by the original golden file."""
+    pv = pixel_values.float()
+    image_processor = getattr(processor, "image_processor", None)
+    if not getattr(image_processor, "do_normalize", False):
+        return pv
+
+    mean = pv.new_tensor(image_processor.image_mean).view(1, -1, 1, 1)
+    std = pv.new_tensor(image_processor.image_std).view(1, -1, 1, 1)
+    return pv * std + mean
+
+
+# ---------------------------------------------------------------------------
+# Pytest fixtures  (session-scoped so the model is loaded only once)
+# ---------------------------------------------------------------------------
+@pytest.fixture(scope="session")
+def env():
+    """Load model, processor, and tokenizer once for the whole test session."""
+    import torch
+    from transformers import AutoModel, AutoProcessor, AutoTokenizer
+
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+    dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32
+
+    print(f"\nLoading model from {MODEL_PATH} on {device} ({dtype})...")
+    model = AutoModel.from_pretrained(
+        MODEL_PATH,
+        trust_remote_code=True,
+        torch_dtype=dtype,
+    ).to(device).eval()
+
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
+    processor = AutoProcessor.from_pretrained(MODEL_PATH, trust_remote_code=True)
+
+    return dict(model=model, tokenizer=tokenizer, processor=processor,
+                device=device, dtype=dtype)
+
+
+@pytest.fixture(scope="session")
+def processed_inputs(env):
+    """Preprocess the test image once for the whole session."""
+    import torch
+
+    image = make_test_image()
+    inputs = env["processor"](
+        images=[image],
+        text=TASK_PROMPT,
+        return_tensors="pt",
+        add_special_tokens=False,
+    ).to(env["device"])
+
+    return inputs, image
+
+
+@pytest.fixture(scope="session")
+def golden():
+    """Load golden data; tests that need it skip if the file is absent."""
+    data = load_golden()
+    if data is None:
+        pytest.skip("golden_outputs.json not found - run: python test_golden.py --capture")
+    return data
+
+
+# ---------------------------------------------------------------------------
+# Layer 1: Image preprocessing
+# (Does not require a model or GPU - fast sanity check on the processor.)
+# ---------------------------------------------------------------------------
+class TestImageProcessing:
+    def test_pixel_values_shape(self, processed_inputs):
+        inputs, _ = processed_inputs
+        pv = inputs["pixel_values"]
+        # Model expects 2048x1648 images.
+        assert list(pv.shape) == [1, 3, 2048, 1648], f"Unexpected shape: {pv.shape}"
+
+    def test_pixel_values_dtype(self, processed_inputs):
+        inputs, _ = processed_inputs
+        # Image preprocessing keeps float32; the model casts internally.
+        assert inputs["pixel_values"].dtype == torch.float32
+
+    def test_pixel_value_range(self, processed_inputs):
+        """Values should be CLIP-normalized after image preprocessing."""
+        pv = processed_inputs[0]["pixel_values"].float()
+        assert pv.min() >= -2.0, f"Pixel values unexpectedly low: {pv.min()}"
+        assert pv.max() <= 2.5, f"Pixel values unexpectedly high: {pv.max()}"
+
+    def test_pixel_values_stats_match_golden(self, processed_inputs, env, golden):
+        pv = _pixel_values_for_golden(env["processor"], processed_inputs[0]["pixel_values"])
+        g = golden["image_processing"]
+
+        assert abs(pv.mean().item() - g["mean"]) < 1e-4, \
+            f"mean changed: {pv.mean().item():.6f} vs golden {g['mean']:.6f}"
+        assert abs(pv.std().item() - g["std"]) < 1e-4, \
+            f"std changed: {pv.std().item():.6f} vs golden {g['std']:.6f}"
+
+    def test_pixel_values_first_values_match_golden(self, processed_inputs, env, golden):
+        """Exact match on the first 20 float values (catches transform-order bugs)."""
+        pv = _pixel_values_for_golden(env["processor"], processed_inputs[0]["pixel_values"])
+        actual = pv.flatten()[:20].tolist()
+        expected = golden["image_processing"]["first_20_values"]
+
+        for i, (a, e) in enumerate(zip(actual, expected)):
+            assert abs(a - e) < 1e-5, f"pixel[{i}] changed: {a} vs {e}"
+
+
+# ---------------------------------------------------------------------------
+# Layer 2: Encoder output
+# ---------------------------------------------------------------------------
+class TestEncoderOutput:
+    @pytest.fixture(scope="class")
+    def encoder_out(self, env, processed_inputs):
+        import torch
+        with torch.no_grad():
+            out = env["model"].encoder(processed_inputs[0]["pixel_values"])
+        return out
+
+    def test_encoder_output_shape(self, encoder_out):
+        # RadioWithNeck outputs (batch, 3201, 1024): patch tokens plus summary token.
+        hs = encoder_out.last_hidden_state
+        assert hs.shape[0] == 1
+        assert hs.shape[2] == 1024, f"Unexpected hidden dim: {hs.shape[2]}"
+
+    def test_encoder_output_stats_match_golden(self, encoder_out, golden):
+        hs = encoder_out.last_hidden_state.float()
+        g = golden["encoder_output"]
+
+        assert abs(hs.mean().item() - g["mean"]) < 0.05, \
+            f"encoder mean changed: {hs.mean().item():.4f} vs {g['mean']:.4f}"
+        assert abs(hs.std().item() - g["std"]) < 0.05, \
+            f"encoder std changed: {hs.std().item():.4f} vs {g['std']:.4f}"
+
+    def test_encoder_output_slice_match_golden(self, encoder_out, golden):
+        """Fixed-position slice: token 0, first 16 hidden dims."""
+        hs = encoder_out.last_hidden_state.float()
+        actual = hs[0, 0, :16].tolist()
+        expected = golden["encoder_output"]["token0_first16"]
+
+        for i, (a, e) in enumerate(zip(actual, expected)):
+            assert abs(a - e) < 0.1, \
+                f"encoder hidden[0,0,{i}] changed: {a:.4f} vs {e:.4f}"
+
+
+# ---------------------------------------------------------------------------
+# Layer 3: Decoder forward pass (logits)
+# ---------------------------------------------------------------------------
+class TestForwardPass:
+    @pytest.fixture(scope="class")
+    def forward_out(self, env, processed_inputs):
+        import torch
+        # Minimal decoder input: just the decoder_start_token (EOS = 2 for mBART)
+        dec_ids = torch.tensor([[2]], device=env["device"])
+        with torch.no_grad():
+            out = env["model"](
+                pixel_values=processed_inputs[0]["pixel_values"],
+                decoder_input_ids=dec_ids,
+                return_dict=True,
+            )
+        return out
+
+    def test_logits_shape(self, forward_out, env):
+        logits = forward_out.logits
+        assert logits.shape[0] == 1
+        assert logits.shape[1] == 1   # one decoder step
+        assert logits.shape[2] == 52352, f"Unexpected vocab size: {logits.shape[2]}"
+
+    def test_top_k_indices_match_golden(self, forward_out, golden):
+        """The TOP_K predicted token IDs should be identical (order matters)."""
+        import torch
+        logits = forward_out.logits[0, -1, :].float()
+        top_k = torch.topk(logits, k=TOP_K)
+
+        actual = top_k.indices.tolist()
+        expected = golden["forward_pass"]["top_k_indices"]
+
+        assert actual == expected, \
+            f"Top-{TOP_K} predicted tokens changed.\n  actual:   {actual}\n  expected: {expected}"
+
+    def test_top_k_values_match_golden(self, forward_out, golden):
+        """Logit magnitudes may drift slightly due to bf16; use a loose tolerance."""
+        import torch
+        logits = forward_out.logits[0, -1, :].float()
+        top_k = torch.topk(logits, k=TOP_K)
+
+        for i, (a, e) in enumerate(zip(top_k.values.tolist(),
+                                       golden["forward_pass"]["top_k_values"])):
+            assert abs(a - e) < 1.0, \
+                f"top-{i+1} logit value changed: {a:.3f} vs {e:.3f}"
+
+
+# ---------------------------------------------------------------------------
+# Layer 4: Generation (greedy, deterministic)
+# ---------------------------------------------------------------------------
+class TestGeneration:
+    @pytest.fixture(scope="class")
+    def gen_out(self, env, processed_inputs):
+        import torch
+        with torch.no_grad():
+            out = env["model"].generate(
+                **processed_inputs[0],
+                max_new_tokens=MAX_NEW_TOKENS_GOLDEN,
+                do_sample=False,
+                num_beams=1,
+            )
+        return out
+
+    def test_generated_token_ids_match_golden(self, gen_out, golden):
+        """Exact token-ID match - the most sensitive regression signal."""
+        actual = gen_out[0].cpu().tolist()
+        expected = golden["generation"]["token_ids"]
+
+        assert actual == expected, (
+            f"Generated token sequence differs from golden.\n"
+            f"  first divergence at index "
+            f"{next((i for i,(a,e) in enumerate(zip(actual,expected)) if a!=e), '?')}\n"
+            f"  actual:   {actual}\n"
+            f"  expected: {expected}"
+        )
+
+    def test_decoded_text_matches_golden(self, gen_out, env, golden):
+        text = env["tokenizer"].decode(gen_out[0], skip_special_tokens=False)
+        assert text == golden["generation"]["decoded_text"], \
+            f"Decoded text differs:\n  actual:   {text!r}\n  expected: {golden['generation']['decoded_text']!r}"
+
+
+# ---------------------------------------------------------------------------
+# Layer 5: Processor (no model or GPU needed - pure preprocessing & text utils)
+# ---------------------------------------------------------------------------
+@pytest.fixture(scope="session")
+def proc():
+    """Load processor + tokenizer only (no model weights, no GPU required)."""
+    from transformers import AutoProcessor, AutoTokenizer
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
+    processor = AutoProcessor.from_pretrained(MODEL_PATH, trust_remote_code=True)
+    return dict(processor=processor, tokenizer=tokenizer)
+
+
+class TestProcessor:
+    # ------------------------------------------------------------------
+    # post_process_generation
+    # ------------------------------------------------------------------
+    def test_post_process_generation_returns_string_for_string_input(self, proc, golden):
+        """String input -> string output."""
+        decoded = golden["generation"]["decoded_text"]
+        result = proc["processor"].post_process_generation(decoded)
+        assert isinstance(result, str)
+
+    def test_post_process_generation_removes_bos_eos(self, proc, golden):
+        """<s> and </s> tokens must be stripped from the output."""
+        decoded = golden["generation"]["decoded_text"]
+        result = proc["processor"].post_process_generation(decoded)
+        assert "<s>" not in result
+        assert "</s>" not in result
+
+    def test_post_process_generation_matches_manual_clean(self, proc, golden):
+        """Exact match against the expected cleaned string."""
+        decoded = golden["generation"]["decoded_text"]
+        expected = decoded.replace("<s>", "").replace("</s>", "").strip()
+        result = proc["processor"].post_process_generation(decoded)
+        assert result == expected
+
+    def test_post_process_generation_list_returns_list(self, proc, golden):
+        """Multi-element list input -> list output of the same length."""
+        decoded = golden["generation"]["decoded_text"]
+        result = proc["processor"].post_process_generation([decoded, decoded])
+        assert isinstance(result, list)
+        assert len(result) == 2
+        assert result[0] == result[1]
+
+    def test_post_process_generation_single_element_list_returns_string(self, proc, golden):
+        """Single-element list input -> scalar string (not a list)."""
+        decoded = golden["generation"]["decoded_text"]
+        result = proc["processor"].post_process_generation([decoded])
+        assert isinstance(result, str)
+
+    # ------------------------------------------------------------------
+    # decode / batch_decode via the processor
+    # ------------------------------------------------------------------
+    def test_decode_via_processor_matches_tokenizer(self, proc, golden):
+        """processor.decode() must give the same result as tokenizer.decode()."""
+        token_ids = golden["generation"]["token_ids"]
+        via_proc = proc["processor"].decode(token_ids, skip_special_tokens=False)
+        via_tok = proc["tokenizer"].decode(token_ids, skip_special_tokens=False)
+        assert via_proc == via_tok
+
+    def test_batch_decode_via_processor(self, proc, golden):
+        """processor.batch_decode() on repeated token lists matches golden decoded text."""
+        token_ids = golden["generation"]["token_ids"]
+        results = proc["processor"].batch_decode(
+            [token_ids, token_ids], skip_special_tokens=False
+        )
+        assert isinstance(results, list)
+        assert len(results) == 2
+        assert results[0] == results[1] == golden["generation"]["decoded_text"]
+
+    # ------------------------------------------------------------------
+    # Image processing edge cases
+    # ------------------------------------------------------------------
+    def test_large_image_resized_to_target(self, proc):
+        """Image larger than 2048x1648 is downscaled to exactly [1, 3, 2048, 1648]."""
+        from PIL import Image
+        large = Image.new("RGB", (4000, 5000), color=(128, 64, 32))
+        out = proc["processor"](images=[large], return_tensors="pt")
+        assert list(out["pixel_values"].shape) == [1, 3, 2048, 1648]
+
+    def test_grayscale_image_converted_to_rgb(self, proc):
+        """Grayscale (mode 'L') image is converted to RGB and produces 3 output channels."""
+        from PIL import Image
+        gray = Image.new("L", (400, 600), color=128)
+        out = proc["processor"](images=[gray], return_tensors="pt")
+        assert list(out["pixel_values"].shape) == [1, 3, 2048, 1648]
+
+    def test_multi_image_batch_first_dim(self, proc):
+        """A batch of N images produces pixel_values with first dimension N."""
+        from PIL import Image
+        imgs = [
+            Image.new("RGB", (400, 600), color=(i * 30, i * 20, i * 10))
+            for i in range(3)
+        ]
+        out = proc["processor"](images=imgs, return_tensors="pt")
+        assert list(out["pixel_values"].shape) == [3, 3, 2048, 1648]
+
+    def test_image_only_input_has_no_input_ids(self, proc):
+        """Passing images without text returns pixel_values and no input_ids key."""
+        from PIL import Image
+        img = Image.new("RGB", (400, 600))
+        out = proc["processor"](images=[img], return_tensors="pt")
+        assert "pixel_values" in out
+        assert "input_ids" not in out
+
+    def test_text_only_input_has_no_pixel_values(self, proc):
+        """Passing text without images returns input_ids and no pixel_values key."""
+        out = proc["processor"](text="hello world", return_tensors="pt")
+        assert "input_ids" in out
+        assert "pixel_values" not in out
+
+
+# ---------------------------------------------------------------------------
+# Capture helper (run as script: python test_golden.py --capture)
+# ---------------------------------------------------------------------------
+def capture(model_path: str = MODEL_PATH):
+    """
+    Run a full inference pass and write golden_outputs.json.
+    Intended to be run once against the pinned dependency set.
+    """
+    import torch
+    import transformers
+    from transformers import AutoModel, AutoProcessor, AutoTokenizer
+
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+    dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32
+
+    print(f"Capturing golden outputs")
+    print(f"  transformers : {transformers.__version__}")
+    print(f"  torch        : {torch.__version__}")
+    print(f"  device       : {device}  dtype={dtype}")
+    print(f"  model_path   : {model_path}")
+
+    model = AutoModel.from_pretrained(
+        model_path, trust_remote_code=True, torch_dtype=dtype
+    ).to(device).eval()
+    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+
+    image = make_test_image()
+    inputs = processor(
+        images=[image],
+        text=TASK_PROMPT,
+        return_tensors="pt",
+        add_special_tokens=False,
+    ).to(device)
+
+    # ---------- image processing ----------
+    pv = inputs["pixel_values"].float()
+    image_data = {
+        "shape": list(pv.shape),
+        "mean": pv.mean().item(),
+        "std": pv.std().item(),
+        "first_20_values": pv.flatten()[:20].tolist(),
+    }
+    print(f"\n[image] shape={image_data['shape']}  mean={image_data['mean']:.4f}  std={image_data['std']:.4f}")
+
+    # ---------- encoder output ----------
+    with torch.no_grad():
+        enc_out = model.encoder(inputs["pixel_values"])
+    hs = enc_out.last_hidden_state.float()
+    encoder_data = {
+        "shape": list(hs.shape),
+        "mean": hs.mean().item(),
+        "std": hs.std().item(),
+        "token0_first16": hs[0, 0, :16].tolist(),
+    }
+    print(f"[encoder] shape={encoder_data['shape']}  mean={encoder_data['mean']:.4f}  std={encoder_data['std']:.4f}")
+
+    # ---------- forward pass (logits) ----------
+    dec_ids = torch.tensor([[2]], device=device)  # decoder_start_token_id
+    with torch.no_grad():
+        fwd_out = model(
+            pixel_values=inputs["pixel_values"],
+            decoder_input_ids=dec_ids,
+            return_dict=True,
+        )
+    logits = fwd_out.logits[0, -1, :].float()
+    top_k = torch.topk(logits, k=TOP_K)
+    forward_data = {
+        "logits_shape": list(fwd_out.logits.shape),
+        "top_k_indices": top_k.indices.tolist(),
+        "top_k_values": top_k.values.tolist(),
+    }
+    top_tokens = [tokenizer.decode([i]) for i in top_k.indices.tolist()]
+    print(f"[forward] top-{TOP_K} tokens: {top_tokens}")
+
+    # ---------- generation ----------
+    with torch.no_grad():
+        gen_out = model.generate(
+            **inputs,
+            max_new_tokens=MAX_NEW_TOKENS_GOLDEN,
+            do_sample=False,
+            num_beams=1,
+        )
+    token_ids = gen_out[0].cpu().tolist()
+    decoded_text = tokenizer.decode(gen_out[0], skip_special_tokens=False)
+    generation_data = {
+        "max_new_tokens": MAX_NEW_TOKENS_GOLDEN,
+        "token_ids": token_ids,
+        "decoded_text": decoded_text,
+    }
+    print(f"[generation] {len(token_ids)} tokens: {decoded_text!r}")
+
+    # ---------- save ----------
+    golden = {
+        "metadata": {
+            "transformers_version": transformers.__version__,
+            "torch_version": torch.__version__,
+            "device": str(device),
+            "dtype": str(dtype),
+            "model_path": model_path,
+        },
+        "image_processing": image_data,
+        "encoder_output": encoder_data,
+        "forward_pass": forward_data,
+        "generation": generation_data,
+    }
+    save_golden(golden)
+    print(f"\nGolden outputs written to {GOLDEN_FILE}")
+    return golden
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Golden reference capture/verify for Nemotron-Parse")
+    parser.add_argument("--capture", action="store_true", help="Capture golden outputs")
+    parser.add_argument("--model-path", default=MODEL_PATH, help="Path to model directory")
+    args = parser.parse_args()
+
+    if args.capture:
+        capture(model_path=args.model_path)
+    else:
+        parser.print_help()
+        print("\nTo run tests:    pytest test_golden.py -v")
+        print("To capture:      python test_golden.py --capture")

test_vllm_golden.py

@@ -0,0 +1,103 @@
+"""
+Golden reference test for NVIDIA-Nemotron-Parse-v1.1 through vLLM.
+
+This mirrors the generation layer in test_golden.py, but exercises the vLLM
+encoder/decoder interface. vLLM returns completion text, while Transformers
+stores the full decoded decoder sequence in golden_outputs.json, so the
+comparison normalizes vLLM output back to the full decoded form.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+pytest.importorskip("vllm", reason="vLLM is not installed")
+
+from test_golden import (
+    GOLDEN_FILE,
+    MAX_NEW_TOKENS_GOLDEN,
+    MODEL_PATH,
+    TASK_PROMPT,
+    make_test_image,
+)
+
+
+def _load_golden_generation() -> str:
+    if not GOLDEN_FILE.exists():
+        pytest.skip("golden_outputs.json not found - run: python test_golden.py --capture")
+    with open(GOLDEN_FILE) as f:
+        return json.load(f)["generation"]["decoded_text"]
+
+
+def _as_full_decoded_text(vllm_text: str, expected_full_text: str) -> str:
+    if vllm_text.startswith(TASK_PROMPT):
+        full_text = vllm_text
+    elif expected_full_text.startswith(TASK_PROMPT):
+        full_text = TASK_PROMPT + vllm_text
+    else:
+        full_text = vllm_text
+
+    # vLLM stops on EOS but does not include that stop token in completion text.
+    if expected_full_text.endswith("</s>") and not full_text.endswith("</s>"):
+        with_eos = full_text + "</s>"
+        if with_eos == expected_full_text:
+            return with_eos
+
+    return full_text
+
+
+def test_vllm_generation_matches_golden():
+    from vllm import LLM, SamplingParams
+
+    expected = _load_golden_generation()
+    image = make_test_image()
+
+    sampling_params = SamplingParams(
+        temperature=0.0,
+        top_k=1,
+        repetition_penalty=1.1,
+        max_tokens=MAX_NEW_TOKENS_GOLDEN,
+        skip_special_tokens=False,
+    )
+    llm = LLM(
+        model=MODEL_PATH,
+        max_num_seqs=2,
+        limit_mm_per_prompt={"image": 1},
+        dtype="bfloat16",
+        trust_remote_code=True,
+        attention_config={"backend": "TRITON_ATTN"},
+    )
+
+    request_names = ["implicit", "explicit_encoder_decoder"]
+    requests = [
+        {
+            "prompt": TASK_PROMPT,
+            "multi_modal_data": {"image": image.copy()},
+        },
+        {
+            "encoder_prompt": {
+                "prompt": "",
+                "multi_modal_data": {"image": image.copy()},
+            },
+            "decoder_prompt": TASK_PROMPT,
+        },
+    ]
+
+    outputs = llm.generate(requests, sampling_params)
+    assert len(outputs) == len(request_names)
+
+    mismatches = []
+    for name, output in zip(request_names, outputs):
+        text = output.outputs[0].text
+        full_text = _as_full_decoded_text(text, expected)
+        if full_text != expected:
+            mismatches.append(
+                f"vLLM {name} generation differs from golden.\n"
+                f"  raw vLLM text: {text!r}\n"
+                f"  normalized:   {full_text!r}\n"
+                f"  expected:     {expected!r}"
+            )
+
+    assert not mismatches, "\n\n".join(mismatches)