JiT-B-16-SIM/pipeline.py

# Copyright 2026 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Hub custom pipeline: JiTPipeline for FD-Loss post-trained JiT (legacy time, velocity Euler)."""

from __future__ import annotations

import importlib.util
import inspect
import json
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union

import torch
from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
from diffusers.utils.torch_utils import randn_tensor

EXAMPLE_DOC_STRING = """
    Examples:
        ```py
        >>> from pathlib import Path
        >>> import torch
        >>> from diffusers import DiffusionPipeline

        >>> model_dir = Path("BiliSakura/FD-Loss-diffusers/JiT-B-16-SIM")
        >>> pipe = DiffusionPipeline.from_pretrained(
        ...     str(model_dir),
        ...     custom_pipeline=str(model_dir / "pipeline.py"),
        ...     trust_remote_code=True,
        ... )
        >>> pipe.to("cuda")
        >>> image = pipe(class_labels="golden retriever", num_inference_steps=1, guidance_scale=3.0).images[0]
        ```
"""

RECOMMENDED_NOISE_BY_SIZE = {256: 1.0, 512: 2.0}


class JiTPipeline(DiffusionPipeline):
    r"""
    Class-conditional JiT pipeline for FD-Loss checkpoints (1-NFE velocity Euler, legacy time convention).

    Args:
        transformer ([`JiTTransformer2DModel`]): JiT backbone.
        scheduler ([`FDLossFlowMatchScheduler`], *optional*): Bundled flow-matching scheduler (`t=1→0`).
            Accepts `generator` in `step()` for reproducible stochastic sampling.
        id2label (`dict[int, str]`, *optional*): ImageNet id → label strings from `model_index.json`.
    """

    model_cpu_offload_seq = "transformer"
    _optional_components = ["scheduler"]

    @staticmethod
    def prepare_extra_step_kwargs(
        scheduler,
        generator=None,
        eta: float | None = None,
    ):
        kwargs = {}
        step_params = set(inspect.signature(scheduler.step).parameters.keys())
        if "generator" in step_params:
            kwargs["generator"] = generator
        if eta is not None and "eta" in step_params:
            kwargs["eta"] = eta
        return kwargs

    @staticmethod
    def _resolve_inference_generator(
        device: Union[str, torch.device],
        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
    ) -> Optional[Union[torch.Generator, List[torch.Generator]]]:
        if generator is None:
            return None
        if isinstance(device, str):
            device = torch.device(device)
        device_type = device.type

        def _relocate(gen: torch.Generator) -> torch.Generator:
            if gen.device.type == device_type:
                return gen
            return torch.Generator(device=device_type).manual_seed(gen.initial_seed())

        if isinstance(generator, list):
            return [_relocate(g) for g in generator]
        return _relocate(generator)

    @staticmethod
    def _coerce_scheduler(scheduler, transformer):
        """Always load the bundled FD-Loss flow scheduler (ignore model_index diffusers schedulers)."""
        variant_path = getattr(transformer.config, "_name_or_path", None)
        if variant_path:
            scheduler_dir = Path(variant_path).resolve().parent / "scheduler"
            module_path = scheduler_dir / "scheduling_flow_match_fd.py"
            config_path = scheduler_dir / "scheduler_config.json"
            if module_path.is_file() and config_path.is_file():
                spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path)
                if spec is not None and spec.loader is not None:
                    module = importlib.util.module_from_spec(spec)
                    spec.loader.exec_module(module)
                    scheduler_cls = getattr(module, "FDLossFlowMatchScheduler")
                    return scheduler_cls.from_pretrained(str(scheduler_dir))
        scheduler_dir = Path(__file__).resolve().parent / "scheduler"
        module_path = scheduler_dir / "scheduling_flow_match_fd.py"
        config_path = scheduler_dir / "scheduler_config.json"
        if module_path.is_file() and config_path.is_file():
            spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path)
            if spec is not None and spec.loader is not None:
                module = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(module)
                scheduler_cls = getattr(module, "FDLossFlowMatchScheduler")
                return scheduler_cls.from_pretrained(str(scheduler_dir))
        raise ValueError("FD-Loss JiT requires a bundled FDLossFlowMatchScheduler under `scheduler/`.")

    def __init__(
        self,
        transformer,
        scheduler=None,
        id2label: Optional[Dict[Union[int, str], str]] = None,
    ):
        super().__init__()
        self.legacy_time_convention = True
        scheduler = self._coerce_scheduler(scheduler, transformer)
        self.register_modules(transformer=transformer, scheduler=scheduler)
        self._id2label = self._normalize_id2label(id2label)
        self.labels = self._build_label2id(self._id2label)
        self._labels_loaded_from_model_index = bool(self._id2label)


    @property
    def id2label(self) -> Dict[int, str]:
        self._ensure_labels_loaded()
        return self._id2label

    def get_label_ids(self, label: Union[str, List[str]]) -> List[int]:
        r"""Map ImageNet English label strings to class ids."""
        self._ensure_labels_loaded()
        if isinstance(label, str):
            label = [label]
        missing = [item for item in label if item not in self.labels]
        if missing:
            preview = ", ".join(list(self.labels.keys())[:8])
            raise ValueError(f"Unknown label(s): {missing}. Examples: {preview}, ...")
        return [self.labels[item] for item in label]

    def _backbone_timestep(self, t: torch.Tensor) -> torch.Tensor:
        if self.legacy_time_convention:
            return 1.0 - t
        return t

    def _cfg_interval(self, interval_min: float, interval_max: float, device: torch.device) -> Tuple[float, float]:
        low = float(self._backbone_timestep(torch.tensor(interval_min, device=device)))
        high = float(self._backbone_timestep(torch.tensor(interval_max, device=device)))
        return min(low, high), max(low, high)

    def _flow_time_tensors(self, t: torch.Tensor, sample: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
        """Broadcast scalar flow time and per-sample backbone timesteps for `sample` (B, C, H, W)."""
        flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype)
        t_view = flow_t.reshape(*([1] * sample.ndim))
        t_bb = self._backbone_timestep(flow_t).expand(sample.shape[0])
        return t_view, t_bb

    def _predict_velocity(
        self,
        sample: torch.Tensor,
        t: torch.Tensor,
        class_labels: torch.Tensor,
        guidance_scale: float,
        cfg_interval: Tuple[float, float],
        t_eps: float,
        interpolate_pos_encoding: bool,
    ) -> torch.Tensor:
        t_view, t_bb = self._flow_time_tensors(t, sample)
        num_classes = int(getattr(self.transformer.config, "num_classes", 1000))

        x_cond = self.transformer(
            sample,
            timestep=t_bb,
            class_labels=class_labels,
            interpolate_pos_encoding=interpolate_pos_encoding,
        ).sample
        v_cond = (sample - x_cond) / t_view.clamp_min(t_eps)

        if guidance_scale <= 1.0:
            return v_cond

        class_null = torch.full_like(class_labels, num_classes)
        x_uncond = self.transformer(
            sample,
            timestep=t_bb,
            class_labels=class_null,
            interpolate_pos_encoding=interpolate_pos_encoding,
        ).sample
        v_uncond = (sample - x_uncond) / t_view.clamp_min(t_eps)

        low, high = cfg_interval
        flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype)
        mask = (flow_t < high) & ((low == 0) | (flow_t > low))
        scale = torch.where(
            mask,
            torch.tensor(guidance_scale, device=sample.device, dtype=sample.dtype),
            torch.tensor(1.0, device=sample.device, dtype=sample.dtype),
        )
        while scale.ndim < v_cond.ndim:
            scale = scale.unsqueeze(-1)
        return v_uncond + scale * (v_cond - v_uncond)

    @staticmethod
    def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]:
        if not id2label:
            return {}
        return {int(key): value for key, value in id2label.items()}

    @staticmethod
    def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]:
        label2id: Dict[str, int] = {}
        for class_id, value in id2label.items():
            for synonym in value.split(","):
                synonym = synonym.strip()
                if synonym:
                    label2id[synonym] = int(class_id)
        return dict(sorted(label2id.items()))

    def _ensure_labels_loaded(self) -> None:
        if self._labels_loaded_from_model_index:
            return
        path = getattr(self.config, "_name_or_path", None)
        if path:
            model_index_path = Path(path).resolve() / "model_index.json"
            if model_index_path.exists():
                raw = json.loads(model_index_path.read_text(encoding="utf-8"))
                id2label = raw.get("id2label")
                if isinstance(id2label, dict):
                    self._id2label = {int(k): v for k, v in id2label.items()}
                    self.labels = self._build_label2id(self._id2label)
        self._labels_loaded_from_model_index = True

    def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]:
        if isinstance(class_labels, int):
            return [class_labels]
        if isinstance(class_labels, str):
            return self.get_label_ids(class_labels)
        if class_labels and isinstance(class_labels[0], str):
            return self.get_label_ids(class_labels)
        return list(class_labels)

    @torch.inference_mode()
    def __call__(
        self,
        class_labels: Union[int, str, List[Union[int, str]]],
        guidance_scale: float = 3.0,
        guidance_interval_min: float = 0.1,
        guidance_interval_max: float = 1.0,
        noise_scale: Optional[float] = None,
        t_eps: float = 5e-2,
        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
        num_inference_steps: int = 1,
        height: Optional[int] = None,
        width: Optional[int] = None,
        interpolate_pos_encoding: bool = True,
        output_type: Optional[str] = "pil",
        return_dict: bool = True,
    ) -> Union[ImagePipelineOutput, Tuple]:
        r"""
        Generate images conditioned on ImageNet class ids or English labels.

        Args:
            class_labels: Class id(s), label string(s), or mixed list.
            guidance_scale: CFG scale (FD-Loss JiT-B default: 3.0).
            guidance_interval_min: Lower flow-time bound for CFG (default 0.1).
            guidance_interval_max: Upper flow-time bound for CFG (default 1.0).
            noise_scale: Initial Gaussian scale (default 1.0 at 256px).
            t_eps: Clamp for flow time in velocity denominator.
            generator: RNG for latent init.
            num_inference_steps: FD-Loss sampling steps (1 = single NFE).
            height: Output height (defaults to native `sample_size`).
            width: Output width (defaults to native `sample_size`).
            interpolate_pos_encoding: Interpolate positional embeddings for non-native sizes.
            output_type: `pil`, `np`, or `pt`.
            return_dict: Return [`ImagePipelineOutput`] if True.
        """
        # 1. Check inputs
        if num_inference_steps < 1:
            raise ValueError("num_inference_steps must be >= 1.")
        if output_type not in {"pil", "np", "pt"}:
            raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.")

        # 2. Call parameters
        class_label_ids = self._normalize_class_labels(class_labels)
        batch_size = len(class_label_ids)
        generator = self._resolve_inference_generator(self._execution_device, generator)
        image_size = int(self.transformer.config.sample_size)
        patch_size = int(self.transformer.config.patch_size)
        height = int(height or image_size)
        width = int(width or image_size)
        if height % patch_size != 0 or width % patch_size != 0:
            raise ValueError(f"height/width must be divisible by patch_size={patch_size}.")
        channels = int(self.transformer.config.in_channels)
        null_class_val = int(getattr(self.transformer.config, "num_classes", 1000))
        if noise_scale is None:
            noise_scale = RECOMMENDED_NOISE_BY_SIZE.get(max(height, width), 1.0)

        # 3. Class labels
        labels = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1)
        labels = labels.clamp(0, null_class_val - 1)
        cfg_interval = self._cfg_interval(guidance_interval_min, guidance_interval_max, labels.device)

        # 4. Latents
        latents = randn_tensor(
            (batch_size, channels, height, width),
            generator=generator,
            device=self._execution_device,
            dtype=self.transformer.dtype,
        ) * noise_scale

        # 5–6. Denoising loop (FD-Loss flow t=1 → 0; generator forwarded to scheduler.step)
        self.scheduler.set_timesteps(num_inference_steps, device=self._execution_device)
        extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator)
        timesteps = self.scheduler.timesteps

        for i in self.progress_bar(range(num_inference_steps)):
            t_cur = timesteps[i]
            velocity = self._predict_velocity(
                latents, t_cur, labels, guidance_scale, cfg_interval, t_eps, interpolate_pos_encoding
            )
            latents = self.scheduler.step(velocity, t_cur, latents, **extra_step_kwargs).prev_sample

        # 8. Output
        images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu()
        if output_type == "pt":
            images = images_pt
        elif output_type == "np":
            images = images_pt.permute(0, 2, 3, 1).numpy()
        else:
            images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy())

        self.maybe_free_model_hooks()
        if not return_dict:
            return (images,)
        return ImagePipelineOutput(images=images)