code/diffusion/generator.py

import torch
from diffusers import AutoencoderKL
import json
import os
from diffusers import FlowMatchEulerDiscreteScheduler
from .transformer import DiffusionTransformer
from .pipeline import ImageGenerationPipeline
import torch.nn as nn
import torch.nn.functional as F
from .autoencoder_kl_qwenimage import AutoencoderKLQwenImage
from inference_profile import InferenceProfile

import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class ToClipMLP(nn.Module):
    def __init__(self, input_dim, output_dim):
        super().__init__()
        #self.activation_fn = ACT2FN[config.hidden_act]
        self.fc1 = nn.Linear(input_dim, 2048)
        self.layer_norm1 = nn.LayerNorm(2048)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(2048, output_dim)
        self.layer_norm2 = nn.LayerNorm(output_dim)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states = self.fc1(hidden_states)
        hidden_states = self.layer_norm1(hidden_states)
        hidden_states = self.relu(hidden_states)
        hidden_states = self.fc2(hidden_states)
        hidden_states = self.layer_norm2(hidden_states)
        return hidden_states

class ConditionedTransformer(nn.Module):
    def __init__(self, transformer, vision_dim=1152, use_identity_mlp=False, text_encoder_norm=False):
        super().__init__()
        self.transformer = transformer
        self.mlp = ToClipMLP(vision_dim, 2560) if not use_identity_mlp else nn.Identity()
        self.mlp.to(dtype=self.dtype)
        # self.mlp_pool = ToClipMLP(vision_dim, 768)
        self.config = self.transformer.config
        self.in_channels = self.transformer.in_channels
        self.text_encoder_norm = text_encoder_norm

        # must be used together
        #if text_encoder_norm or use_identity_mlp:
        #    assert use_identity_mlp and text_encoder_norm

    
    @property
    def dtype(self):
        return next(self.transformer.parameters()).dtype

    def forward(self, hidden_states,
                    timestep,
                    encoder_hidden_states,
                    return_dict,
                    encoder_attention_mask=None,
                    extra_vit_input=None,
                    ref_hidden_states=None,
                    encoder_hidden_states_2=None,
                     **kargs):

        if encoder_hidden_states is not None:
            if isinstance(encoder_hidden_states, list):
                encoder_hidden_states = torch.stack(encoder_hidden_states, dim=0)

            if self.text_encoder_norm:
                encoder_hidden_states = F.normalize(encoder_hidden_states, dim=-1) * 1000.0  # 1000 matches the original text encoder norm
            
            encoder_hidden_states = self.mlp(encoder_hidden_states)
            
            if extra_vit_input is not None:
                encoder_hidden_states = torch.cat((encoder_hidden_states, extra_vit_input), dim=1)

            encoder_hidden_states = list(encoder_hidden_states.unbind(dim=0))
            
        hidden_states = self.transformer(
                    x=hidden_states,
                    cap_feats=encoder_hidden_states,
                    t=timestep,
                    return_dict=False,
                    ref_x=ref_hidden_states,
                    cap_feats_2=encoder_hidden_states_2,
                     **kargs
                )
        return hidden_states

    def enable_gradient_checkpointing(self):
        self.transformer.enable_gradient_checkpointing()

def latent_mean_variance(latents: torch.Tensor, per_channel: bool = True, unbiased_var: bool = False):
    """Compute the mean and variance of latents (shape follows the diffusion
    training ``latents``, usually ``[B, C, H, W]``).

    Args:
        latents: ``[B, C, H, W]``; a ``[B, C, T, H, W]`` tensor with
            ``T==1`` is ``squeeze(2)`` first.
        per_channel: when True aggregate over ``(B, H, W)``, one scalar per
            channel, shape ``[C]``; when False use one global mean/variance.
        unbiased_var: use the unbiased estimator (Bessel).

    Returns:
        ``(mean, variance)``; the std is ``variance.sqrt()`` (use
        ``torch.sqrt(variance.clamp_min(0))`` for numerical stability).
    """
    z = latents
    if z.dim() == 5 and z.shape[2] == 1:
        z = z.squeeze(2)
    if z.dim() != 4:
        raise ValueError(f"Expected 4D latents [B,C,H,W] (or 5D with T=1), got {tuple(z.shape)}")
    if per_channel:
        dims = (0, 2, 3)
        mean_v = z.mean(dim=dims)
        var_v = z.var(dim=dims, unbiased=unbiased_var)
    else:
        mean_v = z.mean()
        var_v = z.var(unbiased=unbiased_var)
    return mean_v, var_v

class ImageGenerator(torch.nn.Module):
    def __init__(self, 
            model_path, 
            vision_dim=2560, 
            scheduler_path=None,
            mlp_state_dict=None,
            torch_dtype=torch.float32,
            device='cpu',
            use_identity_mlp=False,
            text_encoder_norm=False,
            inference_profile=None,
        ):
        super(ImageGenerator, self).__init__()

        if not isinstance(inference_profile, InferenceProfile):
            raise ValueError(
                "inference_profile must be derived from the checkpoint "
                "capability contract (load_checkpoint_capabilities)"
            )
        self.inference_profile = inference_profile

        if device is not None:
            device = torch.device(device)
        else:
            device = torch.device(torch.cuda.current_device())

        self.scheduler_path = scheduler_path

        vae_config_path = os.path.join(model_path, "vae", "config.json")
        assert os.path.exists(vae_config_path)

        with open(vae_config_path, "r") as f:
            vae_config =  json.load(f)
            if "_class_name" in vae_config and vae_config["_class_name"] == "AutoencoderKLQwenImage":
                self.vae = AutoencoderKLQwenImage.from_pretrained(
                    model_path,
                    subfolder="vae",
                    torch_dtype=torch_dtype,
                )
                self.vae_sample_mode = "argmax"
            else:
                self.vae = AutoencoderKL.from_pretrained(
                    model_path,
                    subfolder="vae",
                    torch_dtype=torch_dtype,
                )
                self.vae_sample_mode = "sample"
        
        self.vae.input_channels = 4 if ('input_channels' in self.vae.config and self.vae.config.input_channels == 4) or ('in_channels' in self.vae.config and self.vae.config.in_channels == 4) else 3
        if self.vae.input_channels != self.inference_profile.vae_input_channels:
            raise ValueError(
                "VAE input channels do not match the checkpoint capability "
                f"contract: checkpoint={self.vae.input_channels}, "
                f"capability={self.inference_profile.vae_input_channels}"
            )
        if self.vae_sample_mode != self.inference_profile.vae_sample_mode:
            raise ValueError(
                "VAE sample mode does not match the checkpoint capability "
                f"contract: checkpoint={self.vae_sample_mode}, "
                f"capability={self.inference_profile.vae_sample_mode}"
            )
        
        # self.vae.to(self.torch_type).to(self.device)
        self.vae.requires_grad_(False)

        self.train_model = DiffusionTransformer.from_pretrained(
            model_path, subfolder="transformer",
            torch_dtype=torch_dtype,
            alignment_padding_mode=self.inference_profile.alignment_padding_mode,
            multi_frame_output=self.inference_profile.multi_frame_output,
        )
        if (
            self.train_model.alignment_padding_mode
            != self.inference_profile.alignment_padding_mode
            or self.train_model.multi_frame_output
            != self.inference_profile.multi_frame_output
        ):
            raise ValueError(
                "instantiated Transformer capability does not match the "
                "checkpoint capability contract: "
                f"transformer=({self.train_model.alignment_padding_mode!r}, "
                f"{self.train_model.multi_frame_output!r}), "
                f"capability=({self.inference_profile.alignment_padding_mode!r}, "
                f"{self.inference_profile.multi_frame_output!r})"
            )

        self.train_model = ConditionedTransformer(self.train_model, vision_dim=vision_dim, use_identity_mlp=use_identity_mlp, text_encoder_norm=text_encoder_norm)

        assert mlp_state_dict is not None
        self.train_model.mlp.load_state_dict(mlp_state_dict, strict=True)

        self.noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(self.scheduler_path, subfolder="scheduler")
        self.noise_scheduler.config['use_dynamic_shifting'] = True

        self.pipelines = ImageGenerationPipeline(
            vae=self.vae,
            transformer=self.train_model,  
            text_encoder=None, 
            tokenizer=None,                          
            scheduler=self.noise_scheduler,
        ).to(device)

    @property
    def device(self):
        return next(self.train_model.parameters()).device

    def set_trainable_params(self, trainable_params):
        
        self.vae.requires_grad_(False)

        if trainable_params == 'all':
            self.train_model.requires_grad_(True)
        else:
            self.train_model.requires_grad_(False)
            for name, module in self.train_model.named_modules():
                for trainable_param in trainable_params:
                    if trainable_param in name:
                        for params in module.parameters():
                            params.requires_grad = True

        num_parameters_trainable = 0
        num_parameters = 0
        name_parameters_trainable = []
        for n, p in self.train_model.named_parameters():
            num_parameters += p.data.nelement()
            if not p.requires_grad:
                continue  # frozen weights
            name_parameters_trainable.append(n)
            num_parameters_trainable += p.data.nelement()
        logger.info(f"number of all Diffusion parameters: {num_parameters}, trainable: {num_parameters_trainable}")
    

    def sample(self, encoder_hidden_states, steps=None, cfg=None, cfg_mode=1, seed=42, height=512, width=512, use_dynamic_shifting=False, extra_vit_input=None, ref_x=None, directvlm_hidden_states=None, num_frames_per_prompt=1):
        sampling = self.inference_profile.resolve_sampling_parameters(
            steps=steps,
            cfg=cfg,
        )
        steps = sampling.steps
        cfg = sampling.cfg
        negative_prompt_embeds = None
        if encoder_hidden_states is not None:
            encoder_hidden_states = encoder_hidden_states.to(
                device=self.device, dtype=self.train_model.dtype
            )
            encoder_hidden_states = list(encoder_hidden_states.unbind(dim=0))
            negative_prompt_embeds= [en * 0 for en in encoder_hidden_states]
        
        encoder_hidden_states_2 = directvlm_hidden_states
        negative_prompt_embeds_2 = None
        if encoder_hidden_states_2 is not None:
            encoder_hidden_states_2 = encoder_hidden_states_2.to(
                device=self.device, dtype=self.train_model.dtype
            )
            encoder_hidden_states_2 = list(encoder_hidden_states_2.unbind(dim=0))
            negative_prompt_embeds_2= [en * 0 for en in encoder_hidden_states_2]

        image = self.pipelines(
            prompt_embeds=encoder_hidden_states,
            negative_prompt_embeds=negative_prompt_embeds,
            prompt_embeds_2=encoder_hidden_states_2,
            negative_prompt_embeds_2=negative_prompt_embeds_2,
            guidance_scale=cfg,
            #guidance_scale_mode=cfg_mode,
            generator=torch.manual_seed(seed),
            num_inference_steps=steps,
            height=height,
            width=width,
            max_sequence_length=512,
            device=self.device,
            #extra_vit_input=extra_vit_input,
            ref_hidden_states=ref_x,
            #use_dynamic_shifting=use_dynamic_shifting,
            sample_mode=self.vae_sample_mode,
            num_frames_per_prompt=num_frames_per_prompt,
        ).images

        return image