Spaces:
Running on Zero
Running on Zero
Upload 7 files
Browse files- README.md +120 -43
- app.py +362 -297
- h3_efficiency.py +97 -0
- h3_quantization.py +30 -0
- h3_quick_test.json +30 -0
- requirements-int8.txt +3 -0
README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
title: MiniMax-H3 Studio ·
|
| 3 |
emoji: 🎬
|
| 4 |
colorFrom: blue
|
| 5 |
colorTo: indigo
|
|
@@ -7,7 +7,7 @@ sdk: gradio
|
|
| 7 |
sdk_version: 6.24.0
|
| 8 |
app_file: app.py
|
| 9 |
pinned: true
|
| 10 |
-
short_description: '
|
| 11 |
suggested_hardware: zero-a10g
|
| 12 |
tags:
|
| 13 |
- not-for-all-audiences
|
|
@@ -22,21 +22,25 @@ tags:
|
|
| 22 |
|
| 23 |
# MiniMax-H3 Studio
|
| 24 |
|
| 25 |
-
**One image. One idea.
|
| 26 |
|
| 27 |
-
|
| 28 |
-
|
|
|
|
| 29 |
|
| 30 |
-
**🆕 AI Scene Planner · 🔒
|
| 31 |
|
| 32 |
## Start here
|
| 33 |
|
| 34 |
-
1. Upload
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
The supplied `h3_quick_test.json` is a small, one-clip setup. Import it under Profiles and upload your
|
| 42 |
own image. Importing a profile never starts inference.
|
|
@@ -101,7 +105,7 @@ your original wording is preserved. Missing trigger metadata is not proof that a
|
|
| 101 |
|
| 102 |
**Preparation finishes before the video GPU request:**
|
| 103 |
|
| 104 |
-
- Default limit: **
|
| 105 |
counts too; a zero-strength adapter does not download or load. These are practical size guards,
|
| 106 |
not guarantees of memory fit or model compatibility.
|
| 107 |
- Oversized files are hidden from automatic suggestions. Manual selections pass the same checks.
|
|
@@ -115,30 +119,60 @@ your original wording is preserved. Missing trigger metadata is not proof that a
|
|
| 115 |
Use `.safetensors` adapters for this `transformer_ref` pipeline. Arbitrary `.pt`/`.bin` checkpoints are
|
| 116 |
not accepted by the new bounded safetensors preparation path. WAN/LTX weights are not H3 adapters.
|
| 117 |
|
| 118 |
-
##
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
## Audio, motion and finished-video tools
|
| 144 |
|
|
@@ -175,9 +209,23 @@ history and caches are temporary in this build; download a report before a resta
|
|
| 175 |
|
| 176 |
## Owner setup and token policy
|
| 177 |
|
| 178 |
-
|
| 179 |
-
`
|
| 180 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
|
| 182 |
The dependency file is based on the matching live Space and preserves its torch/diffusers pins; it
|
| 183 |
adds OpenCV and filelock. Gradio is selected by `sdk_version` above, not by a second requirements pin.
|
|
@@ -197,20 +245,49 @@ of unlimited free compute or proof of the origin of any previous account charges
|
|
| 197 |
| `H3_GPU_SIZE` | `xlarge` | Existing large model allocation |
|
| 198 |
| `H3_GPU_DURATION_MIN` / `H3_GPU_DURATION_MAX` | `120` / `1500` | Existing reservation bounds |
|
| 199 |
| `H3_PLACEMENT_ALLOWANCE` | `90` | Existing cold-worker allowance; not silently lowered |
|
| 200 |
-
| `H3_PLACEMENT` | `lazy`
|
|
|
|
| 201 |
| `H3_ATTENTION` | `_native_cudnn` | Base attention; adapter runs use `native` |
|
| 202 |
| `H3_AOTI` | `0` | Existing compilation option; cannot be combined with LoRAs |
|
| 203 |
| `H3_LOKR_RANK` | `32` | CPU LoKr conversion target rank; an approximation when truncated |
|
| 204 |
-
| `H3_LORA_MAX_FILE_MB` / `H3_LORA_MAX_RUN_MB` | `
|
| 205 |
| `H3_LORA_DOWNLOAD_SECONDS` | `600` | Bounded streamed direct download; HF native retries remain separate |
|
| 206 |
| `HF_TOKEN` | unset | Model downloads and existing library/storage access only |
|
| 207 |
| `CIVITAI_TOKEN` | unset | Model downloads and CivitAI metadata/search |
|
| 208 |
| `LORA_LIBRARY_BUCKET` / `LORA_LIBRARY_DIR` | existing configuration | Preserve the shared library location |
|
| 209 |
| `CIVITAI_API_HOST` | unset | Existing search host override |
|
| 210 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
## Validation and attribution
|
| 212 |
|
| 213 |
-
|
|
|
|
|
|
|
| 214 |
and trigger preservation, token isolation, conditioner cache separation, early rejection before
|
| 215 |
compute, no GPU retry, adapter cleanup, Stop/Resume and real ffmpeg joins, mixing and looping.
|
| 216 |
No production GPU generation or paid inference was run. Model quality, actual GPU runtime and the
|
|
|
|
| 1 |
---
|
| 2 |
+
title: MiniMax-H3 Studio · Simple & Efficient
|
| 3 |
emoji: 🎬
|
| 4 |
colorFrom: blue
|
| 5 |
colorTo: indigo
|
|
|
|
| 7 |
sdk_version: 6.24.0
|
| 8 |
app_file: app.py
|
| 9 |
pinned: true
|
| 10 |
+
short_description: 'Simple video, smart scenes, identity & efficient Turbo'
|
| 11 |
suggested_hardware: zero-a10g
|
| 12 |
tags:
|
| 13 |
- not-for-all-audiences
|
|
|
|
| 22 |
|
| 23 |
# MiniMax-H3 Studio
|
| 24 |
|
| 25 |
+
**One image. One idea. Video with sound — with fewer controls in your way.**
|
| 26 |
|
| 27 |
+
In **Simple**, upload a picture, describe the shot, choose its length and press **Generate**.
|
| 28 |
+
**Balanced** is selected for you. Optional tools stay in closed sections; **Everything** reveals the
|
| 29 |
+
manual settings. For several actions, open the scene planner and review the proposed clips and timings.
|
| 30 |
|
| 31 |
+
**🆕 Simple quality presets · 📐 Auto picture shape · 🎬 AI Scene Planner · 🔒 Identity Lock · 🔄 Looping LoRA Refresh · 🔊 Soundtrack mixer**
|
| 32 |
|
| 33 |
## Start here
|
| 34 |
|
| 35 |
+
1. Upload your picture and describe the action, sounds and any dialogue.
|
| 36 |
+
2. Keep **Balanced** for six real denoising steps. Choose **Draft** for four or **Quality** for eight.
|
| 37 |
+
The default length is three seconds; changing quality keeps your chosen length and custom effects.
|
| 38 |
+
3. Press **Generate**. The Auto canvas follows the picture's proportions using a supported size.
|
| 39 |
+
4. For several actions, open **AI Scene Planner**, describe the scene, split it, review it and generate.
|
| 40 |
+
|
| 41 |
+
One Turbo adapter is already active. Prompt writing, scene planning and prompt rewriting are optional;
|
| 42 |
+
ordinary Generate does not automatically call the writing model. More references, Identity Lock,
|
| 43 |
+
LoRA tools, profiles and finishing controls remain available in closed sections.
|
| 44 |
|
| 45 |
The supplied `h3_quick_test.json` is a small, one-clip setup. Import it under Profiles and upload your
|
| 46 |
own image. Importing a profile never starts inference.
|
|
|
|
| 105 |
|
| 106 |
**Preparation finishes before the video GPU request:**
|
| 107 |
|
| 108 |
+
- Default limit: **1536 MiB per source/converted file**, **2048 MiB across selected adapters**. Turbo
|
| 109 |
counts too; a zero-strength adapter does not download or load. These are practical size guards,
|
| 110 |
not guarantees of memory fit or model compatibility.
|
| 111 |
- Oversized files are hidden from automatic suggestions. Manual selections pass the same checks.
|
|
|
|
| 119 |
Use `.safetensors` adapters for this `transformer_ref` pipeline. Arbitrary `.pt`/`.bin` checkpoints are
|
| 120 |
not accepted by the new bounded safetensors preparation path. WAN/LTX weights are not H3 adapters.
|
| 121 |
|
| 122 |
+
## Lower compute without complicating Simple
|
| 123 |
+
|
| 124 |
+
| Quality | Real denoising evaluations | Intended use |
|
| 125 |
+
| --- | ---: | --- |
|
| 126 |
+
| Draft | 4 | Short tests and simple motion |
|
| 127 |
+
| Balanced — default | 6 | Everyday starting point |
|
| 128 |
+
| Quality | 8 | More demanding motion; costs more |
|
| 129 |
+
|
| 130 |
+
The default adapter remains Larry v4 step600 EMA. Its
|
| 131 |
+
[model card](https://huggingface.co/larryvrh/MiniMax-H3-Turbo-Lora) recommends six to eight steps when
|
| 132 |
+
fast motion smears at four. Quality is a setting, not a guarantee for every scene.
|
| 133 |
+
|
| 134 |
+
**Correct step accounting:** the pinned scheduler counts its terminal zero as a schedule point. The
|
| 135 |
+
app now passes `N + 1` points for `N` real evaluations, matching the
|
| 136 |
+
[Turbo inference implementation](https://github.com/ModelTC/Minimax-H3-Turbo/blob/main/inference_minimax_h3.py).
|
| 137 |
+
Thus an older “8 steps” actually performed seven evaluations; the new Balanced performs six. Keeping
|
| 138 |
+
an old numeric step value now performs one more evaluation. Exported profiles use version 3.
|
| 139 |
+
|
| 140 |
+
**Smaller reference processing — requires the companion conditioner update.** With the new endpoint,
|
| 141 |
+
both the Qwen encoder and H3 reference encoder use the same source-preserving policy. A small image
|
| 142 |
+
is no longer enlarged to a 2048-pixel short edge; large images are limited to the output canvas area
|
| 143 |
+
and aligned to the model's 32-pixel grid. For a 512×768 picture and a 512×768 canvas, reference image
|
| 144 |
+
rows drop from 6,144 to 384. That is **16× fewer image-reference rows**, not 16× faster video generation.
|
| 145 |
+
Target video, text, audio, model placement and decoding still cost compute. Smaller references can
|
| 146 |
+
lose fine detail; Everything retains larger output canvases for comparison.
|
| 147 |
+
|
| 148 |
+
The policy is inspired by the Turbo project's
|
| 149 |
+
[reference-resizing guidance](https://github.com/ModelTC/Minimax-H3-Turbo#note-on-reference-image-resizing).
|
| 150 |
+
This package keeps the Ref2VA model and native separate video/audio schedulers. It does not load
|
| 151 |
+
FL2VA weights or apply a second audio shift.
|
| 152 |
+
|
| 153 |
+
The generator discovers the companion's versioned endpoint from public API metadata, validates the
|
| 154 |
+
returned plan and tensor metadata, and uses matching dimensions in both halves. With the old public
|
| 155 |
+
conditioner it retains the old resize policy and canvas table. **Uploading the generator alone does
|
| 156 |
+
not enable the full reference reduction.** A failed efficient request is not silently retried.
|
| 157 |
+
|
| 158 |
+
Other savings: CPU LoRA download/conversion/cache before GPU allocation, one speed adapter when
|
| 159 |
+
switching presets, and a session-scoped conditioning cache. Identical conditioning inputs can reuse
|
| 160 |
+
it when only seed or steps change. Prompt, references, resolved dimensions, frame count, rewriting,
|
| 161 |
+
session, endpoint or resize policy changes invalidate it. The cache holds eight entries for 30 minutes,
|
| 162 |
+
at most 256 MiB each. A changed quality preset may select a different Auto canvas and require encoding.
|
| 163 |
+
|
| 164 |
+
An optional **LightX Ref2VA 4-step** adapter appears in Everything for comparison. It is about 1.29 GiB;
|
| 165 |
+
its CPU download and the total LoRA limit still apply. It is not activated by default and is not stacked
|
| 166 |
+
with another known speed preset. Native sparse attention, a second LTX pass, upscaling and interpolation
|
| 167 |
+
are not added to default generation.
|
| 168 |
+
|
| 169 |
+
**INT8 is an owner opt-in experiment**, described below. The default stays BF16 on xlarge. No GPU
|
| 170 |
+
quality, memory or speed benchmark has been run for this update, and no fixed percentage saving is
|
| 171 |
+
promised. Lower step counts do not translate proportionally to total request time.
|
| 172 |
+
|
| 173 |
+
The estimate includes the video GPU allocation multiplier. Conditioner and optional writer requests
|
| 174 |
+
are additional. A reservation estimate is not a bill; actual usage is governed by
|
| 175 |
+
[Hugging Face ZeroGPU](https://huggingface.co/docs/hub/spaces-zerogpu).
|
| 176 |
|
| 177 |
## Audio, motion and finished-video tools
|
| 178 |
|
|
|
|
| 209 |
|
| 210 |
## Owner setup and token policy
|
| 211 |
|
| 212 |
+
Upload the **contents of `generator/`** to the root of the existing MiniMax-H3 Space. In addition to
|
| 213 |
+
`app.py`, `requirements.txt` and `README.md`, the new `h3_efficiency.py` is required; include
|
| 214 |
+
`h3_quantization.py` and the optional dependency file too. Keep `h3_split_blocks.py`, `h3_aoti.py`,
|
| 215 |
+
`lora_library.py` and the existing runtime files. The generator folder is an update, not a full repository.
|
| 216 |
+
|
| 217 |
+
To enable efficient reference processing:
|
| 218 |
+
|
| 219 |
+
1. Duplicate the public `multimodalart/qwen3vl-conditioner` Space under your account and select ZeroGPU.
|
| 220 |
+
2. Upload the **contents of `conditioner/`** to that Space's root. Its README selects Gradio 6.24.0.
|
| 221 |
+
3. Wait for it to be ready, then set the generator's `H3_CONDITIONER` variable to the new public
|
| 222 |
+
`your-account/your-conditioner-space` ID. It must be publicly callable without an owner token.
|
| 223 |
+
4. Restart the generator after the conditioner is ready. Capability discovery is cached until restart.
|
| 224 |
+
|
| 225 |
+
Both copies of `h3_efficiency.py` must be identical. The conditioner's old endpoints remain available.
|
| 226 |
+
The companion loads the original Qwen conditioner; it is not a paid MiniMax API. It still uses ZeroGPU
|
| 227 |
+
quota, and hosting eligibility/limits are set by Hugging Face. If you cannot host the companion, the
|
| 228 |
+
updated generator can keep using the public legacy conditioner with reduced reference savings.
|
| 229 |
|
| 230 |
The dependency file is based on the matching live Space and preserves its torch/diffusers pins; it
|
| 231 |
adds OpenCV and filelock. Gradio is selected by `sdk_version` above, not by a second requirements pin.
|
|
|
|
| 245 |
| `H3_GPU_SIZE` | `xlarge` | Existing large model allocation |
|
| 246 |
| `H3_GPU_DURATION_MIN` / `H3_GPU_DURATION_MAX` | `120` / `1500` | Existing reservation bounds |
|
| 247 |
| `H3_PLACEMENT_ALLOWANCE` | `90` | Existing cold-worker allowance; not silently lowered |
|
| 248 |
+
| `H3_PLACEMENT` | `lazy` for BF16; `offload` for INT8 | Placement; offload reserves 10 GB for working memory |
|
| 249 |
+
| `H3_QUANTIZATION` | `bf16` | `int8` is an optional, unbenchmarked owner experiment |
|
| 250 |
| `H3_ATTENTION` | `_native_cudnn` | Base attention; adapter runs use `native` |
|
| 251 |
| `H3_AOTI` | `0` | Existing compilation option; cannot be combined with LoRAs |
|
| 252 |
| `H3_LOKR_RANK` | `32` | CPU LoKr conversion target rank; an approximation when truncated |
|
| 253 |
+
| `H3_LORA_MAX_FILE_MB` / `H3_LORA_MAX_RUN_MB` | `1536` / `2048` | LoRA limits in MiB |
|
| 254 |
| `H3_LORA_DOWNLOAD_SECONDS` | `600` | Bounded streamed direct download; HF native retries remain separate |
|
| 255 |
| `HF_TOKEN` | unset | Model downloads and existing library/storage access only |
|
| 256 |
| `CIVITAI_TOKEN` | unset | Model downloads and CivitAI metadata/search |
|
| 257 |
| `LORA_LIBRARY_BUCKET` / `LORA_LIBRARY_DIR` | existing configuration | Preserve the shared library location |
|
| 258 |
| `CIVITAI_API_HOST` | unset | Existing search host override |
|
| 259 |
|
| 260 |
+
## Optional INT8 / large trial — owner only
|
| 261 |
+
|
| 262 |
+
The helper follows the official
|
| 263 |
+
[Diffusers MiniMax-H3 INT8 recipe](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3):
|
| 264 |
+
INT8 weight-only transformer, protected input/output/timing modules, and the existing float32 VAEs.
|
| 265 |
+
Quantization runs at startup, outside the request's GPU job. It still needs host RAM and disk for the
|
| 266 |
+
original checkpoint and conversion. No pre-quantized checkpoint is downloaded automatically.
|
| 267 |
+
|
| 268 |
+
To opt in, append `-r requirements-int8.txt` to the generator's `requirements.txt` and set:
|
| 269 |
+
|
| 270 |
+
```text
|
| 271 |
+
H3_QUANTIZATION=int8
|
| 272 |
+
H3_PLACEMENT=offload
|
| 273 |
+
H3_GPU_SIZE=large
|
| 274 |
+
H3_AOTI=0
|
| 275 |
+
```
|
| 276 |
+
|
| 277 |
+
`requirements-int8.txt` pins torchao 0.17.0 for the existing PyTorch 2.11 build. Test this configuration
|
| 278 |
+
on your own small clip before offering it publicly: TorchAO/PEFT adapters, ZeroGPU workers, total
|
| 279 |
+
memory and performance have not been validated on a real GPU here. The 48 GB large allocation has a
|
| 280 |
+
lower quota multiplier, but slower kernels/offloading could offset that benefit. The conditioner
|
| 281 |
+
still uses xlarge; this switch affects only the generator. BF16 plus large is rejected at startup.
|
| 282 |
+
|
| 283 |
+
Rollback: set `H3_QUANTIZATION=bf16`, `H3_GPU_SIZE=xlarge`, `H3_PLACEMENT=lazy` and restart. There is no
|
| 284 |
+
automatic GPU fallback, extra retry or silent switch to a larger allocation.
|
| 285 |
+
|
| 286 |
## Validation and attribution
|
| 287 |
|
| 288 |
+
39 offline tests passed. Checks cover both Gradio interfaces and their API schemas, exact scheduler
|
| 289 |
+
point counts, shared reference resizing, metadata agreement, quality presets, Simple visibility,
|
| 290 |
+
and the previous Gradio construction and request injection, planner JSON and timing, LoRA rotation
|
| 291 |
and trigger preservation, token isolation, conditioner cache separation, early rejection before
|
| 292 |
compute, no GPU retry, adapter cleanup, Stop/Resume and real ffmpeg joins, mixing and looping.
|
| 293 |
No production GPU generation or paid inference was run. Model quality, actual GPU runtime and the
|
app.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""MiniMax-H3 `ref2va`, split deployment — the denoising half.
|
| 2 |
|
| 3 |
-
This Space holds
|
| 4 |
[`qwen3vl-conditioner`](https://huggingface.co/spaces/multimodalart/qwen3vl-conditioner), which this one calls over the
|
| 5 |
gradio API for every request; `reference_encoder` stays here, next to the autoencoders it runs.
|
| 6 |
"""
|
|
@@ -29,12 +29,14 @@ import spaces
|
|
| 29 |
import gradio as gr
|
| 30 |
|
| 31 |
import lora_library
|
|
|
|
| 32 |
|
| 33 |
MODEL_REPO = os.environ.get("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3")
|
| 34 |
CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
|
| 35 |
# `lazy` moves all 72.16 GiB onto the card on the first GPU call and leaves it there; `offload` hands placement to
|
| 36 |
# `ComponentsManager.enable_auto_cpu_offload`. Startup placement is not an option here — see `load_models`.
|
| 37 |
-
|
|
|
|
| 38 |
# cuDNN's fused attention is 10-20% faster than the SDPA default on this pool and needs nothing installed.
|
| 39 |
# flash-attention 3 is sm90-only and this card is sm120 (the `zero-a10g` flavour name is legacy).
|
| 40 |
ATTENTION = os.environ.get("H3_ATTENTION", "_native_cudnn").lower()
|
|
@@ -46,7 +48,7 @@ MAX_GPU_DURATION = int(os.environ.get("H3_GPU_DURATION_MAX", "1500"))
|
|
| 46 |
# Ceiling on the packed sequence. Above it the card dies inside the rotary embeddings with
|
| 47 |
# `NVML_SUCCESS == r INTERNAL ASSERT FAILED` - not a bug in the code, just out of memory.
|
| 48 |
# 74k rows go through; 165k kill the worker.
|
| 49 |
-
MAX_SEQUENCE = int(os.environ.get("H3_MAX_SEQUENCE", "90000"))
|
| 50 |
# An attached adapter adds its own layers and their activations to the same card, so the ceiling above is not the
|
| 51 |
# ceiling any more. Refusing a request that is over the reduced one is a sentence on screen; letting it through is a
|
| 52 |
# dead worker and a bare "runtime error".
|
|
@@ -83,7 +85,15 @@ CANVASES = {
|
|
| 83 |
"1152x512 · 21:9 fast": (512, 1152),
|
| 84 |
"1536x672 · 21:9 full": (672, 1536),
|
| 85 |
}
|
| 86 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
FPS, FRAMES_PER_CHUNK, LATENTS_PER_CHUNK = 24, 17, 5
|
| 88 |
# It is the *snapped* frame count the ceiling has to hold for: 15 s is 360 frames, which rounds up to 362, i.e.
|
| 89 |
# 15.083 s, and is refused. 14 is the last whole second that survives the snap.
|
|
@@ -93,7 +103,7 @@ MIN_DURATION = 2
|
|
| 93 |
MIN_REFERENCE_VIDEO, MAX_REFERENCE_VIDEO = 2.0, 15.0
|
| 94 |
# `MINIMAX_H3_MAX_REFERENCE_IMAGES`. The slots are built up front and revealed one at a time, because a demo asking
|
| 95 |
# for two subjects should not open with nine boxes.
|
| 96 |
-
MAX_IMAGE_SLOTS, OPEN_IMAGE_SLOTS = 9,
|
| 97 |
|
| 98 |
# How many LoRA slots the UI offers, and the range each strength slider covers. Everything else -
|
| 99 |
# the UI loop, the settings keys, the preset filler, `generate`'s `*lora_fields` tail - is built from
|
|
@@ -140,9 +150,12 @@ LORA_PRESETS = {
|
|
| 140 |
),
|
| 141 |
}
|
| 142 |
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
|
|
|
|
|
|
|
|
|
| 146 |
|
| 147 |
# The lowest step count the model's own schedulers accept; the Turbo LoRAs are tuned for 4.
|
| 148 |
MIN_STEPS = 4
|
|
@@ -186,10 +199,10 @@ def target_rows(height: int, width: int, num_frames: int) -> int:
|
|
| 186 |
return video + round(num_frames / FPS * AUDIO_LATENTS_PER_SECOND) * AUDIO_CHANNELS
|
| 187 |
|
| 188 |
|
| 189 |
-
def reference_rows(references: list[tuple[str, str]], num_frames: int) -> int:
|
| 190 |
"""The rows the reference blocks add, from metadata alone — no decode.
|
| 191 |
|
| 192 |
-
An image
|
| 193 |
own* aspect ratio resolves to, truncated to the generated frame count and snapped **down** to a `17 * n + 5` the
|
| 194 |
VAE encodes without padding; a soundtrack contributes two rows per 1/40 s.
|
| 195 |
"""
|
|
@@ -197,16 +210,14 @@ def reference_rows(references: list[tuple[str, str]], num_frames: int) -> int:
|
|
| 197 |
|
| 198 |
from diffusers.modular_pipelines.minimax_h3.modular_pipeline import resolve_canvas_size
|
| 199 |
|
|
|
|
| 200 |
rows = 0
|
| 201 |
for kind, path in references:
|
| 202 |
if kind == "image":
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
for edge in (height, width)
|
| 208 |
-
]
|
| 209 |
-
rows += (resolved[0] // CANVAS_MULTIPLE) * (resolved[1] // CANVAS_MULTIPLE)
|
| 210 |
continue
|
| 211 |
|
| 212 |
video_seconds, audio_seconds = probe(path)
|
|
@@ -231,17 +242,17 @@ def reference_rows(references: list[tuple[str, str]], num_frames: int) -> int:
|
|
| 231 |
|
| 232 |
|
| 233 |
def get_duration(
|
| 234 |
-
prompt_embeds, text_token_tags, references, height, width, num_frames, steps, seed, loras=(), **_
|
| 235 |
):
|
| 236 |
"""Seconds of GPU to reserve for one request. Takes the arguments of the `@spaces.GPU` function it decorates, and
|
| 237 |
tolerates the `gr.Progress` `spaces` injects."""
|
| 238 |
-
sequence = int(text_token_tags.shape[0]) + reference_rows(references, num_frames) + target_rows(
|
| 239 |
height, width, num_frames
|
| 240 |
)
|
| 241 |
denoise = int(steps) * (STEP_LINEAR * sequence + STEP_QUADRATIC * sequence**2) * SAFETY
|
| 242 |
# The two reference encoders ahead of the loop, and the two decoders plus the mux after it. Both scale with what
|
| 243 |
# they are handed rather than with the step count.
|
| 244 |
-
encode = 5 + reference_rows(references, num_frames) * 1e-3
|
| 245 |
decode = DECODE_BASE + DECODE_PER_DEFAULT_CANVAS * (height * width * num_frames) / DEFAULT_CANVAS_PIXELS
|
| 246 |
total = PLACEMENT_ALLOWANCE + encode + denoise + decode + 10 + LORA_ALLOWANCE * len(loras or ())
|
| 247 |
duration = max(MIN_GPU_DURATION, min(MAX_GPU_DURATION, math.ceil(total)))
|
|
@@ -249,20 +260,20 @@ def get_duration(
|
|
| 249 |
return duration
|
| 250 |
|
| 251 |
|
| 252 |
-
def budget(text_tokens, references, height, width, num_frames, steps, loras=()):
|
| 253 |
"""`(rows, GPU seconds)` for one request, by the same formula as `get_duration`."""
|
| 254 |
-
sequence = int(text_tokens) + reference_rows(references, num_frames) + target_rows(height, width, num_frames)
|
| 255 |
per_step = (STEP_LINEAR * sequence + STEP_QUADRATIC * sequence**2) * SAFETY
|
| 256 |
-
encode = 5 + reference_rows(references, num_frames) * 1e-3
|
| 257 |
decode = DECODE_BASE + DECODE_PER_DEFAULT_CANVAS * (height * width * num_frames) / DEFAULT_CANVAS_PIXELS
|
| 258 |
overhead = PLACEMENT_ALLOWANCE + encode + decode + 10 + LORA_ALLOWANCE * len(loras or ())
|
| 259 |
return sequence, overhead + int(steps) * per_step, per_step, overhead
|
| 260 |
|
| 261 |
|
| 262 |
-
def fits(text_tokens, references, height, width, num_frames, steps, loras=()):
|
| 263 |
"""Stops a request the card or the reservation cannot take, before any GPU time is spent."""
|
| 264 |
sequence, total, per_step, overhead = budget(
|
| 265 |
-
text_tokens, references, height, width, num_frames, steps, loras
|
| 266 |
)
|
| 267 |
ceiling = sequence_ceiling(loras)
|
| 268 |
if sequence <= ceiling and total <= MAX_GPU_DURATION:
|
|
@@ -319,9 +330,16 @@ def load_models() -> str | None:
|
|
| 319 |
|
| 320 |
lower_duration_floor()
|
| 321 |
manager = ComponentsManager()
|
| 322 |
-
blocks = MiniMaxH3Ref2VAGeneratorBlocks()
|
| 323 |
print(f"[ref2va] loading {[c.name for c in blocks.expected_components]} from {MODEL_REPO} ...", flush=True)
|
| 324 |
pipe = blocks.init_pipeline(MODEL_REPO, components_manager=manager, collection="h3")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
pipe.load_components(dtype=torch.bfloat16)
|
| 326 |
|
| 327 |
# Both VAEs first, and explicitly. `set_attention_backend` also sets the registry's *global* backend, which
|
|
@@ -337,10 +355,13 @@ def load_models() -> str | None:
|
|
| 337 |
# are identical field for field and the compiled code carries no weights of either.
|
| 338 |
import h3_aoti
|
| 339 |
|
| 340 |
-
|
|
|
|
|
|
|
|
|
|
| 341 |
|
| 342 |
if PLACEMENT == "offload":
|
| 343 |
-
manager.enable_auto_cpu_offload(device="cuda")
|
| 344 |
_arm_decode_hooks(pipe)
|
| 345 |
|
| 346 |
PIPE, MANAGER = pipe, manager
|
|
@@ -1022,7 +1043,7 @@ def generate(
|
|
| 1022 |
image_9=None,
|
| 1023 |
match=True,
|
| 1024 |
duration=5,
|
| 1025 |
-
steps=
|
| 1026 |
seed=42,
|
| 1027 |
upsample=False,
|
| 1028 |
*lora_fields,
|
|
@@ -1072,6 +1093,10 @@ def generate(
|
|
| 1072 |
requested = 0 if (match and derivable) else snap_frames(duration)
|
| 1073 |
|
| 1074 |
loras, lora_labels = collect_loras(lora_fields, progress)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1075 |
|
| 1076 |
# Before the conditioner spends GPU time: the canvas from the table, worst case for the frame count.
|
| 1077 |
planned_height, planned_width = CANVASES.get(canvas, CANVASES[DEFAULT_CANVAS])
|
|
@@ -1082,7 +1107,7 @@ def generate(
|
|
| 1082 |
planned_width,
|
| 1083 |
requested or snap_frames(min(audio_bearing(references)[0][1], MAX_REFERENCE_VIDEO)),
|
| 1084 |
steps,
|
| 1085 |
-
loras,
|
| 1086 |
)
|
| 1087 |
|
| 1088 |
_runtime_event("conditioner_begin")
|
|
@@ -1090,7 +1115,7 @@ def generate(
|
|
| 1090 |
conditioned = time.time()
|
| 1091 |
try:
|
| 1092 |
prompt_embeds, text_token_tags, metadata, plan = encode_remote(
|
| 1093 |
-
prompt, references, canvas, requested, rewrite_prompt=upsample, session_id=session_id
|
| 1094 |
)
|
| 1095 |
except gr.Error:
|
| 1096 |
raise
|
|
@@ -1107,13 +1132,13 @@ def generate(
|
|
| 1107 |
refined = plan.get("refined_prompt") or ""
|
| 1108 |
|
| 1109 |
# Again, with the exact numbers the conditioner returned.
|
| 1110 |
-
fits(int(text_token_tags.shape[0]), references, height, width, num_frames, steps, loras)
|
| 1111 |
|
| 1112 |
progress(0.1, desc=f"Generating {num_frames / FPS:.1f} s at {width}x{height} ...")
|
| 1113 |
_runtime_event("gpu_request", frames=num_frames, steps=int(steps))
|
| 1114 |
started = time.time()
|
| 1115 |
frames, audio, sampling_rate = _generate(
|
| 1116 |
-
prompt_embeds, text_token_tags, references, height, width, num_frames, steps, seed, loras
|
| 1117 |
)
|
| 1118 |
generate_seconds = time.time() - started
|
| 1119 |
|
|
@@ -1188,7 +1213,7 @@ def generate_with_identity(
|
|
| 1188 |
# themselves are deliberately left out: gradio hands them over as paths into a per-session temporary directory that
|
| 1189 |
# is gone by the next visit, so a saved path would restore as a dead file rather than as the image.
|
| 1190 |
|
| 1191 |
-
SETTINGS_VERSION =
|
| 1192 |
SETTINGS_KEYS = (
|
| 1193 |
["prompt", "upsample", "canvas", "match", "duration", "steps", "seed"]
|
| 1194 |
+ [f"lora_{slot + 1}" for slot in range(LORA_SLOTS)]
|
|
@@ -1595,24 +1620,18 @@ def _fill_lora_slots(files, *current):
|
|
| 1595 |
return [gr.update(value=value) for value in slots]
|
| 1596 |
|
| 1597 |
|
| 1598 |
-
def _add_preset_lora(preset,
|
| 1599 |
-
"""
|
| 1600 |
-
preset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1601 |
|
| 1602 |
-
The Turbo presets are tuned for a specific step range, so the steps slider is moved along with the slot — it is the
|
| 1603 |
-
one output beyond the LoRA fields. A slot already holding the same reference is a no-op, so the button can be
|
| 1604 |
-
pressed twice without duplicating, and a full set of slots is left untouched.
|
| 1605 |
-
"""
|
| 1606 |
-
reference, steps, _, strength = LORA_PRESETS[preset]
|
| 1607 |
-
slots = list(current[:LORA_SLOTS])
|
| 1608 |
-
scales = list(current[LORA_SLOTS:])
|
| 1609 |
-
if reference not in [(value or "").strip() for value in slots]:
|
| 1610 |
-
for index, value in enumerate(slots):
|
| 1611 |
-
if not (value or "").strip():
|
| 1612 |
-
slots[index] = reference
|
| 1613 |
-
scales[index] = strength
|
| 1614 |
-
break
|
| 1615 |
-
return [*slots, *scales, steps]
|
| 1616 |
|
| 1617 |
|
| 1618 |
load_models()
|
|
@@ -1922,7 +1941,7 @@ HERO = """
|
|
| 1922 |
<a href="https://www.minimax.io/blog/minimax-h3" target="_blank" rel="noopener">blog</a> ·
|
| 1923 |
<a href="https://huggingface.co/spaces/multimodalart/minimax-h3" target="_blank" rel="noopener">text / image to video</a></p>
|
| 1924 |
<div class="pills">
|
| 1925 |
-
<span>33B</span><span>joint video + audio</span><span>Turbo
|
| 1926 |
<span>ComfyUI lora accepted</span><span>up to 9 references</span><span>5 custom lora slots</span><span>shared lora library</span><span>trigger words in the prompt</span><span>structured prompt builder</span><span>named profiles</span><span>CivitAI search + links</span><span>scene continuation</span><span>🔒 identity lock</span><span>kohya + LoKr auto-convert</span><span>GPU cost estimate</span><span>clip stitching with audio</span>
|
| 1927 |
</div>
|
| 1928 |
</div>
|
|
@@ -1955,13 +1974,9 @@ diffusers' order, `alpha` is folded into the weights, and `.pt` files are read a
|
|
| 1955 |
to be converted by hand first.
|
| 1956 |
"""
|
| 1957 |
|
| 1958 |
-
TURBO_HELP = """
|
| 1959 |
-
|
| 1960 |
-
|
| 1961 |
-
|
| 1962 |
-
Larryvrh documents strength `1.0` for every build, and that is what the **v4** entries are filled at. The older **v1**
|
| 1963 |
-
line is the one people report over-sharpening and plastic skin on at `1.0`, so those are filled at `0.7` instead. Both
|
| 1964 |
-
are starting points, not rules — if a clip smears, raise it; if it looks over-sharp or plastic, lower it.
|
| 1965 |
"""
|
| 1966 |
|
| 1967 |
PROFILE_HELP = """Profiles keep the prompt, the canvas, the sliders and the lora slots — everything typed rather than
|
|
@@ -3368,7 +3383,7 @@ def _download_lora_source(source, progress=None):
|
|
| 3368 |
return _download_lora_source_unlocked(source, progress)
|
| 3369 |
|
| 3370 |
|
| 3371 |
-
LORA_MAX_FILE_BYTES = _lora_limit("H3_LORA_MAX_FILE_MB",
|
| 3372 |
LORA_MAX_RUN_BYTES = _lora_limit("H3_LORA_MAX_RUN_MB", 2048, 32768) * 1048576
|
| 3373 |
LORA_DOWNLOAD_SECONDS = _lora_limit("H3_LORA_DOWNLOAD_SECONDS", 600, 1800)
|
| 3374 |
def _pick_label(index, item):
|
|
@@ -4250,7 +4265,38 @@ def scene_step(state,request:gr.Request=None,progress=gr.Progress(track_tqdm=Tru
|
|
| 4250 |
values[_SCENE_INDEX['seed']],state['values'][_SCENE_INDEX['image']],min(total,state['index']+1))
|
| 4251 |
|
| 4252 |
|
| 4253 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4254 |
if not session:return None
|
| 4255 |
files=[]
|
| 4256 |
for kind,path in references:
|
|
@@ -4258,17 +4304,29 @@ def _condition_key(session,prompt,references,canvas,frames,rewrite):
|
|
| 4258 |
identity=(hashlib.sha256(Path(path).read_bytes()).hexdigest() if kind=='image'
|
| 4259 |
else [os.path.realpath(path),stat.st_size,stat.st_mtime_ns])
|
| 4260 |
files.append([kind,identity])
|
| 4261 |
-
return hashlib.sha256(json.dumps([session,CONDITIONER_SPACE,prompt,files,canvas,frames,bool(rewrite)],
|
| 4262 |
ensure_ascii=False).encode()).hexdigest()
|
| 4263 |
|
| 4264 |
|
| 4265 |
-
def encode_remote(prompt,references,canvas,num_frames,rewrite_prompt=False,session_id=''):
|
| 4266 |
from gradio_client import handle_file
|
| 4267 |
from safetensors import safe_open
|
| 4268 |
-
key=_condition_key(session_id,prompt,references,canvas,num_frames,rewrite_prompt)
|
|
|
|
| 4269 |
def read(path,plan):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4270 |
with safe_open(path,framework='pt') as handle:
|
| 4271 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4272 |
with _CONDITION_LOCK:
|
| 4273 |
now=time.monotonic()
|
| 4274 |
for k,record in list(_CONDITION_CACHE.items()):
|
|
@@ -4283,9 +4341,14 @@ def encode_remote(prompt,references,canvas,num_frames,rewrite_prompt=False,sessi
|
|
| 4283 |
_CONDITION_CACHE.pop(key,None)
|
| 4284 |
job=None
|
| 4285 |
try:
|
| 4286 |
-
|
| 4287 |
-
kinds=','.join(k for k,_ in references),
|
| 4288 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4289 |
path,plan=job.result(timeout=300)
|
| 4290 |
result=read(path,plan)
|
| 4291 |
except Exception as error:
|
|
@@ -4310,7 +4373,7 @@ def encode_remote(prompt,references,canvas,num_frames,rewrite_prompt=False,sessi
|
|
| 4310 |
|
| 4311 |
|
| 4312 |
@spaces.GPU(duration=get_duration,size=GPU_SIZE)
|
| 4313 |
-
def _generate(prompt_embeds,text_token_tags,references,height,width,num_frames,steps,seed,loras=()):
|
| 4314 |
import torch
|
| 4315 |
state=None;attached=[]
|
| 4316 |
started=time.perf_counter()
|
|
@@ -4322,7 +4385,8 @@ def _generate(prompt_embeds,text_token_tags,references,height,width,num_frames,s
|
|
| 4322 |
_runtime_event('denoising',adapter_files=len(attached))
|
| 4323 |
state=PIPE(prompt_embeds=prompt_embeds.to('cuda'),text_token_tags=text_token_tags,
|
| 4324 |
references=build_references(references),height=height,width=width,num_frames=num_frames,
|
| 4325 |
-
num_inference_steps=
|
|
|
|
| 4326 |
return state.get('videos')[0],state.get('audio')[0].cpu(),state.get('sampling_rate')
|
| 4327 |
except RuntimeError as error:
|
| 4328 |
_runtime_event('gpu_failed',error_type=type(error).__name__)
|
|
@@ -4396,39 +4460,57 @@ def gpu_estimate_studio(canvas,duration,steps,match,audio,video,identity,mode,st
|
|
| 4396 |
active=[ref for ref,scale in zip(refs,scales) if ref and float(scale or 0)!=0]
|
| 4397 |
if mode=='Extra H3 reference' and identity and float(strength)>0 and identity not in images:images.append(identity)
|
| 4398 |
try:
|
| 4399 |
-
references=collect(images,audio,video)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4400 |
seconds=float(duration)
|
| 4401 |
carried=audio_bearing(references)
|
| 4402 |
if match and len(carried)==1:seconds=carried[0][1]
|
| 4403 |
frames=snap_frames(seconds)
|
| 4404 |
-
rows,total,_,_=budget(TEXT_TOKEN_ALLOWANCE,references,height,width,frames,steps,active)
|
| 4405 |
if rows>sequence_ceiling(active):return '🚫 Too large for this model. Shorten the clip, choose a smaller canvas or remove a reference.'
|
| 4406 |
if total>MAX_GPU_DURATION:return '🚫 This request exceeds the configured runtime limit. Lower steps, duration or canvas size.'
|
| 4407 |
durations=_scene_durations(timing,_scene_count(count),duration)
|
| 4408 |
-
reservations=[max(MIN_GPU_DURATION,min(MAX_GPU_DURATION,math.ceil(budget(TEXT_TOKEN_ALLOWANCE,references,height,width,snap_frames(d),steps,active)[1]))) for d in durations]
|
| 4409 |
reservation=max(MIN_GPU_DURATION,min(MAX_GPU_DURATION,math.ceil(total)))
|
| 4410 |
factor=2 if GPU_SIZE=='xlarge' else 1
|
| 4411 |
-
return (f'**
|
| 4412 |
f'Scene estimate: ~{sum(reservations)*factor} quota-seconds for {len(durations)} clips. '
|
| 4413 |
'The remote conditioner and optional AI writer are additional requests; their quota is not included. '
|
| 4414 |
-
'This is an estimate, not a bill.
|
| 4415 |
'References for later clips may change the estimate.')
|
| 4416 |
except Exception as e:return f'Estimate unavailable ({type(e).__name__}); check uploads and per-clip timings.'
|
| 4417 |
|
| 4418 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4419 |
def budget_recipe(recipe,*fields):
|
|
|
|
|
|
|
| 4420 |
refs=list(fields[:LORA_SLOTS]);scales=list(fields[LORA_SLOTS:]);presets=list(LORA_PRESETS.values())
|
| 4421 |
turbo_urls={p[0] for p in presets}
|
| 4422 |
for i,ref in enumerate(refs):
|
| 4423 |
if ref in turbo_urls:refs[i]=''
|
| 4424 |
if recipe=='Original quality · 28 steps':
|
| 4425 |
-
return (
|
| 4426 |
try:i=refs.index('')
|
| 4427 |
-
except ValueError:raise gr.Error('The
|
| 4428 |
-
fast=recipe.startswith('Draft')
|
|
|
|
| 4429 |
refs[i]=presets[0][0];scales[i]=1.0
|
| 4430 |
-
return (
|
| 4431 |
-
'
|
| 4432 |
|
| 4433 |
|
| 4434 |
def _profile_updates(payload):
|
|
@@ -4566,7 +4648,7 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 4566 |
gr.HTML("""<div class="studio-hero"><small>MINIMAX-H3 · IMAGE + MOTION + SOUND</small>
|
| 4567 |
<h1>MiniMax-H3 Studio</h1><p>One idea → a planned scene, made one clip at a time.</p>
|
| 4568 |
<div class="studio-features"><span>🆕 AI Scene Planner · 64 clips</span><span>🔒 CPU Identity Lock</span>
|
| 4569 |
-
<span>⚡
|
| 4570 |
<p>Simple for quick creation. Pro for all controls. Downloads and LoRA conversion finish before GPU generation.</p></div>""")
|
| 4571 |
with gr.Accordion("Runtime diagnostics", open=False):
|
| 4572 |
diagnostics_button = gr.Button("Check runtime / download report")
|
|
@@ -4593,7 +4675,7 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 4593 |
)
|
| 4594 |
upsample = gr.Checkbox(label="✨ Upsample prompt", value=False, visible=False)
|
| 4595 |
|
| 4596 |
-
with gr.
|
| 4597 |
gr.Markdown(
|
| 4598 |
"**Picture in, a few words above, one press.** The description is "
|
| 4599 |
"written from your first reference picture, wrapped in the labelled "
|
|
@@ -4602,12 +4684,12 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 4602 |
"changed with a tick."
|
| 4603 |
)
|
| 4604 |
basic_space = gr.Textbox(
|
| 4605 |
-
value=REMOTE_SPACE, label="🛰️
|
| 4606 |
placeholder="owner/space-name",
|
| 4607 |
info="A chat Space of your own, shown your first picture. Empty it "
|
| 4608 |
"and only the structured builder runs, on your own words.",
|
| 4609 |
)
|
| 4610 |
-
basic_btn = gr.Button("✨
|
| 4611 |
elem_id="ir-btn")
|
| 4612 |
basic_status = gr.Markdown("Nothing done yet.")
|
| 4613 |
basic_pick = gr.CheckboxGroup(
|
|
@@ -4625,7 +4707,7 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 4625 |
gr.Markdown(
|
| 4626 |
"H3 was trained on the output of a preprocessor that rewrites a request into labelled "
|
| 4627 |
"sections, and MiniMax call that structure *critical to the quality of the final output*. "
|
| 4628 |
-
"
|
| 4629 |
"in the prompt box above, set the pieces below, and press **Build**.\n\n"
|
| 4630 |
"**Dialogue has to be verbatim.** Speech is generated together with the picture, so naming "
|
| 4631 |
"that someone speaks without giving the words produces correct mouth shapes with nothing in "
|
|
@@ -4656,180 +4738,155 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 4656 |
with gr.Row(elem_classes="chip-row"):
|
| 4657 |
chip_buttons_b = [gr.Button(text, size="sm", variant="secondary") for text in CHIPS[4:]]
|
| 4658 |
|
| 4659 |
-
#
|
| 4660 |
with gr.Group(elem_classes="panel"):
|
| 4661 |
-
|
| 4662 |
-
|
| 4663 |
-
|
| 4664 |
-
with gr.
|
| 4665 |
-
|
| 4666 |
-
|
| 4667 |
-
|
| 4668 |
-
|
| 4669 |
-
|
| 4670 |
-
|
| 4671 |
-
|
| 4672 |
-
|
| 4673 |
-
|
| 4674 |
-
|
| 4675 |
-
|
| 4676 |
-
|
| 4677 |
-
|
| 4678 |
-
|
| 4679 |
-
|
| 4680 |
-
|
| 4681 |
-
visible=index < OPEN_IMAGE_SLOTS,
|
| 4682 |
-
)
|
| 4683 |
-
for index in range(MAX_IMAGE_SLOTS)
|
| 4684 |
-
]
|
| 4685 |
-
add_image = gr.Button("+ Add another image", size="sm", variant="secondary")
|
| 4686 |
-
identity_ref = gr.Image(
|
| 4687 |
-
label="🔒 identity face (optional)",
|
| 4688 |
-
type="filepath",
|
| 4689 |
-
height=160,
|
| 4690 |
-
)
|
| 4691 |
-
identity_mode = gr.Radio(["Off", "CPU face protection", "Extra H3 reference"],
|
| 4692 |
-
value="CPU face protection", label="Identity Lock method")
|
| 4693 |
-
identity_strength = gr.Slider(0, 1, value=.65, step=.05, label="Face correction strength",
|
| 4694 |
-
info="0 disables it. CPU mode protects the starting frame without an extra H3 reference.")
|
| 4695 |
-
gr.Markdown(
|
| 4696 |
-
"<sub>The identity portrait stays fixed across continuation clips. It remains the original "
|
| 4697 |
-
"portrait across clips. CPU mode corrects an aligned face gently; it does not lock every generated frame. "
|
| 4698 |
-
"Extra H3 reference adds GPU work. Leave the portrait empty to use the original scene image.</sub>",
|
| 4699 |
-
elem_classes="turbo-blurb",
|
| 4700 |
-
)
|
| 4701 |
-
with gr.Tab("🔊 Audio"):
|
| 4702 |
-
audio = gr.Audio(label="A voice or a piece of music", type="filepath")
|
| 4703 |
-
with gr.Tab("🎥 Video"):
|
| 4704 |
-
video = gr.Video(label="Motion & camera, 2–15 s. Its soundtrack comes along.")
|
| 4705 |
|
| 4706 |
# ---------------- speed ----------------
|
| 4707 |
with gr.Group(elem_classes="panel"):
|
| 4708 |
gr.Markdown("### ⚡ Speed & cost")
|
| 4709 |
-
budget_choice = gr.Dropdown(["Balanced
|
| 4710 |
-
value="Balanced
|
| 4711 |
-
budget_button = gr.Button("
|
| 4712 |
-
budget_status = gr.Markdown("
|
| 4713 |
-
gr.
|
| 4714 |
-
|
| 4715 |
-
|
| 4716 |
-
|
| 4717 |
-
|
| 4718 |
-
|
| 4719 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4720 |
)
|
| 4721 |
-
lora_preset_add = gr.Button("⚡ add to a free slot", variant="secondary", scale=1,
|
| 4722 |
-
elem_id="turbo-btn")
|
| 4723 |
-
turbo_blurb = gr.Markdown(
|
| 4724 |
-
LORA_PRESETS[list(LORA_PRESETS)[0]][2], elem_classes="turbo-blurb"
|
| 4725 |
-
)
|
| 4726 |
|
| 4727 |
-
|
| 4728 |
-
|
| 4729 |
-
with gr.
|
| 4730 |
-
gr.
|
| 4731 |
-
|
| 4732 |
-
gr.
|
| 4733 |
-
|
| 4734 |
-
|
| 4735 |
-
|
| 4736 |
-
with gr.Row():
|
| 4737 |
-
search_query = gr.Textbox(label="Search", placeholder="e.g. dance, rain, camera move",
|
| 4738 |
-
scale=3)
|
| 4739 |
-
search_base = gr.Dropdown(H3_BASE_MODELS, value=H3_BASE_MODELS[0], label="Base model",
|
| 4740 |
-
allow_custom_value=True, scale=2)
|
| 4741 |
-
with gr.Row():
|
| 4742 |
-
search_nsfw = gr.Checkbox(value=True, label="Include NSFW results")
|
| 4743 |
-
search_btn = gr.Button("🔍 Search", variant="secondary", elem_id="search-btn")
|
| 4744 |
-
search_pick = gr.Dropdown(choices=[], label="Pick a file")
|
| 4745 |
-
with gr.Row():
|
| 4746 |
-
search_slot = gr.Dropdown([f"lora {i}" for i in range(1, LORA_SLOTS + 1)],
|
| 4747 |
-
value="lora 1", label="Into slot", scale=2)
|
| 4748 |
-
search_put_btn = gr.Button("⬇️ Put it in", variant="primary", scale=1,
|
| 4749 |
-
elem_id="search-put")
|
| 4750 |
-
# Below the picker on purpose: a wall of results above it would push the
|
| 4751 |
-
# controls off the screen, which is exactly what happened the first time.
|
| 4752 |
-
with gr.Accordion("📋 The results", open=False):
|
| 4753 |
-
search_results = gr.Markdown("No search yet.")
|
| 4754 |
-
search_state = gr.State({})
|
| 4755 |
-
|
| 4756 |
-
lora_references, lora_scales = [], []
|
| 4757 |
-
for slot in range(LORA_SLOTS):
|
| 4758 |
-
with gr.Row():
|
| 4759 |
-
lora_references.append(
|
| 4760 |
-
gr.Textbox(label=f"lora {slot + 1}", placeholder="owner/repo", scale=3, value=list(LORA_PRESETS.values())[0][0] if slot == 0 else "")
|
| 4761 |
)
|
| 4762 |
-
|
| 4763 |
-
gr.
|
| 4764 |
-
|
| 4765 |
-
|
| 4766 |
-
|
| 4767 |
-
|
| 4768 |
-
|
| 4769 |
-
|
| 4770 |
-
|
| 4771 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4772 |
)
|
| 4773 |
-
|
| 4774 |
-
|
| 4775 |
-
|
| 4776 |
-
|
| 4777 |
-
|
| 4778 |
-
|
| 4779 |
-
|
| 4780 |
-
|
| 4781 |
-
|
| 4782 |
-
|
| 4783 |
-
|
| 4784 |
-
|
| 4785 |
-
|
| 4786 |
-
|
| 4787 |
-
|
| 4788 |
-
|
| 4789 |
-
|
| 4790 |
-
|
| 4791 |
-
|
| 4792 |
-
|
| 4793 |
-
|
| 4794 |
-
|
| 4795 |
-
|
| 4796 |
-
value=8, visible=False)
|
| 4797 |
-
with gr.Row(visible=False) as pro_seed:
|
| 4798 |
-
seed = gr.Number(label="Seed", value=42, precision=0, scale=3)
|
| 4799 |
-
seed_dice = gr.Button("🎲 roll", variant="secondary", scale=1, elem_id="seed-dice")
|
| 4800 |
-
randomize_seed = gr.Checkbox(
|
| 4801 |
-
label="🎲 Randomize seed on every run",
|
| 4802 |
-
value=True,
|
| 4803 |
-
visible=False,
|
| 4804 |
-
info="A new seed is drawn each time Generate is pressed, and lands in the box above.",
|
| 4805 |
-
)
|
| 4806 |
|
| 4807 |
-
|
| 4808 |
-
|
| 4809 |
-
|
| 4810 |
-
|
| 4811 |
-
|
| 4812 |
-
|
| 4813 |
-
|
| 4814 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4815 |
)
|
| 4816 |
-
|
| 4817 |
-
|
| 4818 |
-
|
| 4819 |
-
|
| 4820 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4821 |
)
|
| 4822 |
-
profile_save = gr.Button("💾 save", variant="primary", scale=1, elem_id="profile-save")
|
| 4823 |
-
profile_delete = gr.Button("🗑️ delete the selected profile", variant="secondary",
|
| 4824 |
-
elem_id="profile-delete")
|
| 4825 |
-
profile_status = gr.Markdown("")
|
| 4826 |
-
gr.Markdown("---")
|
| 4827 |
-
gr.Markdown("**Take it with you (.json file)**")
|
| 4828 |
-
save = gr.Button("⬇️ export the current settings to .json", size="sm")
|
| 4829 |
-
settings_download = gr.File(label="Your settings", visible=False, interactive=False)
|
| 4830 |
-
settings_upload = gr.File(
|
| 4831 |
-
label="📤 import a settings .json", file_types=[".json"], type="filepath"
|
| 4832 |
-
)
|
| 4833 |
|
| 4834 |
# ---------------- output ----------------
|
| 4835 |
with gr.Column(scale=6):
|
|
@@ -4843,58 +4900,60 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 4843 |
with gr.Group(elem_classes="panel"):
|
| 4844 |
scene_progress = gr.HTML(_scene_progress(0, 1, phase="ready"))
|
| 4845 |
result = gr.Video(label="🎞️ Video + soundtrack", height=560)
|
| 4846 |
-
gr.
|
| 4847 |
-
|
| 4848 |
-
|
| 4849 |
-
|
| 4850 |
-
|
| 4851 |
-
|
| 4852 |
-
|
| 4853 |
-
|
| 4854 |
-
|
| 4855 |
-
|
| 4856 |
-
|
| 4857 |
-
|
| 4858 |
-
|
| 4859 |
-
|
| 4860 |
-
|
| 4861 |
-
|
| 4862 |
-
with gr.Accordion("Review or edit the clip prompts and timings", open=False) as pro_perclip:
|
| 4863 |
-
scene_prompts = gr.Textbox(
|
| 4864 |
-
label="One line per clip", lines=6, max_lines=64,
|
| 4865 |
-
placeholder=("line 1 = clip 1, line 2 = clip 2, and so on\n"
|
| 4866 |
-
"she turns toward the window\n"
|
| 4867 |
-
"she smiles and looks down\n"
|
| 4868 |
-
"…"),
|
| 4869 |
-
info="Leave a line empty — or the whole box — and that clip uses the "
|
| 4870 |
-
"main prompt, unchanged.",
|
| 4871 |
)
|
| 4872 |
-
|
| 4873 |
-
|
| 4874 |
-
|
| 4875 |
-
|
| 4876 |
-
|
| 4877 |
-
|
| 4878 |
-
|
| 4879 |
-
|
| 4880 |
-
|
| 4881 |
-
|
| 4882 |
-
|
| 4883 |
-
|
| 4884 |
-
|
| 4885 |
-
|
| 4886 |
-
|
| 4887 |
-
|
| 4888 |
-
|
| 4889 |
-
|
| 4890 |
-
|
| 4891 |
-
|
| 4892 |
-
|
| 4893 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4894 |
# An output, so it can be revealed only for a request that asked for a rewrite.
|
| 4895 |
with gr.Accordion("Upsampled prompt", open=False, visible=False) as upsampled_panel:
|
| 4896 |
upsampled = gr.Textbox(show_label=False, lines=8, interactive=False)
|
| 4897 |
-
with gr.Accordion("🎞️
|
| 4898 |
gr.Markdown(
|
| 4899 |
"Add clips to a queue and they are joined into a single file, **soundtrack "
|
| 4900 |
"included**. Three clips make one long video for no extra GPU time. Everything "
|
|
@@ -5040,8 +5099,13 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 5040 |
[edited_video, edited_file], api_name=False, concurrency_id="h3-heavy", concurrency_limit=1)
|
| 5041 |
loop_button.click(loop_finished, [result, merged_video, edit_target, loop_count], [edited_video, edited_file],
|
| 5042 |
api_name=False, concurrency_id="h3-heavy", concurrency_limit=1)
|
|
|
|
|
|
|
| 5043 |
budget_button.click(budget_recipe, [budget_choice, *lora_references, *lora_scales],
|
| 5044 |
[canvas, duration, steps, *lora_references, *lora_scales, budget_status], api_name=False)
|
|
|
|
|
|
|
|
|
|
| 5045 |
|
| 5046 |
# Searching CivitAI, and dropping a result into a slot.
|
| 5047 |
search_btn.click(
|
|
@@ -5140,11 +5204,12 @@ with gr.Blocks(title="MiniMax-H3 Studio · AI Scenes · Identity · Turbo", dele
|
|
| 5140 |
_planned.success(_picker_refresh, _PICK_INPUTS, _PICK_OUTPUTS, api_name=False)
|
| 5141 |
|
| 5142 |
_PRO_ONLY = [pro_builder, pro_chips, upsample, pro_loratab, pro_profiles,
|
| 5143 |
-
steps, pro_seed, randomize_seed
|
|
|
|
| 5144 |
|
| 5145 |
def _switch_mode(mode):
|
| 5146 |
pro = str(mode) == "pro"
|
| 5147 |
-
return [gr.update(visible=pro) for _ in _PRO_ONLY] + [gr.update(visible=
|
| 5148 |
|
| 5149 |
ui_mode.change(_switch_mode, [ui_mode], _PRO_ONLY + [basic_panel], api_name=False)
|
| 5150 |
|
|
|
|
| 1 |
"""MiniMax-H3 `ref2va`, split deployment — the denoising half.
|
| 2 |
|
| 3 |
+
This Space holds `transformer_ref` (bfloat16 by default) and both autoencoders. Text encoding runs in
|
| 4 |
[`qwen3vl-conditioner`](https://huggingface.co/spaces/multimodalart/qwen3vl-conditioner), which this one calls over the
|
| 5 |
gradio API for every request; `reference_encoder` stays here, next to the autoencoders it runs.
|
| 6 |
"""
|
|
|
|
| 29 |
import gradio as gr
|
| 30 |
|
| 31 |
import lora_library
|
| 32 |
+
from h3_efficiency import (PROTOCOL, efficient_blocks, reference_size, scheduler_points, validate_resize_mode)
|
| 33 |
|
| 34 |
MODEL_REPO = os.environ.get("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3")
|
| 35 |
CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
|
| 36 |
# `lazy` moves all 72.16 GiB onto the card on the first GPU call and leaves it there; `offload` hands placement to
|
| 37 |
# `ComponentsManager.enable_auto_cpu_offload`. Startup placement is not an option here — see `load_models`.
|
| 38 |
+
QUANTIZATION = os.environ.get("H3_QUANTIZATION", "bf16").lower()
|
| 39 |
+
PLACEMENT = os.environ.get("H3_PLACEMENT", "offload" if QUANTIZATION == "int8" else "lazy").lower()
|
| 40 |
# cuDNN's fused attention is 10-20% faster than the SDPA default on this pool and needs nothing installed.
|
| 41 |
# flash-attention 3 is sm90-only and this card is sm120 (the `zero-a10g` flavour name is legacy).
|
| 42 |
ATTENTION = os.environ.get("H3_ATTENTION", "_native_cudnn").lower()
|
|
|
|
| 48 |
# Ceiling on the packed sequence. Above it the card dies inside the rotary embeddings with
|
| 49 |
# `NVML_SUCCESS == r INTERNAL ASSERT FAILED` - not a bug in the code, just out of memory.
|
| 50 |
# 74k rows go through; 165k kill the worker.
|
| 51 |
+
MAX_SEQUENCE = int(os.environ.get("H3_MAX_SEQUENCE", "45000" if GPU_SIZE == "large" else "90000"))
|
| 52 |
# An attached adapter adds its own layers and their activations to the same card, so the ceiling above is not the
|
| 53 |
# ceiling any more. Refusing a request that is over the reduced one is a sentence on screen; letting it through is a
|
| 54 |
# dead worker and a bare "runtime error".
|
|
|
|
| 85 |
"1152x512 · 21:9 fast": (512, 1152),
|
| 86 |
"1536x672 · 21:9 full": (672, 1536),
|
| 87 |
}
|
| 88 |
+
LEGACY_CANVASES = dict(CANVASES)
|
| 89 |
+
AUTO_CANVAS = "Auto · match my picture"
|
| 90 |
+
CANVASES = {AUTO_CANVAS: (544, 960), **CANVASES,
|
| 91 |
+
"736x416 · 16:9 draft": (416, 736), "416x736 · 9:16 draft": (736, 416),
|
| 92 |
+
"864x480 · 16:9 balanced": (480, 864), "480x864 · 9:16 balanced": (864, 480),
|
| 93 |
+
"512x768 · 2:3 balanced": (768, 512), "768x512 · 3:2 balanced": (512, 768),
|
| 94 |
+
"448x672 · 2:3 draft": (672, 448), "672x448 · 3:2 draft": (448, 672),
|
| 95 |
+
"704x1056 · 2:3 quality": (1056, 704), "1056x704 · 3:2 quality": (704, 1056)}
|
| 96 |
+
DEFAULT_CANVAS = AUTO_CANVAS
|
| 97 |
FPS, FRAMES_PER_CHUNK, LATENTS_PER_CHUNK = 24, 17, 5
|
| 98 |
# It is the *snapped* frame count the ceiling has to hold for: 15 s is 360 frames, which rounds up to 362, i.e.
|
| 99 |
# 15.083 s, and is refused. 14 is the last whole second that survives the snap.
|
|
|
|
| 103 |
MIN_REFERENCE_VIDEO, MAX_REFERENCE_VIDEO = 2.0, 15.0
|
| 104 |
# `MINIMAX_H3_MAX_REFERENCE_IMAGES`. The slots are built up front and revealed one at a time, because a demo asking
|
| 105 |
# for two subjects should not open with nine boxes.
|
| 106 |
+
MAX_IMAGE_SLOTS, OPEN_IMAGE_SLOTS = 9, 1
|
| 107 |
|
| 108 |
# How many LoRA slots the UI offers, and the range each strength slider covers. Everything else -
|
| 109 |
# the UI loop, the settings keys, the preset filler, `generate`'s `*lora_fields` tail - is built from
|
|
|
|
| 150 |
),
|
| 151 |
}
|
| 152 |
|
| 153 |
+
LORA_PRESETS["LightX Ref2VA Turbo · 4 real steps · 1.29 GiB"] = (
|
| 154 |
+
"lightx2v/Minimax-h3-Turbo/minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors",
|
| 155 |
+
4, "A reference-specific distilled adapter. Use four real evaluations and the match reference policy. Compare quality with the default before switching your usual setup.", 1.0)
|
| 156 |
+
|
| 157 |
+
# H3 already steps video and audio on separate schedules. UI steps below mean
|
| 158 |
+
# actual transformer evaluations; scheduler_points() adds the terminal sigma point.
|
| 159 |
|
| 160 |
# The lowest step count the model's own schedulers accept; the Turbo LoRAs are tuned for 4.
|
| 161 |
MIN_STEPS = 4
|
|
|
|
| 199 |
return video + round(num_frames / FPS * AUDIO_LATENTS_PER_SECOND) * AUDIO_CHANNELS
|
| 200 |
|
| 201 |
|
| 202 |
+
def reference_rows(references: list[tuple[str, str]], num_frames: int, height=544, width=960, reference_resize_mode="legacy") -> int:
|
| 203 |
"""The rows the reference blocks add, from metadata alone — no decode.
|
| 204 |
|
| 205 |
+
An image follows the negotiated reference policy and is encoded as a single frame; a video is put on the canvas *its
|
| 206 |
own* aspect ratio resolves to, truncated to the generated frame count and snapped **down** to a `17 * n + 5` the
|
| 207 |
VAE encodes without padding; a soundtrack contributes two rows per 1/40 s.
|
| 208 |
"""
|
|
|
|
| 210 |
|
| 211 |
from diffusers.modular_pipelines.minimax_h3.modular_pipeline import resolve_canvas_size
|
| 212 |
|
| 213 |
+
canvas_size = (width, height)
|
| 214 |
rows = 0
|
| 215 |
for kind, path in references:
|
| 216 |
if kind == "image":
|
| 217 |
+
with Image.open(path) as picture:
|
| 218 |
+
source_size = picture.size
|
| 219 |
+
resolved_width, resolved_height = reference_size(source_size, canvas_size, reference_resize_mode)
|
| 220 |
+
rows += (resolved_height // CANVAS_MULTIPLE) * (resolved_width // CANVAS_MULTIPLE)
|
|
|
|
|
|
|
|
|
|
| 221 |
continue
|
| 222 |
|
| 223 |
video_seconds, audio_seconds = probe(path)
|
|
|
|
| 242 |
|
| 243 |
|
| 244 |
def get_duration(
|
| 245 |
+
prompt_embeds, text_token_tags, references, height, width, num_frames, steps, seed, loras=(), reference_resize_mode="legacy", **_
|
| 246 |
):
|
| 247 |
"""Seconds of GPU to reserve for one request. Takes the arguments of the `@spaces.GPU` function it decorates, and
|
| 248 |
tolerates the `gr.Progress` `spaces` injects."""
|
| 249 |
+
sequence = int(text_token_tags.shape[0]) + reference_rows(references, num_frames, height, width, reference_resize_mode) + target_rows(
|
| 250 |
height, width, num_frames
|
| 251 |
)
|
| 252 |
denoise = int(steps) * (STEP_LINEAR * sequence + STEP_QUADRATIC * sequence**2) * SAFETY
|
| 253 |
# The two reference encoders ahead of the loop, and the two decoders plus the mux after it. Both scale with what
|
| 254 |
# they are handed rather than with the step count.
|
| 255 |
+
encode = 5 + reference_rows(references, num_frames, height, width, reference_resize_mode) * 1e-3
|
| 256 |
decode = DECODE_BASE + DECODE_PER_DEFAULT_CANVAS * (height * width * num_frames) / DEFAULT_CANVAS_PIXELS
|
| 257 |
total = PLACEMENT_ALLOWANCE + encode + denoise + decode + 10 + LORA_ALLOWANCE * len(loras or ())
|
| 258 |
duration = max(MIN_GPU_DURATION, min(MAX_GPU_DURATION, math.ceil(total)))
|
|
|
|
| 260 |
return duration
|
| 261 |
|
| 262 |
|
| 263 |
+
def budget(text_tokens, references, height, width, num_frames, steps, loras=(), reference_resize_mode="legacy"):
|
| 264 |
"""`(rows, GPU seconds)` for one request, by the same formula as `get_duration`."""
|
| 265 |
+
sequence = int(text_tokens) + reference_rows(references, num_frames, height, width, reference_resize_mode) + target_rows(height, width, num_frames)
|
| 266 |
per_step = (STEP_LINEAR * sequence + STEP_QUADRATIC * sequence**2) * SAFETY
|
| 267 |
+
encode = 5 + reference_rows(references, num_frames, height, width, reference_resize_mode) * 1e-3
|
| 268 |
decode = DECODE_BASE + DECODE_PER_DEFAULT_CANVAS * (height * width * num_frames) / DEFAULT_CANVAS_PIXELS
|
| 269 |
overhead = PLACEMENT_ALLOWANCE + encode + decode + 10 + LORA_ALLOWANCE * len(loras or ())
|
| 270 |
return sequence, overhead + int(steps) * per_step, per_step, overhead
|
| 271 |
|
| 272 |
|
| 273 |
+
def fits(text_tokens, references, height, width, num_frames, steps, loras=(), reference_resize_mode="legacy"):
|
| 274 |
"""Stops a request the card or the reservation cannot take, before any GPU time is spent."""
|
| 275 |
sequence, total, per_step, overhead = budget(
|
| 276 |
+
text_tokens, references, height, width, num_frames, steps, loras, reference_resize_mode
|
| 277 |
)
|
| 278 |
ceiling = sequence_ceiling(loras)
|
| 279 |
if sequence <= ceiling and total <= MAX_GPU_DURATION:
|
|
|
|
| 330 |
|
| 331 |
lower_duration_floor()
|
| 332 |
manager = ComponentsManager()
|
| 333 |
+
blocks = efficient_blocks(MiniMaxH3Ref2VAGeneratorBlocks)()
|
| 334 |
print(f"[ref2va] loading {[c.name for c in blocks.expected_components]} from {MODEL_REPO} ...", flush=True)
|
| 335 |
pipe = blocks.init_pipeline(MODEL_REPO, components_manager=manager, collection="h3")
|
| 336 |
+
if QUANTIZATION not in ("bf16", "int8"):
|
| 337 |
+
raise ValueError("H3_QUANTIZATION must be bf16 or int8.")
|
| 338 |
+
if GPU_SIZE == "large" and QUANTIZATION != "int8":
|
| 339 |
+
raise ValueError("The unquantized H3 transformer cannot fit large. Use xlarge, or explicitly enable experimental int8.")
|
| 340 |
+
if QUANTIZATION == "int8":
|
| 341 |
+
from h3_quantization import load_int8_transformer
|
| 342 |
+
pipe.update_components(transformer_ref=load_int8_transformer(MODEL_REPO))
|
| 343 |
pipe.load_components(dtype=torch.bfloat16)
|
| 344 |
|
| 345 |
# Both VAEs first, and explicitly. `set_attention_backend` also sets the registry's *global* backend, which
|
|
|
|
| 355 |
# are identical field for field and the compiled code carries no weights of either.
|
| 356 |
import h3_aoti
|
| 357 |
|
| 358 |
+
if QUANTIZATION == "bf16":
|
| 359 |
+
h3_aoti.maybe_load(pipe.transformer_ref)
|
| 360 |
+
elif os.environ.get("H3_AOTI", "0") == "1":
|
| 361 |
+
raise ValueError("The existing AoTI artifact does not support the experimental INT8 transformer.")
|
| 362 |
|
| 363 |
if PLACEMENT == "offload":
|
| 364 |
+
manager.enable_auto_cpu_offload(device="cuda", memory_reserve_margin="10GB")
|
| 365 |
_arm_decode_hooks(pipe)
|
| 366 |
|
| 367 |
PIPE, MANAGER = pipe, manager
|
|
|
|
| 1043 |
image_9=None,
|
| 1044 |
match=True,
|
| 1045 |
duration=5,
|
| 1046 |
+
steps=6,
|
| 1047 |
seed=42,
|
| 1048 |
upsample=False,
|
| 1049 |
*lora_fields,
|
|
|
|
| 1093 |
requested = 0 if (match and derivable) else snap_frames(duration)
|
| 1094 |
|
| 1095 |
loras, lora_labels = collect_loras(lora_fields, progress)
|
| 1096 |
+
efficient = _efficient_conditioner_available()
|
| 1097 |
+
reference_resize_mode = "match" if efficient else "legacy"
|
| 1098 |
+
canvas = _resolve_canvas(canvas, images[0], steps, efficient)
|
| 1099 |
+
progress(0, desc="Preparing compact references" if efficient else "Preparing references with the existing conditioner")
|
| 1100 |
|
| 1101 |
# Before the conditioner spends GPU time: the canvas from the table, worst case for the frame count.
|
| 1102 |
planned_height, planned_width = CANVASES.get(canvas, CANVASES[DEFAULT_CANVAS])
|
|
|
|
| 1107 |
planned_width,
|
| 1108 |
requested or snap_frames(min(audio_bearing(references)[0][1], MAX_REFERENCE_VIDEO)),
|
| 1109 |
steps,
|
| 1110 |
+
loras, reference_resize_mode,
|
| 1111 |
)
|
| 1112 |
|
| 1113 |
_runtime_event("conditioner_begin")
|
|
|
|
| 1115 |
conditioned = time.time()
|
| 1116 |
try:
|
| 1117 |
prompt_embeds, text_token_tags, metadata, plan = encode_remote(
|
| 1118 |
+
prompt, references, canvas, requested, rewrite_prompt=upsample, session_id=session_id, reference_resize_mode=reference_resize_mode
|
| 1119 |
)
|
| 1120 |
except gr.Error:
|
| 1121 |
raise
|
|
|
|
| 1132 |
refined = plan.get("refined_prompt") or ""
|
| 1133 |
|
| 1134 |
# Again, with the exact numbers the conditioner returned.
|
| 1135 |
+
fits(int(text_token_tags.shape[0]), references, height, width, num_frames, steps, loras, reference_resize_mode)
|
| 1136 |
|
| 1137 |
progress(0.1, desc=f"Generating {num_frames / FPS:.1f} s at {width}x{height} ...")
|
| 1138 |
_runtime_event("gpu_request", frames=num_frames, steps=int(steps))
|
| 1139 |
started = time.time()
|
| 1140 |
frames, audio, sampling_rate = _generate(
|
| 1141 |
+
prompt_embeds, text_token_tags, references, height, width, num_frames, steps, seed, loras, reference_resize_mode
|
| 1142 |
)
|
| 1143 |
generate_seconds = time.time() - started
|
| 1144 |
|
|
|
|
| 1213 |
# themselves are deliberately left out: gradio hands them over as paths into a per-session temporary directory that
|
| 1214 |
# is gone by the next visit, so a saved path would restore as a dead file rather than as the image.
|
| 1215 |
|
| 1216 |
+
SETTINGS_VERSION = 3
|
| 1217 |
SETTINGS_KEYS = (
|
| 1218 |
["prompt", "upsample", "canvas", "match", "duration", "steps", "seed"]
|
| 1219 |
+ [f"lora_{slot + 1}" for slot in range(LORA_SLOTS)]
|
|
|
|
| 1620 |
return [gr.update(value=value) for value in slots]
|
| 1621 |
|
| 1622 |
|
| 1623 |
+
def _add_preset_lora(preset,*current):
|
| 1624 |
+
"""One acceleration adapter at a time; preserve unrelated effect slots."""
|
| 1625 |
+
reference,steps,_,strength=LORA_PRESETS[preset]
|
| 1626 |
+
slots=list(current[:LORA_SLOTS]);scales=list(current[LORA_SLOTS:])
|
| 1627 |
+
speed_refs={value[0] for value in LORA_PRESETS.values()}
|
| 1628 |
+
for i,value in enumerate(slots):
|
| 1629 |
+
if str(value or '').strip() in speed_refs:slots[i]=''
|
| 1630 |
+
try:index=next(i for i,value in enumerate(slots) if not str(value or '').strip())
|
| 1631 |
+
except StopIteration:raise gr.Error('Free one effect slot before choosing a speed adapter. Your settings were kept.')
|
| 1632 |
+
slots[index]=reference;scales[index]=strength
|
| 1633 |
+
return [*slots,*scales,steps]
|
| 1634 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1635 |
|
| 1636 |
|
| 1637 |
load_models()
|
|
|
|
| 1941 |
<a href="https://www.minimax.io/blog/minimax-h3" target="_blank" rel="noopener">blog</a> ·
|
| 1942 |
<a href="https://huggingface.co/spaces/multimodalart/minimax-h3" target="_blank" rel="noopener">text / image to video</a></p>
|
| 1943 |
<div class="pills">
|
| 1944 |
+
<span>33B</span><span>joint video + audio</span><span>Turbo: 4–8 real steps</span>
|
| 1945 |
<span>ComfyUI lora accepted</span><span>up to 9 references</span><span>5 custom lora slots</span><span>shared lora library</span><span>trigger words in the prompt</span><span>structured prompt builder</span><span>named profiles</span><span>CivitAI search + links</span><span>scene continuation</span><span>🔒 identity lock</span><span>kohya + LoKr auto-convert</span><span>GPU cost estimate</span><span>clip stitching with audio</span>
|
| 1946 |
</div>
|
| 1947 |
</div>
|
|
|
|
| 1974 |
to be converted by hand first.
|
| 1975 |
"""
|
| 1976 |
|
| 1977 |
+
TURBO_HELP = """Steps count actual model evaluations. Balanced uses **6**, Draft **4**, and Quality **8**.
|
| 1978 |
+
Video and audio keep their own schedules. Model transfer, conditioning and decoding also take time.
|
| 1979 |
+
One speed LoRA is enough; changing a quality preset replaces only known speed adapters.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1980 |
"""
|
| 1981 |
|
| 1982 |
PROFILE_HELP = """Profiles keep the prompt, the canvas, the sliders and the lora slots — everything typed rather than
|
|
|
|
| 3383 |
return _download_lora_source_unlocked(source, progress)
|
| 3384 |
|
| 3385 |
|
| 3386 |
+
LORA_MAX_FILE_BYTES = _lora_limit("H3_LORA_MAX_FILE_MB", 1536, 16384) * 1048576
|
| 3387 |
LORA_MAX_RUN_BYTES = _lora_limit("H3_LORA_MAX_RUN_MB", 2048, 32768) * 1048576
|
| 3388 |
LORA_DOWNLOAD_SECONDS = _lora_limit("H3_LORA_DOWNLOAD_SECONDS", 600, 1800)
|
| 3389 |
def _pick_label(index, item):
|
|
|
|
| 4265 |
values[_SCENE_INDEX['seed']],state['values'][_SCENE_INDEX['image']],min(total,state['index']+1))
|
| 4266 |
|
| 4267 |
|
| 4268 |
+
_CONDITIONER_EFFICIENT = None
|
| 4269 |
+
|
| 4270 |
+
|
| 4271 |
+
def _efficient_conditioner_available(cached_only=False):
|
| 4272 |
+
global _CONDITIONER_EFFICIENT
|
| 4273 |
+
if _CONDITIONER_EFFICIENT is not None:return _CONDITIONER_EFFICIENT
|
| 4274 |
+
if cached_only:return False
|
| 4275 |
+
# Client construction reads public configuration only; it starts no compute.
|
| 4276 |
+
client=conditioner()
|
| 4277 |
+
config=getattr(client,'config',{}) or {}
|
| 4278 |
+
_CONDITIONER_EFFICIENT=any(str(d.get('api_name','')).lstrip('/')=='encode_ref2va_efficient'
|
| 4279 |
+
for d in config.get('dependencies',[]))
|
| 4280 |
+
return _CONDITIONER_EFFICIENT
|
| 4281 |
+
|
| 4282 |
+
|
| 4283 |
+
def _resolve_canvas(canvas,image,steps,efficient=False):
|
| 4284 |
+
choices={k:v for k,v in (CANVASES if efficient else LEGACY_CANVASES).items() if k!=AUTO_CANVAS}
|
| 4285 |
+
if canvas!=AUTO_CANVAS:
|
| 4286 |
+
if canvas not in choices:
|
| 4287 |
+
raise gr.Error('This canvas needs the included efficient conditioner. Select Auto or a legacy canvas.')
|
| 4288 |
+
return canvas
|
| 4289 |
+
ratio=16/9
|
| 4290 |
+
if image:
|
| 4291 |
+
with Image.open(image) as im:
|
| 4292 |
+
im=ImageOps.exif_transpose(im);ratio=im.width/im.height
|
| 4293 |
+
area=300000 if int(steps)<=4 else (700000 if int(steps)>=8 else 450000)
|
| 4294 |
+
# Aspect ratio dominates; area selects the quality level within that family.
|
| 4295 |
+
return min(choices,key=lambda k:4*abs(math.log((choices[k][1]/choices[k][0])/ratio))
|
| 4296 |
+
+abs(math.log((choices[k][0]*choices[k][1])/area)))
|
| 4297 |
+
|
| 4298 |
+
|
| 4299 |
+
def _condition_key(session,prompt,references,canvas,frames,rewrite,reference_resize_mode="legacy"):
|
| 4300 |
if not session:return None
|
| 4301 |
files=[]
|
| 4302 |
for kind,path in references:
|
|
|
|
| 4304 |
identity=(hashlib.sha256(Path(path).read_bytes()).hexdigest() if kind=='image'
|
| 4305 |
else [os.path.realpath(path),stat.st_size,stat.st_mtime_ns])
|
| 4306 |
files.append([kind,identity])
|
| 4307 |
+
return hashlib.sha256(json.dumps([session,CONDITIONER_SPACE,PROTOCOL,reference_resize_mode,prompt,files,canvas,frames,bool(rewrite)],
|
| 4308 |
ensure_ascii=False).encode()).hexdigest()
|
| 4309 |
|
| 4310 |
|
| 4311 |
+
def encode_remote(prompt,references,canvas,num_frames,rewrite_prompt=False,session_id='',reference_resize_mode='legacy'):
|
| 4312 |
from gradio_client import handle_file
|
| 4313 |
from safetensors import safe_open
|
| 4314 |
+
key=_condition_key(session_id,prompt,references,canvas,num_frames,rewrite_prompt,reference_resize_mode)
|
| 4315 |
+
validate_resize_mode(reference_resize_mode)
|
| 4316 |
def read(path,plan):
|
| 4317 |
+
if reference_resize_mode == 'match':
|
| 4318 |
+
if plan.get('reference_protocol') != PROTOCOL or plan.get('reference_resize_mode') != 'match':
|
| 4319 |
+
raise gr.Error('Conditioner returned an incompatible reference plan. No video GPU request was sent.')
|
| 4320 |
+
if (int(plan.get('height', 0)), int(plan.get('width', 0))) != CANVASES[canvas]:
|
| 4321 |
+
raise gr.Error('Conditioner changed the requested canvas. No video GPU request was sent.')
|
| 4322 |
with safe_open(path,framework='pt') as handle:
|
| 4323 |
+
metadata=handle.metadata() or {}
|
| 4324 |
+
if reference_resize_mode=='match':
|
| 4325 |
+
expected={'reference_protocol':PROTOCOL,'reference_resize_mode':'match',
|
| 4326 |
+
'height':str(plan['height']),'width':str(plan['width']),'num_frames':str(plan['num_frames'])}
|
| 4327 |
+
if any(str(metadata.get(k,''))!=str(v) for k,v in expected.items()):
|
| 4328 |
+
raise gr.Error('Conditioning file metadata does not match its reference plan. No video GPU request was sent.')
|
| 4329 |
+
return handle.get_tensor('prompt_embeds'),handle.get_tensor('text_token_tags'),metadata,dict(plan)
|
| 4330 |
with _CONDITION_LOCK:
|
| 4331 |
now=time.monotonic()
|
| 4332 |
for k,record in list(_CONDITION_CACHE.items()):
|
|
|
|
| 4341 |
_CONDITION_CACHE.pop(key,None)
|
| 4342 |
job=None
|
| 4343 |
try:
|
| 4344 |
+
fields=dict(prompt=prompt,media=[handle_file(p) for _,p in references],
|
| 4345 |
+
kinds=','.join(k for k,_ in references),num_frames=num_frames,rewrite_prompt=bool(rewrite_prompt))
|
| 4346 |
+
if reference_resize_mode == 'match':
|
| 4347 |
+
fields.update(height=CANVASES[canvas][0],width=CANVASES[canvas][1],reference_resize_mode='match',
|
| 4348 |
+
api_name='/encode_ref2va_efficient')
|
| 4349 |
+
else:
|
| 4350 |
+
fields.update(canvas=canvas,api_name='/encode_ref2va')
|
| 4351 |
+
job=conditioner().submit(**fields)
|
| 4352 |
path,plan=job.result(timeout=300)
|
| 4353 |
result=read(path,plan)
|
| 4354 |
except Exception as error:
|
|
|
|
| 4373 |
|
| 4374 |
|
| 4375 |
@spaces.GPU(duration=get_duration,size=GPU_SIZE)
|
| 4376 |
+
def _generate(prompt_embeds,text_token_tags,references,height,width,num_frames,steps,seed,loras=(),reference_resize_mode="legacy"):
|
| 4377 |
import torch
|
| 4378 |
state=None;attached=[]
|
| 4379 |
started=time.perf_counter()
|
|
|
|
| 4385 |
_runtime_event('denoising',adapter_files=len(attached))
|
| 4386 |
state=PIPE(prompt_embeds=prompt_embeds.to('cuda'),text_token_tags=text_token_tags,
|
| 4387 |
references=build_references(references),height=height,width=width,num_frames=num_frames,
|
| 4388 |
+
num_inference_steps=scheduler_points(steps),reference_resize_mode=reference_resize_mode,
|
| 4389 |
+
generator=torch.Generator('cpu').manual_seed(int(seed)))
|
| 4390 |
return state.get('videos')[0],state.get('audio')[0].cpu(),state.get('sampling_rate')
|
| 4391 |
except RuntimeError as error:
|
| 4392 |
_runtime_event('gpu_failed',error_type=type(error).__name__)
|
|
|
|
| 4460 |
active=[ref for ref,scale in zip(refs,scales) if ref and float(scale or 0)!=0]
|
| 4461 |
if mode=='Extra H3 reference' and identity and float(strength)>0 and identity not in images:images.append(identity)
|
| 4462 |
try:
|
| 4463 |
+
references=collect(images,audio,video)
|
| 4464 |
+
efficient = _efficient_conditioner_available(cached_only=True)
|
| 4465 |
+
policy = 'match' if efficient else 'legacy'
|
| 4466 |
+
canvas = _resolve_canvas(canvas, images[0] if images else None, steps, efficient)
|
| 4467 |
+
height,width=CANVASES[canvas]
|
| 4468 |
seconds=float(duration)
|
| 4469 |
carried=audio_bearing(references)
|
| 4470 |
if match and len(carried)==1:seconds=carried[0][1]
|
| 4471 |
frames=snap_frames(seconds)
|
| 4472 |
+
rows,total,_,_=budget(TEXT_TOKEN_ALLOWANCE,references,height,width,frames,steps,active,policy)
|
| 4473 |
if rows>sequence_ceiling(active):return '🚫 Too large for this model. Shorten the clip, choose a smaller canvas or remove a reference.'
|
| 4474 |
if total>MAX_GPU_DURATION:return '🚫 This request exceeds the configured runtime limit. Lower steps, duration or canvas size.'
|
| 4475 |
durations=_scene_durations(timing,_scene_count(count),duration)
|
| 4476 |
+
reservations=[max(MIN_GPU_DURATION,min(MAX_GPU_DURATION,math.ceil(budget(TEXT_TOKEN_ALLOWANCE,references,height,width,snap_frames(d),steps,active,policy)[1]))) for d in durations]
|
| 4477 |
reservation=max(MIN_GPU_DURATION,min(MAX_GPU_DURATION,math.ceil(total)))
|
| 4478 |
factor=2 if GPU_SIZE=='xlarge' else 1
|
| 4479 |
+
return (f'**Estimated allowance for one clip: ~{reservation*factor} seconds** · {width}×{height} · {int(steps)} real steps\n\n'
|
| 4480 |
f'Scene estimate: ~{sum(reservations)*factor} quota-seconds for {len(durations)} clips. '
|
| 4481 |
'The remote conditioner and optional AI writer are additional requests; their quota is not included. '
|
| 4482 |
+
f'This is an estimate, not a bill. Reference preparation: {policy}. Identical inputs can reuse conditioning. '
|
| 4483 |
'References for later clips may change the estimate.')
|
| 4484 |
except Exception as e:return f'Estimate unavailable ({type(e).__name__}); check uploads and per-clip timings.'
|
| 4485 |
|
| 4486 |
|
| 4487 |
+
def quality_indicator(steps,*fields):
|
| 4488 |
+
refs=list(fields[:LORA_SLOTS]);scales=list(fields[LORA_SLOTS:])
|
| 4489 |
+
presets=list(LORA_PRESETS.values())
|
| 4490 |
+
known={preset[0] for preset in presets}
|
| 4491 |
+
active=[(ref,float(scale or 0)) for ref,scale in zip(refs,scales) if ref in known and float(scale or 0)!=0]
|
| 4492 |
+
label={4:'Draft · quickest',6:'Balanced · recommended',8:'Quality · more detail'}.get(float(steps))
|
| 4493 |
+
if active!=[(presets[0][0],1.0)]:label=None
|
| 4494 |
+
note=f'{int(steps)} real steps. ' + ('Custom settings are active.' if label is None else 'Quality preset active; your length and custom effects are kept.')
|
| 4495 |
+
return gr.update(value=label,label='Quality' if label else 'Quality · custom settings'),note
|
| 4496 |
+
|
| 4497 |
+
|
| 4498 |
def budget_recipe(recipe,*fields):
|
| 4499 |
+
if not recipe:
|
| 4500 |
+
return (*[gr.update() for _ in range(3+2*LORA_SLOTS)],'Custom settings kept.')
|
| 4501 |
refs=list(fields[:LORA_SLOTS]);scales=list(fields[LORA_SLOTS:]);presets=list(LORA_PRESETS.values())
|
| 4502 |
turbo_urls={p[0] for p in presets}
|
| 4503 |
for i,ref in enumerate(refs):
|
| 4504 |
if ref in turbo_urls:refs[i]=''
|
| 4505 |
if recipe=='Original quality · 28 steps':
|
| 4506 |
+
return (gr.update(),gr.update(),28,*refs,*scales,'Original model: 28 real evaluations.')
|
| 4507 |
try:i=refs.index('')
|
| 4508 |
+
except ValueError:raise gr.Error('The quality preset needs a free LoRA slot; your settings were kept.')
|
| 4509 |
+
fast=recipe.startswith('Draft');quality=recipe.startswith('Quality')
|
| 4510 |
+
count=4 if fast else (8 if quality else 6)
|
| 4511 |
refs[i]=presets[0][0];scales[i]=1.0
|
| 4512 |
+
return (gr.update(),gr.update(),count,*refs,*scales,
|
| 4513 |
+
f'{count} real steps. Auto canvas follows your picture. Duration and custom effects are kept.')
|
| 4514 |
|
| 4515 |
|
| 4516 |
def _profile_updates(payload):
|
|
|
|
| 4648 |
gr.HTML("""<div class="studio-hero"><small>MINIMAX-H3 · IMAGE + MOTION + SOUND</small>
|
| 4649 |
<h1>MiniMax-H3 Studio</h1><p>One idea → a planned scene, made one clip at a time.</p>
|
| 4650 |
<div class="studio-features"><span>🆕 AI Scene Planner · 64 clips</span><span>🔒 CPU Identity Lock</span>
|
| 4651 |
+
<span>⚡ Balanced 6-step presets</span><span>🔄 LoRA Refresh + trigger sync</span><span>🔊 Native audio + soundtrack mixer</span></div>
|
| 4652 |
<p>Simple for quick creation. Pro for all controls. Downloads and LoRA conversion finish before GPU generation.</p></div>""")
|
| 4653 |
with gr.Accordion("Runtime diagnostics", open=False):
|
| 4654 |
diagnostics_button = gr.Button("Check runtime / download report")
|
|
|
|
| 4675 |
)
|
| 4676 |
upsample = gr.Checkbox(label="✨ Upsample prompt", value=False, visible=False)
|
| 4677 |
|
| 4678 |
+
with gr.Accordion("✨ Help with the prompt & effects", open=False) as basic_panel:
|
| 4679 |
gr.Markdown(
|
| 4680 |
"**Picture in, a few words above, one press.** The description is "
|
| 4681 |
"written from your first reference picture, wrapped in the labelled "
|
|
|
|
| 4684 |
"changed with a tick."
|
| 4685 |
)
|
| 4686 |
basic_space = gr.Textbox(
|
| 4687 |
+
value=REMOTE_SPACE, label="🛰️ Writing Space", lines=1, visible=False,
|
| 4688 |
placeholder="owner/space-name",
|
| 4689 |
info="A chat Space of your own, shown your first picture. Empty it "
|
| 4690 |
"and only the structured builder runs, on your own words.",
|
| 4691 |
)
|
| 4692 |
+
basic_btn = gr.Button("✨ Improve my description", variant="secondary",
|
| 4693 |
elem_id="ir-btn")
|
| 4694 |
basic_status = gr.Markdown("Nothing done yet.")
|
| 4695 |
basic_pick = gr.CheckboxGroup(
|
|
|
|
| 4707 |
gr.Markdown(
|
| 4708 |
"H3 was trained on the output of a preprocessor that rewrites a request into labelled "
|
| 4709 |
"sections, and MiniMax call that structure *critical to the quality of the final output*. "
|
| 4710 |
+
"Generate automatically adds the required structure. Describe the shot "
|
| 4711 |
"in the prompt box above, set the pieces below, and press **Build**.\n\n"
|
| 4712 |
"**Dialogue has to be verbatim.** Speech is generated together with the picture, so naming "
|
| 4713 |
"that someone speaks without giving the words produces correct mouth shapes with nothing in "
|
|
|
|
| 4738 |
with gr.Row(elem_classes="chip-row"):
|
| 4739 |
chip_buttons_b = [gr.Button(text, size="sm", variant="secondary") for text in CHIPS[4:]]
|
| 4740 |
|
| 4741 |
+
# One picture is the complete default reference UI.
|
| 4742 |
with gr.Group(elem_classes="panel"):
|
| 4743 |
+
images = [gr.Image(label="🖼️ Your picture", type="filepath", height=260)]
|
| 4744 |
+
with gr.Accordion("More references & Identity Lock", open=False) as extra_references:
|
| 4745 |
+
gr.Markdown("The first picture supplies identity for the scene. Add more media only when it helps your shot.")
|
| 4746 |
+
with gr.Tabs():
|
| 4747 |
+
with gr.Tab("Pictures & identity"):
|
| 4748 |
+
with gr.Row():
|
| 4749 |
+
images.extend(gr.Image(label=f"Reference {index+1}",type="filepath",height=180,
|
| 4750 |
+
min_width=160,visible=False) for index in range(1,MAX_IMAGE_SLOTS))
|
| 4751 |
+
add_image=gr.Button("+ Add another picture",size="sm")
|
| 4752 |
+
identity_ref=gr.Image(label="Identity portrait (optional)",type="filepath",height=160)
|
| 4753 |
+
identity_mode=gr.Radio(["Off","CPU face protection","Extra H3 reference"],
|
| 4754 |
+
value="CPU face protection",label="Identity method",visible=False)
|
| 4755 |
+
identity_strength=gr.Slider(0,1,value=.65,step=.05,label="Face correction strength",visible=False)
|
| 4756 |
+
gr.Markdown("Leave the portrait empty to use the scene’s original picture. CPU protection guides continuation frames; it cannot guarantee identity in every frame.")
|
| 4757 |
+
with gr.Tab("Audio reference"):
|
| 4758 |
+
audio=gr.Audio(label="Voice or music",type="filepath")
|
| 4759 |
+
with gr.Tab("Motion reference"):
|
| 4760 |
+
video=gr.Video(label="Motion or camera reference, 2–15 seconds")
|
| 4761 |
+
duration=gr.Slider(label="Clip length (seconds)",minimum=MIN_DURATION,maximum=MAX_UI_DURATION,
|
| 4762 |
+
step=.1,value=3)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4763 |
|
| 4764 |
# ---------------- speed ----------------
|
| 4765 |
with gr.Group(elem_classes="panel"):
|
| 4766 |
gr.Markdown("### ⚡ Speed & cost")
|
| 4767 |
+
budget_choice = gr.Dropdown(["Balanced · recommended", "Draft · quickest", "Quality · more detail"],
|
| 4768 |
+
value="Balanced · recommended", label="Quality")
|
| 4769 |
+
budget_button = gr.Button("Reapply quality preset", variant="secondary", visible=False)
|
| 4770 |
+
budget_status = gr.Markdown("Balanced uses 6 real steps. Picture shape is automatic; sound is generated with the video.")
|
| 4771 |
+
with gr.Group(visible=False) as pro_speed:
|
| 4772 |
+
gr.Markdown(TURBO_HELP, elem_classes="turbo-blurb")
|
| 4773 |
+
with gr.Row():
|
| 4774 |
+
lora_preset = gr.Dropdown(
|
| 4775 |
+
label="Preset",
|
| 4776 |
+
choices=list(LORA_PRESETS),
|
| 4777 |
+
value=list(LORA_PRESETS)[0],
|
| 4778 |
+
scale=4,
|
| 4779 |
+
)
|
| 4780 |
+
lora_preset_add = gr.Button("⚡ Use this speed adapter", variant="secondary", scale=1,
|
| 4781 |
+
elem_id="turbo-btn")
|
| 4782 |
+
turbo_blurb = gr.Markdown(
|
| 4783 |
+
LORA_PRESETS[list(LORA_PRESETS)[0]][2], elem_classes="turbo-blurb"
|
| 4784 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4785 |
|
| 4786 |
+
with gr.Accordion("LoRA library, profiles & output settings", open=False) as studio_tools:
|
| 4787 |
+
# ---------------- the rest, in tabs ----------------
|
| 4788 |
+
with gr.Tabs():
|
| 4789 |
+
with gr.Tab(f"⭐ Custom lora ({LORA_SLOTS} slots)", visible=False) as pro_loratab:
|
| 4790 |
+
gr.Markdown(LORA_HELP)
|
| 4791 |
+
with gr.Accordion("🔍 Search CivitAI", open=False):
|
| 4792 |
+
gr.Markdown(
|
| 4793 |
+
"Search CivitAI without leaving the Space, then drop a result straight into a slot. "
|
| 4794 |
+
"Each hit shows its size, downloads and trigger words."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4795 |
)
|
| 4796 |
+
with gr.Row():
|
| 4797 |
+
search_query = gr.Textbox(label="Search", placeholder="e.g. dance, rain, camera move",
|
| 4798 |
+
scale=3)
|
| 4799 |
+
search_base = gr.Dropdown(H3_BASE_MODELS, value=H3_BASE_MODELS[0], label="Base model",
|
| 4800 |
+
allow_custom_value=True, scale=2)
|
| 4801 |
+
with gr.Row():
|
| 4802 |
+
search_nsfw = gr.Checkbox(value=True, label="Include NSFW results")
|
| 4803 |
+
search_btn = gr.Button("🔍 Search", variant="secondary", elem_id="search-btn")
|
| 4804 |
+
search_pick = gr.Dropdown(choices=[], label="Pick a file")
|
| 4805 |
+
with gr.Row():
|
| 4806 |
+
search_slot = gr.Dropdown([f"lora {i}" for i in range(1, LORA_SLOTS + 1)],
|
| 4807 |
+
value="lora 1", label="Into slot", scale=2)
|
| 4808 |
+
search_put_btn = gr.Button("⬇️ Put it in", variant="primary", scale=1,
|
| 4809 |
+
elem_id="search-put")
|
| 4810 |
+
# Below the picker on purpose: a wall of results above it would push the
|
| 4811 |
+
# controls off the screen, which is exactly what happened the first time.
|
| 4812 |
+
with gr.Accordion("📋 The results", open=False):
|
| 4813 |
+
search_results = gr.Markdown("No search yet.")
|
| 4814 |
+
search_state = gr.State({})
|
| 4815 |
+
|
| 4816 |
+
lora_references, lora_scales = [], []
|
| 4817 |
+
for slot in range(LORA_SLOTS):
|
| 4818 |
+
with gr.Row():
|
| 4819 |
+
lora_references.append(
|
| 4820 |
+
gr.Textbox(label=f"lora {slot + 1}", placeholder="owner/repo", scale=3, value=list(LORA_PRESETS.values())[0][0] if slot == 0 else "")
|
| 4821 |
)
|
| 4822 |
+
lora_scales.append(
|
| 4823 |
+
gr.Slider(
|
| 4824 |
+
label="Strength",
|
| 4825 |
+
minimum=LORA_MIN_SCALE,
|
| 4826 |
+
maximum=LORA_MAX_SCALE,
|
| 4827 |
+
step=0.05,
|
| 4828 |
+
# Most H3 adapters on CivitAI are written up for 0.5, so an empty slot starts
|
| 4829 |
+
# there rather than at 1.0. A Turbo preset overwrites it with its own number.
|
| 4830 |
+
value=1.0 if slot == 0 else DEFAULT_LORA_SCALE,
|
| 4831 |
+
scale=2,
|
| 4832 |
+
)
|
| 4833 |
+
)
|
| 4834 |
+
with gr.Row():
|
| 4835 |
+
lora_identify_btn = gr.Button("🔎 name the links", variant="secondary",
|
| 4836 |
+
elem_id="lora-identify")
|
| 4837 |
+
with gr.Accordion("📋 The named links", open=False):
|
| 4838 |
+
lora_names = gr.Markdown("Nothing in the slots yet.", elem_classes="turbo-blurb")
|
| 4839 |
+
lora_upload = gr.File(
|
| 4840 |
+
label="Drop .safetensors here to fill the slots",
|
| 4841 |
+
file_count="multiple",
|
| 4842 |
+
file_types=[".safetensors"],
|
| 4843 |
+
type="filepath",
|
| 4844 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4845 |
|
| 4846 |
+
lora_library.library_tab(lora_slots=lora_references, scale_slots=lora_scales,
|
| 4847 |
+
prompt_box=prompt)
|
| 4848 |
+
|
| 4849 |
+
with gr.Tab("🎛️ Output"):
|
| 4850 |
+
canvas = gr.Dropdown(label="Canvas", choices=list(CANVASES), value=DEFAULT_CANVAS)
|
| 4851 |
+
match = gr.Checkbox(label="Match the reference soundtrack", value=True, visible=False)
|
| 4852 |
+
steps = gr.Slider(label="Steps", minimum=MIN_STEPS, maximum=40, step=1,
|
| 4853 |
+
value=6, visible=False)
|
| 4854 |
+
with gr.Row(visible=False) as pro_seed:
|
| 4855 |
+
seed = gr.Number(label="Seed", value=42, precision=0, scale=3)
|
| 4856 |
+
seed_dice = gr.Button("🎲 roll", variant="secondary", scale=1, elem_id="seed-dice")
|
| 4857 |
+
randomize_seed = gr.Checkbox(
|
| 4858 |
+
label="🎲 Randomize seed on every run",
|
| 4859 |
+
value=True,
|
| 4860 |
+
visible=False,
|
| 4861 |
+
info="A new seed is drawn each time Generate is pressed, and lands in the box above.",
|
| 4862 |
)
|
| 4863 |
+
|
| 4864 |
+
with gr.Tab("💾 Profiles", visible=False) as pro_profiles:
|
| 4865 |
+
gr.Markdown(PROFILE_HELP)
|
| 4866 |
+
with gr.Row():
|
| 4867 |
+
profile_picker = gr.Dropdown(
|
| 4868 |
+
label="Saved profiles",
|
| 4869 |
+
choices=[NO_PROFILE, *list_profiles()],
|
| 4870 |
+
value=NO_PROFILE,
|
| 4871 |
+
scale=3,
|
| 4872 |
+
)
|
| 4873 |
+
profile_load = gr.Button("📂 load", variant="secondary", scale=1, elem_id="profile-load")
|
| 4874 |
+
profile_refresh = gr.Button("🔄", variant="secondary", scale=1, min_width=60, elem_id="profile-refresh")
|
| 4875 |
+
with gr.Row():
|
| 4876 |
+
profile_name = gr.Textbox(
|
| 4877 |
+
label="Save as", placeholder="e.g. neon street, turbo 8 steps", max_lines=1, scale=3
|
| 4878 |
+
)
|
| 4879 |
+
profile_save = gr.Button("💾 save", variant="primary", scale=1, elem_id="profile-save")
|
| 4880 |
+
profile_delete = gr.Button("🗑️ delete the selected profile", variant="secondary",
|
| 4881 |
+
elem_id="profile-delete")
|
| 4882 |
+
profile_status = gr.Markdown("")
|
| 4883 |
+
gr.Markdown("---")
|
| 4884 |
+
gr.Markdown("**Take it with you (.json file)**")
|
| 4885 |
+
save = gr.Button("⬇️ export the current settings to .json", size="sm")
|
| 4886 |
+
settings_download = gr.File(label="Your settings", visible=False, interactive=False)
|
| 4887 |
+
settings_upload = gr.File(
|
| 4888 |
+
label="📤 import a settings .json", file_types=[".json"], type="filepath"
|
| 4889 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4890 |
|
| 4891 |
# ---------------- output ----------------
|
| 4892 |
with gr.Column(scale=6):
|
|
|
|
| 4900 |
with gr.Group(elem_classes="panel"):
|
| 4901 |
scene_progress = gr.HTML(_scene_progress(0, 1, phase="ready"))
|
| 4902 |
result = gr.Video(label="🎞️ Video + soundtrack", height=560)
|
| 4903 |
+
with gr.Accordion("🆕 Make a longer scene · AI planner", open=False) as scene_tools:
|
| 4904 |
+
gr.Markdown("### 🆕 AI Scene Planner")
|
| 4905 |
+
scene_idea = gr.Textbox(label="The whole scene", lines=3,
|
| 4906 |
+
placeholder="The character turns, crouches, then claps while crouching.")
|
| 4907 |
+
with gr.Row():
|
| 4908 |
+
auto_count = gr.Checkbox(value=True, label="Choose clip count automatically", visible=False)
|
| 4909 |
+
auto_seconds = gr.Checkbox(value=True, label="Choose each clip’s duration automatically", visible=False)
|
| 4910 |
+
planner_button = gr.Button("🪄 Split into clips", variant="primary")
|
| 4911 |
+
planner_status = gr.HTML("Describe the actions; planning is a separate optional writer request.")
|
| 4912 |
+
scene_pick = gr.CheckboxGroup(choices=[], value=[], visible=False, label="Scene LoRAs · shared with prompt")
|
| 4913 |
+
scene_pick_note = gr.Markdown("")
|
| 4914 |
+
scene_links = gr.Markdown("")
|
| 4915 |
+
scene_refresh = gr.Button("🔄 Refresh relevant LoRAs ↻")
|
| 4916 |
+
chain_count = gr.Slider(
|
| 4917 |
+
1, 64, value=3, step=1, label="How many clips in a row", visible=False,
|
| 4918 |
+
info="1–64 clips; each is a separate generation request.",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4919 |
)
|
| 4920 |
+
with gr.Accordion("Review or edit the clip prompts and timings", open=False) as pro_perclip:
|
| 4921 |
+
scene_prompts = gr.Textbox(
|
| 4922 |
+
label="One line per clip", lines=6, max_lines=64,
|
| 4923 |
+
placeholder=("line 1 = clip 1, line 2 = clip 2, and so on\n"
|
| 4924 |
+
"she turns toward the window\n"
|
| 4925 |
+
"she smiles and looks down\n"
|
| 4926 |
+
"…"),
|
| 4927 |
+
info="Leave a line empty — or the whole box — and that clip uses the "
|
| 4928 |
+
"main prompt, unchanged.",
|
| 4929 |
+
)
|
| 4930 |
+
scene_seconds = gr.Textbox(label="Seconds per clip", placeholder="3, 2.33, 3.75",
|
| 4931 |
+
info="One number per clip; blank uses the main duration. Times follow H3’s supported frame grid.")
|
| 4932 |
+
plan_preview = gr.HTML("")
|
| 4933 |
+
with gr.Row():
|
| 4934 |
+
chain_btn = gr.Button("🎬 Make the whole scene and join it",
|
| 4935 |
+
variant="primary", elem_id="extend-btn")
|
| 4936 |
+
chain_stop_btn = gr.Button("⏹ Stop", variant="stop")
|
| 4937 |
+
chain_from = gr.Number(
|
| 4938 |
+
value=1, precision=0, minimum=1, maximum=64, label="Start at clip", visible=False,
|
| 4939 |
+
info="Leave it at 1. After a stop it points at the clip that did not get "
|
| 4940 |
+
"made, so pressing 🎬 again carries on with the right prompt "
|
| 4941 |
+
"line instead of starting the sheet over.",
|
| 4942 |
+
)
|
| 4943 |
+
extend_btn = gr.Button("➕ Just one more clip", variant="secondary")
|
| 4944 |
+
gr.Markdown(
|
| 4945 |
+
"Press 🎬 once and leave it. Each clip starts on the last frame of the one "
|
| 4946 |
+
"before it. Completed clips stay queued; joining runs once at the end with sound. "
|
| 4947 |
+
"Stop keeps finished clips. Identity Lock can reduce drift; it cannot guarantee an identical face. \n"
|
| 4948 |
+
"➕ does the same thing one clip at a time, for when you want to change "
|
| 4949 |
+
"something in between.",
|
| 4950 |
+
elem_classes="turbo-blurb",
|
| 4951 |
+
)
|
| 4952 |
+
|
| 4953 |
# An output, so it can be revealed only for a request that asked for a rewrite.
|
| 4954 |
with gr.Accordion("Upsampled prompt", open=False, visible=False) as upsampled_panel:
|
| 4955 |
upsampled = gr.Textbox(show_label=False, lines=8, interactive=False)
|
| 4956 |
+
with gr.Accordion("🎞️ Joined scene & finished clips", open=False):
|
| 4957 |
gr.Markdown(
|
| 4958 |
"Add clips to a queue and they are joined into a single file, **soundtrack "
|
| 4959 |
"included**. Three clips make one long video for no extra GPU time. Everything "
|
|
|
|
| 5099 |
[edited_video, edited_file], api_name=False, concurrency_id="h3-heavy", concurrency_limit=1)
|
| 5100 |
loop_button.click(loop_finished, [result, merged_video, edit_target, loop_count], [edited_video, edited_file],
|
| 5101 |
api_name=False, concurrency_id="h3-heavy", concurrency_limit=1)
|
| 5102 |
+
budget_choice.input(budget_recipe, [budget_choice, *lora_references, *lora_scales],
|
| 5103 |
+
[canvas, duration, steps, *lora_references, *lora_scales, budget_status], api_name=False)
|
| 5104 |
budget_button.click(budget_recipe, [budget_choice, *lora_references, *lora_scales],
|
| 5105 |
[canvas, duration, steps, *lora_references, *lora_scales, budget_status], api_name=False)
|
| 5106 |
+
for field in (steps, *lora_references, *lora_scales):
|
| 5107 |
+
field.change(quality_indicator, [steps, *lora_references, *lora_scales],
|
| 5108 |
+
[budget_choice, budget_status], queue=False, show_progress='hidden', api_name=False)
|
| 5109 |
|
| 5110 |
# Searching CivitAI, and dropping a result into a slot.
|
| 5111 |
search_btn.click(
|
|
|
|
| 5204 |
_planned.success(_picker_refresh, _PICK_INPUTS, _PICK_OUTPUTS, api_name=False)
|
| 5205 |
|
| 5206 |
_PRO_ONLY = [pro_builder, pro_chips, upsample, pro_loratab, pro_profiles,
|
| 5207 |
+
steps, pro_seed, randomize_seed, basic_space, pro_speed, budget_button,
|
| 5208 |
+
identity_mode, identity_strength, auto_count, auto_seconds, chain_count, chain_from]
|
| 5209 |
|
| 5210 |
def _switch_mode(mode):
|
| 5211 |
pro = str(mode) == "pro"
|
| 5212 |
+
return [gr.update(visible=pro) for _ in _PRO_ONLY] + [gr.update(visible=True)]
|
| 5213 |
|
| 5214 |
ui_mode.change(_switch_mode, [ui_mode], _PRO_ONLY + [basic_panel], api_name=False)
|
| 5215 |
|
h3_efficiency.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared CPU policy for the H3 generator and its optional efficient conditioner.
|
| 2 |
+
|
| 3 |
+
No model loads, networking, CUDA initialization or monkey-patches at import.
|
| 4 |
+
The setup extension uses the pinned Diffusers validation and media normalization.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import math
|
| 9 |
+
from PIL import Image
|
| 10 |
+
|
| 11 |
+
PROTOCOL = "h3-reference-budget-v1"
|
| 12 |
+
RESIZE_MODES = ("legacy", "match")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def validate_resize_mode(mode):
|
| 16 |
+
if mode not in RESIZE_MODES:
|
| 17 |
+
raise ValueError("Unsupported H3 reference resize policy.")
|
| 18 |
+
return mode
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def scheduler_points(evaluations):
|
| 22 |
+
n = float(evaluations)
|
| 23 |
+
if not math.isfinite(n) or not n.is_integer() or not 4 <= n <= 40:
|
| 24 |
+
raise ValueError("H3 steps must be an integer from 4 to 40.")
|
| 25 |
+
# The pinned MiniMaxH3Scheduler includes terminal zero in the point count.
|
| 26 |
+
return int(n) + 1
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def reference_size(size, canvas, mode="match", multiple=32):
|
| 30 |
+
validate_resize_mode(mode)
|
| 31 |
+
width, height = map(int, size)
|
| 32 |
+
target_width, target_height = map(int, canvas)
|
| 33 |
+
if min(width, height, target_width, target_height) <= 0:
|
| 34 |
+
raise ValueError("Image and canvas dimensions must be positive.")
|
| 35 |
+
if max(width / height, height / width) > 4:
|
| 36 |
+
raise ValueError("H3 image references must have an aspect ratio between 1:4 and 4:1.")
|
| 37 |
+
if mode == "legacy":
|
| 38 |
+
scale = 2048 / min(width, height)
|
| 39 |
+
return tuple(max(multiple, round(edge * scale / multiple) * multiple) for edge in (width, height))
|
| 40 |
+
scale = min(1.0, math.sqrt(target_width * target_height / (width * height)))
|
| 41 |
+
# Floor rather than round: the policy never exceeds its source or area cap,
|
| 42 |
+
# except the mandatory minimum cell for tiny (<32 px) images.
|
| 43 |
+
return tuple(max(multiple, int(edge * scale) // multiple * multiple) for edge in (width, height))
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def resize_reference(image, canvas, mode="match"):
|
| 47 |
+
size = reference_size(image.size, canvas, mode)
|
| 48 |
+
return image if image.size == size else image.resize(size, Image.Resampling.LANCZOS)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def efficient_blocks(original_blocks_type):
|
| 52 |
+
"""Create a local block class; never mutate the upstream or global pipeline."""
|
| 53 |
+
from diffusers.modular_pipelines.minimax_h3.before_encoder import MiniMaxH3Ref2VASetupStep
|
| 54 |
+
from diffusers.modular_pipelines.modular_pipeline_utils import InputParam
|
| 55 |
+
from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference
|
| 56 |
+
|
| 57 |
+
class BudgetReferenceSetup(MiniMaxH3Ref2VASetupStep):
|
| 58 |
+
@property
|
| 59 |
+
def inputs(self):
|
| 60 |
+
return [*super().inputs, InputParam(
|
| 61 |
+
name="reference_resize_mode", type_hint=str, default="legacy",
|
| 62 |
+
description="Negotiated reference policy; must match the conditioning encoder.")]
|
| 63 |
+
|
| 64 |
+
def __call__(self, components, state):
|
| 65 |
+
# Preserve all upstream validation, audio handling and frame-grid rules.
|
| 66 |
+
# Its image enlargement is CPU-only; the replacement below occurs
|
| 67 |
+
# before either the Qwen encoder or the video VAE consumes the images.
|
| 68 |
+
mode = validate_resize_mode(self.get_block_state(state).reference_resize_mode)
|
| 69 |
+
components, state = super().__call__(components, state)
|
| 70 |
+
if mode == "match":
|
| 71 |
+
block = self.get_block_state(state)
|
| 72 |
+
canvas = (block.width, block.height)
|
| 73 |
+
block.normalized_references = [
|
| 74 |
+
MiniMaxH3ImageReference(image=resize_reference(original.image, canvas, mode))
|
| 75 |
+
if original.kind == "image" else normalized
|
| 76 |
+
for original, normalized in zip(block.references, block.normalized_references)
|
| 77 |
+
]
|
| 78 |
+
self.set_block_state(state, block)
|
| 79 |
+
return components, state
|
| 80 |
+
|
| 81 |
+
class EfficientBlocks(original_blocks_type):
|
| 82 |
+
block_classes = [BudgetReferenceSetup if cls is MiniMaxH3Ref2VASetupStep else cls
|
| 83 |
+
for cls in original_blocks_type.block_classes]
|
| 84 |
+
|
| 85 |
+
return EfficientBlocks
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def validate_canvas(height, width):
|
| 89 |
+
raw = (float(height), float(width))
|
| 90 |
+
if any(not math.isfinite(x) or not x.is_integer() for x in raw):
|
| 91 |
+
raise ValueError("Canvas dimensions must be whole numbers.")
|
| 92 |
+
height, width = map(int, raw)
|
| 93 |
+
if min(height, width) < 256 or max(height, width) > 1536 or height % 32 or width % 32:
|
| 94 |
+
raise ValueError("H3 canvas must use multiples of 32, between 256 and 1536 pixels.")
|
| 95 |
+
if height * width > 1_200_000 or max(height / width, width / height) > 4:
|
| 96 |
+
raise ValueError("Canvas exceeds the supported area or aspect ratio.")
|
| 97 |
+
return height, width
|
h3_quantization.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Owner-opt-in INT8 loading. Conversion is at startup, never inside a GPU job.
|
| 2 |
+
|
| 3 |
+
Based on the official MiniMax-H3 Diffusers memory recipe. Kept separate because
|
| 4 |
+
GPU/PEFT/ZeroGPU compatibility must be measured on the owner's actual runtime.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def load_int8_transformer(repo):
|
| 9 |
+
import torch
|
| 10 |
+
try:
|
| 11 |
+
from diffusers import MiniMaxH3Transformer3DModel, TorchAoConfig
|
| 12 |
+
from torchao.quantization import Int8WeightOnlyConfig
|
| 13 |
+
except ImportError as error:
|
| 14 |
+
raise RuntimeError(
|
| 15 |
+
"INT8 needs the optional requirements-int8.txt dependencies. "
|
| 16 |
+
"Use H3_QUANTIZATION=bf16 for the existing supported path."
|
| 17 |
+
) from error
|
| 18 |
+
model = MiniMaxH3Transformer3DModel.from_pretrained(
|
| 19 |
+
repo, subfolder="transformer_ref", dtype=torch.bfloat16,
|
| 20 |
+
quantization_config=TorchAoConfig(
|
| 21 |
+
Int8WeightOnlyConfig(version=2),
|
| 22 |
+
modules_to_not_convert=[
|
| 23 |
+
"proj_in", "audio_proj_in", "context_embedder", "time_embedder", "time_proj",
|
| 24 |
+
"token_refiner", "norm_out", "proj_out", "audio_proj_out",
|
| 25 |
+
],
|
| 26 |
+
),
|
| 27 |
+
low_cpu_mem_usage=False,
|
| 28 |
+
)
|
| 29 |
+
model.requires_grad_(False)
|
| 30 |
+
return model
|
h3_quick_test.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": 3,
|
| 3 |
+
"prompt": "The same subject gently turns their head toward the camera, then holds a relaxed expression. Keep the same clothing and background. Quiet room ambience, no speech or music.",
|
| 4 |
+
"upsample": false,
|
| 5 |
+
"canvas": "Auto \u00b7 match my picture",
|
| 6 |
+
"match": false,
|
| 7 |
+
"duration": 2,
|
| 8 |
+
"steps": 4,
|
| 9 |
+
"seed": 42,
|
| 10 |
+
"randomize_seed": false,
|
| 11 |
+
"scene_idea": "",
|
| 12 |
+
"scene_prompts": "",
|
| 13 |
+
"scene_seconds": "",
|
| 14 |
+
"clip_count": 1,
|
| 15 |
+
"auto_count": false,
|
| 16 |
+
"auto_seconds": false,
|
| 17 |
+
"identity_mode": "CPU face protection",
|
| 18 |
+
"identity_strength": 0.65,
|
| 19 |
+
"dialogue_language": "English",
|
| 20 |
+
"lora_1": "larryvrh/MiniMax-H3-Turbo-Lora/minimax_h3_turbo_v4_step600_ema.safetensors",
|
| 21 |
+
"lora_1_scale": 1,
|
| 22 |
+
"lora_2": "",
|
| 23 |
+
"lora_2_scale": 0.5,
|
| 24 |
+
"lora_3": "",
|
| 25 |
+
"lora_3_scale": 0.5,
|
| 26 |
+
"lora_4": "",
|
| 27 |
+
"lora_4_scale": 0.5,
|
| 28 |
+
"lora_5": "",
|
| 29 |
+
"lora_5_scale": 0.5
|
| 30 |
+
}
|
requirements-int8.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Optional owner experiment; torch 2.11 compatibility per pytorch/ao#2919.
|
| 2 |
+
# Add -r requirements-int8.txt to requirements.txt only when enabling INT8.
|
| 3 |
+
torchao==0.17.0
|