multimodalart HF Staff commited on
Commit
2ed3187
·
verified ·
1 Parent(s): 58fa180

Revert to original (undo accidental overwrite)

Browse files
Files changed (1) hide show
  1. app.py +459 -144
app.py CHANGED
@@ -1,195 +1,510 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import random
2
 
 
3
  import spaces
4
  import torch
 
5
  import gradio as gr
6
- from diffusers import DiffusionPipeline
7
 
8
- BASE_MODEL = "krea/Krea-2-Turbo"
9
- CUSTOM_PIPELINE = "ostris/Krea2OstrisEdit"
 
10
 
11
- # Instruction-based, identity-preserving edit LoRA. This IS the edit brain for
12
- # the pipeline's dual conditioning (in-context VAE tokens + Qwen3-VL grounding),
13
- # so it replaces the Ostris style-reference LoRA rather than stacking on it.
 
14
  LORA_REPO = "conradlocke/krea2-identity-edit"
15
- LORA_WEIGHT = "krea2_identity_edit_v1_1.safetensors" # v1.1 recommended (full rank)
16
-
17
  DTYPE = torch.bfloat16
18
- MAX_SEED = 2**31 - 1
19
- TARGET_PX = 1024 # ~1MP target; keep well under the LoRA's ~1.5-2MP ceiling
20
 
 
 
 
 
 
 
 
 
 
 
21
 
22
- pipe = DiffusionPipeline.from_pretrained(
23
- BASE_MODEL,
24
- custom_pipeline=CUSTOM_PIPELINE,
25
- torch_dtype=DTYPE,
26
- trust_remote_code=True,
 
 
 
 
27
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  pipe.to("cuda")
29
- pipe.load_lora_weights(LORA_REPO, weight_name=LORA_WEIGHT)
30
-
31
-
32
- def _match_source_size(image, base=TARGET_PX, mult=16, lo=512, hi=1536):
33
- """Output AR must match the source (training pairs are same-size); target
34
- ~base*base area, preserve aspect ratio, snap to multiples of 16."""
35
- w, h = image.size
36
- ar = w / h
37
- th = (base * base / ar) ** 0.5
38
- tw = th * ar
39
- tw = max(lo, min(hi, round(tw / mult) * mult))
40
- th = max(lo, min(hi, round(th / mult) * mult))
41
- return int(tw), int(th)
42
-
43
-
44
- @spaces.GPU(duration=90, size="large")
45
- def generate(
46
- prompt,
47
- reference_image,
48
- use_reference,
49
- lora_scale=1.0,
50
- steps=8,
51
- guidance=1.0,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
  seed=0,
53
  randomize_seed=True,
54
  progress=gr.Progress(track_tqdm=True),
55
  ):
56
- if not prompt or not prompt.strip():
57
- raise gr.Error("Enter an edit instruction, e.g. 'put a red party hat on it'.")
58
- if use_reference and reference_image is None:
59
- raise gr.Error("Upload a photo to edit, or disable the reference option.")
60
 
61
- if randomize_seed or seed is None:
62
  seed = random.randint(0, MAX_SEED)
63
  seed = int(seed)
64
 
65
- lora_scale = 1.0 if lora_scale is None else float(lora_scale)
66
- steps = 8 if steps is None else int(steps)
67
- guidance = 1.0 if guidance is None else float(guidance)
 
 
 
 
 
 
68
 
69
- # Match output size/AR to the source image; fall back to square otherwise.
70
- if use_reference and reference_image is not None:
71
- width, height = _match_source_size(reference_image)
72
  else:
73
- width = height = TARGET_PX
74
-
75
- generator = torch.Generator("cuda").manual_seed(seed)
76
-
77
- pipe_kwargs = dict(
78
- prompt=prompt.strip(),
79
- num_inference_steps=steps,
80
- guidance_scale=guidance,
81
- width=width,
82
- height=height,
83
- generator=generator,
84
- attention_kwargs={"scale": lora_scale},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
  )
86
- if use_reference and reference_image is not None:
87
- pipe_kwargs["image"] = reference_image
88
 
89
- image = pipe(**pipe_kwargs).images[0]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
  return image, seed
91
 
92
 
 
 
 
93
  CSS = """
94
- #page { max-width: 1100px; margin: 0 auto; padding: 4px 8px 32px; }
95
- #header { padding: 24px 4px 18px; border-bottom: 1px solid #e5e5e5; margin-bottom: 20px; }
96
- #header h1 { font-size: 32px; font-weight: 700; margin: 0 0 6px; letter-spacing: -0.02em; }
97
- #header .subtitle { font-size: 15px; color: #666; margin: 0; max-width: 72ch; line-height: 1.5; }
98
- #header .links { margin-top: 12px; display: flex; gap: 16px; }
99
- #header .links a {
100
- font-size: 13px; color: #888; text-decoration: none;
101
- border: 1px solid #ddd; border-radius: 6px; padding: 4px 10px;
102
- }
103
- #header .links a:hover { color: #333; border-color: #aaa; }
104
- footer { display: none !important; }
105
  """
106
 
107
- HEADER = """
108
- <div id="header">
109
- <h1>Krea 2 Identity Edit ✏️</h1>
110
- <p class="subtitle">
111
- Instruction-based, <b>identity-preserving</b> image editing. Upload a photo and give a
112
- plain-language instruction — "put this person at a night market", "add a red party hat",
113
- "make it snow" — and the
114
- <a href="https://huggingface.co/conradlocke/krea2-identity-edit" target="_blank">Krea2 Identity Edit LoRA</a>
115
- edits it while keeping the face, outfit, and untouched parts of the scene fixed. Built on
116
- <a href="https://huggingface.co/krea/Krea-2-Turbo" target="_blank">Krea 2 Turbo</a>
117
- via the
118
- <a href="https://huggingface.co/ostris/Krea2OstrisEdit" target="_blank">Krea2OstrisEdit</a>
119
- pipeline.
120
- </p>
121
- <div class="links">
122
- <a href="https://huggingface.co/conradlocke/krea2-identity-edit" target="_blank">Edit LoRA ↗</a>
123
- <a href="https://huggingface.co/ostris/Krea2OstrisEdit" target="_blank">Pipeline ↗</a>
124
- <a href="https://huggingface.co/krea/Krea-2-Turbo" target="_blank">Base model ↗</a>
125
- </div>
126
- </div>
127
  """
128
 
129
- with gr.Blocks(title="Krea 2 Identity Edit") as demo:
130
- with gr.Column(elem_id="page"):
131
- gr.HTML(HEADER)
132
- with gr.Row(equal_height=False):
133
- with gr.Column(scale=1):
134
- prompt = gr.Textbox(
 
 
135
  label="Edit instruction",
 
136
  lines=2,
137
- placeholder="e.g. 'put a red party hat on it' or 'place this person on a beach at sunset'",
138
- )
139
- use_reference = gr.Checkbox(
140
- value=True,
141
- label="Use uploaded photo",
142
- info="On: edit your photo. Off: plain text-to-image from the instruction.",
143
- )
144
- reference_image = gr.Image(
145
- label="Photo to edit",
146
- type="pil",
147
- height=300,
148
  )
149
- generate_btn = gr.Button("Edit", variant="primary", size="lg")
150
- with gr.Accordion("Advanced", open=False):
 
 
 
 
 
 
151
  lora_scale = gr.Slider(
152
- 0.0, 1.5, value=1.0, step=0.05,
153
- label="Edit LoRA strength",
154
- info="1.0 recommended",
 
 
 
 
 
 
 
155
  )
156
- steps = gr.Slider(1, 30, value=8, step=1, label="Steps",
157
- info="8–12 for most edits; ~20 for large removals")
158
- guidance = gr.Slider(
159
- 0.0, 6.0, value=1.0, step=0.1,
160
- label="Guidance (CFG)",
161
- info="1.0 for most edits; ~3.0 for deletions",
162
  )
163
  with gr.Row():
164
- seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
165
- randomize_seed = gr.Checkbox(value=True, label="Randomize seed")
166
- gr.Markdown(
167
- "<small>Output size auto-matches your photo's aspect ratio (~1MP). "
168
- "For two-person edits the LoRA works best ≤1.5MP.</small>"
169
- )
170
- with gr.Column(scale=1):
171
- result = gr.Image(label="Result", format="png", height=420)
172
- used_seed = gr.Number(label="Seed used", visible=True, interactive=False)
173
 
174
- inputs = [
175
- prompt, reference_image, use_reference,
176
- lora_scale, steps, guidance, seed, randomize_seed,
177
- ]
178
- outputs = [result, used_seed]
179
 
180
  gr.Examples(
181
  examples=[
182
- ["place this on a wooden table with soft window light", "examples/style_ref_clean.png", True],
183
- ["make the background a golden sunset", "examples/style_ref_clean.png", True],
 
184
  ],
185
- inputs=[prompt, reference_image, use_reference],
186
- outputs=outputs,
187
- fn=generate,
188
  cache_examples=True,
189
  cache_mode="lazy",
190
  )
191
 
192
- gr.on([generate_btn.click, prompt.submit], generate, inputs, outputs)
 
 
 
 
 
193
 
194
  if __name__ == "__main__":
195
- demo.launch(theme=gr.themes.Citrus(), css=CSS)
 
1
+ """Krea 2 Identity Edit — instruction-based, identity-preserving image editing.
2
+
3
+ This Space runs the community LoRA `conradlocke/krea2-identity-edit` on top of
4
+ `krea/Krea-2-Turbo` (the distilled 8-step checkpoint). The LoRA is trained with a
5
+ *dual conditioning* recipe that stock text-to-image `Krea2Pipeline` does not
6
+ provide, so this app reproduces the two custom pieces from the reference
7
+ ComfyUI-Krea2Edit node pack (https://github.com/lbouaraba/comfyui-krea2edit) in
8
+ plain diffusers / PyTorch:
9
+
10
+ 1. Krea2EditGroundedEncode — the instruction is encoded *together with the
11
+ source image* through the Qwen3-VL text encoder (vision tokens inserted via
12
+ the image-grounded chat template), and the 12 selected decoder layers are
13
+ tapped, exactly like the text-only Krea 2 path but with the image grounding
14
+ the semantics ("the man on the left", "the sign in the back").
15
+
16
+ 2. Krea2EditModelPatch — the VAE-encoded SOURCE latent is prepended to the
17
+ transformer sequence as a block of *clean* tokens, distinguished from the
18
+ noisy target purely by the 3-axis RoPE frame index (source frame = 1,
19
+ target frame = 0, h/w aligned). The sequence becomes
20
+ [text | source(frame=1) | target(frame=0)] and only the target tokens are
21
+ kept as the velocity prediction — mirroring ai-toolkit's
22
+ `predict_velocity_edit`.
23
+
24
+ Everything runs on ZeroGPU: modules go on CUDA at module scope, inference is
25
+ wrapped in @spaces.GPU, no torch.compile, no CPU offload.
26
+ """
27
+
28
+ import os
29
+
30
+ # Disable Gradio server-side rendering: SSR + the gradio_client / ZeroGPU
31
+ # interaction has been observed to surface as opaque CUDA allocator asserts.
32
+ os.environ.setdefault("GRADIO_SSR_MODE", "False")
33
+
34
  import random
35
 
36
+ import numpy as np
37
  import spaces
38
  import torch
39
+ from torch.nn.attention import sdpa_kernel, SDPBackend
40
  import gradio as gr
41
+ from PIL import Image
42
 
43
+ from diffusers import Krea2Pipeline
44
+ from diffusers.pipelines.krea2.pipeline_krea2 import retrieve_timesteps
45
+ from transformers import AutoProcessor
46
 
47
+ # --------------------------------------------------------------------------------------
48
+ # Constants
49
+ # --------------------------------------------------------------------------------------
50
+ BASE_MODEL = "krea/Krea-2-Turbo"
51
  LORA_REPO = "conradlocke/krea2-identity-edit"
52
+ LORA_WEIGHT = "krea2_identity_edit_v1_1.safetensors"
 
53
  DTYPE = torch.bfloat16
54
+ MAX_SEED = np.iinfo(np.int32).max
 
55
 
56
+ # Turbo recipe for "most edits" (add / recolor / restyle / re-stage), per the LoRA card:
57
+ # Turbo, 8 steps, CFG 1.0 (== guidance disabled in the Krea convention).
58
+ DEFAULT_STEPS = 8
59
+ DEFAULT_GUIDANCE = 0.0 # Krea convention: 0.0 disables guidance
60
+ DEFAULT_GROUNDING_PX = 768 # trained dial 512-1536; 768 balanced, 1024+ for people
61
+ DEFAULT_LORA_SCALE = 1.0
62
+ MAX_MEGAPIXELS = 1.0 # card allows <=2MP; we cap at 1MP because the edit
63
+ # path prepends the source latent (~2x image tokens).
64
+ # Inference runs under torch.inference_mode(), so no
65
+ # autograd buffers are held.
66
 
67
+ # The image-grounded instruction template from ComfyUI-Krea2Edit. The system
68
+ # prefix is byte-identical to the diffusers Krea 2 text template; the difference
69
+ # is the <|vision_start|><|image_pad|><|vision_end|> block inserted before the
70
+ # instruction so the VLM grounds the edit on the source image.
71
+ GROUNDED_TEMPLATE = (
72
+ "<|im_start|>system\nDescribe the image by detailing the color, shape, size, "
73
+ "texture, quantity, text, spatial relationships of the objects and background:"
74
+ "<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
75
+ "{}<|im_end|>\n<|im_start|>assistant\n"
76
  )
77
+
78
+ # --------------------------------------------------------------------------------------
79
+ # Load the pipeline (module scope, eager CUDA placement — required for ZeroGPU)
80
+ # --------------------------------------------------------------------------------------
81
+ pipe = Krea2Pipeline.from_pretrained(BASE_MODEL, torch_dtype=DTYPE)
82
+
83
+
84
+ def _convert_lora_keys(state_dict):
85
+ """Convert the ai-toolkit / ComfyUI Krea2 LoRA naming to the diffusers
86
+ `Krea2Transformer2DModel` module names so `load_lora_adapter` matches.
87
+
88
+ ai-toolkit keys look like `diffusion_model.blocks.0.attn.wq.lora_A.weight`;
89
+ diffusers expects `transformer_blocks.0.attn.to_q.lora_A.weight` (and
90
+ `txtfusion` -> `text_fusion`, `mlp` -> `ff`, `wo` -> `to_out.0`, etc.).
91
+ """
92
+ attn_map = {
93
+ ".attn.wq.": ".attn.to_q.",
94
+ ".attn.wk.": ".attn.to_k.",
95
+ ".attn.wv.": ".attn.to_v.",
96
+ ".attn.wo.": ".attn.to_out.0.",
97
+ ".attn.gate.": ".attn.to_gate.",
98
+ ".mlp.gate.": ".ff.gate.",
99
+ ".mlp.up.": ".ff.up.",
100
+ ".mlp.down.": ".ff.down.",
101
+ }
102
+ out = {}
103
+ for k, v in state_dict.items():
104
+ nk = k
105
+ if nk.startswith("diffusion_model."):
106
+ nk = nk[len("diffusion_model."):]
107
+ nk = nk.replace("blocks.", "transformer_blocks.", 1) if nk.startswith("blocks.") else nk
108
+ nk = nk.replace("txtfusion.", "text_fusion.", 1) if nk.startswith("txtfusion.") else nk
109
+ for a, b in attn_map.items():
110
+ if a in nk:
111
+ nk = nk.replace(a, b)
112
+ out[nk] = v
113
+ return out
114
+
115
+
116
+ # Load the identity-edit LoRA onto the transformer (Krea 2 LoRAs load through the
117
+ # transformer's adapter API). The weights ship in ai-toolkit naming, so convert
118
+ # the keys to the diffusers convention first.
119
+ from safetensors.torch import load_file as _load_safetensors
120
+ from huggingface_hub import hf_hub_download as _hf_hub_download
121
+
122
+ _lora_path = _hf_hub_download(LORA_REPO, LORA_WEIGHT)
123
+ _lora_sd = _convert_lora_keys(_load_safetensors(_lora_path))
124
+ # Keys are already in diffusers module-path form (transformer_blocks.N.attn.to_q...),
125
+ # so bypass the default prefix="transformer" filter (which would drop every key
126
+ # because "transformer_blocks." does not start with "transformer.").
127
+ pipe.transformer.load_lora_adapter(_lora_sd, adapter_name="default", prefix=None)
128
+ pipe.transformer.set_adapters("default", weights=DEFAULT_LORA_SCALE)
129
+ # Fuse the LoRA into the base weights at strength 1.0 (the card's recommended
130
+ # setting) and unload the adapter. This bakes the delta into the linear weights so
131
+ # inference runs through plain nn.Linear rather than peft's LoRA layer wrapper —
132
+ # faster, and it sidesteps a ZeroGPU caching-allocator assert triggered inside the
133
+ # peft LoRA forward. The adjustable lora_scale dial is therefore fixed at 1.0.
134
+ pipe.transformer.fuse_lora(lora_scale=DEFAULT_LORA_SCALE)
135
+ pipe.transformer.unload_lora()
136
+
137
  pipe.to("cuda")
138
+
139
+ # A Qwen3-VL processor for the grounded (image + text) encode. The Krea 2 repo
140
+ # ships only a text tokenizer; the vision-side preprocessing (image mean/std,
141
+ # 16px patch, spatial-merge=2) comes from the Qwen3-VL processor. We reuse the
142
+ # Krea 2 tokenizer's special tokens by aligning the template above with the
143
+ # processor's <|image_pad|> expansion.
144
+ try:
145
+ processor = AutoProcessor.from_pretrained("Qwen/Qwen3-VL-4B-Instruct")
146
+ except Exception as e: # pragma: no cover - surfaced in logs if it happens
147
+ print(f"[warn] could not load Qwen3-VL processor, falling back: {e}")
148
+ processor = None
149
+
150
+ # VAE latent normalization stats (the transformer works in normalized latent
151
+ # space; randn target latents already live there, so the source must be
152
+ # normalized the same way as the pipeline: (z - mean) / std).
153
+ _LATENTS_MEAN = (
154
+ torch.tensor(pipe.vae.config.latents_mean).view(1, pipe.vae.config.z_dim, 1, 1, 1)
155
+ )
156
+ _LATENTS_STD = (
157
+ torch.tensor(pipe.vae.config.latents_std).view(1, pipe.vae.config.z_dim, 1, 1, 1)
158
+ )
159
+
160
+
161
+ # --------------------------------------------------------------------------------------
162
+ # Grounded instruction encoding (semantic path)
163
+ # --------------------------------------------------------------------------------------
164
+ def _grounded_encode(instruction: str, source: Image.Image, grounding_px: int):
165
+ """Encode the instruction grounded on the source image through Qwen3-VL.
166
+
167
+ Returns (prompt_embeds, prompt_embeds_mask) shaped like the diffusers
168
+ Krea 2 text conditioning: (1, seq, num_text_layers, text_hidden_dim) and
169
+ (1, seq).
170
+ """
171
+ device = pipe._execution_device
172
+ select_layers = pipe.text_encoder_select_layers
173
+ prefix_idx = pipe.prompt_template_encode_start_idx # 34: drop the system prefix
174
+
175
+ # Cap the longest side fed to the VLM (the LoRA trained with 384-768px jitter).
176
+ img = source.convert("RGB")
177
+ if grounding_px and max(img.size) > grounding_px:
178
+ s = grounding_px / max(img.size)
179
+ img = img.resize(
180
+ (max(16, round(img.size[0] * s)), max(16, round(img.size[1] * s))),
181
+ Image.LANCZOS,
182
+ )
183
+
184
+ text = GROUNDED_TEMPLATE.format(instruction or "")
185
+
186
+ inputs = processor(
187
+ text=[text],
188
+ images=[img],
189
+ padding=True,
190
+ return_tensors="pt",
191
+ ).to(device)
192
+
193
+ te_kwargs = dict(
194
+ input_ids=inputs["input_ids"],
195
+ attention_mask=inputs.get("attention_mask"),
196
+ pixel_values=inputs.get("pixel_values"),
197
+ image_grid_thw=inputs.get("image_grid_thw"),
198
+ output_hidden_states=True,
199
+ )
200
+ # Qwen3-VL needs mm_token_type_ids (returned by the processor) to compute
201
+ # multimodal M-RoPE positions when image tokens are present.
202
+ if inputs.get("mm_token_type_ids") is not None:
203
+ te_kwargs["mm_token_type_ids"] = inputs["mm_token_type_ids"]
204
+ outputs = pipe.text_encoder(**te_kwargs)
205
+ hidden_states = torch.stack(
206
+ [outputs.hidden_states[i] for i in select_layers], dim=2
207
+ ) # (1, seq, num_layers, dim)
208
+
209
+ attention_mask = inputs.get("attention_mask")
210
+ if attention_mask is None:
211
+ attention_mask = torch.ones(
212
+ hidden_states.shape[:2], device=device, dtype=torch.bool
213
+ )
214
+ else:
215
+ attention_mask = attention_mask.bool()
216
+
217
+ # Drop the system-prefix tokens (identical prefix to the text-only path).
218
+ hidden_states = hidden_states[:, prefix_idx:]
219
+ attention_mask = attention_mask[:, prefix_idx:]
220
+ return hidden_states.to(DTYPE), attention_mask
221
+
222
+
223
+ def _text_only_encode(instruction: str):
224
+ """Fallback text-only encode if the processor is unavailable."""
225
+ return pipe.encode_prompt(prompt=instruction, device=pipe._execution_device)
226
+
227
+
228
+ # --------------------------------------------------------------------------------------
229
+ # Source-preservation forward (appearance path)
230
+ # --------------------------------------------------------------------------------------
231
+ def _encode_source_latent(source: Image.Image, height: int, width: int):
232
+ """VAE-encode the source image to a packed, normalized latent block matching
233
+ the target grid, ready to prepend to the transformer sequence."""
234
+ device = pipe._execution_device
235
+ # Preprocess to the target resolution (training pairs are same-size; the card
236
+ # says match output AR to the source, which we enforce upstream).
237
+ px = pipe.image_processor.preprocess(source.convert("RGB"), height=height, width=width)
238
+ px = px.unsqueeze(2).to(device=device, dtype=pipe.vae.dtype) # (B,C,1,H,W)
239
+
240
+ latent = pipe.vae.encode(px).latent_dist.mode() # (B, z, 1, lh, lw), unnormalized
241
+ mean = _LATENTS_MEAN.to(latent.device, latent.dtype)
242
+ std = _LATENTS_STD.to(latent.device, latent.dtype)
243
+ latent = (latent - mean) / std # normalized latent space (matches pipeline)
244
+ latent = latent[:, :, 0] # (B, z, lh, lw)
245
+
246
+ b, c, lh, lw = latent.shape
247
+ packed = pipe._pack_latents(latent, b, c, lh, lw) # (B, lh*lw/p^2, c*p*p)
248
+ return packed.to(DTYPE)
249
+
250
+
251
+ def _edit_position_ids(text_seq_len, grid_h, grid_w, n_src, device):
252
+ """Build (text + n_src*grid + grid, 3) rotary coords:
253
+ text @ (0,0,0); each source block @ frame=(i+1) with (h,w); target @ frame=0.
254
+ """
255
+ text_ids = torch.zeros(text_seq_len, 3, device=device)
256
+
257
+ def _img_ids(frame):
258
+ ids = torch.zeros(grid_h, grid_w, 3, device=device)
259
+ ids[..., 0] = frame
260
+ ids[..., 1] = torch.arange(grid_h, device=device)[:, None]
261
+ ids[..., 2] = torch.arange(grid_w, device=device)[None, :]
262
+ return ids.reshape(grid_h * grid_w, 3)
263
+
264
+ blocks = [text_ids]
265
+ blocks += [_img_ids(i + 1) for i in range(n_src)] # sources frame=1..N
266
+ blocks += [_img_ids(0)] # target frame=0
267
+ return torch.cat(blocks, dim=0)
268
+
269
+
270
+ def _edit_transformer_forward(latents, src_packed, prompt_embeds, prompt_mask,
271
+ timestep, position_ids):
272
+ """Run the Krea 2 transformer with the source latent block prepended, keeping
273
+ only the target tokens out. Reproduces ComfyUI-Krea2Edit's krea2_edit_forward
274
+ against the diffusers Krea2Transformer2DModel (img_in == m.first,
275
+ transformer_blocks == m.blocks, text_fusion/txt_in == m.txtfusion/m.txtmlp,
276
+ rotary_emb == m.pe_embedder, final_layer == m.last)."""
277
+ m = pipe.transformer
278
+ combined_img = torch.cat([src_packed, latents], dim=1) # [source | target] packed
279
+
280
+ temb = m.time_embed(timestep, dtype=latents.dtype)
281
+ temb_mod = m.time_mod_proj(torch.nn.functional.gelu(temb, approximate="tanh"))
282
+
283
+ # Text fusion + projection (attention mask over text only; all image tokens valid).
284
+ text_attn_mask = prompt_mask[:, None, None, :] if prompt_mask is not None else None
285
+ enc = m.text_fusion(prompt_embeds, attention_mask=text_attn_mask)
286
+ enc = m.txt_in(enc)
287
+
288
+ img = m.img_in(combined_img)
289
+ hidden = torch.cat([enc, img], dim=1) # [text | source | target]
290
+
291
+ image_rotary_emb = m.rotary_emb(position_ids)
292
+
293
+ # Single prompt, batch=1: the grounded encode uses padding=True on one
294
+ # sequence, so there is no padding and every token is valid. Passing
295
+ # attention_mask=None lets SDPA pick the flash / mem-efficient kernels
296
+ # (a dense boolean mask would force the score-materializing math path and
297
+ # OOM given the doubled token count). If padding is ever introduced (batched
298
+ # or CFG with unequal lengths), trim to the shorter length instead of masking.
299
+ attention_mask = None
300
+
301
+ # Prefer the flash / mem-efficient SDPA kernels (they never materialize the
302
+ # full seq x seq score matrix, which the MATH kernel would OOM on given the
303
+ # doubled token count from prepending the source latent).
304
+ with sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]):
305
+ for block in m.transformer_blocks:
306
+ hidden = block(hidden, temb_mod, image_rotary_emb, attention_mask)
307
+
308
+ text_seq_len = enc.shape[1]
309
+ tgt_len = latents.shape[1]
310
+ hidden = hidden[:, text_seq_len:] # drop text -> [source | target]
311
+ hidden = hidden[:, -tgt_len:] # keep target tokens only
312
+ return m.final_layer(hidden, temb)
313
+
314
+
315
+ # --------------------------------------------------------------------------------------
316
+ # Sizing helpers
317
+ # --------------------------------------------------------------------------------------
318
+ def _target_size(source: Image.Image):
319
+ """Match the output AR to the source (card requirement) and cap at ~2MP,
320
+ snapping each side to a multiple of vae_scale_factor * patch_size."""
321
+ multiple = pipe.vae_scale_factor * pipe.patch_size # 8 * 2 = 16
322
+ w, h = source.size
323
+ mp = (w * h) / 1e6
324
+ if mp > MAX_MEGAPIXELS:
325
+ s = (MAX_MEGAPIXELS / mp) ** 0.5
326
+ w, h = round(w * s), round(h * s)
327
+ w = max(multiple, (w // multiple) * multiple)
328
+ h = max(multiple, (h // multiple) * multiple)
329
+ return h, w
330
+
331
+
332
+ # --------------------------------------------------------------------------------------
333
+ # Inference
334
+ # --------------------------------------------------------------------------------------
335
+ @spaces.GPU
336
+ @torch.inference_mode()
337
+ def edit(
338
+ source_image,
339
+ instruction,
340
+ grounding_px=DEFAULT_GROUNDING_PX,
341
+ lora_scale=DEFAULT_LORA_SCALE,
342
+ steps=DEFAULT_STEPS,
343
+ guidance_scale=DEFAULT_GUIDANCE,
344
  seed=0,
345
  randomize_seed=True,
346
  progress=gr.Progress(track_tqdm=True),
347
  ):
348
+ if source_image is None:
349
+ raise gr.Error("Please upload a source image to edit.")
350
+ if not instruction or not instruction.strip():
351
+ raise gr.Error("Please describe the edit you want (e.g. 'put this person at a night market').")
352
 
353
+ if randomize_seed:
354
  seed = random.randint(0, MAX_SEED)
355
  seed = int(seed)
356
 
357
+ device = pipe._execution_device
358
+ source = source_image if isinstance(source_image, Image.Image) else Image.fromarray(source_image)
359
+ source = source.convert("RGB")
360
+
361
+ # The identity-edit LoRA is fused into the base weights at load time (strength
362
+ # 1.0, the card's recommended setting), so lora_scale is fixed and not re-applied
363
+ # per call.
364
+
365
+ height, width = _target_size(source)
366
 
367
+ # --- Semantic path: grounded instruction encode -----------------------------------
368
+ if processor is not None:
369
+ prompt_embeds, prompt_mask = _grounded_encode(instruction.strip(), source, int(grounding_px))
370
  else:
371
+ prompt_embeds, prompt_mask = _text_only_encode(instruction.strip())
372
+
373
+ do_cfg = float(guidance_scale) > 0
374
+ if do_cfg:
375
+ # At CFG > 1 the card says ground the negative too: empty prompt, same image.
376
+ if processor is not None:
377
+ neg_embeds, neg_mask = _grounded_encode("", source, int(grounding_px))
378
+ else:
379
+ neg_embeds, neg_mask = _text_only_encode("")
380
+
381
+ # --- Appearance path: encode + pack the source latent -----------------------------
382
+ src_packed = _encode_source_latent(source, height, width)
383
+
384
+ # --- Prepare noisy target latents -------------------------------------------------
385
+ num_channels_latents = pipe.transformer.config.in_channels // (pipe.patch_size ** 2)
386
+ generator = torch.Generator(device=device).manual_seed(seed)
387
+ latents = pipe.prepare_latents(
388
+ 1, num_channels_latents, height, width, DTYPE, device, generator, None
389
+ )
390
+
391
+ grid_h = height // (pipe.vae_scale_factor * pipe.patch_size)
392
+ grid_w = width // (pipe.vae_scale_factor * pipe.patch_size)
393
+ position_ids = _edit_position_ids(prompt_embeds.shape[1], grid_h, grid_w, 1, device)
394
+
395
+ # --- Timesteps (distilled schedule: fixed mu = 1.15) ------------------------------
396
+ sigmas = np.linspace(1.0, 1 / int(steps), int(steps))
397
+ timesteps, num_steps = retrieve_timesteps(
398
+ pipe.scheduler, int(steps), device, sigmas=sigmas, mu=1.15
399
  )
 
 
400
 
401
+ # --- Denoising loop ---------------------------------------------------------------
402
+ pipe.scheduler.set_begin_index(0)
403
+ for t in progress.tqdm(timesteps, desc="Editing"):
404
+ timestep = (t / pipe.scheduler.config.num_train_timesteps).expand(latents.shape[0]).to(latents.dtype)
405
+
406
+ noise_pred = _edit_transformer_forward(
407
+ latents, src_packed, prompt_embeds, prompt_mask, timestep, position_ids
408
+ )
409
+ if do_cfg:
410
+ neg_position_ids = _edit_position_ids(neg_embeds.shape[1], grid_h, grid_w, 1, device)
411
+ neg_pred = _edit_transformer_forward(
412
+ latents, src_packed, neg_embeds, neg_mask, timestep, neg_position_ids
413
+ )
414
+ noise_pred = noise_pred + float(guidance_scale) * (noise_pred - neg_pred)
415
+
416
+ latents = pipe.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
417
+
418
+ # --- Decode -----------------------------------------------------------------------
419
+ latents = pipe._unpack_latents(latents, height, width).to(pipe.vae.dtype)
420
+ mean = _LATENTS_MEAN.to(latents.device, latents.dtype)
421
+ std = _LATENTS_STD.to(latents.device, latents.dtype)
422
+ latents = latents * std + mean # denorm matches pipeline: normalized * std + mean
423
+ image = pipe.vae.decode(latents, return_dict=False)[0][:, :, 0]
424
+ image = pipe.image_processor.postprocess(image, output_type="pil")[0]
425
+
426
  return image, seed
427
 
428
 
429
+ # --------------------------------------------------------------------------------------
430
+ # UI
431
+ # --------------------------------------------------------------------------------------
432
  CSS = """
433
+ #col-container { max-width: 1100px; margin: 0 auto; }
 
 
 
 
 
 
 
 
 
 
434
  """
435
 
436
+ DESCRIPTION = """
437
+ # 🧬 Krea 2 Identity Edit
438
+
439
+ Instruction-based, **identity-preserving** image editing on
440
+ [Krea 2 Turbo](https://huggingface.co/krea/Krea-2-Turbo) with the community LoRA
441
+ [`conradlocke/krea2-identity-edit`](https://huggingface.co/conradlocke/krea2-identity-edit).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
442
  """
443
 
444
+ with gr.Blocks(css=CSS, title="Krea 2 Identity Edit") as demo:
445
+ with gr.Column(elem_id="col-container"):
446
+ gr.Markdown(DESCRIPTION)
447
+
448
+ with gr.Row(equal_height=True):
449
+ with gr.Column():
450
+ source_image = gr.Image(label="Source image", type="pil", height=420)
451
+ instruction = gr.Textbox(
452
  label="Edit instruction",
453
+ placeholder="e.g. create a photo of this person at a night market",
454
  lines=2,
 
 
 
 
 
 
 
 
 
 
 
455
  )
456
+ run_button = gr.Button("Edit", variant="primary", size="lg")
457
+
458
+ with gr.Accordion("Advanced settings", open=False):
459
+ grounding_px = gr.Slider(
460
+ label="Grounding resolution (px)",
461
+ minimum=512, maximum=1536, step=64, value=DEFAULT_GROUNDING_PX,
462
+ info="Lower = stronger edit adherence; higher = stronger identity/likeness. Try 1024+ for people.",
463
+ )
464
  lora_scale = gr.Slider(
465
+ label="LoRA strength (fused at 1.0)",
466
+ minimum=0.0, maximum=1.5, step=0.05, value=DEFAULT_LORA_SCALE,
467
+ interactive=False,
468
+ info="The identity-edit LoRA is fused into the base weights at the "
469
+ "card-recommended strength 1.0.",
470
+ )
471
+ steps = gr.Slider(
472
+ label="Steps",
473
+ minimum=4, maximum=28, step=1, value=DEFAULT_STEPS,
474
+ info="Turbo default is 8. For removals/large deletions try more steps + guidance > 0.",
475
  )
476
+ guidance_scale = gr.Slider(
477
+ label="Guidance (0 = off, Turbo default)",
478
+ minimum=0.0, maximum=5.0, step=0.5, value=DEFAULT_GUIDANCE,
479
+ info="Krea convention: 0 disables guidance. Raise (e.g. 3) for removals/large edits.",
 
 
480
  )
481
  with gr.Row():
482
+ seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
483
+ randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
 
 
 
 
 
 
 
484
 
485
+ with gr.Column():
486
+ result = gr.Image(label="Edited image", height=420)
487
+ used_seed = gr.Number(label="Seed used", interactive=False)
 
 
488
 
489
  gr.Examples(
490
  examples=[
491
+ ["examples/woman.jpg", "create a photo of this person at a busy night market at night"],
492
+ ["examples/businessman_suit.jpg", "change the suit jacket to a red leather jacket"],
493
+ ["examples/man_beach.jpg", "make it a vintage film photo with warm golden-hour light"],
494
  ],
495
+ inputs=[source_image, instruction],
496
+ outputs=[result, used_seed],
497
+ fn=edit,
498
  cache_examples=True,
499
  cache_mode="lazy",
500
  )
501
 
502
+ gr.on(
503
+ triggers=[run_button.click, instruction.submit],
504
+ fn=edit,
505
+ inputs=[source_image, instruction, grounding_px, lora_scale, steps, guidance_scale, seed, randomize_seed],
506
+ outputs=[result, used_seed],
507
+ )
508
 
509
  if __name__ == "__main__":
510
+ demo.launch(theme=gr.themes.Citrus(), css=CSS, show_error=True)