Add GeoNeXt-SVD backend source

backend-svd/Dockerfile

@@ -0,0 +1,29 @@
+FROM nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    HF_HOME=/data/.huggingface
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        git python3 python3-pip python3-dev libgl1 libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+RUN git clone --depth 1 https://github.com/Creative-Intelligence-Studio/GeoNeXt.git /app/GeoNeXt
+
+RUN python3 -m pip install --upgrade pip \
+    && python3 -m pip install torch==2.3.1 torchvision==0.18.1 \
+       --index-url https://download.pytorch.org/whl/cu121 \
+    && python3 -m pip install -r /app/GeoNeXt/requirements-svd.txt \
+    && python3 -m pip install \
+       gradio==4.44.1 pydantic==2.10.6 fastapi==0.115.6 \
+    && python3 -m pip install -e /app/GeoNeXt
+
+COPY app.py /app/app.py
+
+EXPOSE 7860
+
+CMD ["python3", "/app/app.py"]
+

backend-svd/README.md

@@ -0,0 +1,15 @@
+---
+title: GeoNeXt-SVD
+emoji: 🌐
+colorFrom: indigo
+colorTo: purple
+sdk: docker
+app_port: 7860
+pinned: false
+license: apache-2.0
+---
+
+# GeoNeXt-SVD
+
+Official demo for **Video Generative Models as Geometry Learner**.
+

backend-svd/app.py

@@ -0,0 +1,209 @@
+import importlib.util
+import os
+import sys
+import threading
+from contextlib import nullcontext
+from pathlib import Path
+
+os.environ["NO_PROXY"] = "localhost,127.0.0.1,0.0.0.0"
+os.environ["no_proxy"] = os.environ["NO_PROXY"]
+
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import AutoencoderKL, UNetSpatioTemporalConditionModel
+from huggingface_hub import snapshot_download
+from PIL import Image
+
+
+ROOT = Path("/app/GeoNeXt")
+if not ROOT.exists():
+    ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "GeoNeXt-SVD"))
+
+
+def load_module(name, path):
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+svd = load_module("geonext_svd_inference", ROOT / "GeoNeXt-SVD" / "inference.py")
+pipeline_module = load_module("geonext_svd_pipeline", ROOT / "GeoNeXt-SVD" / "pipeline.py")
+GeoNeXtPipeline = pipeline_module.GeoNeXtPipeline
+
+device = torch.device("cuda")
+dtype = torch.float16
+checkpoint_root = snapshot_download(
+    repo_id="happy0612/GeoNeXt",
+    allow_patterns="GeoNeXt-SVD/**",
+)
+checkpoint = str(Path(checkpoint_root) / "GeoNeXt-SVD")
+
+vae = AutoencoderKL.from_pretrained(
+    "stabilityai/sd-vae-ft-mse",
+    torch_dtype=dtype,
+    subfolder=None,
+)
+unet = UNetSpatioTemporalConditionModel.from_pretrained(
+    checkpoint,
+    subfolder="unet",
+    torch_dtype=dtype,
+    low_cpu_mem_usage=False,
+)
+pipe = GeoNeXtPipeline.from_pretrained(
+    svd.SVD_BASE_MODEL,
+    unet=unet,
+    vae=vae,
+    variant="fp16",
+    torch_dtype=dtype,
+    low_cpu_mem_usage=False,
+).to(device)
+pipe.set_progress_bar_config(disable=True)
+
+inference_lock = threading.Lock()
+
+
+@torch.inference_mode()
+def predict(image, steps):
+    if image is None:
+        raise gr.Error("Please upload an image first.")
+
+    original = Image.fromarray(np.asarray(image, dtype=np.uint8), mode="RGB")
+    resized = svd._resize(original, 768, "long")
+    width, height = resized.size
+    width = max(64, round(width / 64) * 64)
+    height = max(64, round(height / 64) * 64)
+    resized = resized.resize((width, height), Image.Resampling.BICUBIC)
+    generator = torch.Generator(device=device).manual_seed(0)
+
+    context = torch.autocast("cuda", dtype=dtype)
+    with inference_lock, context:
+        prediction = pipe(
+            resized,
+            num_frames=3,
+            width=width,
+            height=height,
+            min_guidance_scale=1.0,
+            max_guidance_scale=1.2,
+            noise_aug_strength=0.0,
+            decode_chunk_size=8,
+            generator=generator,
+            motion_bucket_id=127,
+            fps=7,
+            num_inference_steps=int(steps),
+        )
+
+    depth = prediction.geo_res[0].mean(dim=1).squeeze().float().cpu().numpy()
+    normal = prediction.geo_res[1].squeeze().permute(1, 2, 0).float().cpu().numpy()
+    depth = np.asarray(
+        Image.fromarray(depth, mode="F").resize(original.size, Image.Resampling.BILINEAR)
+    )
+    normal = np.stack(
+        [
+            np.asarray(
+                Image.fromarray(normal[..., channel], mode="F").resize(
+                    original.size, Image.Resampling.BILINEAR
+                )
+            )
+            for channel in range(3)
+        ],
+        axis=-1,
+    )
+
+    from utils.visualization import depth_to_vis, normal_to_vis
+
+    depth_vis = depth_to_vis(np.clip(depth, 0.0, 1.0), reverse_color=True)
+    normal_vis = normal_to_vis(np.clip(normal, -1.0, 1.0))
+    return np.asarray(depth_vis), np.asarray(normal_vis)
+
+
+header = """
+<div id="geonext-header">
+  <h1>GeoNeXt</h1>
+  <h3>Video Generative Models as Geometry Learner</h3>
+  <p>Predict monocular depth and surface normals with video generative priors.</p>
+  <div class="geonext-links">
+    <a href="https://huggingface.co/spaces/happy0612/GeoNeXt">🌐 Unified Demo</a>
+    <a href="https://arxiv.org/abs/2608.28549" target="_blank">📄 Paper</a>
+    <a href="https://happy-hsy.github.io/projects/GeoNeXt/" target="_blank">🌐 Project Page</a>
+    <a href="https://github.com/Creative-Intelligence-Studio/GeoNeXt" target="_blank">💻 GitHub</a>
+    <a href="https://huggingface.co/happy0612/GeoNeXt" target="_blank">🤗 Model Checkpoints</a>
+  </div>
+</div>
+"""
+
+css = """
+.gradio-container { max-width: 1180px !important; margin: 0 auto !important; }
+#geonext-header { text-align: center; padding: 1.5rem 0 1.1rem; }
+#geonext-header h1 { font-size: 2.7rem; line-height: 1; margin: 0 0 0.55rem; }
+#geonext-header h3 { font-size: 1.25rem; font-weight: 600; margin: 0 0 0.45rem; }
+#geonext-header p { color: var(--body-text-color-subdued); margin: 0 0 1rem; }
+.geonext-links { display: flex; justify-content: center; flex-wrap: wrap; gap: 0.55rem; }
+.geonext-links a { border: 1px solid var(--border-color-primary); border-radius: 999px; color: var(--body-text-color); padding: 0.4rem 0.8rem; text-decoration: none !important; }
+.geonext-links a:hover { border-color: var(--color-accent); color: var(--color-accent); }
+#run-button { min-height: 46px; font-weight: 700; }
+#input-panel, #output-panel { min-width: 0; }
+"""
+
+examples = [
+    str(path)
+    for path in sorted((ROOT / "assets" / "input").glob("*"))
+    if path.suffix.lower() in {".jpg", ".jpeg", ".png"}
+]
+
+with gr.Blocks(
+    theme=gr.themes.Soft(primary_hue="indigo", neutral_hue="slate"),
+    css=css,
+) as demo:
+    gr.HTML(header)
+    with gr.Row():
+        with gr.Column(scale=5, elem_id="input-panel"):
+            input_image = gr.Image(
+                type="numpy", image_mode="RGB", label="Input Image", height=520
+            )
+        with gr.Column(scale=7, elem_id="output-panel"):
+            with gr.Tabs():
+                with gr.Tab("Depth"):
+                    depth_output = gr.Image(
+                        type="numpy", label="Predicted Depth", format="png",
+                        height=520, interactive=False,
+                    )
+                with gr.Tab("Surface Normal"):
+                    normal_output = gr.Image(
+                        type="numpy", label="Predicted Surface Normal", format="png",
+                        height=520, interactive=False,
+                    )
+
+    inference_steps = gr.Slider(
+        minimum=1,
+        maximum=5,
+        value=5,
+        step=1,
+        label="Inference Steps",
+        info="More steps may improve quality but take longer.",
+    )
+    run_button = gr.Button(
+        "Run GeoNeXt-SVD", variant="primary", elem_id="run-button"
+    )
+
+    run_button.click(
+        fn=predict,
+        inputs=[input_image, inference_steps],
+        outputs=[depth_output, normal_output],
+        concurrency_limit=1,
+        api_name="predict",
+    )
+    if examples:
+        gr.Examples(examples=examples, inputs=input_image, label="Try an example")
+
+
+if __name__ == "__main__":
+    is_space = bool(os.environ.get("SPACE_ID"))
+    demo.queue(default_concurrency_limit=1).launch(
+        server_name="0.0.0.0" if is_space else "127.0.0.1",
+        server_port=int(os.environ.get("PORT", "7860")),
+        share=not is_space,
+    )