"""Gradio demo: signed stereo matching with six backbones. Upload (or pick) a rectified stereo pair, choose a model, and get the corrected signed disparity -- optionally next to the released model's prediction, which cannot represent content behind the screen plane. """ import os import sys import numpy as np import gradio as gr sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import engine try: # ZeroGPU on Spaces; a no-op elsewhere import spaces _gpu = spaces.GPU except Exception: def _gpu(fn): return fn KEY = {v: k for k, v in engine.NAMES.items()} CHOICES = [engine.NAMES[k] for k in ("fs", "raft", "igev")] @_gpu def run(left, right, model_name, show_orig): if left is None or right is None: raise gr.Error("Please provide both left and right images.") fam = KEY[model_name] d_ours = engine.predict(fam, "ours", left, right) vmax = max(8.0, float(np.percentile(np.abs(d_ours), 99))) behind = 100.0 * float((d_ours < -0.5).mean()) ours = gr.update(value=engine.colorize(d_ours, vmax), label=f"{model_name} + ours — {behind:.0f}% behind the screen") md = (f"**{model_name} (corrected)** — disparity range " f"[{d_ours.min():.1f}, {d_ours.max():.1f}] px, {behind:.0f}% behind the screen " f"(blue). Shared color scale ±{vmax:.0f} px.") if show_orig: d_zs = engine.predict(fam, "zs", left, right) zs = gr.update(value=engine.colorize(d_zs, vmax), visible=True, label=f"{model_name} zero-shot (released)") md += (f"\n\n**Zero-shot** — range [{d_zs.min():.1f}, {d_zs.max():.1f}] px: " "the released model reports a non-negative match everywhere, so " "behind-screen content is forced to the wrong side.") else: zs = gr.update(value=None, visible=False) return ours, zs, md def build_examples(): ex_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples") rows = [] if os.path.isdir(ex_dir): for name in sorted(os.listdir(ex_dir)): L = os.path.join(ex_dir, name, "left.jpg") R = os.path.join(ex_dir, name, "right.jpg") if os.path.exists(L) and os.path.exists(R): rows.append([L, R, engine.NAMES["fs"], True]) return rows CLIP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "clips") VIDEO_CHOICES = [engine.NAMES[k] for k in ("sav", "dynamic", "bida")] def _load_clip(name): import glob d = os.path.join(CLIP_DIR, name) L = [np.asarray(__import__("PIL.Image", fromlist=["Image"]).open(p)) for p in sorted(glob.glob(f"{d}/left_*.jpg"))] R = [np.asarray(__import__("PIL.Image", fromlist=["Image"]).open(p)) for p in sorted(glob.glob(f"{d}/right_*.jpg"))] return L, R def _write_video(frames, path, fps=8): import imageio.v2 as imageio with imageio.get_writer(path, fps=fps, codec="libx264", quality=7) as w: for f in frames: w.append_data(f) return path @_gpu def run_video(clip_video, model_name, show_orig): import tempfile fam = KEY[model_name] if not clip_video: raise gr.Error("Pick one of the example clips below.") clip_name = os.path.splitext(os.path.basename(str(clip_video)))[0] L, R = _load_clip(clip_name) if not L: raise gr.Error( f"Unknown clip '{clip_name}'. This demo runs on the predefined " "stereo clips — pick one from the examples below.") tmp = tempfile.mkdtemp() d_ours = engine.predict_clip(fam, "ours", L, R) vmax = max(8.0, float(np.percentile(np.abs(d_ours), 99))) behind = 100.0 * float((d_ours < -0.5).mean()) _write_video([engine.colorize(f, vmax) for f in d_ours], f"{tmp}/ours.mp4") md = (f"**{model_name} (corrected)** — {d_ours.shape[0]} frames, range " f"[{d_ours.min():.1f}, {d_ours.max():.1f}] px, {behind:.0f}% behind the " f"screen. Shared color scale ±{vmax:.0f} px.") if show_orig: d_zs = engine.predict_clip(fam, "zs", L, R) _write_video([engine.colorize(f, vmax) for f in d_zs], f"{tmp}/zs.mp4") zs = gr.update(value=f"{tmp}/zs.mp4", visible=True, label=f"{model_name} zero-shot (released)") md += (f"\n\n**Zero-shot** — range [{d_zs.min():.1f}, {d_zs.max():.1f}] px.") else: zs = gr.update(value=None, visible=False) return (gr.update(value=f"{tmp}/ours.mp4", label=f"{model_name} + ours — {behind:.0f}% behind"), zs, md) with gr.Blocks(title="Unmuzzling Stereo") as demo: gr.Markdown( "# Unmuzzling Stereo — signed disparity for content behind the screen\n" "[Code](https://github.com/shijianjian/ZDPShift) · " "[Weights](https://huggingface.co/shijianjian/ZDPShift) · " "[Dataset](https://huggingface.co/datasets/shijianjian/ZDPShift) · " "[Project page](https://shijianjian.github.io/ZDPShift/)\n\n" "Released stereo matchers cannot report negative disparity. The corrected " "models — fine-tuned on horizontally translated SceneFlow only — recover " "it with no architectural change. Red = in front of the screen plane, " "blue = behind it.") with gr.Tab("Image matchers"): with gr.Row(): left = gr.Image(label="Left view", type="numpy") right = gr.Image(label="Right view", type="numpy") with gr.Row(): model = gr.Dropdown(CHOICES, value=CHOICES[0], label="Backbone") show_orig = gr.Checkbox(value=True, label="Also run the released (zero-shot) model") btn = gr.Button("Run", variant="primary") with gr.Row(): out_ours = gr.Image(label="Corrected (ours)", interactive=False) out_zs = gr.Image(label="Zero-shot (released)", interactive=False) stats = gr.Markdown() btn.click(run, [left, right, model, show_orig], [out_ours, out_zs, stats], api_name="run") ex = build_examples() if ex: gr.Examples([[L, R] for L, R, *_ in ex], [left, right], label="Stereo pairs (CC-BY renders + film shots)") with gr.Tab("Video matchers"): import glob as _glob previews = sorted(_glob.glob(os.path.join(CLIP_DIR, "*.mp4"))) with gr.Row(): v_clip = gr.Video(label="Stereo clip (left view preview)", value=previews[0] if previews else None, interactive=False) v_ours = gr.Video(label="Corrected (ours)", interactive=False) v_zs = gr.Video(label="Zero-shot (released)", interactive=False) with gr.Row(): vmodel = gr.Dropdown(VIDEO_CHOICES, value=VIDEO_CHOICES[0], label="Video backbone") vshow = gr.Checkbox(value=True, label="Also run the released (zero-shot) model") vbtn = gr.Button("Run clip", variant="primary") vstats = gr.Markdown() vbtn.click(run_video, [v_clip, vmodel, vshow], [v_ours, v_zs, vstats], api_name="run_video") if previews: gr.Examples([[p] for p in previews], [v_clip], label="Stereo clip demos (12 frames each)") if __name__ == "__main__": demo.launch()