# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. import gc import os import shutil import sys import time import json import base64 import threading from datetime import datetime from io import BytesIO os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" import cv2 import gradio as gr import numpy as np import spaces import torch from PIL import Image from pillow_heif import register_heif_opener register_heif_opener() import rerun as rr try: import rerun.blueprint as rrb except ImportError: rrb = None from gradio_rerun import Rerun sys.path.append("mapanything/") from mapanything.utils.geometry import depthmap_to_world_frame, points_to_normals from mapanything.utils.hf_utils.hf_helpers import initialize_mapanything_model from mapanything.utils.hf_utils.visual_util import predictions_to_glb from mapanything.utils.image import load_images, rgb # --------------------------------------------------------------------------- # MapAnything Configuration # --------------------------------------------------------------------------- high_level_config = { "path": "configs/train.yaml", "hf_model_name": "facebook/map-anything", "model_str": "mapanything", "config_overrides": [ "machine=aws", "model=mapanything", "model/task=images_only", "model.encoder.uses_torch_hub=false", ], "checkpoint_name": "model.safetensors", "config_name": "config.json", "trained_with_amp": True, "trained_with_amp_dtype": "bf16", "data_norm_type": "dinov2", "patch_size": 14, "resolution": 518, } model = None _model_lock = threading.Lock() # --------------------------------------------------------------------------- # Example scenes discovery # --------------------------------------------------------------------------- def get_scene_info(examples_dir="examples"): import glob scenes = [] if not os.path.exists(examples_dir): return scenes for scene_folder in sorted(os.listdir(examples_dir)): scene_path = os.path.join(examples_dir, scene_folder) if not os.path.isdir(scene_path): continue if scene_folder.startswith("WAI-Logo"): continue image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.tiff", "*.tif"] image_files = [] for ext in image_extensions: image_files.extend(glob.glob(os.path.join(scene_path, ext))) image_files.extend(glob.glob(os.path.join(scene_path, ext.upper()))) if image_files: image_files = sorted(image_files) scenes.append({ "name": scene_folder, "path": scene_path, "thumbnail": image_files[0], "num_images": len(image_files), "image_files": image_files, }) return scenes def build_example_thumbnails_html(scenes): if not scenes: return "" html = '
' for i, scene in enumerate(scenes): try: with open(scene["thumbnail"], "rb") as f: thumb_b64 = base64.b64encode(f.read()).decode() ext = os.path.splitext(scene["thumbnail"])[1].lower() mime = "image/jpeg" if ext in (".jpg", ".jpeg") else "image/png" src = f"data:{mime};base64,{thumb_b64}" except Exception: src = "" html += f'''
{scene["name"]}
{scene["num_images"]} images
''' html += '
' return html # --------------------------------------------------------------------------- # Core helpers # --------------------------------------------------------------------------- def colorize_depth(depth_map, mask=None): if depth_map is None: return None import matplotlib.pyplot as plt depth_normalized = depth_map.copy() valid_mask = depth_normalized > 0 if mask is not None: valid_mask = valid_mask & mask if valid_mask.sum() > 0: valid_depths = depth_normalized[valid_mask] p5 = np.percentile(valid_depths, 5) p95 = np.percentile(valid_depths, 95) if p95 > p5: depth_normalized[valid_mask] = (depth_normalized[valid_mask] - p5) / (p95 - p5) else: depth_normalized[valid_mask] = 0.5 colormap = plt.cm.turbo_r colored = colormap(np.clip(depth_normalized, 0, 1)) colored = (colored[:, :, :3] * 255).astype(np.uint8) colored[~valid_mask] = [255, 255, 255] return colored def colorize_normal(normal_map, mask=None): if normal_map is None: return None normal_vis = normal_map.copy() if mask is not None: normal_vis[~mask] = [0, 0, 0] normal_vis = (normal_vis + 1.0) / 2.0 normal_vis = (np.clip(normal_vis, 0, 1) * 255).astype(np.uint8) return normal_vis def process_predictions_for_visualization( predictions, views, config, filter_black_bg=False, filter_white_bg=False ): processed_data = {} for view_idx, view in enumerate(views): image = rgb(view["img"], norm_type=config["data_norm_type"]) pred_pts3d = predictions["world_points"][view_idx] mask = predictions["final_mask"][view_idx].copy() if filter_black_bg: vc = image[0] * 255 if image[0].max() <= 1.0 else image[0] mask = mask & (vc.sum(axis=2) >= 16) if filter_white_bg: vc = image[0] * 255 if image[0].max() <= 1.0 else image[0] mask = mask & ~((vc[:, :, 0] > 240) & (vc[:, :, 1] > 240) & (vc[:, :, 2] > 240)) depth = predictions["depth"][view_idx].squeeze() normals, _ = points_to_normals(pred_pts3d, mask=mask) processed_data[view_idx] = { "image": image[0], "points3d": pred_pts3d, "depth": depth, "normal": normals, "mask": mask, } return processed_data def get_view_data(processed_data, view_index): if processed_data is None or len(processed_data) == 0: return None keys = list(processed_data.keys()) idx = max(0, min(view_index, len(keys) - 1)) return processed_data[keys[idx]] def get_depth_vis(processed_data, view_index): vd = get_view_data(processed_data, view_index) if vd is None or vd["depth"] is None: return None return colorize_depth(vd["depth"], mask=vd.get("mask")) def get_normal_vis(processed_data, view_index): vd = get_view_data(processed_data, view_index) if vd is None or vd["normal"] is None: return None return colorize_normal(vd["normal"], mask=vd.get("mask")) def get_measure_image(processed_data, view_index): vd = get_view_data(processed_data, view_index) if vd is None: return None, [] image = vd["image"].copy() if image.dtype != np.uint8: image = (image * 255).astype(np.uint8) if image.max() <= 1.0 else image.astype(np.uint8) if vd["mask"] is not None: invalid = ~vd["mask"] if invalid.any(): overlay = np.array([255, 220, 220], dtype=np.uint8) alpha = 0.5 for c in range(3): image[:, :, c] = np.where( invalid, (1 - alpha) * image[:, :, c] + alpha * overlay[c], image[:, :, c], ).astype(np.uint8) return image, [] def image_to_b64(img_array): if img_array is None: return "" if img_array.dtype != np.uint8: if img_array.max() <= 1.0: img_array = (img_array * 255).astype(np.uint8) else: img_array = img_array.astype(np.uint8) pil = Image.fromarray(img_array) buf = BytesIO() pil.save(buf, format="PNG") return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() # --------------------------------------------------------------------------- # File handling # --------------------------------------------------------------------------- def handle_uploads(files, s_time_interval=1.0): gc.collect() torch.cuda.empty_cache() timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") target_dir = f"input_images_{timestamp}" target_dir_images = os.path.join(target_dir, "images") if os.path.exists(target_dir): shutil.rmtree(target_dir) os.makedirs(target_dir_images) image_paths = [] video_extensions = [".mp4", ".avi", ".mov", ".mkv", ".wmv", ".flv", ".webm", ".m4v", ".3gp"] if files is not None: for file_data in files: file_path = file_data["name"] if isinstance(file_data, dict) and "name" in file_data else str(file_data) file_ext = os.path.splitext(file_path)[1].lower() if file_ext in video_extensions: vs = cv2.VideoCapture(file_path) fps = vs.get(cv2.CAP_PROP_FPS) frame_interval = max(1, int(fps * s_time_interval)) count = 0 frame_num = 0 while True: gotit, frame = vs.read() if not gotit: break count += 1 if count % frame_interval == 0: base_name = os.path.splitext(os.path.basename(file_path))[0] img_path = os.path.join(target_dir_images, f"{base_name}_{frame_num:06}.png") cv2.imwrite(img_path, frame) image_paths.append(img_path) frame_num += 1 vs.release() elif file_ext in [".heic", ".heif"]: try: with Image.open(file_path) as img: if img.mode not in ("RGB", "L"): img = img.convert("RGB") base_name = os.path.splitext(os.path.basename(file_path))[0] dst_path = os.path.join(target_dir_images, f"{base_name}.jpg") img.save(dst_path, "JPEG", quality=95) image_paths.append(dst_path) except Exception: dst_path = os.path.join(target_dir_images, os.path.basename(file_path)) shutil.copy(file_path, dst_path) image_paths.append(dst_path) else: dst_path = os.path.join(target_dir_images, os.path.basename(file_path)) shutil.copy(file_path, dst_path) image_paths.append(dst_path) image_paths = sorted(image_paths) return target_dir, image_paths def load_example_scene(scene_name, examples_dir="examples"): scenes = get_scene_info(examples_dir) selected = None for s in scenes: if s["name"] == scene_name: selected = s break if selected is None: return "None", [], "Scene not found" target_dir, image_paths = handle_uploads( [{"name": p} for p in selected["image_files"]], 1.0 ) return target_dir, image_paths, f"Loaded scene '{scene_name}' with {selected['num_images']} images. Click Reconstruct to begin." # --------------------------------------------------------------------------- # Model inference # --------------------------------------------------------------------------- @spaces.GPU(duration=120) def run_model(target_dir, apply_mask=True, mask_edges=True, filter_black_bg=False, filter_white_bg=False): global model device = torch.device("cuda" if torch.cuda.is_available() else "cpu") with _model_lock: if model is None: model = initialize_mapanything_model(high_level_config, device) else: model = model.to(device) model.eval() image_folder_path = os.path.join(target_dir, "images") views = load_images(image_folder_path) if len(views) == 0: raise ValueError("No images found.") outputs = model.infer(views, apply_mask=apply_mask, mask_edges=True, memory_efficient_inference=False) extrinsics, intrinsics, world_pts, depths, images, masks = [], [], [], [], [], [] for pred in outputs: depthmap = pred["depth_z"][0].squeeze(-1) K = pred["intrinsics"][0] pose = pred["camera_poses"][0] pts3d, valid = depthmap_to_world_frame(depthmap, K, pose) msk = pred["mask"][0].squeeze(-1).cpu().numpy().astype(bool) if "mask" in pred else np.ones_like(depthmap.cpu().numpy(), dtype=bool) msk = msk & valid.cpu().numpy() extrinsics.append(pose.cpu().numpy()) intrinsics.append(K.cpu().numpy()) world_pts.append(pts3d.cpu().numpy()) depths.append(depthmap.cpu().numpy()) images.append(pred["img_no_norm"][0].cpu().numpy()) masks.append(msk) predictions = { "extrinsic": np.stack(extrinsics), "intrinsic": np.stack(intrinsics), "world_points": np.stack(world_pts), "depth": np.stack(depths)[..., np.newaxis] if np.stack(depths).ndim == 3 else np.stack(depths), "images": np.stack(images), "final_mask": np.stack(masks), } processed_data = process_predictions_for_visualization( predictions, views, high_level_config, filter_black_bg, filter_white_bg ) torch.cuda.empty_cache() return predictions, processed_data # --------------------------------------------------------------------------- # Rerun 3D logging # --------------------------------------------------------------------------- def log_predictions_to_rerun( predictions, filter_by_frames="All", show_cam=True, filter_black_bg=False, filter_white_bg=False ): rr.init("MapAnything", spawn=False) stream = rr.binary_stream() num_views = predictions["world_points"].shape[0] if filter_by_frames != "All": try: idx = int(filter_by_frames.split(":")[0]) frame_indices = [idx] except Exception: frame_indices = list(range(num_views)) else: frame_indices = list(range(num_views)) all_points = [] all_colors = [] for vi in frame_indices: pts = predictions["world_points"][vi] img = predictions["images"][vi] mask = predictions["final_mask"][vi] H, W = mask.shape[:2] if img.max() <= 1.0: img_u8 = (img * 255).astype(np.uint8) else: img_u8 = img.astype(np.uint8) if filter_black_bg: mask = mask & (img_u8.reshape(H, W, 3).sum(axis=2) >= 16) if filter_white_bg: c = img_u8.reshape(H, W, 3) mask = mask & ~((c[:, :, 0] > 240) & (c[:, :, 1] > 240) & (c[:, :, 2] > 240)) flat_pts = pts.reshape(-1, 3) flat_colors = img_u8.reshape(-1, 3) flat_mask = mask.reshape(-1) valid_pts = flat_pts[flat_mask] valid_colors = flat_colors[flat_mask] all_points.append(valid_pts) all_colors.append(valid_colors) if show_cam: extrinsic = predictions["extrinsic"][vi] intrinsic = predictions["intrinsic"][vi] rotation = extrinsic[:3, :3] translation = extrinsic[:3, 3] rr.log( f"world/camera_{vi}", rr.Transform3D( translation=translation, mat3x3=rotation, ), ) fx = float(intrinsic[0, 0]) fy = float(intrinsic[1, 1]) cx = float(intrinsic[0, 2]) cy = float(intrinsic[1, 2]) rr.log( f"world/camera_{vi}/image", rr.Pinhole( resolution=[W, H], focal_length=[fx, fy], principal_point=[cx, cy], ), ) rr.log( f"world/camera_{vi}/image/rgb", rr.Image(img_u8.reshape(H, W, 3)), ) if all_points: combined_pts = np.concatenate(all_points, axis=0) combined_colors = np.concatenate(all_colors, axis=0) rr.log( "world/pointcloud", rr.Points3D( positions=combined_pts, colors=combined_colors, radii=np.full(len(combined_pts), 0.005), ), ) return stream.read() # --------------------------------------------------------------------------- # Reconstruction pipeline # --------------------------------------------------------------------------- @spaces.GPU(duration=120) def gradio_reconstruct( target_dir, frame_filter="All", show_cam=True, filter_black_bg=False, filter_white_bg=False, apply_mask=True, ): if not target_dir or target_dir == "None" or not os.path.isdir(target_dir): return None, "No valid target directory. Please upload files first.", None, "All", None, None, None, None, "View 1", "View 1", "View 1" gc.collect() torch.cuda.empty_cache() target_dir_images = os.path.join(target_dir, "images") all_files = sorted(os.listdir(target_dir_images)) if os.path.isdir(target_dir_images) else [] all_files_labeled = [f"{i}: {fn}" for i, fn in enumerate(all_files)] frame_choices = ["All"] + all_files_labeled with torch.no_grad(): predictions, processed_data = run_model( target_dir, apply_mask, filter_black_bg=filter_black_bg, filter_white_bg=filter_white_bg, ) np.savez(os.path.join(target_dir, "predictions.npz"), **predictions) if frame_filter is None: frame_filter = "All" rerun_data = log_predictions_to_rerun( predictions, filter_by_frames=frame_filter, show_cam=show_cam, filter_black_bg=filter_black_bg, filter_white_bg=filter_white_bg, ) depth_vis = get_depth_vis(processed_data, 0) normal_vis = get_normal_vis(processed_data, 0) measure_img, _ = get_measure_image(processed_data, 0) num_views = len(processed_data) view_choices = [f"View {i + 1}" for i in range(num_views)] del predictions gc.collect() torch.cuda.empty_cache() log_msg = f"Reconstruction complete ({len(all_files)} frames)." return ( rerun_data, log_msg, processed_data, gr.Dropdown(choices=frame_choices, value=frame_filter, interactive=True), depth_vis, normal_vis, measure_img, "", gr.Dropdown(choices=view_choices, value="View 1"), gr.Dropdown(choices=view_choices, value="View 1"), gr.Dropdown(choices=view_choices, value="View 1"), ) def update_rerun_visualization( target_dir, frame_filter, show_cam, filter_black_bg, filter_white_bg ): if not target_dir or target_dir == "None" or not os.path.isdir(target_dir): return None, "No reconstruction available." predictions_path = os.path.join(target_dir, "predictions.npz") if not os.path.exists(predictions_path): return None, "No reconstruction found. Click Reconstruct first." loaded = np.load(predictions_path, allow_pickle=True) predictions = {key: loaded[key] for key in loaded.keys()} rerun_data = log_predictions_to_rerun( predictions, filter_by_frames=frame_filter or "All", show_cam=show_cam, filter_black_bg=filter_black_bg, filter_white_bg=filter_white_bg, ) return rerun_data, "Visualization updated." def update_all_views( target_dir, filter_black_bg, filter_white_bg, processed_data, d_sel, n_sel, m_sel ): if not target_dir or target_dir == "None" or not os.path.isdir(target_dir): return processed_data, None, None, None, [] predictions_path = os.path.join(target_dir, "predictions.npz") if not os.path.exists(predictions_path): return processed_data, None, None, None, [] try: loaded = np.load(predictions_path, allow_pickle=True) predictions = {k: loaded[k] for k in loaded.keys()} image_folder = os.path.join(target_dir, "images") views = load_images(image_folder) new_data = process_predictions_for_visualization( predictions, views, high_level_config, filter_black_bg, filter_white_bg ) di = max(0, int(d_sel.split()[1]) - 1) if d_sel else 0 ni = max(0, int(n_sel.split()[1]) - 1) if n_sel else 0 mi = max(0, int(m_sel.split()[1]) - 1) if m_sel else 0 return ( new_data, get_depth_vis(new_data, di), get_normal_vis(new_data, ni), get_measure_image(new_data, mi)[0], [], ) except Exception as e: print(f"Error updating views: {e}") return processed_data, None, None, None, [] # --------------------------------------------------------------------------- # Measure logic # --------------------------------------------------------------------------- def measure_click(processed_data, measure_points, view_selector, evt: gr.SelectData): try: if processed_data is None or len(processed_data) == 0: return None, [], "No data available" try: vi = int(view_selector.split()[1]) - 1 except Exception: vi = 0 vi = max(0, min(vi, len(processed_data) - 1)) vd = list(processed_data.values())[vi] if vd is None: return None, [], "No view data" px, py = evt.index[0], evt.index[1] if vd["mask"] is not None and 0 <= py < vd["mask"].shape[0] and 0 <= px < vd["mask"].shape[1]: if not vd["mask"][py, px]: img, _ = get_measure_image(processed_data, vi) return img, measure_points, "Cannot measure on masked areas (shown in grey overlay)." measure_points.append((px, py)) image, _ = get_measure_image(processed_data, vi) if image is None: return None, [], "No image" image = image.copy() pts3d = vd["points3d"] for p in measure_points: if 0 <= p[0] < image.shape[1] and 0 <= p[1] < image.shape[0]: image = cv2.circle(image, p, radius=5, color=(255, 0, 0), thickness=2) depth_text = "" for i, p in enumerate(measure_points): if vd["depth"] is not None and 0 <= p[1] < vd["depth"].shape[0] and 0 <= p[0] < vd["depth"].shape[1]: d = vd["depth"][p[1], p[0]] depth_text += f"P{i + 1} depth: {d:.2f}m | " if len(measure_points) == 2: p1, p2 = measure_points if all(0 <= p1[c] < image.shape[1 - c] for c in range(2)) and all(0 <= p2[c] < image.shape[1 - c] for c in range(2)): image = cv2.line(image, p1, p2, color=(255, 0, 0), thickness=2) dist_text = "Distance: N/A" if pts3d is not None: try: p1_3d = pts3d[p1[1], p1[0]] p2_3d = pts3d[p2[1], p2[0]] distance = np.linalg.norm(p1_3d - p2_3d) dist_text = f"Distance: {distance:.2f}m" except Exception: pass measure_points = [] return image, measure_points, depth_text + dist_text else: return image, measure_points, depth_text + "Click a second point to measure distance." except Exception as e: return None, [], f"Error: {e}" # --------------------------------------------------------------------------- # Navigation helpers # --------------------------------------------------------------------------- def nav_depth(processed_data, selector, direction): if not processed_data: return "View 1", None try: cur = int(selector.split()[1]) - 1 except Exception: cur = 0 nv = (cur + direction) % len(processed_data) return f"View {nv + 1}", get_depth_vis(processed_data, nv) def nav_normal(processed_data, selector, direction): if not processed_data: return "View 1", None try: cur = int(selector.split()[1]) - 1 except Exception: cur = 0 nv = (cur + direction) % len(processed_data) return f"View {nv + 1}", get_normal_vis(processed_data, nv) def nav_measure(processed_data, selector, direction): if not processed_data: return "View 1", None, [] try: cur = int(selector.split()[1]) - 1 except Exception: cur = 0 nv = (cur + direction) % len(processed_data) img, pts = get_measure_image(processed_data, nv) return f"View {nv + 1}", img, pts # --------------------------------------------------------------------------- # Upload handlers # --------------------------------------------------------------------------- def on_files_upload(files, interval): if not files: return "None", [], "No files uploaded." target_dir, paths = handle_uploads(files, interval) return target_dir, paths, f"Uploaded {len(paths)} images. Click Reconstruct to process." def on_example_load(scene_name): return load_example_scene(scene_name) # --------------------------------------------------------------------------- # CSS # --------------------------------------------------------------------------- css = r""" @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap'); *{box-sizing:border-box;margin:0;padding:0} body,.gradio-container{ background:#0f0f13!important; font-family:'Inter',system-ui,-apple-system,sans-serif!important; font-size:14px!important; color:#e4e4e7!important; min-height:100vh; } .dark body,.dark .gradio-container{ background:#0f0f13!important; color:#e4e4e7!important; } footer{display:none!important} .hidden-component{ display:none!important; height:0!important; overflow:hidden!important; margin:0!important; padding:0!important; } /* Main Shell */ .app-shell{ background:#18181b; border:1px solid #27272a; border-radius:16px; margin:12px auto; max-width:1500px; overflow:hidden; box-shadow:0 25px 50px -12px rgba(0,0,0,.6), 0 0 0 1px rgba(255,255,255,.03); } /* Header */ .app-header{ background:linear-gradient(135deg,#18181b 0%,#1e1e24 100%); border-bottom:1px solid #27272a; padding:14px 24px; display:flex; align-items:center; justify-content:space-between; } .app-header-left{ display:flex; align-items:center; gap:12px; } .app-logo{ width:36px;height:36px; background:linear-gradient(135deg,#6366f1,#8b5cf6,#a78bfa); border-radius:10px; display:flex;align-items:center;justify-content:center; font-size:14px;font-weight:800;color:#fff; box-shadow:0 4px 12px rgba(99,102,241,.35); letter-spacing:-0.5px; } .app-title{ font-size:18px;font-weight:700; background:linear-gradient(135deg,#e4e4e7,#a1a1aa); -webkit-background-clip:text; -webkit-text-fill-color:transparent; letter-spacing:-.3px; } .app-badge{ font-size:11px;font-weight:600; padding:3px 10px;border-radius:20px; background:rgba(99,102,241,.15); color:#818cf8; border:1px solid rgba(99,102,241,.25); letter-spacing:.3px; } /* Toolbar */ .app-toolbar{ background:#18181b; border-bottom:1px solid #27272a; padding:8px 16px; display:flex;gap:4px;align-items:center;flex-wrap:wrap; } .tb-sep{ width:1px;height:28px;background:#27272a;margin:0 8px; } .modern-tb-btn{ display:inline-flex;align-items:center;justify-content:center;gap:6px; min-width:32px;height:34px; background:transparent; border:1px solid transparent; border-radius:8px;cursor:pointer; font-size:13px;font-weight:600;padding:0 12px; font-family:'Inter',sans-serif; color:#a1a1aa!important; transition:all .15s ease; } .modern-tb-btn:hover{ background:rgba(99,102,241,.15); color:#ffffff!important; border-color:rgba(99,102,241,.3); } .modern-tb-btn:active,.modern-tb-btn.active{ background:rgba(99,102,241,.25); color:#ffffff!important; border-color:rgba(99,102,241,.45); } .modern-tb-btn .tb-icon{font-size:15px;line-height:1} .modern-tb-btn .tb-label{font-size:13px;font-weight:600} /* Main Layout */ .app-main-row{ display:flex;gap:0;flex:1;overflow:hidden; min-height:600px; } .app-main-left{ flex:1;display:flex;flex-direction:column;min-width:0; border-right:1px solid #27272a; } .app-main-right{ width:420px;display:flex;flex-direction:column;flex-shrink:0; background:#18181b;overflow-y:auto; } /* Upload Area */ .upload-area{ position:relative; background:#09090b; min-height:200px; display:flex;align-items:center;justify-content:center; cursor:pointer; border-bottom:1px solid #27272a; } .upload-click-area{ display:flex;flex-direction:column;align-items:center;justify-content:center; cursor:pointer;padding:30px 44px; border:2px dashed #3f3f46;border-radius:16px; background:rgba(99,102,241,.03); transition:all .2s ease; } .upload-click-area:hover{ background:rgba(99,102,241,.08); border-color:#6366f1;transform:scale(1.02); } .upload-click-area:active{ background:rgba(99,102,241,.12);transform:scale(.98); } .upload-click-area svg{width:64px;height:64px;margin-bottom:8px} .upload-hint{font-size:13px;color:#52525b;margin-top:4px} /* Gallery Grid */ .gallery-grid{ display:flex;flex-wrap:wrap;gap:6px; padding:10px 12px; background:#09090b; border-bottom:1px solid #27272a; max-height:200px; overflow-y:auto; } .gallery-grid:empty{display:none} .gallery-thumb{ width:80px;height:60px;object-fit:cover; border-radius:6px;border:1px solid #27272a; cursor:pointer;transition:all .15s; } .gallery-thumb:hover{border-color:#6366f1;transform:scale(1.05)} /* Tabs */ .tab-bar{ display:flex;gap:0; border-bottom:1px solid #27272a; background:#18181b; overflow-x:auto; } .tab-btn{ padding:10px 20px; font-size:13px;font-weight:600; color:#71717a; background:transparent; border:none;border-bottom:2px solid transparent; cursor:pointer; font-family:'Inter',sans-serif; transition:all .15s; white-space:nowrap; } .tab-btn:hover{color:#a1a1aa;background:rgba(99,102,241,.05)} .tab-btn.active{ color:#c7d2fe; border-bottom-color:#6366f1; background:rgba(99,102,241,.08); } .tab-content{display:none;flex:1;flex-direction:column;overflow:hidden} .tab-content.active{display:flex} /* Viewer Panel */ .viewer-panel{ flex:1;background:#09090b; display:flex;align-items:center;justify-content:center; position:relative;overflow:hidden; min-height:400px; } .viewer-placeholder{ color:#3f3f46;font-size:13px;text-align:center;padding:20px; } .viewer-panel img{ max-width:100%;max-height:100%;object-fit:contain; } /* Nav Row */ .nav-row{ display:flex;align-items:center;gap:8px; padding:8px 12px; background:#18181b; border-bottom:1px solid #27272a; } .nav-btn{ display:inline-flex;align-items:center;justify-content:center; height:30px;padding:0 14px; background:rgba(99,102,241,.1); border:1px solid rgba(99,102,241,.2); border-radius:6px;cursor:pointer; font-size:12px;font-weight:600;color:#c7d2fe; font-family:'Inter',sans-serif; transition:all .15s; } .nav-btn:hover{ background:rgba(99,102,241,.2);border-color:rgba(99,102,241,.35); color:#fff; } .nav-select{ flex:1; background:#09090b;border:1px solid #27272a;border-radius:6px; padding:5px 10px;font-size:12px;color:#e4e4e7; font-family:'JetBrains Mono',monospace; outline:none; } .nav-select:focus{border-color:#6366f1} /* Panel Cards */ .panel-card{border-bottom:1px solid #27272a} .panel-card-title{ padding:12px 20px;font-size:12px;font-weight:600; color:#71717a;text-transform:uppercase;letter-spacing:.8px; border-bottom:1px solid rgba(39,39,42,.6); } .panel-card-body{ padding:16px 20px;display:flex;flex-direction:column;gap:10px; } /* Settings Group */ .settings-group{ border:1px solid #27272a;border-radius:10px; margin:12px 16px;overflow:hidden; } .settings-group-title{ font-size:12px;font-weight:600;color:#71717a; text-transform:uppercase;letter-spacing:.8px; padding:10px 16px;border-bottom:1px solid #27272a; background:rgba(24,24,27,.5); } .settings-group-body{ padding:14px 16px;display:flex;flex-direction:column;gap:10px; } .checkbox-row{ display:flex;align-items:center;gap:8px; font-size:13px;color:#a1a1aa; } .checkbox-row input[type="checkbox"]{ accent-color:#6366f1;width:16px;height:16px;cursor:pointer; } .checkbox-row label{color:#a1a1aa;font-size:13px;cursor:pointer} .slider-row{ display:flex;align-items:center;gap:10px;min-height:28px; } .slider-row label{ font-size:13px;font-weight:500;color:#a1a1aa;min-width:90px;flex-shrink:0; } .slider-row input[type="range"]{ flex:1;-webkit-appearance:none;appearance:none; height:6px;background:#27272a;border-radius:3px;outline:none; } .slider-row input[type="range"]::-webkit-slider-thumb{ -webkit-appearance:none;appearance:none; width:16px;height:16px; background:linear-gradient(135deg,#6366f1,#7c3aed); border-radius:50%;cursor:pointer; box-shadow:0 2px 6px rgba(99,102,241,.4); } .slider-row .slider-val{ min-width:40px;text-align:right; font-family:'JetBrains Mono',monospace;font-size:12px; padding:3px 8px;background:#09090b; border:1px solid #27272a;border-radius:6px;color:#a1a1aa; } /* Primary Button */ .btn-primary{ display:flex;align-items:center;justify-content:center;gap:8px; width:100%; background:linear-gradient(135deg,#6366f1,#7c3aed); border:none;border-radius:10px; padding:12px 24px;cursor:pointer; font-size:15px;font-weight:600; font-family:'Inter',sans-serif;color:#fff; transition:all .2s ease; box-shadow:0 4px 16px rgba(99,102,241,.3), inset 0 1px 0 rgba(255,255,255,.1); letter-spacing:-.2px; } .btn-primary:hover{ background:linear-gradient(135deg,#7c7cf5,#8b5cf6); box-shadow:0 6px 24px rgba(99,102,241,.45); transform:translateY(-1px); } .btn-primary:active{ transform:translateY(0); box-shadow:0 2px 8px rgba(99,102,241,.3); } .btn-primary:disabled{ opacity:.5;cursor:not-allowed;transform:none; } .btn-secondary{ display:flex;align-items:center;justify-content:center;gap:6px; width:100%; background:transparent; border:1px solid #3f3f46;border-radius:8px; padding:8px 16px;cursor:pointer; font-size:13px;font-weight:600; font-family:'Inter',sans-serif;color:#a1a1aa; transition:all .15s; } .btn-secondary:hover{ background:rgba(255,255,255,.05);border-color:#52525b;color:#e4e4e7; } /* Loader */ .modern-loader{ display:none;position:absolute; top:0;left:0;right:0;bottom:0; background:rgba(9,9,11,.92);z-index:15; flex-direction:column;align-items:center;justify-content:center;gap:16px; backdrop-filter:blur(4px); } .modern-loader.active{display:flex} .loader-spinner{ width:36px;height:36px; border:3px solid #27272a;border-top-color:#6366f1; border-radius:50%;animation:spin .8s linear infinite; } @keyframes spin{to{transform:rotate(360deg)}} .loader-text{font-size:13px;color:#a1a1aa;font-weight:500} .loader-bar-track{ width:200px;height:4px;background:#27272a;border-radius:2px;overflow:hidden; } .loader-bar-fill{ height:100%; background:linear-gradient(90deg,#6366f1,#8b5cf6,#6366f1); background-size:200% 100%; animation:shimmer 1.5s ease-in-out infinite;border-radius:2px; } @keyframes shimmer{ 0%{background-position:200% 0}100%{background-position:-200% 0} } /* Toast */ .toast-notification{ position:fixed;top:24px;left:50%; transform:translateX(-50%) translateY(-120%); z-index:9999;padding:10px 24px;border-radius:10px; font-family:'Inter',sans-serif;font-size:14px;font-weight:600; display:flex;align-items:center;gap:8px; box-shadow:0 8px 24px rgba(0,0,0,.5); transition:transform .35s cubic-bezier(.34,1.56,.64,1),opacity .35s ease; opacity:0;pointer-events:none; } .toast-notification.visible{ transform:translateX(-50%) translateY(0);opacity:1;pointer-events:auto; } .toast-notification.error{ background:linear-gradient(135deg,#dc2626,#b91c1c);color:#fff; border:1px solid rgba(255,255,255,.15); } .toast-notification.success{ background:linear-gradient(135deg,#059669,#047857);color:#fff; border:1px solid rgba(255,255,255,.15); } .toast-notification.info{ background:linear-gradient(135deg,#2563eb,#1d4ed8);color:#fff; border:1px solid rgba(255,255,255,.15); } /* Status Bar */ .app-statusbar{ background:#18181b;border-top:1px solid #27272a; padding:6px 20px;display:flex;gap:12px;height:34px;align-items:center; } .sb-section{ padding:0 12px;flex:1;display:flex;align-items:center; font-family:'JetBrains Mono',monospace;font-size:12px;color:#52525b; overflow:hidden;white-space:nowrap; } .sb-section.sb-fixed{ flex:0 0 auto;min-width:90px;text-align:center;justify-content:center; padding:3px 12px;background:rgba(99,102,241,.08);border-radius:6px; color:#818cf8;font-weight:500; } /* Log Panel */ .log-panel{ background:#09090b;border-bottom:1px solid #27272a; padding:8px 16px; font-family:'JetBrains Mono',monospace; font-size:12px;color:#71717a; min-height:32px; display:flex;align-items:center; } .log-panel .log-dot{ width:6px;height:6px;border-radius:50%; margin-right:10px;flex-shrink:0; } .log-panel .log-dot.idle{background:#52525b} .log-panel .log-dot.ok{background:#22c55e} .log-panel .log-dot.busy{background:#eab308;animation:pulse 1s infinite} @keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}} /* Measure info */ .measure-info{ padding:10px 16px;background:rgba(99,102,241,.05); border-bottom:1px solid #27272a; font-size:13px;color:#a1a1aa;line-height:1.7; } .measure-info b{color:#c7d2fe;font-weight:600} .measure-result{ padding:10px 16px; font-family:'JetBrains Mono',monospace; font-size:13px;color:#a1a1aa; background:#09090b; border-top:1px solid #27272a; min-height:36px; } /* Example thumbs */ .example-thumb:hover{ border-color:#6366f1!important; box-shadow:0 4px 12px rgba(99,102,241,.2); transform:translateY(-2px); } /* Rerun viewer override */ .rerun-viewer-wrapper{ flex:1;min-height:400px; background:#09090b; } /* Scrollbars */ ::-webkit-scrollbar{width:8px;height:8px} ::-webkit-scrollbar-track{background:#09090b} ::-webkit-scrollbar-thumb{background:#27272a;border-radius:4px} ::-webkit-scrollbar-thumb:hover{background:#3f3f46} /* Hide gradio components used as state */ #gradio-hidden-group{ position:absolute!important; left:-9999px!important;top:-9999px!important; width:1px!important;height:1px!important; opacity:0.01!important;pointer-events:none!important; overflow:hidden!important; } @media(max-width:900px){ .app-main-row{flex-direction:column} .app-main-right{width:100%} .app-main-left{border-right:none;border-bottom:1px solid #27272a} } """ # --------------------------------------------------------------------------- # JavaScript # --------------------------------------------------------------------------- scenes_data = get_scene_info("examples") example_thumbs_html = build_example_thumbnails_html(scenes_data) main_js = r""" () => { if (window.__maInitDone) return; window.__maInitDone = true; // ---- Toast ---- let toastTimer = null; function showToast(msg, type) { let t = document.getElementById('app-toast'); if (!t) { t = document.createElement('div'); t.id = 'app-toast'; t.className = 'toast-notification'; t.innerHTML = ''; document.body.appendChild(t); } t.className = 'toast-notification ' + (type || 'info'); t.querySelector('.toast-icon').textContent = type === 'error' ? '\u2717' : type === 'success' ? '\u2713' : '\u2139'; t.querySelector('.toast-text').textContent = msg; if (toastTimer) clearTimeout(toastTimer); void t.offsetWidth; t.classList.add('visible'); toastTimer = setTimeout(() => t.classList.remove('visible'), 3500); } window.__showToast = showToast; // ---- Tabs ---- function initTabs() { const btns = document.querySelectorAll('.tab-btn[data-tab]'); btns.forEach(btn => { btn.addEventListener('click', () => { const group = btn.closest('.tab-container'); if (!group) return; group.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('active')); group.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active')); btn.classList.add('active'); const target = group.querySelector('#' + btn.dataset.tab); if (target) target.classList.add('active'); }); }); } initTabs(); // ---- File Upload ---- const fileInput = document.getElementById('ma-file-input'); const uploadArea = document.getElementById('ma-upload-area'); const uploadPrompt = document.getElementById('ma-upload-prompt'); const galleryGrid = document.getElementById('ma-gallery-grid'); if (uploadArea && fileInput) { uploadArea.addEventListener('click', () => fileInput.click()); uploadArea.addEventListener('dragover', e => { e.preventDefault(); uploadArea.style.outline = '2px solid #6366f1'; uploadArea.style.outlineOffset = '-2px'; }); uploadArea.addEventListener('dragleave', e => { e.preventDefault(); uploadArea.style.outline = ''; }); uploadArea.addEventListener('drop', e => { e.preventDefault(); uploadArea.style.outline = ''; if (e.dataTransfer.files.length) { fileInput.files = e.dataTransfer.files; fileInput.dispatchEvent(new Event('change', {bubbles: true})); } }); } // ---- Gradio value setter ---- function setGradioValue(containerId, value) { const container = document.getElementById(containerId); if (!container) return; const els = container.querySelectorAll('input, textarea'); els.forEach(el => { if (el.type === 'file' || el.type === 'range' || el.type === 'checkbox') return; const proto = el.tagName === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; const ns = Object.getOwnPropertyDescriptor(proto, 'value'); if (ns && ns.set) { ns.set.call(el, value); el.dispatchEvent(new Event('input', {bubbles:true, composed:true})); el.dispatchEvent(new Event('change', {bubbles:true, composed:true})); } }); } function setGradioCheckbox(containerId, checked) { const container = document.getElementById(containerId); if (!container) return; const cb = container.querySelector('input[type="checkbox"]'); if (cb && cb.checked !== checked) cb.click(); } // ---- Sync custom controls to Gradio hidden components ---- function syncSlider(customId, gradioId) { const slider = document.getElementById(customId); const valSpan = document.getElementById(customId + '-val'); if (!slider) return; slider.addEventListener('input', () => { if (valSpan) valSpan.textContent = parseFloat(slider.value).toFixed( slider.step && slider.step.includes('.') ? 1 : 0 ); const container = document.getElementById(gradioId); if (!container) return; container.querySelectorAll('input[type="range"],input[type="number"]').forEach(el => { const ns = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value'); if (ns && ns.set) { ns.set.call(el, slider.value); el.dispatchEvent(new Event('input', {bubbles:true, composed:true})); el.dispatchEvent(new Event('change', {bubbles:true, composed:true})); } }); }); } syncSlider('custom-interval', 'gradio-interval'); // Checkboxes ['show-cam', 'filter-black', 'filter-white', 'apply-mask'].forEach(name => { const el = document.getElementById('custom-' + name); if (!el) return; el.addEventListener('change', () => { setGradioCheckbox('gradio-' + name, el.checked); }); }); // ---- Loaders ---- function showLoader(id) { const l = document.getElementById(id); if (l) l.classList.add('active'); } function hideLoader(id) { const l = document.getElementById(id); if (l) l.classList.remove('active'); } window.__showLoader = showLoader; window.__hideLoader = hideLoader; function setLogStatus(text, state) { const logEl = document.getElementById('log-text'); const dotEl = document.getElementById('log-dot'); if (logEl) logEl.textContent = text; if (dotEl) { dotEl.className = 'log-dot'; dotEl.classList.add(state || 'idle'); } } window.__setLogStatus = setLogStatus; // ---- Reconstruct button ---- const reconstructBtn = document.getElementById('custom-reconstruct-btn'); if (reconstructBtn) { reconstructBtn.addEventListener('click', () => { showLoader('rerun-loader'); setLogStatus('Reconstructing...', 'busy'); const sb = document.querySelector('.sb-section.sb-fixed'); if (sb) sb.textContent = 'Processing...'; setTimeout(() => { const gradioBtn = document.getElementById('gradio-reconstruct-btn'); if (gradioBtn) { const btn = gradioBtn.querySelector('button'); if (btn) btn.click(); else gradioBtn.click(); } }, 150); }); } // ---- Clear button ---- const clearBtn = document.getElementById('custom-clear-btn'); if (clearBtn) { clearBtn.addEventListener('click', () => { if (galleryGrid) galleryGrid.innerHTML = ''; if (uploadPrompt) uploadPrompt.style.display = ''; setLogStatus('Ready', 'idle'); // Click gradio clear const gradioClearBtn = document.getElementById('gradio-clear-btn'); if (gradioClearBtn) { const btn = gradioClearBtn.querySelector('button'); if (btn) btn.click(); } }); } // ---- Example scenes ---- window.__loadExampleScene = function(sceneName) { setGradioValue('gradio-example-name', sceneName); setLogStatus('Loading example: ' + sceneName + '...', 'busy'); setTimeout(() => { const btn = document.getElementById('gradio-load-example-btn'); if (btn) { const b = btn.querySelector('button'); if (b) b.click(); else btn.click(); } }, 150); }; // ---- View navigation buttons ---- ['depth', 'normal', 'measure'].forEach(tab => { const prevBtn = document.getElementById('nav-prev-' + tab); const nextBtn = document.getElementById('nav-next-' + tab); if (prevBtn) { prevBtn.addEventListener('click', () => { const gradioBtn = document.getElementById('gradio-prev-' + tab); if (gradioBtn) { const b = gradioBtn.querySelector('button'); if (b) b.click(); } }); } if (nextBtn) { nextBtn.addEventListener('click', () => { const gradioBtn = document.getElementById('gradio-next-' + tab); if (gradioBtn) { const b = gradioBtn.querySelector('button'); if (b) b.click(); } }); } }); // ---- Watch gradio outputs and sync to custom UI ---- function watchOutputs() { // Log output const logGradio = document.getElementById('gradio-log-output'); if (logGradio) { const obs = new MutationObserver(() => { const md = logGradio.querySelector('.markdown-text, .prose, p, span'); if (md && md.textContent) { const text = md.textContent.trim(); if (text) { setLogStatus(text, text.toLowerCase().includes('error') ? 'idle' : 'ok'); const sb = document.querySelector('.sb-section.sb-fixed'); if (sb) sb.textContent = text.includes('complete') || text.includes('Success') ? 'Done' : 'Ready'; hideLoader('rerun-loader'); } } }); obs.observe(logGradio, {childList:true, subtree:true, characterData:true}); } // Depth image syncImageOutput('gradio-depth-image', 'custom-depth-img'); syncImageOutput('gradio-normal-image', 'custom-normal-img'); syncImageOutput('gradio-measure-image', 'custom-measure-img'); // Measure text const measureTextGradio = document.getElementById('gradio-measure-text'); if (measureTextGradio) { const obs = new MutationObserver(() => { const md = measureTextGradio.querySelector('.markdown-text, .prose, p, span'); const target = document.getElementById('custom-measure-result'); if (md && target) target.textContent = md.textContent || ''; }); obs.observe(measureTextGradio, {childList:true, subtree:true, characterData:true}); } // View selectors ['depth', 'normal', 'measure'].forEach(tab => { const gradioSel = document.getElementById('gradio-' + tab + '-selector'); const customSel = document.getElementById('custom-' + tab + '-select'); if (gradioSel && customSel) { const obs = new MutationObserver(() => { // Sync dropdown options and value from gradio to custom const gradioInput = gradioSel.querySelector('input'); if (gradioInput && gradioInput.value) { customSel.value = gradioInput.value; } // Sync options const options = gradioSel.querySelectorAll('li, option, [role="option"]'); if (options.length > 0) { const currentOptions = new Set(Array.from(customSel.options).map(o => o.value)); const newOptions = Array.from(options).map(o => o.textContent.trim()).filter(Boolean); if (newOptions.length > 0 && !newOptions.every(o => currentOptions.has(o))) { customSel.innerHTML = ''; newOptions.forEach(text => { const opt = document.createElement('option'); opt.value = text; opt.textContent = text; customSel.appendChild(opt); }); } } }); obs.observe(gradioSel, {childList:true, subtree:true, attributes:true, characterData:true}); } }); // Gallery sync from gradio-gallery const gradioGallery = document.getElementById('gradio-gallery'); if (gradioGallery && galleryGrid) { const gobs = new MutationObserver(() => { const imgs = gradioGallery.querySelectorAll('img'); if (imgs.length > 0) { galleryGrid.innerHTML = ''; imgs.forEach(img => { const thumb = document.createElement('img'); thumb.src = img.src; thumb.className = 'gallery-thumb'; galleryGrid.appendChild(thumb); }); if (uploadPrompt) uploadPrompt.style.display = 'none'; } }); gobs.observe(gradioGallery, {childList:true, subtree:true, attributes:true, attributeFilter:['src']}); } } function syncImageOutput(gradioId, customImgId) { const gradioContainer = document.getElementById(gradioId); const customImg = document.getElementById(customImgId); if (!gradioContainer || !customImg) return; const obs = new MutationObserver(() => { const img = gradioContainer.querySelector('img'); if (img && img.src && customImg.src !== img.src) { customImg.src = img.src; customImg.style.display = 'block'; const ph = customImg.parentElement.querySelector('.viewer-placeholder'); if (ph) ph.style.display = 'none'; } }); obs.observe(gradioContainer, {childList:true, subtree:true, attributes:true, attributeFilter:['src']}); } watchOutputs(); // Custom select change -> gradio ['depth', 'normal', 'measure'].forEach(tab => { const sel = document.getElementById('custom-' + tab + '-select'); if (!sel) return; sel.addEventListener('change', () => { setGradioValue('gradio-' + tab + '-selector', sel.value); // Trigger the gradio dropdown change const gradioSel = document.getElementById('gradio-' + tab + '-selector'); if (gradioSel) { const input = gradioSel.querySelector('input'); if (input) { const ns = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value'); if (ns && ns.set) { ns.set.call(input, sel.value); input.dispatchEvent(new Event('input', {bubbles:true, composed:true})); input.dispatchEvent(new Event('change', {bubbles:true, composed:true})); } } } }); }); } """ # --------------------------------------------------------------------------- # Build Gradio App # --------------------------------------------------------------------------- with gr.Blocks() as demo: # -- Hidden Gradio components for state and data binding -- with gr.Group(elem_id="gradio-hidden-group"): target_dir_state = gr.Textbox(value="None", elem_id="gradio-target-dir", elem_classes="hidden-component", container=False) example_name_state = gr.Textbox(value="", elem_id="gradio-example-name", elem_classes="hidden-component", container=False) is_example_state = gr.Textbox(value="None", elem_id="gradio-is-example", elem_classes="hidden-component", container=False) log_output = gr.Markdown("Ready", elem_id="gradio-log-output", elem_classes="hidden-component", container=False) unified_upload = gr.File(file_count="multiple", elem_id="gradio-file-upload", elem_classes="hidden-component", container=False, file_types=["image", "video"]) interval_slider = gr.Slider(minimum=0.1, maximum=5.0, value=1.0, step=0.1, elem_id="gradio-interval", elem_classes="hidden-component", container=False) show_cam_cb = gr.Checkbox(value=True, elem_id="gradio-show-cam", elem_classes="hidden-component", container=False) filter_black_cb = gr.Checkbox(value=False, elem_id="gradio-filter-black", elem_classes="hidden-component", container=False) filter_white_cb = gr.Checkbox(value=False, elem_id="gradio-filter-white", elem_classes="hidden-component", container=False) apply_mask_cb = gr.Checkbox(value=True, elem_id="gradio-apply-mask", elem_classes="hidden-component", container=False) frame_filter_dd = gr.Dropdown(choices=["All"], value="All", elem_id="gradio-frame-filter", elem_classes="hidden-component", container=False, allow_custom_value=True) image_gallery = gr.Gallery(elem_id="gradio-gallery", elem_classes="hidden-component", container=False) depth_image = gr.Image(type="numpy", elem_id="gradio-depth-image", elem_classes="hidden-component", container=False, format="png") normal_image = gr.Image(type="numpy", elem_id="gradio-normal-image", elem_classes="hidden-component", container=False, format="png") measure_image = gr.Image(type="numpy", elem_id="gradio-measure-image", elem_classes="hidden-component", container=False, format="png", interactive=False, sources=[]) measure_text_md = gr.Markdown("", elem_id="gradio-measure-text", elem_classes="hidden-component", container=False) depth_selector = gr.Dropdown(choices=["View 1"], value="View 1", elem_id="gradio-depth-selector", elem_classes="hidden-component", container=False, allow_custom_value=True) normal_selector = gr.Dropdown(choices=["View 1"], value="View 1", elem_id="gradio-normal-selector", elem_classes="hidden-component", container=False, allow_custom_value=True) measure_selector = gr.Dropdown(choices=["View 1"], value="View 1", elem_id="gradio-measure-selector", elem_classes="hidden-component", container=False, allow_custom_value=True) processed_data_state = gr.State(value=None) measure_points_state = gr.State(value=[]) reconstruct_btn = gr.Button("Reconstruct", elem_id="gradio-reconstruct-btn", elem_classes="hidden-component") clear_btn = gr.Button("Clear", elem_id="gradio-clear-btn", elem_classes="hidden-component") load_example_btn = gr.Button("Load Example", elem_id="gradio-load-example-btn", elem_classes="hidden-component") prev_depth_btn = gr.Button("Prev", elem_id="gradio-prev-depth", elem_classes="hidden-component") next_depth_btn = gr.Button("Next", elem_id="gradio-next-depth", elem_classes="hidden-component") prev_normal_btn = gr.Button("Prev", elem_id="gradio-prev-normal", elem_classes="hidden-component") next_normal_btn = gr.Button("Next", elem_id="gradio-next-normal", elem_classes="hidden-component") prev_measure_btn = gr.Button("Prev", elem_id="gradio-prev-measure", elem_classes="hidden-component") next_measure_btn = gr.Button("Next", elem_id="gradio-next-measure", elem_classes="hidden-component") # -- Rerun Viewer (hidden, data source only) -- rerun_viewer = Rerun( label="Rerun 3D Viewer", elem_id="gradio-rerun-viewer", streaming=True, ) # -- Main HTML UI -- gr.HTML(f"""
MapAnything 3D Reconstruction
MapAnything -- Metric 3D Reconstruction
Ready. Upload images or video to begin.
Running reconstruction...
3D point cloud will appear here after reconstruction
Depth map will appear after reconstruction
Normal map will appear after reconstruction
How to measure: Click two points on the image to measure the 3D distance between them. Grey areas indicate regions without depth data.
Measurement view will appear after reconstruction
Input
Click or drag files to upload
Images, videos (MP4, AVI, MOV), HEIC
Reconstruction Settings
1.0
Visualization Options
Example Scenes
{example_thumbs_html if example_thumbs_html else '
No example scenes found
'}
MapAnything -- Metric 3D Reconstruction from Images
Ready
""") # -- Load JavaScript -- demo.load(fn=None, js=main_js) # -- Wire custom file input to Gradio file upload -- # This JavaScript bridges the custom file input to the hidden Gradio file component file_bridge_js = r""" () => { const customInput = document.getElementById('ma-file-input'); const gradioUpload = document.getElementById('gradio-file-upload'); if (!customInput || !gradioUpload) { setTimeout(arguments.callee, 500); return; } customInput.addEventListener('change', () => { // Find the actual Gradio file input inside the hidden component const gradioInput = gradioUpload.querySelector('input[type="file"]'); if (gradioInput && customInput.files.length > 0) { // Create a DataTransfer to set files const dt = new DataTransfer(); Array.from(customInput.files).forEach(f => dt.items.add(f)); gradioInput.files = dt.files; gradioInput.dispatchEvent(new Event('change', {bubbles: true})); } customInput.value = ''; }); } """ demo.load(fn=None, js=file_bridge_js) # -- Wire Rerun viewer into the 3D tab -- rerun_mount_js = r""" () => { function mountRerun() { const rerunContainer = document.getElementById('gradio-rerun-viewer'); const targetPanel = document.querySelector('#tab-3d .rerun-viewer-wrapper'); if (!rerunContainer || !targetPanel) { setTimeout(mountRerun, 500); return; } // Move rerun viewer into the custom panel rerunContainer.style.display = ''; rerunContainer.style.width = '100%'; rerunContainer.style.height = '100%'; rerunContainer.style.position = 'absolute'; rerunContainer.style.top = '0'; rerunContainer.style.left = '0'; rerunContainer.style.zIndex = '5'; targetPanel.style.position = 'relative'; targetPanel.appendChild(rerunContainer); // Hide placeholder when rerun has content const placeholder = document.getElementById('rerun-placeholder'); const obs = new MutationObserver(() => { const hasCanvas = rerunContainer.querySelector('canvas, iframe, .rerun'); if (hasCanvas && placeholder) placeholder.style.display = 'none'; }); obs.observe(rerunContainer, {childList: true, subtree: true}); } mountRerun(); } """ demo.load(fn=None, js=rerun_mount_js) # -- Measure image click bridge -- # The measure_image gr.Image is hidden; we need clicks on custom-measure-img # to route through the Gradio component measure_bridge_js = r""" () => { function bridgeMeasure() { const customImg = document.getElementById('custom-measure-img'); const gradioMeasure = document.getElementById('gradio-measure-image'); if (!customImg || !gradioMeasure) { setTimeout(bridgeMeasure, 500); return; } // Forward clicks from custom image to the gradio image component customImg.addEventListener('click', (e) => { const rect = customImg.getBoundingClientRect(); const x = e.clientX - rect.left; const y = e.clientY - rect.top; const scaleX = customImg.naturalWidth / rect.width; const scaleY = customImg.naturalHeight / rect.height; const imgX = Math.round(x * scaleX); const imgY = Math.round(y * scaleY); // Find the gradio image element and simulate click const gradioImg = gradioMeasure.querySelector('img'); if (gradioImg) { const gRect = gradioImg.getBoundingClientRect(); // We need to compute position relative to the gradio image const gScaleX = gradioImg.naturalWidth ? rect.width / gradioImg.naturalWidth : 1; const gScaleY = gradioImg.naturalHeight ? rect.height / gradioImg.naturalHeight : 1; const clickEvent = new MouseEvent('click', { clientX: gRect.left + imgX / scaleX, clientY: gRect.top + imgY / scaleY, bubbles: true, cancelable: true, }); gradioImg.dispatchEvent(clickEvent); } }); } bridgeMeasure(); } """ demo.load(fn=None, js=measure_bridge_js) # ----------------------------------------------------------------------- # Event Wiring # ----------------------------------------------------------------------- # File upload unified_upload.change( fn=on_files_upload, inputs=[unified_upload, interval_slider], outputs=[target_dir_state, image_gallery, log_output], ) # Reconstruct reconstruct_btn.click( fn=lambda: (None, "Reconstructing..."), inputs=[], outputs=[rerun_viewer, log_output], ).then( fn=gradio_reconstruct, inputs=[ target_dir_state, frame_filter_dd, show_cam_cb, filter_black_cb, filter_white_cb, apply_mask_cb, ], outputs=[ rerun_viewer, log_output, processed_data_state, frame_filter_dd, depth_image, normal_image, measure_image, measure_text_md, depth_selector, normal_selector, measure_selector, ], ).then( fn=lambda: "False", outputs=[is_example_state], ) # Clear clear_btn.click( fn=lambda: ("None", None, None, None, "Ready", None, None, None, "", [], None), outputs=[ target_dir_state, rerun_viewer, image_gallery, unified_upload, log_output, depth_image, normal_image, measure_image, measure_text_md, measure_points_state, processed_data_state, ], ) # Load example load_example_btn.click( fn=lambda name: load_example_scene(name) if name else ("None", [], "No scene selected"), inputs=[example_name_state], outputs=[target_dir_state, image_gallery, log_output], ) # Visualization updates on checkbox changes for cb_component in [show_cam_cb, filter_black_cb, filter_white_cb]: cb_component.change( fn=update_rerun_visualization, inputs=[target_dir_state, frame_filter_dd, show_cam_cb, filter_black_cb, filter_white_cb], outputs=[rerun_viewer, log_output], ) filter_black_cb.change( fn=update_all_views, inputs=[target_dir_state, filter_black_cb, filter_white_cb, processed_data_state, depth_selector, normal_selector, measure_selector], outputs=[processed_data_state, depth_image, normal_image, measure_image, measure_points_state], ) filter_white_cb.change( fn=update_all_views, inputs=[target_dir_state, filter_black_cb, filter_white_cb, processed_data_state, depth_selector, normal_selector, measure_selector], outputs=[processed_data_state, depth_image, normal_image, measure_image, measure_points_state], ) frame_filter_dd.change( fn=update_rerun_visualization, inputs=[target_dir_state, frame_filter_dd, show_cam_cb, filter_black_cb, filter_white_cb], outputs=[rerun_viewer, log_output], ) # Depth navigation prev_depth_btn.click( fn=lambda pd, sel: nav_depth(pd, sel, -1), inputs=[processed_data_state, depth_selector], outputs=[depth_selector, depth_image], ) next_depth_btn.click( fn=lambda pd, sel: nav_depth(pd, sel, 1), inputs=[processed_data_state, depth_selector], outputs=[depth_selector, depth_image], ) depth_selector.change( fn=lambda pd, sel: get_depth_vis(pd, int(sel.split()[1]) - 1) if pd and sel else None, inputs=[processed_data_state, depth_selector], outputs=[depth_image], ) # Normal navigation prev_normal_btn.click( fn=lambda pd, sel: nav_normal(pd, sel, -1), inputs=[processed_data_state, normal_selector], outputs=[normal_selector, normal_image], ) next_normal_btn.click( fn=lambda pd, sel: nav_normal(pd, sel, 1), inputs=[processed_data_state, normal_selector], outputs=[normal_selector, normal_image], ) normal_selector.change( fn=lambda pd, sel: get_normal_vis(pd, int(sel.split()[1]) - 1) if pd and sel else None, inputs=[processed_data_state, normal_selector], outputs=[normal_image], ) # Measure navigation prev_measure_btn.click( fn=lambda pd, sel: nav_measure(pd, sel, -1), inputs=[processed_data_state, measure_selector], outputs=[measure_selector, measure_image, measure_points_state], ) next_measure_btn.click( fn=lambda pd, sel: nav_measure(pd, sel, 1), inputs=[processed_data_state, measure_selector], outputs=[measure_selector, measure_image, measure_points_state], ) measure_selector.change( fn=lambda pd, sel: get_measure_image(pd, int(sel.split()[1]) - 1) if pd and sel else (None, []), inputs=[processed_data_state, measure_selector], outputs=[measure_image, measure_points_state], ) # Measure click measure_image.select( fn=measure_click, inputs=[processed_data_state, measure_points_state, measure_selector], outputs=[measure_image, measure_points_state, measure_text_md], ) if __name__ == "__main__": demo.queue(max_size=20).launch( css=css, show_error=True, share=True, ssr_mode=False, mcp_server=True, )