Show the body mesh and project it onto the source clip

The canonical world gains the translucent SMPL-X mesh (exoego pattern:
static faces with albedo alpha 90, streamed vertices) with the joints
and bones reading through it, and the moving source camera — pose
recovered per frame by a rigid Kabsch fit of the incam body onto the
canonical one, K_fullimg as its pinhole. The source video moves under
world/camera/video so the viewer reprojects the mesh and skeleton onto
the person natively (arkitscenes technique; world content listed
explicitly in the pane's contents). Result layout matches the approved
reference: wide 3D pane and source on top, six clean views in one row.
Median reprojection error against ViTPose observations: ~10 px at
704-pixel width, with a 1e-7 m rigid-fit residual.

Implemented by Codex against the reviewed spec.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

fdanyone/rerun_streaming.py

@@ -44,6 +44,7 @@ from fdanyone.viz import (
     log_body,
     log_result,
     log_smplx_params,
+    log_source_camera,
     log_status,
     log_video_stream,
     motion_blueprint,
@@ -53,7 +54,7 @@ from fdanyone.viz import (
 
 LOGGER: logging.Logger = logging.getLogger("fdanyone.app")
 
-APPLICATION_ID: str = "4danyone-rerun-v2"
+APPLICATION_ID: str = "4danyone-rerun-v3"
 """Versioned per blueprint change: the viewer persists blueprints by app id,
 so a stale layout from an earlier deploy would otherwise shadow a new one."""
 
@@ -338,7 +339,7 @@ def _motion_hook(
             for index, box in enumerate(boxes.numpy()[: INFERENCE.num_frames]):
                 set_frame_on_recording(recording, index, session.spec.fps)
                 recording.log(
-                    "source/bboxes",
+                    "world/camera/video/bboxes",
                     rr.Boxes2D(
                         array=box.reshape(1, 4), array_format=rr.Box2DFormat.XYXY, colors=BOX_COLOR
                     ),
@@ -348,7 +349,7 @@ def _motion_hook(
             for index, frame in enumerate(keypoints.numpy()[: INFERENCE.num_frames]):
                 set_frame_on_recording(recording, index, session.spec.fps)
                 recording.log(
-                    "source/keypoints",
+                    "world/camera/video/keypoints",
                     rr.Points2D(positions=frame[:, :2], radii=6.0, colors=KEYPOINT_COLOR),
                 )
         log_status(recording, f"motion: {stage}")
@@ -448,7 +449,7 @@ def _decode_source_stills(spec: RunSpec) -> dict[int, RgbFrame]:
 def source_phase(recording: rr.RecordingStream, spec: RunSpec) -> str:
     """Log the source clip on the frame timeline; CPU only."""
 
-    log_video_stream(recording, "source/video", spec.video_path)
+    log_video_stream(recording, "world/camera/video", spec.video_path)
     log_status(recording, "prepare: source clip logged")
     return "Recovering motion with GVHMR."
 
@@ -500,8 +501,15 @@ def motion_phase(recording: rr.RecordingStream, session: Session) -> Iterator[st
 
     body: BodyMotion = load_body_motion(session.prepared.motion_dir, device="cpu")
     log_body(recording, body, session.spec.fps)
+    reprojection_error: float = log_source_camera(
+        recording, body, session.prepared.motion_dir, session.spec.fps
+    )
     log_smplx_params(recording, session.prepared.motion_dir, session.spec.fps)
-    log_status(recording, "motion: SMPL-X body and parameters logged")
+    log_status(
+        recording,
+        f"motion: SMPL-X body, parameters, and source camera logged "
+        f"(median reprojection error {reprojection_error:.2f} px)",
+    )
     yield "Generating six views."
 
 
@@ -561,11 +569,11 @@ def smoke_motion_phase(recording: rr.RecordingStream, session: Session) -> Itera
         set_frame_on_recording(recording, index, session.spec.fps)
         x0: float = 180.0 + 2.0 * index
         recording.log(
-            "source/bboxes",
+            "world/camera/video/bboxes",
             rr.Boxes2D(array=[[x0, 300.0, x0 + 300.0, 1100.0]], array_format=rr.Box2DFormat.XYXY),
         )
         keypoints: Float[np.ndarray, "17 2"] = rng.uniform((x0, 350.0), (x0 + 300.0, 1050.0), (17, 2))
-        recording.log("source/keypoints", rr.Points2D(keypoints))
+        recording.log("world/camera/video/keypoints", rr.Points2D(keypoints))
         log_status(recording, f"[smoke] motion frame {index}")
         time.sleep(SMOKE_DELAY)
         yield f"[smoke] motion frame {index}"

fdanyone/viz.py

@@ -17,11 +17,11 @@ if TYPE_CHECKING:
     import rerun as rr
     import rerun.blueprint as rrb
     import torch
-    from jaxtyping import Float
+    from jaxtyping import Bool, Float, Int
 
     from fdanyone.motion.body import BodyMotion
 
-APPLICATION_ID = "4danyone"
+APPLICATION_ID: str = "4danyone-rerun-v3"
 FRAME_TIMELINE = "frame"
 TIME_TIMELINE = "time"
 
@@ -113,7 +113,7 @@ def log_video_stream(recording: rr.RecordingStream, entity: str, video: Path) ->
 
 
 def log_body(recording: rr.RecordingStream, body: BodyMotion, fps: Fraction) -> None:
-    """Log the posed SMPL-X joints and skeleton over the whole clip."""
+    """Log the posed SMPL-X mesh, joints, and skeleton over the whole clip."""
 
     import numpy as np
     import rerun as rr
@@ -127,9 +127,18 @@ def log_body(recording: rr.RecordingStream, body: BodyMotion, fps: Fraction) ->
     recording.log(
         "world/body/skeleton", rr.LineStrips3D.from_fields(radii=0.008, colors=BONE_COLOR), static=True
     )
+    recording.log(
+        "world/body/mesh",
+        rr.Mesh3D.from_fields(
+            triangle_indices=body.faces,
+            albedo_factor=(66, 135, 245, 90),
+        ),
+        static=True,
+    )
     for index in range(min(len(body.joints), INFERENCE.num_frames)):
         set_frame_on_recording(recording, index, fps)
         joints: Float[np.ndarray, "joints 3"] = body.joints[index]
+        vertices: Float[np.ndarray, "vertices 3"] = body.vertices[index]
         recording.log("world/body/joints", rr.Points3D.from_fields(positions=joints))
         recording.log(
             "world/body/skeleton",
@@ -137,6 +146,114 @@ def log_body(recording: rr.RecordingStream, body: BodyMotion, fps: Fraction) ->
                 strips=np.stack([joints[[child, parent]] for child, parent in bones], axis=0)
             ),
         )
+        recording.log("world/body/mesh", rr.Mesh3D.from_fields(vertex_positions=vertices))
+
+
+def log_source_camera(
+    recording: rr.RecordingStream,
+    body: BodyMotion,
+    motion_dir: Path,
+    fps: Fraction,
+) -> float:
+    """Log the moving source camera and return its median 2D joint error."""
+
+    import numpy as np
+    import rerun as rr
+    import smplx
+    import torch
+
+    from fdanyone.config import INFERENCE
+    from fdanyone.motion.body import NUM_SMPLX_SKELETON_JOINTS, _smplx_model_path
+    from fdanyone.motion.result import MotionResult
+
+    motion: MotionResult = MotionResult.load(motion_dir)
+    num_frames: int = min(motion.num_frames, len(body.joints), INFERENCE.num_frames)
+    body_model: torch.nn.Module = smplx.create(
+        model_path=str(_smplx_model_path()),
+        model_type="smplx",
+        gender="neutral",
+        num_betas=10,
+        num_pca_comps=12,
+        flat_hand_mean=False,
+        use_pca=True,
+        batch_size=motion.num_frames,
+    ).to("cpu")
+    parameters: dict[str, torch.Tensor] = motion.smpl_params_incam
+    with torch.inference_mode():
+        incam_joints_tensor: Float[torch.Tensor, "frames all_joints 3"] = body_model(
+            betas=parameters["betas"],
+            global_orient=parameters["global_orient"],
+            body_pose=parameters["body_pose"],
+            transl=parameters["transl"],
+        ).joints.detach()
+    incam_joints: Float[np.ndarray, "frames joints 3"] = (
+        incam_joints_tensor[:, :NUM_SMPLX_SKELETON_JOINTS].cpu().numpy().astype(np.float64)
+    )
+    # BodyMotion is load_body_motion's global-parameter pass after canonicalization.
+    canonical_joints: Float[np.ndarray, "frames joints 3"] = body.joints[:num_frames].astype(np.float64)
+    intrinsics: Float[np.ndarray, "frames 3 3"] = motion.K_fullimg[:num_frames].numpy().astype(np.float64)
+    observed_keypoints: Float[np.ndarray, "frames 17 3"] = (
+        motion.observed_keypoints_2d[:num_frames].numpy().astype(np.float64)
+    )
+    # COCO17 body order after the five facial keypoints.
+    coco_body_joints: Int[np.ndarray, "12"] = np.asarray(
+        [16, 17, 18, 19, 20, 21, 1, 2, 4, 5, 7, 8]
+    )
+    pixel_errors: list[float] = []
+
+    recording.log(
+        "world/camera",
+        rr.Pinhole(
+            image_from_camera=intrinsics[0],
+            resolution=(motion.image_width, motion.image_height),
+            camera_xyz=rr.ViewCoordinates.RDF,
+            image_plane_distance=0.35,
+        ),
+        static=True,
+    )
+    for index in range(num_frames):
+        source_joints: Float[np.ndarray, "joints 3"] = incam_joints[index]
+        target_joints: Float[np.ndarray, "joints 3"] = canonical_joints[index]
+        source_center: Float[np.ndarray, "3"] = source_joints.mean(axis=0)
+        target_center: Float[np.ndarray, "3"] = target_joints.mean(axis=0)
+        source_centered: Float[np.ndarray, "joints 3"] = source_joints - source_center
+        target_centered: Float[np.ndarray, "joints 3"] = target_joints - target_center
+        covariance: Float[np.ndarray, "3 3"] = source_centered.T @ target_centered
+        decomposition: tuple[
+            Float[np.ndarray, "3 3"], Float[np.ndarray, "3"], Float[np.ndarray, "3 3"]
+        ] = np.linalg.svd(covariance)
+        left_vectors: Float[np.ndarray, "3 3"] = decomposition[0]
+        right_vectors_t: Float[np.ndarray, "3 3"] = decomposition[2]
+        determinant_fix: Float[np.ndarray, "3"] = np.ones(3, dtype=np.float64)
+        orientation: float = float(np.linalg.det(left_vectors @ right_vectors_t))
+        determinant_fix[-1] = -1.0 if orientation < 0.0 else 1.0
+        camera_to_world_row: Float[np.ndarray, "3 3"] = (
+            left_vectors @ np.diag(determinant_fix) @ right_vectors_t
+        )
+        camera_to_world: Float[np.ndarray, "3 3"] = camera_to_world_row.T
+        translation: Float[np.ndarray, "3"] = target_center - source_center @ camera_to_world_row
+
+        set_frame_on_recording(recording, index, fps)
+        recording.log(
+            "world/camera",
+            rr.Transform3D(translation=translation, mat3x3=camera_to_world),
+        )
+
+        body_joints: Float[np.ndarray, "12 3"] = target_joints[coco_body_joints]
+        camera_joints: Float[np.ndarray, "12 3"] = (body_joints - translation) @ camera_to_world
+        pixels_homogeneous: Float[np.ndarray, "12 3"] = camera_joints @ intrinsics[index].T
+        projected_pixels: Float[np.ndarray, "12 2"] = (
+            pixels_homogeneous[:, :2] / pixels_homogeneous[:, 2:3]
+        )
+        observed_body: Float[np.ndarray, "12 3"] = observed_keypoints[index, 5:]
+        frame_errors: Float[np.ndarray, "12"] = np.linalg.norm(
+            projected_pixels - observed_body[:, :2], axis=1
+        )
+        visible: Bool[np.ndarray, "12"] = observed_body[:, 2] > 0.0
+        pixel_errors.extend(float(value) for value in frame_errors[visible])
+
+    median_error: float = float(np.median(np.asarray(pixel_errors, dtype=np.float64)))
+    return median_error
 
 
 def log_smplx_params(recording: rr.RecordingStream, motion_dir: Path, fps: Fraction) -> None:
@@ -208,8 +325,8 @@ def motion_blueprint() -> rrb.Blueprint:
 
     return rrb.Blueprint(
         rrb.Horizontal(
-            rrb.Spatial2DView(origin="source", contents=["source/video"], name="Source"),
-            rrb.Spatial2DView(origin="source", name="Detections"),
+            rrb.Spatial2DView(origin="world/camera", contents=["$origin/video"], name="Source"),
+            rrb.Spatial2DView(origin="world/camera", name="Detections"),
             rrb.TextLogView(origin="log", name="Progress"),
             column_shares=[1.0, 1.0, 1.0],
         ),
@@ -247,16 +364,20 @@ def result_blueprint(fps: Fraction) -> rrb.Blueprint:
         rrb.Vertical(
             rrb.Horizontal(
                 rrb.Spatial3DView(origin="world", name="Canonical world"),
-                rrb.Spatial2DView(origin="source", contents=["source/video"], name="Source"),
+                rrb.Spatial2DView(
+                    origin="world/camera",
+                    contents=["$origin/**", "/world/body/**"],
+                    name="Source",
+                ),
+                column_shares=[3.0, 1.0],
             ),
-            rrb.Grid(
+            rrb.Horizontal(
                 *(
-                    rrb.Spatial2DView(origin=f"world/cameras/dense/{index:02d}/image", name=f"View {index:02d}")
+                    rrb.Spatial2DView(origin=f"world/cameras/dense/{index:02d}/image")
                     for index in range(VIEWS)
                 ),
-                grid_columns=3,
             ),
-            row_shares=[1.0, 1.0],
+            row_shares=[2.0, 1.0],
         ),
         rrb.TimePanel(
             timeline=TIME_TIMELINE,

tests/test_app_helpers.py

@@ -14,6 +14,7 @@ from pathlib import Path
 import numpy as np
 import pytest
 import rerun as rr
+import rerun.blueprint as rrb
 import torch
 
 os.environ.setdefault("FDANYONE_SKIP_LOAD", "1")
@@ -157,11 +158,37 @@ def test_pump_reraises_a_worker_failure_on_the_caller_thread() -> None:
 
 
 def test_blueprints_build_for_every_phase() -> None:
-    """A malformed blueprint only fails when it is sent, so build all three."""
+    """Blueprints expose every phase in the approved camera-centered layout."""
+
+    assert streaming.APPLICATION_ID == "4danyone-rerun-v3"
+    assert viz.APPLICATION_ID == "4danyone-rerun-v3"
+
+    motion: rrb.Blueprint = viz.motion_blueprint()
+    source: rrb.Spatial2DView = motion.root_container.contents[0]  # pyrefly: ignore
+    detections: rrb.Spatial2DView = motion.root_container.contents[1]  # pyrefly: ignore
+    assert (source.origin, source.contents) == ("world/camera", ["$origin/video"])
+    assert detections.origin == "world/camera"
 
-    assert viz.motion_blueprint() is not None
     assert viz.diffusion_blueprint() is not None
-    assert viz.result_blueprint(Fraction(25, 1)) is not None
+
+    result: rrb.Blueprint = viz.result_blueprint(Fraction(25, 1))
+    top: rrb.Horizontal = result.root_container.contents[0]  # pyrefly: ignore
+    dense: rrb.Horizontal = result.root_container.contents[1]  # pyrefly: ignore
+    world: rrb.Spatial3DView = top.contents[0]  # pyrefly: ignore
+    result_source: rrb.Spatial2DView = top.contents[1]  # pyrefly: ignore
+    assert result.root_container.row_shares == [2.0, 1.0]
+    assert top.column_shares == [3.0, 1.0]
+    assert (world.origin, world.name) == ("world", "Canonical world")
+    assert (result_source.origin, result_source.contents) == (
+        "world/camera",
+        ["$origin/**", "/world/body/**"],
+    )
+    assert type(dense).__name__ == "Horizontal"
+    assert tuple(view.origin for view in dense.contents) == tuple(
+        f"world/cameras/dense/{index:02d}/image" for index in range(viz.VIEWS)
+    )
+    assert result.time_panel.play_state == "playing"
+    assert result.time_panel.loop_mode == "all"
 
 
 # ---------------------------------------------------------------------------