Give the ViTPose keypoints a COCO-17 annotation context

Named, side-colored joints (left blue, right orange) with the COCO
skeleton drawn between them, low-confidence joints filtered out, labels
on hover only — the stage previously rendered seventeen identical dots.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

fdanyone/rerun_streaming.py

@@ -38,9 +38,9 @@ from fdanyone.video import choose_canonical_fps
 from fdanyone.viz import (
     BOX_COLOR,
     DIFFUSION_TIMELINE,
-    KEYPOINT_COLOR,
     VIEWS,
     body_blueprint,
+    coco17_annotation_context,
     diffusion_blueprint,
     log_body,
     log_result,
@@ -362,6 +362,10 @@ def pump(session: Session, work: Callable[[], T]) -> Iterator[str]:
     return outcome[0]
 
 
+KEYPOINT_CONFIDENCE: float = 0.3
+"""ViTPose confidence below which a joint is occlusion noise, not a detection."""
+
+
 def _motion_hook(
     recording: rr.RecordingStream, session: Session
 ) -> Callable[[str, dict[str, object]], None]:
@@ -381,11 +385,23 @@ def _motion_hook(
                 )
         elif stage == "keypoints_2d":
             keypoints: Float[torch.Tensor, "frames joints 3"] = payload["kp2d"]  # pyrefly: ignore
+            # Named, side-colored joints with the COCO skeleton drawn between
+            # them; the class ids resolve against this static context.
+            recording.log("world/camera/video/keypoints", coco17_annotation_context(), static=True)
             for index, frame in enumerate(keypoints.numpy()[: INFERENCE.num_frames]):
                 set_frame_on_recording(recording, index, session.spec.fps)
+                confident: np.ndarray = frame[:, 2] > KEYPOINT_CONFIDENCE
                 recording.log(
                     "world/camera/video/keypoints",
-                    rr.Points2D(positions=frame[:, :2], radii=6.0, colors=KEYPOINT_COLOR),
+                    rr.Points2D(
+                        positions=frame[confident, :2],
+                        radii=6.0,
+                        class_ids=0,
+                        keypoint_ids=np.flatnonzero(confident),
+                        # Names on seventeen joints at once bury the clip;
+                        # hovering a joint still shows its label.
+                        show_labels=False,
+                    ),
                 )
         log_status(recording, f"motion: {stage}")
         # Only the ``:start`` announcements become user-facing labels; the

fdanyone/viz.py

@@ -42,6 +42,68 @@ KEYPOINT_COLOR: tuple[int, int, int] = (248, 129, 81)
 JOINT_COLOR: tuple[int, int, int] = (255, 212, 59)
 BONE_COLOR: tuple[int, int, int] = (116, 192, 252)
 
+COCO17_KEYPOINTS: tuple[str, ...] = (
+    "nose",
+    "left_eye",
+    "right_eye",
+    "left_ear",
+    "right_ear",
+    "left_shoulder",
+    "right_shoulder",
+    "left_elbow",
+    "right_elbow",
+    "left_wrist",
+    "right_wrist",
+    "left_hip",
+    "right_hip",
+    "left_knee",
+    "right_knee",
+    "left_ankle",
+    "right_ankle",
+)
+"""ViTPose's COCO-17 joint order; GVHMR's ``kp2d`` rows follow it."""
+
+COCO17_CONNECTIONS: tuple[tuple[int, int], ...] = (
+    (0, 1), (0, 2), (1, 3), (2, 4),
+    (3, 5), (4, 6), (5, 6), (5, 7), (6, 8), (7, 9), (8, 10),
+    (5, 11), (6, 12), (11, 12), (11, 13), (12, 14), (13, 15), (14, 16),
+)
+"""The COCO skeleton over those joints: face, arms, torso, legs."""
+
+LEFT_COLOR: tuple[int, int, int] = (77, 171, 247)
+RIGHT_COLOR: tuple[int, int, int] = (255, 146, 43)
+CENTER_COLOR: tuple[int, int, int] = (255, 212, 59)
+
+
+def coco17_annotation_context():
+    """Class description that names, colors, and connects the ViTPose joints.
+
+    Left joints are blue and right joints orange, so a crossed detection reads
+    instantly; without this the keypoints were seventeen identical dots.
+    """
+
+    import rerun as rr
+
+    def joint_color(name: str) -> tuple[int, int, int]:
+        if name.startswith("left"):
+            return LEFT_COLOR
+        if name.startswith("right"):
+            return RIGHT_COLOR
+        return CENTER_COLOR
+
+    return rr.AnnotationContext(
+        [
+            rr.ClassDescription(
+                info=rr.AnnotationInfo(id=0, label="Person", color=KEYPOINT_COLOR),
+                keypoint_annotations=[
+                    rr.AnnotationInfo(id=index, label=name, color=joint_color(name))
+                    for index, name in enumerate(COCO17_KEYPOINTS)
+                ],
+                keypoint_connections=COCO17_CONNECTIONS,
+            )
+        ]
+    )
+
 
 def set_frame(index: int, fps: Fraction) -> None:
     """Put the next log calls on the shared frame and seconds timelines."""

tests/test_app_helpers.py

@@ -306,6 +306,15 @@ def test_a_stream_per_link_adds_no_framing_the_viewer_did_not_already_get(tmp_pa
         assert payload.startswith(RRD_MAGIC), f"read {row} is not a self-contained document"
 
 
+def test_coco17_annotation_context_matches_the_vitpose_layout() -> None:
+    """Seventeen named joints, and every skeleton edge points at one of them."""
+
+    assert len(viz.COCO17_KEYPOINTS) == 17
+    assert all(0 <= a < 17 and 0 <= b < 17 for a, b in viz.COCO17_CONNECTIONS)
+    context: rr.AnnotationContext = viz.coco17_annotation_context()
+    assert context is not None
+
+
 def test_format_status_counts_known_stages_and_passes_failures_through() -> None:
     """The banner shows x/x plus elapsed for the fixed run shape, nothing else."""