Spaces:
Running on Zero
Running on Zero
Commit ·
982b332
1
Parent(s): 131fa87
Reveal the recovered body while the generator loads
Browse filesThe mesh and camera streamed at the motion tail but stayed invisible:
the motion layout has no 3D pane and the diffusion grid replaced it
seconds later, leaving a minutes-long model-loading window staring at
stale detections. The motion tail now switches to a body layout — the
canonical world looping beside the projected source overlay — and the
denoise hook brings in the preview grid when the first step actually
lands.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
- fdanyone/rerun_streaming.py +11 -2
- fdanyone/viz.py +25 -0
- tests/test_app_helpers.py +7 -0
fdanyone/rerun_streaming.py
CHANGED
|
@@ -40,6 +40,7 @@ from fdanyone.viz import (
|
|
| 40 |
DIFFUSION_TIMELINE,
|
| 41 |
KEYPOINT_COLOR,
|
| 42 |
VIEWS,
|
|
|
|
| 43 |
diffusion_blueprint,
|
| 44 |
log_body,
|
| 45 |
log_result,
|
|
@@ -371,6 +372,8 @@ def _denoise_hook(
|
|
| 371 |
check_stop(session.spec)
|
| 372 |
if PREVIEW_DECODER is None:
|
| 373 |
return
|
|
|
|
|
|
|
| 374 |
plan: tuple[tuple[int, int], ...] = preview_slice_plan(x0_hat.shape[2])
|
| 375 |
recording.set_time(DIFFUSION_TIMELINE, sequence=step_index)
|
| 376 |
with torch.inference_mode():
|
|
@@ -510,6 +513,9 @@ def motion_phase(recording: rr.RecordingStream, session: Session) -> Iterator[st
|
|
| 510 |
f"motion: SMPL-X body, parameters, and source camera logged "
|
| 511 |
f"(median reprojection error {reprojection_error:.2f} px)",
|
| 512 |
)
|
|
|
|
|
|
|
|
|
|
| 513 |
yield "Generating six views."
|
| 514 |
|
| 515 |
|
|
@@ -524,7 +530,8 @@ def generate_phase(recording: rr.RecordingStream, session: Session) -> Iterator[
|
|
| 524 |
raise FourDAnyoneError("The motion phase did not finish.")
|
| 525 |
prepared: PreparedRun = session.prepared
|
| 526 |
session.source_frames = _decode_source_stills(session.spec)
|
| 527 |
-
|
|
|
|
| 528 |
yield "Generating six views."
|
| 529 |
|
| 530 |
def work() -> dict:
|
|
@@ -577,16 +584,18 @@ def smoke_motion_phase(recording: rr.RecordingStream, session: Session) -> Itera
|
|
| 577 |
log_status(recording, f"[smoke] motion frame {index}")
|
| 578 |
time.sleep(SMOKE_DELAY)
|
| 579 |
yield f"[smoke] motion frame {index}"
|
|
|
|
| 580 |
|
| 581 |
|
| 582 |
def smoke_generate_phase(recording: rr.RecordingStream, session: Session) -> Iterator[str]:
|
| 583 |
"""Synthetic diffusion phase: gradient frames sharpening per denoising step."""
|
| 584 |
|
| 585 |
rng: np.random.Generator = np.random.default_rng(1)
|
| 586 |
-
recording.send_blueprint(diffusion_blueprint(), make_active=True)
|
| 587 |
yield "[smoke] diffusion"
|
| 588 |
for step in range(4):
|
| 589 |
check_stop(session.spec)
|
|
|
|
|
|
|
| 590 |
recording.set_time(DIFFUSION_TIMELINE, sequence=step)
|
| 591 |
for view in range(VIEWS):
|
| 592 |
noise: UInt8[np.ndarray, "160 88 3"] = rng.integers(
|
|
|
|
| 40 |
DIFFUSION_TIMELINE,
|
| 41 |
KEYPOINT_COLOR,
|
| 42 |
VIEWS,
|
| 43 |
+
body_blueprint,
|
| 44 |
diffusion_blueprint,
|
| 45 |
log_body,
|
| 46 |
log_result,
|
|
|
|
| 372 |
check_stop(session.spec)
|
| 373 |
if PREVIEW_DECODER is None:
|
| 374 |
return
|
| 375 |
+
if step_index == 0:
|
| 376 |
+
recording.send_blueprint(diffusion_blueprint(), make_active=True)
|
| 377 |
plan: tuple[tuple[int, int], ...] = preview_slice_plan(x0_hat.shape[2])
|
| 378 |
recording.set_time(DIFFUSION_TIMELINE, sequence=step_index)
|
| 379 |
with torch.inference_mode():
|
|
|
|
| 513 |
f"motion: SMPL-X body, parameters, and source camera logged "
|
| 514 |
f"(median reprojection error {reprojection_error:.2f} px)",
|
| 515 |
)
|
| 516 |
+
# Reveal the recovered body right away: it loops here through the
|
| 517 |
+
# generator's model-loading window instead of hiding until the result.
|
| 518 |
+
recording.send_blueprint(body_blueprint(), make_active=True)
|
| 519 |
yield "Generating six views."
|
| 520 |
|
| 521 |
|
|
|
|
| 530 |
raise FourDAnyoneError("The motion phase did not finish.")
|
| 531 |
prepared: PreparedRun = session.prepared
|
| 532 |
session.source_frames = _decode_source_stills(session.spec)
|
| 533 |
+
# The body layout from the motion tail stays up while models load; the
|
| 534 |
+
# denoise hook switches to the preview grid when the first step lands.
|
| 535 |
yield "Generating six views."
|
| 536 |
|
| 537 |
def work() -> dict:
|
|
|
|
| 584 |
log_status(recording, f"[smoke] motion frame {index}")
|
| 585 |
time.sleep(SMOKE_DELAY)
|
| 586 |
yield f"[smoke] motion frame {index}"
|
| 587 |
+
recording.send_blueprint(body_blueprint(), make_active=True)
|
| 588 |
|
| 589 |
|
| 590 |
def smoke_generate_phase(recording: rr.RecordingStream, session: Session) -> Iterator[str]:
|
| 591 |
"""Synthetic diffusion phase: gradient frames sharpening per denoising step."""
|
| 592 |
|
| 593 |
rng: np.random.Generator = np.random.default_rng(1)
|
|
|
|
| 594 |
yield "[smoke] diffusion"
|
| 595 |
for step in range(4):
|
| 596 |
check_stop(session.spec)
|
| 597 |
+
if step == 0:
|
| 598 |
+
recording.send_blueprint(diffusion_blueprint(), make_active=True)
|
| 599 |
recording.set_time(DIFFUSION_TIMELINE, sequence=step)
|
| 600 |
for view in range(VIEWS):
|
| 601 |
noise: UInt8[np.ndarray, "160 88 3"] = rng.integers(
|
fdanyone/viz.py
CHANGED
|
@@ -334,6 +334,31 @@ def motion_blueprint() -> rrb.Blueprint:
|
|
| 334 |
)
|
| 335 |
|
| 336 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
def diffusion_blueprint() -> rrb.Blueprint:
|
| 338 |
"""A grid of per-view previews that fills in as the denoiser steps."""
|
| 339 |
|
|
|
|
| 334 |
)
|
| 335 |
|
| 336 |
|
| 337 |
+
def body_blueprint() -> rrb.Blueprint:
|
| 338 |
+
"""The recovered body looping in 3D while the generator loads its models."""
|
| 339 |
+
|
| 340 |
+
import rerun.blueprint as rrb
|
| 341 |
+
|
| 342 |
+
return rrb.Blueprint(
|
| 343 |
+
rrb.Horizontal(
|
| 344 |
+
rrb.Spatial3DView(origin="world", name="Canonical world"),
|
| 345 |
+
rrb.Spatial2DView(
|
| 346 |
+
origin="world/camera",
|
| 347 |
+
contents=["$origin/**", "/world/body/**"],
|
| 348 |
+
name="Source",
|
| 349 |
+
),
|
| 350 |
+
rrb.TextLogView(origin="log", name="Progress"),
|
| 351 |
+
column_shares=[2.0, 1.0, 1.0],
|
| 352 |
+
),
|
| 353 |
+
rrb.TimePanel(
|
| 354 |
+
timeline=TIME_TIMELINE,
|
| 355 |
+
play_state="playing",
|
| 356 |
+
loop_mode="all",
|
| 357 |
+
state="collapsed",
|
| 358 |
+
),
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
|
| 362 |
def diffusion_blueprint() -> rrb.Blueprint:
|
| 363 |
"""A grid of per-view previews that fills in as the denoiser steps."""
|
| 364 |
|
tests/test_app_helpers.py
CHANGED
|
@@ -171,6 +171,13 @@ def test_blueprints_build_for_every_phase() -> None:
|
|
| 171 |
|
| 172 |
assert viz.diffusion_blueprint() is not None
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
result: rrb.Blueprint = viz.result_blueprint(Fraction(25, 1))
|
| 175 |
top: rrb.Horizontal = result.root_container.contents[0] # pyrefly: ignore
|
| 176 |
dense: rrb.Horizontal = result.root_container.contents[1] # pyrefly: ignore
|
|
|
|
| 171 |
|
| 172 |
assert viz.diffusion_blueprint() is not None
|
| 173 |
|
| 174 |
+
body: rrb.Blueprint = viz.body_blueprint()
|
| 175 |
+
body_world: rrb.Spatial3DView = body.root_container.contents[0] # pyrefly: ignore
|
| 176 |
+
body_source: rrb.Spatial2DView = body.root_container.contents[1] # pyrefly: ignore
|
| 177 |
+
assert (body_world.origin, body_world.name) == ("world", "Canonical world")
|
| 178 |
+
assert body_source.contents == ["$origin/**", "/world/body/**"]
|
| 179 |
+
assert body.time_panel.play_state == "playing"
|
| 180 |
+
|
| 181 |
result: rrb.Blueprint = viz.result_blueprint(Fraction(25, 1))
|
| 182 |
top: rrb.Horizontal = result.root_container.contents[0] # pyrefly: ignore
|
| 183 |
dense: rrb.Horizontal = result.root_container.contents[1] # pyrefly: ignore
|