Spaces:
Running
Running
RemiFabre commited on
Commit Β·
8e1482d
1
Parent(s): 86f06fb
Add deep pipeline integration tests for hardware suite
Browse files5 new tests in TestFullPipeline class:
- test_recording_produces_correct_json: verify frame count, timestamps, structure
- test_recording_with_audio_produces_wav: verify WAV duration matches recording
- test_record_replay_full_lifecycle: record β verify β replay β delete
- test_stop_cancels_queued_recording: immediate cancel of queued recording
- test_recording_transitions_through_phases: observe queued β countdown β recording β idle
These test the full pipeline that the unit tests can't reach β the handoff
between HTTP API and robot worker thread where the freeze bug lived.
Hardware test count: 6 β 11
- tests/run_tests.py +1 -0
- tests/test_hardware.py +207 -1
tests/run_tests.py
CHANGED
|
@@ -58,6 +58,7 @@ TEST_CLASS_DESCRIPTIONS: dict[str, tuple[str, str]] = {
|
|
| 58 |
"TestHardwareStartup": ("hardware", "Robot reaches idle after startup"),
|
| 59 |
"TestHardwareRecording": ("hardware", "Record and verify motion capture"),
|
| 60 |
"TestHardwarePlayback": ("hardware", "Playback with and without audio"),
|
|
|
|
| 61 |
}
|
| 62 |
|
| 63 |
|
|
|
|
| 58 |
"TestHardwareStartup": ("hardware", "Robot reaches idle after startup"),
|
| 59 |
"TestHardwareRecording": ("hardware", "Record and verify motion capture"),
|
| 60 |
"TestHardwarePlayback": ("hardware", "Playback with and without audio"),
|
| 61 |
+
"TestFullPipeline": ("hardware", "Full record β verify files β replay β delete lifecycle"),
|
| 62 |
}
|
| 63 |
|
| 64 |
|
tests/test_hardware.py
CHANGED
|
@@ -16,11 +16,13 @@ Or via the test runner:
|
|
| 16 |
import json
|
| 17 |
import threading
|
| 18 |
import time
|
|
|
|
|
|
|
| 19 |
|
| 20 |
import pytest
|
| 21 |
import uvicorn
|
| 22 |
|
| 23 |
-
from marionette.main import create_app
|
| 24 |
|
| 25 |
# Skip all tests in this module unless -m hardware is specified
|
| 26 |
pytestmark = pytest.mark.hardware
|
|
@@ -258,3 +260,207 @@ class TestHardwarePlayback:
|
|
| 258 |
# Verify it's gone
|
| 259 |
state = httpx.get(f"{base_url}/api/state", timeout=5).json()
|
| 260 |
assert not any(m["id"] == move_id for m in state["moves"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
import json
|
| 17 |
import threading
|
| 18 |
import time
|
| 19 |
+
import wave
|
| 20 |
+
from pathlib import Path
|
| 21 |
|
| 22 |
import pytest
|
| 23 |
import uvicorn
|
| 24 |
|
| 25 |
+
from marionette.main import create_app, COUNTDOWN_SECONDS, MOTION_SAMPLE_RATE
|
| 26 |
|
| 27 |
# Skip all tests in this module unless -m hardware is specified
|
| 28 |
pytestmark = pytest.mark.hardware
|
|
|
|
| 260 |
# Verify it's gone
|
| 261 |
state = httpx.get(f"{base_url}/api/state", timeout=5).json()
|
| 262 |
assert not any(m["id"] == move_id for m in state["moves"])
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
class TestFullPipeline:
|
| 266 |
+
"""End-to-end pipeline tests: record β verify files β replay β delete.
|
| 267 |
+
|
| 268 |
+
These tests exercise the full lifecycle and check that output files
|
| 269 |
+
have the expected structure, durations, and frame counts.
|
| 270 |
+
"""
|
| 271 |
+
|
| 272 |
+
def test_recording_produces_correct_json(self, base_url: str, hw_marionette):
|
| 273 |
+
"""Record 3s silent, verify JSON has timestamps, frames at ~100Hz."""
|
| 274 |
+
import httpx
|
| 275 |
+
|
| 276 |
+
duration = 3.0
|
| 277 |
+
resp = httpx.post(
|
| 278 |
+
f"{base_url}/api/record",
|
| 279 |
+
json={"duration": duration, "record_audio": False, "label": "pipeline-json"},
|
| 280 |
+
timeout=5,
|
| 281 |
+
)
|
| 282 |
+
assert resp.status_code == 200
|
| 283 |
+
move_id = resp.json()["move_id"]
|
| 284 |
+
|
| 285 |
+
_wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + duration + 10)
|
| 286 |
+
|
| 287 |
+
# Read the actual JSON file from disk
|
| 288 |
+
json_path = hw_marionette._dataset_dir / f"{move_id}.json"
|
| 289 |
+
assert json_path.exists(), f"JSON file not found: {json_path}"
|
| 290 |
+
|
| 291 |
+
data = json.loads(json_path.read_text())
|
| 292 |
+
timestamps = data["time"]
|
| 293 |
+
frames = data["set_target_data"]
|
| 294 |
+
|
| 295 |
+
# Frame count should be close to duration * 100Hz
|
| 296 |
+
expected_frames = int(duration * MOTION_SAMPLE_RATE)
|
| 297 |
+
assert len(frames) > expected_frames * 0.8, (
|
| 298 |
+
f"Too few frames: {len(frames)} (expected ~{expected_frames})"
|
| 299 |
+
)
|
| 300 |
+
assert len(timestamps) == len(frames)
|
| 301 |
+
|
| 302 |
+
# Timestamps should span close to the requested duration
|
| 303 |
+
actual_duration = timestamps[-1] - timestamps[0]
|
| 304 |
+
assert actual_duration > duration * 0.8, (
|
| 305 |
+
f"Recorded duration too short: {actual_duration:.2f}s (expected ~{duration}s)"
|
| 306 |
+
)
|
| 307 |
+
|
| 308 |
+
# Each frame must have head pose and antennas
|
| 309 |
+
for frame in frames[:3]: # check first few
|
| 310 |
+
assert "head" in frame
|
| 311 |
+
assert "antennas" in frame
|
| 312 |
+
|
| 313 |
+
# Cleanup
|
| 314 |
+
httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
|
| 315 |
+
|
| 316 |
+
def test_recording_with_audio_produces_wav(self, base_url: str, hw_marionette):
|
| 317 |
+
"""Record 3s with mic, verify WAV file duration matches recording."""
|
| 318 |
+
import httpx
|
| 319 |
+
|
| 320 |
+
duration = 3.0
|
| 321 |
+
resp = httpx.post(
|
| 322 |
+
f"{base_url}/api/record",
|
| 323 |
+
json={"duration": duration, "record_audio": True, "label": "pipeline-audio"},
|
| 324 |
+
timeout=5,
|
| 325 |
+
)
|
| 326 |
+
assert resp.status_code == 200
|
| 327 |
+
move_id = resp.json()["move_id"]
|
| 328 |
+
|
| 329 |
+
_wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + duration + 10)
|
| 330 |
+
|
| 331 |
+
# Verify WAV exists and has reasonable duration
|
| 332 |
+
wav_path = hw_marionette._dataset_dir / f"{move_id}.wav"
|
| 333 |
+
assert wav_path.exists(), f"WAV file not found: {wav_path}"
|
| 334 |
+
|
| 335 |
+
with wave.open(str(wav_path), "rb") as wf:
|
| 336 |
+
wav_duration = wf.getnframes() / wf.getframerate()
|
| 337 |
+
assert wav_duration > duration * 0.7, (
|
| 338 |
+
f"WAV too short: {wav_duration:.2f}s (expected ~{duration}s)"
|
| 339 |
+
)
|
| 340 |
+
assert wav_duration < duration + 2.0, (
|
| 341 |
+
f"WAV too long: {wav_duration:.2f}s (expected ~{duration}s)"
|
| 342 |
+
)
|
| 343 |
+
|
| 344 |
+
# Also verify the JSON file reports has_audio
|
| 345 |
+
state = httpx.get(f"{base_url}/api/state", timeout=5).json()
|
| 346 |
+
move = next((m for m in state["moves"] if m["id"] == move_id), None)
|
| 347 |
+
assert move is not None
|
| 348 |
+
assert move["has_audio"] is True
|
| 349 |
+
|
| 350 |
+
# Cleanup
|
| 351 |
+
httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
|
| 352 |
+
|
| 353 |
+
def test_record_replay_full_lifecycle(self, base_url: str, hw_marionette):
|
| 354 |
+
"""Record β verify β replay β verify playback completes β delete."""
|
| 355 |
+
import httpx
|
| 356 |
+
|
| 357 |
+
duration = 2.0
|
| 358 |
+
|
| 359 |
+
# Step 1: Record
|
| 360 |
+
resp = httpx.post(
|
| 361 |
+
f"{base_url}/api/record",
|
| 362 |
+
json={"duration": duration, "record_audio": False, "label": "lifecycle"},
|
| 363 |
+
timeout=5,
|
| 364 |
+
)
|
| 365 |
+
assert resp.status_code == 200
|
| 366 |
+
move_id = resp.json()["move_id"]
|
| 367 |
+
|
| 368 |
+
state = _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + duration + 10)
|
| 369 |
+
assert any(m["id"] == move_id for m in state["moves"]), "Move not in list after recording"
|
| 370 |
+
|
| 371 |
+
# Step 2: Replay
|
| 372 |
+
resp = httpx.post(
|
| 373 |
+
f"{base_url}/api/play",
|
| 374 |
+
json={"move_id": move_id},
|
| 375 |
+
timeout=5,
|
| 376 |
+
)
|
| 377 |
+
assert resp.status_code == 200
|
| 378 |
+
|
| 379 |
+
state = _wait_for_mode(base_url, "idle", timeout=duration + 15)
|
| 380 |
+
assert state["mode"] == "idle"
|
| 381 |
+
|
| 382 |
+
# Step 3: Delete and verify cleanup
|
| 383 |
+
json_path = hw_marionette._dataset_dir / f"{move_id}.json"
|
| 384 |
+
assert json_path.exists()
|
| 385 |
+
|
| 386 |
+
resp = httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
|
| 387 |
+
assert resp.status_code == 200
|
| 388 |
+
assert not json_path.exists(), "JSON file should be deleted"
|
| 389 |
+
|
| 390 |
+
state = httpx.get(f"{base_url}/api/state", timeout=5).json()
|
| 391 |
+
assert not any(m["id"] == move_id for m in state["moves"])
|
| 392 |
+
|
| 393 |
+
def test_stop_cancels_queued_recording(self, base_url: str):
|
| 394 |
+
"""Submit a recording, immediately stop it β verify cancel works."""
|
| 395 |
+
import httpx
|
| 396 |
+
|
| 397 |
+
resp = httpx.post(
|
| 398 |
+
f"{base_url}/api/record",
|
| 399 |
+
json={"duration": 10.0, "record_audio": False, "label": "cancel-test"},
|
| 400 |
+
timeout=5,
|
| 401 |
+
)
|
| 402 |
+
assert resp.status_code == 200
|
| 403 |
+
|
| 404 |
+
# Immediately try to stop (may be queued or in countdown)
|
| 405 |
+
time.sleep(0.1)
|
| 406 |
+
resp = httpx.post(f"{base_url}/api/record/stop", timeout=5)
|
| 407 |
+
assert resp.status_code == 200
|
| 408 |
+
data = resp.json()
|
| 409 |
+
assert data["stopped"] is True
|
| 410 |
+
|
| 411 |
+
# Should return to idle
|
| 412 |
+
_wait_for_mode(base_url, "idle", timeout=10)
|
| 413 |
+
|
| 414 |
+
def test_recording_transitions_through_phases(self, base_url: str):
|
| 415 |
+
"""Verify the recording goes through queued β countdown β recording β idle."""
|
| 416 |
+
import httpx
|
| 417 |
+
|
| 418 |
+
duration = 3.0
|
| 419 |
+
observed_modes = set()
|
| 420 |
+
|
| 421 |
+
resp = httpx.post(
|
| 422 |
+
f"{base_url}/api/record",
|
| 423 |
+
json={"duration": duration, "record_audio": False, "label": "phases"},
|
| 424 |
+
timeout=5,
|
| 425 |
+
)
|
| 426 |
+
assert resp.status_code == 200
|
| 427 |
+
|
| 428 |
+
# Poll rapidly to observe phase transitions
|
| 429 |
+
deadline = time.time() + COUNTDOWN_SECONDS + duration + 10
|
| 430 |
+
while time.time() < deadline:
|
| 431 |
+
try:
|
| 432 |
+
state = httpx.get(f"{base_url}/api/state", timeout=2).json()
|
| 433 |
+
observed_modes.add(state["mode"])
|
| 434 |
+
|
| 435 |
+
# When in recording mode, verify timing fields are present
|
| 436 |
+
if state["mode"] == "recording":
|
| 437 |
+
assert state["phase_start_at"] is not None, "recording phase_start_at is null"
|
| 438 |
+
assert state["phase_end_at"] is not None, "recording phase_end_at is null"
|
| 439 |
+
|
| 440 |
+
# When in countdown, verify timing fields
|
| 441 |
+
if state["mode"] == "countdown":
|
| 442 |
+
assert state["phase_start_at"] is not None, "countdown phase_start_at is null"
|
| 443 |
+
assert state["phase_end_at"] is not None, "countdown phase_end_at is null"
|
| 444 |
+
|
| 445 |
+
if state["mode"] == "idle" and "phases" in (state.get("message") or ""):
|
| 446 |
+
break # Recording completed
|
| 447 |
+
except Exception:
|
| 448 |
+
pass
|
| 449 |
+
time.sleep(0.15)
|
| 450 |
+
|
| 451 |
+
# We should have seen at least countdown and recording phases
|
| 452 |
+
assert "countdown" in observed_modes, (
|
| 453 |
+
f"Never saw countdown mode. Observed: {observed_modes}"
|
| 454 |
+
)
|
| 455 |
+
assert "recording" in observed_modes, (
|
| 456 |
+
f"Never saw recording mode. Observed: {observed_modes}"
|
| 457 |
+
)
|
| 458 |
+
assert "idle" in observed_modes, (
|
| 459 |
+
f"Never returned to idle. Observed: {observed_modes}"
|
| 460 |
+
)
|
| 461 |
+
|
| 462 |
+
# Cleanup
|
| 463 |
+
state = httpx.get(f"{base_url}/api/state", timeout=5).json()
|
| 464 |
+
move = next((m for m in state["moves"] if "phases" in m.get("label", "")), None)
|
| 465 |
+
if move:
|
| 466 |
+
httpx.delete(f"{base_url}/api/moves/{move['id']}", timeout=5)
|