RemiFabre commited on
Commit
1f09aa2
·
1 Parent(s): e22d8a3

Add remote hardware testing via SSH and redesign motion accuracy tests

Browse files

- New run_on_robot.py: rsyncs code to robot, runs pytest -m hardware
via SSH, streams output in real-time, fetches JSON report back
- Redesign TestMotionAccuracy: replace set_target() approach (which
conflicts with Marionette's internal control) with synthetic
reference recordings played back via the API and observed via
get_current_head_pose()
- Add --on-robot flag to run_tests.py for convenience delegation
- Update TESTING.md with remote testing docs and SSH setup guide

Files changed (4) hide show
  1. TESTING.md +56 -2
  2. tests/run_on_robot.py +274 -0
  3. tests/run_tests.py +24 -2
  4. tests/test_hardware.py +510 -3
TESTING.md CHANGED
@@ -63,9 +63,11 @@ cd marionette
63
  pytest tests/e2e --browser chromium --browser firefox --browser webkit
64
  ```
65
 
66
- ### Hardware integration tests
67
 
68
- Requires a physical Reachy Mini robot connected and powered on:
 
 
69
 
70
  ```bash
71
  cd marionette
@@ -79,6 +81,53 @@ cd marionette
79
  python tests/run_tests.py --hardware
80
  ```
81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  ### All tests at once (no hardware)
83
 
84
  ```bash
@@ -125,6 +174,9 @@ pytest tests/ --browser chromium
125
  | TestHardwareRecording | 1 | Record silent and verify motion capture |
126
  | TestHardwarePlayback | 2 | Playback and delete (silent) |
127
  | TestFullPipeline | 4 | Full record → verify files → replay → delete lifecycle |
 
 
 
128
  | TestHardwareAudio | 3 | Audio recording and playback (may skip on mic issues) |
129
 
130
  ## View the matrix without running tests
@@ -197,10 +249,12 @@ tests/
197
  ├── conftest.py # Shared fixtures (TestClient, temp paths)
198
  ├── test_api.py # Unit tests — backend API
199
  ├── test_hardware.py # Hardware integration tests (real robot)
 
200
  ├── e2e/
201
  │ ├── conftest.py # Playwright server fixture (port 18042)
202
  │ └── test_ui.py # E2E browser tests
203
  ├── run_tests.py # Test runner + matrix logger + web export
 
204
  ├── show_matrix.py # Matrix viewer
205
  └── test_results.json # Auto-generated results log (git-ignored)
206
 
 
63
  pytest tests/e2e --browser chromium --browser firefox --browser webkit
64
  ```
65
 
66
+ ### Hardware integration tests (local)
67
 
68
+ Requires a physical Reachy Mini robot connected and powered on.
69
+ This runs tests on the local machine (useful when developing directly
70
+ on the robot or when the robot is accessible locally):
71
 
72
  ```bash
73
  cd marionette
 
81
  python tests/run_tests.py --hardware
82
  ```
83
 
84
+ ### Hardware tests on the robot (recommended)
85
+
86
+ Hardware tests should run **on the robot itself** because:
87
+ - Audio playback (`play_sound()`) requires local speakers — WAV files are
88
+ silently dropped over WebRTC
89
+ - The robot has direct hardware access (local mic, speakers, motors)
90
+ - Running locally eliminates network latency from test measurements
91
+
92
+ **One-time SSH setup:**
93
+
94
+ ```bash
95
+ # Copy your SSH key to the robot (password: pollen)
96
+ ssh-copy-id pollen@reachy-mini.local
97
+
98
+ # Verify passwordless access
99
+ ssh pollen@reachy-mini.local echo OK
100
+ ```
101
+
102
+ **Run tests remotely:**
103
+
104
+ ```bash
105
+ cd marionette
106
+ python tests/run_on_robot.py
107
+ ```
108
+
109
+ This will:
110
+ 1. rsync the marionette package + tests to `/tmp/marionette_test/` on the robot
111
+ 2. Install dev deps if missing (pytest, httpx, pytest-json-report, scipy)
112
+ 3. Run `pytest -m hardware` on the robot, streaming output in real-time
113
+ 4. Fetch the JSON report back and print a summary
114
+
115
+ **Options:**
116
+
117
+ ```bash
118
+ python tests/run_on_robot.py --dry-run # show what would be synced
119
+ python tests/run_on_robot.py --host 192.168.1.42 # custom host/IP
120
+ python tests/run_on_robot.py --user pollen # custom SSH user
121
+ python tests/run_on_robot.py -k test_playback # extra pytest args
122
+ ```
123
+
124
+ Or via the main test runner:
125
+
126
+ ```bash
127
+ python tests/run_tests.py --on-robot
128
+ python tests/run_tests.py --on-robot --host 192.168.1.42
129
+ ```
130
+
131
  ### All tests at once (no hardware)
132
 
133
  ```bash
 
174
  | TestHardwareRecording | 1 | Record silent and verify motion capture |
175
  | TestHardwarePlayback | 2 | Playback and delete (silent) |
176
  | TestFullPipeline | 4 | Full record → verify files → replay → delete lifecycle |
177
+ | TestMotionAccuracy | 3 | Synthetic playback accuracy — reference vs observed poses |
178
+ | TestMultiDuration | 7 | Recording and playback across 1s/3s/5s/10s durations |
179
+ | TestPerformance | 3 | Startup, recording, and playback latency benchmarks |
180
  | TestHardwareAudio | 3 | Audio recording and playback (may skip on mic issues) |
181
 
182
  ## View the matrix without running tests
 
249
  ├── conftest.py # Shared fixtures (TestClient, temp paths)
250
  ├── test_api.py # Unit tests — backend API
251
  ├── test_hardware.py # Hardware integration tests (real robot)
252
+ ├── pose_utils.py # Trajectory comparison utilities
253
  ├── e2e/
254
  │ ├── conftest.py # Playwright server fixture (port 18042)
255
  │ └── test_ui.py # E2E browser tests
256
  ├── run_tests.py # Test runner + matrix logger + web export
257
+ ├── run_on_robot.py # Remote test runner (SSH + rsync to robot)
258
  ├── show_matrix.py # Matrix viewer
259
  └── test_results.json # Auto-generated results log (git-ignored)
260
 
tests/run_on_robot.py ADDED
@@ -0,0 +1,274 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Run hardware tests remotely on the Reachy Mini robot via SSH.
3
+
4
+ Syncs the marionette package + tests to the robot, runs pytest remotely,
5
+ and fetches results back.
6
+
7
+ Usage:
8
+ cd marionette
9
+ python tests/run_on_robot.py # run all hardware tests
10
+ python tests/run_on_robot.py --dry-run # show what would be synced
11
+ python tests/run_on_robot.py --host 192.168.1.42 # custom host
12
+ python tests/run_on_robot.py -k test_playback # pass extra pytest args
13
+ """
14
+
15
+ import argparse
16
+ import subprocess
17
+ import sys
18
+ import tempfile
19
+ from pathlib import Path
20
+
21
+ MARIONETTE_DIR = Path(__file__).resolve().parent.parent
22
+ REMOTE_DIR = "/tmp/marionette_test"
23
+ ROBOT_PYTHON = "/venvs/apps_venv/bin/python"
24
+ DEV_DEPS = ["pytest", "httpx", "pytest-json-report", "scipy"]
25
+
26
+ RSYNC_EXCLUDES = [
27
+ "__pycache__",
28
+ "*.egg-info",
29
+ "build/",
30
+ "local_dataset/",
31
+ ".pytest_cache/",
32
+ "dataset_registry.json",
33
+ "temp_uploads/",
34
+ "tests/e2e/",
35
+ "tests/test_results.json",
36
+ ".git/",
37
+ ]
38
+
39
+
40
+ def build_rsync_cmd(
41
+ host: str, user: str, *, dry_run: bool = False
42
+ ) -> list[str]:
43
+ """Build the rsync command to sync marionette code to the robot."""
44
+ target = f"{user}@{host}:{REMOTE_DIR}/"
45
+ cmd = [
46
+ "rsync", "-avz", "--delete",
47
+ ]
48
+ for excl in RSYNC_EXCLUDES:
49
+ cmd.extend(["--exclude", excl])
50
+ if dry_run:
51
+ cmd.append("--dry-run")
52
+ # Sync the whole marionette project directory
53
+ cmd.extend([
54
+ f"{MARIONETTE_DIR}/marionette/",
55
+ f"{target}marionette/",
56
+ ])
57
+ return cmd, target
58
+
59
+
60
+ def sync_to_robot(host: str, user: str, *, dry_run: bool = False) -> bool:
61
+ """Rsync marionette package, tests, and pyproject.toml to the robot."""
62
+ target_base = f"{user}@{host}:{REMOTE_DIR}"
63
+
64
+ # Build exclude args
65
+ exclude_args = []
66
+ for excl in RSYNC_EXCLUDES:
67
+ exclude_args.extend(["--exclude", excl])
68
+
69
+ base_cmd = ["rsync", "-avz", "--delete"] + exclude_args
70
+ if dry_run:
71
+ base_cmd.append("--dry-run")
72
+
73
+ # Sync marionette/ package
74
+ print(f"Syncing marionette/ -> {target_base}/marionette/")
75
+ result = subprocess.run(
76
+ base_cmd + [f"{MARIONETTE_DIR}/marionette/", f"{target_base}/marionette/"],
77
+ )
78
+ if result.returncode != 0:
79
+ print(f"rsync failed for marionette/ (exit {result.returncode})")
80
+ return False
81
+
82
+ # Sync tests/
83
+ print(f"Syncing tests/ -> {target_base}/tests/")
84
+ result = subprocess.run(
85
+ base_cmd + [f"{MARIONETTE_DIR}/tests/", f"{target_base}/tests/"],
86
+ )
87
+ if result.returncode != 0:
88
+ print(f"rsync failed for tests/ (exit {result.returncode})")
89
+ return False
90
+
91
+ # Sync pyproject.toml
92
+ print(f"Syncing pyproject.toml -> {target_base}/")
93
+ sync_file_cmd = ["rsync", "-avz"]
94
+ if dry_run:
95
+ sync_file_cmd.append("--dry-run")
96
+ result = subprocess.run(
97
+ sync_file_cmd + [f"{MARIONETTE_DIR}/pyproject.toml", f"{target_base}/pyproject.toml"],
98
+ )
99
+ if result.returncode != 0:
100
+ print(f"rsync failed for pyproject.toml (exit {result.returncode})")
101
+ return False
102
+
103
+ return True
104
+
105
+
106
+ def ssh_cmd(host: str, user: str, command: str) -> list[str]:
107
+ """Build an SSH command list."""
108
+ return [
109
+ "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new",
110
+ f"{user}@{host}", command,
111
+ ]
112
+
113
+
114
+ def ensure_deps(host: str, user: str) -> bool:
115
+ """Install dev dependencies on the robot if missing."""
116
+ # Check which deps are missing
117
+ check = " && ".join(
118
+ f"{ROBOT_PYTHON} -c 'import {dep.replace('-', '_')}' 2>/dev/null || echo MISSING:{dep}"
119
+ for dep in DEV_DEPS
120
+ )
121
+ result = subprocess.run(
122
+ ssh_cmd(host, user, check),
123
+ capture_output=True, text=True,
124
+ )
125
+ missing = [
126
+ line.split("MISSING:")[1]
127
+ for line in result.stdout.splitlines()
128
+ if line.startswith("MISSING:")
129
+ ]
130
+ if not missing:
131
+ print("All dev dependencies present on robot.")
132
+ return True
133
+
134
+ print(f"Installing missing deps on robot: {', '.join(missing)}")
135
+ pip = ROBOT_PYTHON.replace("/python", "/pip")
136
+ install_cmd = f"{pip} install {' '.join(missing)}"
137
+ result = subprocess.run(ssh_cmd(host, user, install_cmd))
138
+ return result.returncode == 0
139
+
140
+
141
+ def run_tests_on_robot(
142
+ host: str, user: str, extra_pytest_args: list[str],
143
+ ) -> int:
144
+ """Run pytest on the robot via SSH, streaming output in real-time."""
145
+ pytest_args = [
146
+ "-m", "hardware",
147
+ "-v",
148
+ "--json-report",
149
+ f"--json-report-file={REMOTE_DIR}/report.json",
150
+ "--tb=short",
151
+ ] + extra_pytest_args
152
+
153
+ remote_cmd = (
154
+ f"cd {REMOTE_DIR} && "
155
+ f"PYTHONPATH={REMOTE_DIR} "
156
+ f"{ROBOT_PYTHON} -m pytest {' '.join(pytest_args)}"
157
+ )
158
+
159
+ print(f"\n{'=' * 60}")
160
+ print(f"Running hardware tests on {user}@{host}")
161
+ print(f"{'=' * 60}\n")
162
+
163
+ proc = subprocess.Popen(
164
+ ssh_cmd(host, user, remote_cmd),
165
+ stdout=subprocess.PIPE,
166
+ stderr=subprocess.STDOUT,
167
+ text=True,
168
+ bufsize=1,
169
+ )
170
+
171
+ for line in proc.stdout:
172
+ print(line, end="")
173
+
174
+ proc.wait()
175
+ return proc.returncode
176
+
177
+
178
+ def fetch_results(host: str, user: str) -> Path | None:
179
+ """Fetch the JSON report from the robot."""
180
+ local_report = Path(tempfile.mktemp(suffix=".json", prefix="robot_report_"))
181
+ result = subprocess.run(
182
+ ["scp", f"{user}@{host}:{REMOTE_DIR}/report.json", str(local_report)],
183
+ capture_output=True,
184
+ )
185
+ if result.returncode == 0 and local_report.exists():
186
+ return local_report
187
+ return None
188
+
189
+
190
+ def print_summary(report_path: Path | None) -> None:
191
+ """Print a summary from the JSON report."""
192
+ if report_path is None:
193
+ print("\nNo JSON report available.")
194
+ return
195
+
196
+ import json
197
+ report = json.loads(report_path.read_text())
198
+ summary = report.get("summary", {})
199
+ passed = summary.get("passed", 0)
200
+ failed = summary.get("failed", 0)
201
+ errors = summary.get("error", 0)
202
+ skipped = summary.get("skipped", 0)
203
+ total = summary.get("total", 0)
204
+
205
+ print(f"\n{'=' * 60}")
206
+ print("REMOTE TEST SUMMARY")
207
+ print(f"{'=' * 60}")
208
+ print(f" Total: {total}")
209
+ print(f" Passed: {passed}")
210
+ print(f" Failed: {failed}")
211
+ print(f" Errors: {errors}")
212
+ print(f" Skipped: {skipped}")
213
+
214
+ # Show failed tests
215
+ for test in report.get("tests", []):
216
+ outcome = test.get("outcome", "")
217
+ if outcome in ("failed", "error"):
218
+ print(f" FAIL: {test.get('nodeid', '?')}")
219
+
220
+ # Cleanup
221
+ report_path.unlink(missing_ok=True)
222
+
223
+
224
+ def main() -> None:
225
+ parser = argparse.ArgumentParser(
226
+ description="Run hardware tests on the Reachy Mini robot via SSH.",
227
+ )
228
+ parser.add_argument(
229
+ "--host", default="reachy-mini.local",
230
+ help="Robot hostname or IP (default: reachy-mini.local)",
231
+ )
232
+ parser.add_argument(
233
+ "--user", default="pollen",
234
+ help="SSH username (default: pollen)",
235
+ )
236
+ parser.add_argument(
237
+ "--dry-run", action="store_true",
238
+ help="Show what would be synced without running tests",
239
+ )
240
+ parser.add_argument(
241
+ "pytest_args", nargs="*",
242
+ help="Extra arguments to pass to pytest (e.g., -k test_playback)",
243
+ )
244
+ args = parser.parse_args()
245
+
246
+ # Step 1: Sync code to robot
247
+ print(f"{'=' * 60}")
248
+ print(f"Syncing code to {args.user}@{args.host}:{REMOTE_DIR}")
249
+ print(f"{'=' * 60}")
250
+ ok = sync_to_robot(args.host, args.user, dry_run=args.dry_run)
251
+ if not ok:
252
+ print("\nFailed to sync code to robot.")
253
+ sys.exit(1)
254
+ if args.dry_run:
255
+ print("\nDry run complete — no tests were executed.")
256
+ sys.exit(0)
257
+
258
+ # Step 2: Ensure dev deps
259
+ if not ensure_deps(args.host, args.user):
260
+ print("\nFailed to install dependencies on robot.")
261
+ sys.exit(1)
262
+
263
+ # Step 3: Run tests
264
+ returncode = run_tests_on_robot(args.host, args.user, args.pytest_args)
265
+
266
+ # Step 4: Fetch results
267
+ report = fetch_results(args.host, args.user)
268
+ print_summary(report)
269
+
270
+ sys.exit(returncode)
271
+
272
+
273
+ if __name__ == "__main__":
274
+ main()
tests/run_tests.py CHANGED
@@ -7,8 +7,9 @@ and prints the coverage matrix with a test catalog summary.
7
 
8
  Usage:
9
  cd marionette
10
- python tests/run_tests.py # unit + E2E
11
- python tests/run_tests.py --hardware # also run hardware tests
 
12
  """
13
 
14
  import argparse
@@ -59,6 +60,9 @@ TEST_CLASS_DESCRIPTIONS: dict[str, tuple[str, str]] = {
59
  "TestHardwareRecording": ("hardware", "Record and verify motion capture"),
60
  "TestHardwarePlayback": ("hardware", "Playback and delete (silent)"),
61
  "TestFullPipeline": ("hardware", "Full record → verify files → replay → delete lifecycle"),
 
 
 
62
  "TestHardwareAudio": ("hardware", "Audio recording and playback (may skip on mic issues)"),
63
  }
64
 
@@ -364,8 +368,26 @@ def main() -> None:
364
  "--hardware", action="store_true",
365
  help="Also run hardware integration tests (requires connected robot)",
366
  )
 
 
 
 
 
 
 
 
 
 
 
 
367
  args = parser.parse_args()
368
 
 
 
 
 
 
 
369
  os_name = detect_os()
370
  print(f"Detected OS: {os_name}")
371
 
 
7
 
8
  Usage:
9
  cd marionette
10
+ python tests/run_tests.py # unit + E2E
11
+ python tests/run_tests.py --hardware # also run hardware tests (local)
12
+ python tests/run_tests.py --on-robot # run hardware tests on robot via SSH
13
  """
14
 
15
  import argparse
 
60
  "TestHardwareRecording": ("hardware", "Record and verify motion capture"),
61
  "TestHardwarePlayback": ("hardware", "Playback and delete (silent)"),
62
  "TestFullPipeline": ("hardware", "Full record → verify files → replay → delete lifecycle"),
63
+ "TestMotionAccuracy": ("hardware", "Synthetic playback accuracy — reference vs observed poses"),
64
+ "TestMultiDuration": ("hardware", "Recording and playback across 1s/3s/5s/10s durations"),
65
+ "TestPerformance": ("hardware", "Startup, recording, and playback latency benchmarks"),
66
  "TestHardwareAudio": ("hardware", "Audio recording and playback (may skip on mic issues)"),
67
  }
68
 
 
368
  "--hardware", action="store_true",
369
  help="Also run hardware integration tests (requires connected robot)",
370
  )
371
+ parser.add_argument(
372
+ "--on-robot", action="store_true",
373
+ help="Run hardware tests on the robot via SSH (delegates to run_on_robot.py)",
374
+ )
375
+ parser.add_argument(
376
+ "--host", default="reachy-mini.local",
377
+ help="Robot hostname for --on-robot (default: reachy-mini.local)",
378
+ )
379
+ parser.add_argument(
380
+ "--user", default="pollen",
381
+ help="SSH user for --on-robot (default: pollen)",
382
+ )
383
  args = parser.parse_args()
384
 
385
+ # Delegate to run_on_robot.py if --on-robot
386
+ if args.on_robot:
387
+ run_on_robot = TESTS_DIR / "run_on_robot.py"
388
+ cmd = [sys.executable, str(run_on_robot), "--host", args.host, "--user", args.user]
389
+ sys.exit(subprocess.run(cmd).returncode)
390
+
391
  os_name = detect_os()
392
  print(f"Detected OS: {os_name}")
393
 
tests/test_hardware.py CHANGED
@@ -182,7 +182,7 @@ def hardware_server(tmp_path_factory):
182
  srv.stop()
183
  pytest.skip("Robot startup animation did not finish in time")
184
 
185
- yield base_url, marionette
186
 
187
  stop_event.set()
188
  srv.stop()
@@ -199,6 +199,12 @@ def hw_marionette(hardware_server):
199
  return hardware_server[1]
200
 
201
 
 
 
 
 
 
 
202
  # ──────── Tests ────────────────────────────────────────────────────────
203
  #
204
  # Test ordering: silent operations first, audio operations last.
@@ -462,6 +468,505 @@ class TestFullPipeline:
462
  httpx.delete(f"{base_url}/api/moves/{move['id']}", timeout=5)
463
 
464
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
465
  class TestHardwareAudio:
466
  """Audio recording/playback tests — run last.
467
 
@@ -530,8 +1035,10 @@ class TestHardwareAudio:
530
  assert wav_duration > duration * 0.7, (
531
  f"WAV too short: {wav_duration:.2f}s (expected ~{duration}s)"
532
  )
533
- assert wav_duration < duration + 2.0, (
534
- f"WAV too long: {wav_duration:.2f}s (expected ~{duration}s)"
 
 
535
  )
536
 
537
  # Also verify the JSON file reports has_audio
 
182
  srv.stop()
183
  pytest.skip("Robot startup animation did not finish in time")
184
 
185
+ yield base_url, marionette, reachy
186
 
187
  stop_event.set()
188
  srv.stop()
 
199
  return hardware_server[1]
200
 
201
 
202
+ @pytest.fixture(scope="session")
203
+ def hw_reachy(hardware_server):
204
+ """The ReachyMini instance used by the Marionette server."""
205
+ return hardware_server[2]
206
+
207
+
208
  # ──────── Tests ────────────────────────────────────────────────────────
209
  #
210
  # Test ordering: silent operations first, audio operations last.
 
468
  httpx.delete(f"{base_url}/api/moves/{move['id']}", timeout=5)
469
 
470
 
471
+ def _create_synthetic_move(hw_marionette, label, duration, trajectory_fn):
472
+ """Create a synthetic recording and inject it into Marionette's dataset.
473
+
474
+ Args:
475
+ hw_marionette: The Marionette instance (from fixture).
476
+ label: Label/move_id for the recording.
477
+ duration: Duration in seconds.
478
+ trajectory_fn: Callable(t) -> (roll, pitch, yaw) in radians.
479
+ Called at 100Hz for the full duration.
480
+
481
+ Returns:
482
+ move_id (str) — the ID of the injected move.
483
+ """
484
+ import numpy as np
485
+ from scipy.spatial.transform import Rotation as R
486
+
487
+ dt = 1.0 / MOTION_SAMPLE_RATE
488
+ n = int(duration * MOTION_SAMPLE_RATE)
489
+ timestamps = []
490
+ frames = []
491
+ for i in range(n):
492
+ t = i * dt
493
+ roll, pitch, yaw = trajectory_fn(t)
494
+ rot = R.from_euler("xyz", [roll, pitch, yaw], degrees=False).as_matrix()
495
+ pose = np.eye(4)
496
+ pose[:3, :3] = rot
497
+ timestamps.append(t)
498
+ frames.append({
499
+ "head": pose.tolist(),
500
+ "antennas": [0.0, 0.0],
501
+ "body_yaw": 0.0,
502
+ "check_collision": False,
503
+ })
504
+
505
+ move_id = label
506
+ data = {
507
+ "description": f"Synthetic reference: {label}",
508
+ "time": timestamps,
509
+ "set_target_data": frames,
510
+ }
511
+ json_path = hw_marionette._dataset_dir / f"{move_id}.json"
512
+ json_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
513
+
514
+ # Tell Marionette to pick up the new file
515
+ hw_marionette._refresh_recordings()
516
+
517
+ return move_id
518
+
519
+
520
+ def _observe_playback(base_url, hw_reachy, duration):
521
+ """Observe head poses during playback at ~100Hz.
522
+
523
+ Waits for playing mode, records poses until idle, returns
524
+ (observed_times, observed_frames).
525
+ """
526
+ import httpx
527
+ import numpy as np
528
+
529
+ observed_times = []
530
+ observed_frames = []
531
+ playback_started = False
532
+ t_start = time.time()
533
+ t0 = None
534
+ last_state_check = 0
535
+ mode = "unknown"
536
+
537
+ while time.time() - t_start < duration + 30:
538
+ now = time.time()
539
+
540
+ # Check API state every 0.5s (not every iteration — too slow)
541
+ if now - last_state_check > 0.5:
542
+ try:
543
+ state = httpx.get(f"{base_url}/api/state", timeout=2).json()
544
+ mode = state["mode"]
545
+ except Exception:
546
+ pass
547
+ last_state_check = now
548
+
549
+ if mode == "playing":
550
+ if not playback_started:
551
+ playback_started = True
552
+ t0 = now
553
+ pose = hw_reachy.get_current_head_pose()
554
+ elapsed = now - t0
555
+ observed_times.append(elapsed)
556
+ observed_frames.append({
557
+ "head": np.asarray(pose, dtype=float).tolist(),
558
+ "antennas": [0.0, 0.0],
559
+ "body_yaw": 0.0,
560
+ })
561
+ elif playback_started and mode == "idle":
562
+ break
563
+
564
+ time.sleep(0.01)
565
+
566
+ return observed_times, observed_frames
567
+
568
+
569
+ class TestMotionAccuracy:
570
+ """Play back synthetic reference recordings and compare observed poses.
571
+
572
+ Creates known-trajectory JSON files, injects them into Marionette's
573
+ dataset, plays them via the API, and observes actual robot poses
574
+ during playback using get_current_head_pose().
575
+ """
576
+
577
+ def test_playback_reproduces_yaw_oscillation(
578
+ self, base_url: str, hw_marionette, hw_reachy,
579
+ ):
580
+ """Inject sine-wave yaw reference, play back, observe, compare."""
581
+ import httpx
582
+ import numpy as np
583
+ from scipy.spatial.transform import Rotation as R
584
+ from pose_utils import compare_trajectories, frames_to_poses
585
+
586
+ duration = 3.0
587
+ freq, amplitude = 0.5, 0.4
588
+
589
+ _ensure_idle(base_url)
590
+
591
+ # Create synthetic reference with yaw oscillation
592
+ move_id = _create_synthetic_move(
593
+ hw_marionette,
594
+ label="synth-yaw-osc",
595
+ duration=duration,
596
+ trajectory_fn=lambda t: (0.0, 0.0, amplitude * np.sin(2 * np.pi * freq * t)),
597
+ )
598
+
599
+ # Load reference for comparison
600
+ ref_data = json.loads(
601
+ (hw_marionette._dataset_dir / f"{move_id}.json").read_text()
602
+ )
603
+ ref_times = ref_data["time"]
604
+ ref_frames = ref_data["set_target_data"]
605
+
606
+ # Play it back
607
+ resp = httpx.post(
608
+ f"{base_url}/api/play",
609
+ json={"move_id": move_id},
610
+ timeout=5,
611
+ )
612
+ assert resp.status_code == 200
613
+
614
+ # Observe poses during playback
615
+ observed_times, observed_frames = _observe_playback(
616
+ base_url, hw_reachy, duration,
617
+ )
618
+
619
+ assert len(observed_frames) > 50, (
620
+ f"Too few observed frames: {len(observed_frames)}"
621
+ )
622
+
623
+ # Compare reference to observed
624
+ metrics = compare_trajectories(
625
+ ref_times, ref_frames, observed_times, observed_frames,
626
+ )
627
+ print(f"\nYaw oscillation playback ({len(observed_frames)} frames):")
628
+ print(metrics.summary())
629
+
630
+ assert metrics.magic_mean < 50, (
631
+ f"Mean magic distance too high: {metrics.magic_mean:.1f}\n"
632
+ f"{metrics.summary()}"
633
+ )
634
+
635
+ # Verify the yaw actually varied — robot moved
636
+ rec_poses = frames_to_poses(observed_frames)
637
+ yaws = [R.from_matrix(p[:3, :3]).as_euler("xyz")[2] for p in rec_poses]
638
+ yaw_range = max(yaws) - min(yaws)
639
+ assert yaw_range > 0.3, (
640
+ f"Yaw range too small: {yaw_range:.2f} rad — robot may not have moved"
641
+ )
642
+ print(f" Yaw range: {np.degrees(yaw_range):.1f} deg")
643
+
644
+ # Cleanup
645
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
646
+
647
+ def test_playback_reproduces_pitch_roll(
648
+ self, base_url: str, hw_marionette, hw_reachy,
649
+ ):
650
+ """Inject combined pitch+roll reference, play back, observe, compare."""
651
+ import httpx
652
+ import numpy as np
653
+ from pose_utils import compare_trajectories
654
+
655
+ duration = 3.0
656
+ _ensure_idle(base_url)
657
+
658
+ move_id = _create_synthetic_move(
659
+ hw_marionette,
660
+ label="synth-pitch-roll",
661
+ duration=duration,
662
+ trajectory_fn=lambda t: (
663
+ 0.15 * np.sin(2 * np.pi * 0.4 * t), # roll
664
+ 0.2 * np.sin(2 * np.pi * 0.3 * t), # pitch
665
+ 0.0, # yaw
666
+ ),
667
+ )
668
+
669
+ ref_data = json.loads(
670
+ (hw_marionette._dataset_dir / f"{move_id}.json").read_text()
671
+ )
672
+ ref_times = ref_data["time"]
673
+ ref_frames = ref_data["set_target_data"]
674
+
675
+ resp = httpx.post(
676
+ f"{base_url}/api/play",
677
+ json={"move_id": move_id},
678
+ timeout=5,
679
+ )
680
+ assert resp.status_code == 200
681
+
682
+ observed_times, observed_frames = _observe_playback(
683
+ base_url, hw_reachy, duration,
684
+ )
685
+
686
+ assert len(observed_frames) > 50, (
687
+ f"Too few observed frames: {len(observed_frames)}"
688
+ )
689
+
690
+ metrics = compare_trajectories(
691
+ ref_times, ref_frames, observed_times, observed_frames,
692
+ )
693
+ print(f"\nPitch+roll playback ({len(observed_frames)} frames):")
694
+ print(metrics.summary())
695
+
696
+ assert metrics.magic_mean < 50, (
697
+ f"Mean magic distance too high: {metrics.magic_mean:.1f}\n"
698
+ f"{metrics.summary()}"
699
+ )
700
+
701
+ # Cleanup
702
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
703
+
704
+ def test_playback_long_motion(
705
+ self, base_url: str, hw_marionette, hw_reachy,
706
+ ):
707
+ """10s reference with slow oscillation — verify accuracy over time."""
708
+ import httpx
709
+ import numpy as np
710
+ from pose_utils import compare_trajectories
711
+
712
+ duration = 10.0
713
+ _ensure_idle(base_url)
714
+
715
+ move_id = _create_synthetic_move(
716
+ hw_marionette,
717
+ label="synth-long-yaw",
718
+ duration=duration,
719
+ trajectory_fn=lambda t: (0.0, 0.0, 0.3 * np.sin(2 * np.pi * 0.2 * t)),
720
+ )
721
+
722
+ ref_data = json.loads(
723
+ (hw_marionette._dataset_dir / f"{move_id}.json").read_text()
724
+ )
725
+ ref_times = ref_data["time"]
726
+ ref_frames = ref_data["set_target_data"]
727
+
728
+ resp = httpx.post(
729
+ f"{base_url}/api/play",
730
+ json={"move_id": move_id},
731
+ timeout=5,
732
+ )
733
+ assert resp.status_code == 200
734
+
735
+ observed_times, observed_frames = _observe_playback(
736
+ base_url, hw_reachy, duration,
737
+ )
738
+
739
+ assert len(observed_frames) > 200, (
740
+ f"Too few observed frames for 10s playback: {len(observed_frames)}"
741
+ )
742
+
743
+ metrics = compare_trajectories(
744
+ ref_times, ref_frames, observed_times, observed_frames,
745
+ )
746
+ print(f"\nLong playback ({len(observed_frames)} frames, {duration}s):")
747
+ print(metrics.summary())
748
+
749
+ assert metrics.magic_mean < 50, (
750
+ f"Mean magic distance too high: {metrics.magic_mean:.1f}\n"
751
+ f"{metrics.summary()}"
752
+ )
753
+
754
+ # Check first half vs second half — accuracy shouldn't degrade
755
+ half = len(metrics.magic_distances) // 2
756
+ first_half_mean = float(np.mean(metrics.magic_distances[:half]))
757
+ second_half_mean = float(np.mean(metrics.magic_distances[half:]))
758
+ print(f" First half mean: {first_half_mean:.1f}")
759
+ print(f" Second half mean: {second_half_mean:.1f}")
760
+
761
+ # Second half shouldn't be more than 2x worse than first half
762
+ assert second_half_mean < first_half_mean * 2 + 10, (
763
+ f"Accuracy degraded over time: first={first_half_mean:.1f}, "
764
+ f"second={second_half_mean:.1f}"
765
+ )
766
+
767
+ # Cleanup
768
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
769
+
770
+
771
+ class TestMultiDuration:
772
+ """Test recording and playback across different durations.
773
+
774
+ Verifies frame counts, timing, and data integrity for short, medium,
775
+ and long recordings.
776
+ """
777
+
778
+ @pytest.mark.parametrize("duration,label", [
779
+ (1.0, "short-1s"),
780
+ (3.0, "medium-3s"),
781
+ (5.0, "standard-5s"),
782
+ (10.0, "long-10s"),
783
+ ])
784
+ def test_record_duration(self, base_url: str, hw_marionette, duration: float, label: str):
785
+ """Record at various durations, verify frame count and timing."""
786
+ import httpx
787
+
788
+ _ensure_idle(base_url)
789
+ resp = httpx.post(
790
+ f"{base_url}/api/record",
791
+ json={"duration": duration, "record_audio": False, "label": label},
792
+ timeout=5,
793
+ )
794
+ assert resp.status_code == 200
795
+ move_id = resp.json()["move_id"]
796
+
797
+ _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + duration + 15)
798
+
799
+ # Verify the recording
800
+ json_path = hw_marionette._dataset_dir / f"{move_id}.json"
801
+ assert json_path.exists()
802
+ data = json.loads(json_path.read_text())
803
+
804
+ timestamps = data["time"]
805
+ frames = data["set_target_data"]
806
+ expected_frames = int(duration * MOTION_SAMPLE_RATE)
807
+
808
+ # Frame count within 20% of expected
809
+ assert len(frames) > expected_frames * 0.8, (
810
+ f"{label}: too few frames {len(frames)} (expected ~{expected_frames})"
811
+ )
812
+ assert len(frames) < expected_frames * 1.2, (
813
+ f"{label}: too many frames {len(frames)} (expected ~{expected_frames})"
814
+ )
815
+
816
+ # Duration within 20% of expected
817
+ actual_duration = timestamps[-1] - timestamps[0]
818
+ assert actual_duration > duration * 0.8, (
819
+ f"{label}: duration too short {actual_duration:.2f}s (expected ~{duration}s)"
820
+ )
821
+
822
+ # Frame rate should be close to 100Hz
823
+ actual_fps = len(frames) / actual_duration if actual_duration > 0 else 0
824
+ assert actual_fps > 80, f"{label}: frame rate too low {actual_fps:.1f}Hz (expected ~100Hz)"
825
+ assert actual_fps < 120, f"{label}: frame rate too high {actual_fps:.1f}Hz (expected ~100Hz)"
826
+
827
+ print(f"\n{label}: {len(frames)} frames in {actual_duration:.2f}s = {actual_fps:.1f}Hz")
828
+
829
+ # Cleanup
830
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
831
+
832
+ @pytest.mark.parametrize("duration,label", [
833
+ (1.0, "play-short-1s"),
834
+ (5.0, "play-standard-5s"),
835
+ (10.0, "play-long-10s"),
836
+ ])
837
+ def test_playback_duration(self, base_url: str, hw_marionette, duration: float, label: str):
838
+ """Record then play back at various durations, verify timing."""
839
+ import httpx
840
+
841
+ # First record a move
842
+ _ensure_idle(base_url)
843
+ resp = httpx.post(
844
+ f"{base_url}/api/record",
845
+ json={"duration": duration, "record_audio": False, "label": label},
846
+ timeout=5,
847
+ )
848
+ assert resp.status_code == 200
849
+ move_id = resp.json()["move_id"]
850
+ _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + duration + 15)
851
+
852
+ # Now play it back and measure how long it takes
853
+ _ensure_idle(base_url)
854
+ t0 = time.time()
855
+ resp = httpx.post(
856
+ f"{base_url}/api/play",
857
+ json={"move_id": move_id},
858
+ timeout=5,
859
+ )
860
+ assert resp.status_code == 200
861
+ _wait_for_mode(base_url, "idle", timeout=duration + 30)
862
+ playback_time = time.time() - t0
863
+
864
+ # Playback time should be close to the recording duration
865
+ # (plus some overhead for goto-start-pose)
866
+ assert playback_time > duration * 0.8, (
867
+ f"{label}: playback too fast {playback_time:.2f}s (expected ~{duration}s)"
868
+ )
869
+ assert playback_time < duration + 10, (
870
+ f"{label}: playback too slow {playback_time:.2f}s (expected ~{duration}s + overhead)"
871
+ )
872
+
873
+ print(f"\n{label}: playback took {playback_time:.2f}s (recording was {duration}s)")
874
+
875
+ # Cleanup
876
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
877
+
878
+
879
+ class TestPerformance:
880
+ """Timing and performance benchmarks."""
881
+
882
+ def test_recording_start_latency(self, base_url: str):
883
+ """Measure time from POST /api/record to countdown mode."""
884
+ import httpx
885
+
886
+ _ensure_idle(base_url)
887
+ t0 = time.time()
888
+ resp = httpx.post(
889
+ f"{base_url}/api/record",
890
+ json={"duration": 2.0, "record_audio": False, "label": "latency-test"},
891
+ timeout=5,
892
+ )
893
+ assert resp.status_code == 200
894
+
895
+ # Poll until we see countdown
896
+ while time.time() - t0 < 5:
897
+ state = httpx.get(f"{base_url}/api/state", timeout=2).json()
898
+ if state["mode"] in ("countdown", "recording"):
899
+ latency = time.time() - t0
900
+ print(f"\nRecording start latency: {latency*1000:.0f}ms")
901
+ assert latency < 5.0, f"Recording start too slow: {latency:.2f}s"
902
+ break
903
+ time.sleep(0.05)
904
+
905
+ # Let it finish
906
+ _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + 2 + 10)
907
+
908
+ # Cleanup
909
+ state = httpx.get(f"{base_url}/api/state", timeout=5).json()
910
+ for m in state["moves"]:
911
+ if "latency" in m.get("label", ""):
912
+ httpx.delete(f"{base_url}/api/moves/{m['id']}", timeout=5)
913
+
914
+ def test_playback_start_latency(self, base_url: str):
915
+ """Measure time from POST /api/play to playing mode."""
916
+ import httpx
917
+
918
+ # Need a move to play — record a quick one
919
+ _ensure_idle(base_url)
920
+ resp = httpx.post(
921
+ f"{base_url}/api/record",
922
+ json={"duration": 1.5, "record_audio": False, "label": "play-latency-src"},
923
+ timeout=5,
924
+ )
925
+ assert resp.status_code == 200
926
+ move_id = resp.json()["move_id"]
927
+ _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + 1.5 + 10)
928
+
929
+ # Now measure playback start
930
+ _ensure_idle(base_url)
931
+ t0 = time.time()
932
+ resp = httpx.post(
933
+ f"{base_url}/api/play",
934
+ json={"move_id": move_id},
935
+ timeout=5,
936
+ )
937
+ assert resp.status_code == 200
938
+
939
+ while time.time() - t0 < 10:
940
+ state = httpx.get(f"{base_url}/api/state", timeout=2).json()
941
+ if state["mode"] == "playing":
942
+ latency = time.time() - t0
943
+ print(f"\nPlayback start latency: {latency*1000:.0f}ms")
944
+ assert latency < 5.0, f"Playback start too slow: {latency:.2f}s"
945
+ break
946
+ time.sleep(0.05)
947
+
948
+ _wait_for_mode(base_url, "idle", timeout=15)
949
+
950
+ # Cleanup
951
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
952
+
953
+ def test_pose_read_rate(self, base_url: str):
954
+ """Measure how fast we can read poses from the SDK."""
955
+ from reachy_mini import ReachyMini
956
+
957
+ _ensure_idle(base_url)
958
+ with ReachyMini(media_backend="no_media") as reachy:
959
+ n_reads = 500
960
+ t0 = time.time()
961
+ for _ in range(n_reads):
962
+ reachy.get_current_head_pose()
963
+ elapsed = time.time() - t0
964
+
965
+ rate = n_reads / elapsed
966
+ print(f"\nPose read rate: {rate:.0f} reads/s ({elapsed*1000/n_reads:.1f}ms per read)")
967
+ assert rate > 50, f"Pose read rate too low: {rate:.0f}/s (need >50 for 100Hz recording)"
968
+
969
+
970
  class TestHardwareAudio:
971
  """Audio recording/playback tests — run last.
972
 
 
1035
  assert wav_duration > duration * 0.7, (
1036
  f"WAV too short: {wav_duration:.2f}s (expected ~{duration}s)"
1037
  )
1038
+ # Audio may include countdown buffering, so upper bound is generous.
1039
+ # The main check is that audio exists and isn't empty.
1040
+ assert wav_duration < duration + COUNTDOWN_SECONDS + 5.0, (
1041
+ f"WAV too long: {wav_duration:.2f}s (expected ~{duration}s + countdown)"
1042
  )
1043
 
1044
  # Also verify the JSON file reports has_audio