RemiFabre commited on
Commit
dc8a097
·
1 Parent(s): f851c34

Add critical hardware tests for real-world breakage scenarios

Browse files

- TestStopDuringPlayback: stop mid-playback and during goto-start-pose
- TestStopDuringRecording: stop mid-recording, verify partial save
- TestMultipleRecordPlayCycles: 5 back-to-back record/play cycles
- TestPlaybackWithCorruptFile: deleted/truncated JSON recovery

These cover the most common user interactions that were previously
untested on real hardware: interrupting playback, cancelling recordings,
repeated usage sessions, and filesystem inconsistencies.

Total: 95 unit + 44 E2E + 37 hardware = 176 tests.

Files changed (3) hide show
  1. TESTING.md +4 -0
  2. tests/run_tests.py +4 -0
  3. tests/test_hardware.py +267 -0
TESTING.md CHANGED
@@ -196,6 +196,10 @@ pytest tests/ --browser chromium
196
  | TestRecordingRoundTrip | 2 | Record → playback fidelity and timing verification |
197
  | TestAntennaAndBodyYaw | 2 | Verify antenna and body_yaw data in recordings |
198
  | TestPlaybackAntennas | 1 | Synthetic antenna oscillation playback |
 
 
 
 
199
  | TestHardwareAudio | 3 | Audio recording and playback (may skip on mic issues) |
200
 
201
  ## View the matrix without running tests
 
196
  | TestRecordingRoundTrip | 2 | Record → playback fidelity and timing verification |
197
  | TestAntennaAndBodyYaw | 2 | Verify antenna and body_yaw data in recordings |
198
  | TestPlaybackAntennas | 1 | Synthetic antenna oscillation playback |
199
+ | TestStopDuringPlayback | 2 | Stop mid-playback and during goto-start-pose |
200
+ | TestStopDuringRecording | 1 | Stop mid-recording, verify partial save |
201
+ | TestMultipleRecordPlayCycles | 1 | 5 back-to-back record/play cycles |
202
+ | TestPlaybackWithCorruptFile | 2 | Deleted/corrupt JSON during playback |
203
  | TestHardwareAudio | 3 | Audio recording and playback (may skip on mic issues) |
204
 
205
  ## View the matrix without running tests
tests/run_tests.py CHANGED
@@ -83,6 +83,10 @@ TEST_CLASS_DESCRIPTIONS: dict[str, tuple[str, str]] = {
83
  "TestRecordingRoundTrip": ("hardware", "Record → playback fidelity and timing verification"),
84
  "TestAntennaAndBodyYaw": ("hardware", "Verify antenna and body_yaw data in recordings"),
85
  "TestPlaybackAntennas": ("hardware", "Synthetic antenna oscillation playback"),
 
 
 
 
86
  }
87
 
88
 
 
83
  "TestRecordingRoundTrip": ("hardware", "Record → playback fidelity and timing verification"),
84
  "TestAntennaAndBodyYaw": ("hardware", "Verify antenna and body_yaw data in recordings"),
85
  "TestPlaybackAntennas": ("hardware", "Synthetic antenna oscillation playback"),
86
+ "TestStopDuringPlayback": ("hardware", "Stop mid-playback and during goto-start-pose"),
87
+ "TestStopDuringRecording": ("hardware", "Stop mid-recording, verify partial save"),
88
+ "TestMultipleRecordPlayCycles": ("hardware", "5 back-to-back record/play cycles"),
89
+ "TestPlaybackWithCorruptFile": ("hardware", "Deleted/corrupt JSON during playback"),
90
  }
91
 
92
 
tests/test_hardware.py CHANGED
@@ -1211,6 +1211,273 @@ class TestPlaybackAntennas:
1211
  httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
1212
 
1213
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1214
  class TestHardwareAudio:
1215
  """Audio recording/playback tests — run last.
1216
 
 
1211
  httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
1212
 
1213
 
1214
+ class TestStopDuringPlayback:
1215
+ """Stop playback mid-stream — the #1 user-initiated interruption."""
1216
+
1217
+ def test_stop_during_playback_returns_to_idle(self, base_url: str, hw_marionette):
1218
+ """Start playback, stop mid-way, verify robot returns to idle gracefully."""
1219
+ import httpx
1220
+
1221
+ _ensure_idle(base_url)
1222
+
1223
+ # Record a 5s move so we have time to stop mid-playback
1224
+ resp = httpx.post(
1225
+ f"{base_url}/api/record",
1226
+ json={"duration": 5.0, "record_audio": False, "label": "stop-play-test"},
1227
+ timeout=5,
1228
+ )
1229
+ assert resp.status_code == 200
1230
+ move_id = resp.json()["move_id"]
1231
+ _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + 5 + 10)
1232
+
1233
+ # Start playback
1234
+ resp = httpx.post(
1235
+ f"{base_url}/api/play",
1236
+ json={"move_id": move_id},
1237
+ timeout=5,
1238
+ )
1239
+ assert resp.status_code == 200
1240
+
1241
+ # Wait until we're actually in playing mode
1242
+ _wait_for_mode(base_url, "playing", timeout=15)
1243
+ # Let it play for 1-2 seconds
1244
+ time.sleep(1.5)
1245
+
1246
+ # Stop mid-playback
1247
+ resp = httpx.post(f"{base_url}/api/play/stop", timeout=5)
1248
+ assert resp.status_code == 200
1249
+
1250
+ # Should return to idle
1251
+ state = _wait_for_mode(base_url, "idle", timeout=10)
1252
+ assert state["mode"] == "idle"
1253
+
1254
+ # Cleanup
1255
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
1256
+
1257
+ def test_stop_during_goto_start_pose(self, base_url: str, hw_marionette):
1258
+ """Stop playback during the initial goto-start-pose transition."""
1259
+ import httpx
1260
+ import numpy as np
1261
+
1262
+ _ensure_idle(base_url)
1263
+
1264
+ # Create synthetic move with head far from neutral — long goto transition
1265
+ move_id = _create_synthetic_move(
1266
+ hw_marionette,
1267
+ label="stop-goto-test",
1268
+ duration=5.0,
1269
+ trajectory_fn=lambda t: (0.0, 0.0, 0.4), # constant yaw offset
1270
+ )
1271
+
1272
+ # Start playback — goto_pose_scaled will take time
1273
+ resp = httpx.post(
1274
+ f"{base_url}/api/play",
1275
+ json={"move_id": move_id},
1276
+ timeout=5,
1277
+ )
1278
+ assert resp.status_code == 200
1279
+
1280
+ # Stop immediately (likely still in goto phase)
1281
+ time.sleep(0.3)
1282
+ resp = httpx.post(f"{base_url}/api/play/stop", timeout=5)
1283
+ assert resp.status_code == 200
1284
+
1285
+ state = _wait_for_mode(base_url, "idle", timeout=15)
1286
+ assert state["mode"] == "idle"
1287
+
1288
+ # Cleanup
1289
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
1290
+
1291
+
1292
+ class TestStopDuringRecording:
1293
+ """Stop recording mid-capture — verify partial data is saved correctly."""
1294
+
1295
+ def test_stop_mid_recording_saves_partial(self, base_url: str, hw_marionette):
1296
+ """Record 10s, stop after 2-3s, verify partial JSON is saved with valid data."""
1297
+ import httpx
1298
+
1299
+ _ensure_idle(base_url)
1300
+ resp = httpx.post(
1301
+ f"{base_url}/api/record",
1302
+ json={"duration": 10.0, "record_audio": False, "label": "stop-partial"},
1303
+ timeout=5,
1304
+ )
1305
+ assert resp.status_code == 200
1306
+ move_id = resp.json()["move_id"]
1307
+
1308
+ # Wait for actual recording mode (not countdown)
1309
+ _wait_for_mode(base_url, "recording", timeout=COUNTDOWN_SECONDS + 5)
1310
+ # Let it record for 2 seconds
1311
+ time.sleep(2.0)
1312
+
1313
+ # Stop recording
1314
+ resp = httpx.post(f"{base_url}/api/record/stop", timeout=5)
1315
+ assert resp.status_code == 200
1316
+
1317
+ state = _wait_for_mode(base_url, "idle", timeout=10)
1318
+ assert state["mode"] == "idle"
1319
+
1320
+ # Verify partial recording was saved with reasonable data
1321
+ json_path = hw_marionette._dataset_dir / f"{move_id}.json"
1322
+ assert json_path.exists(), "Partial recording should be saved"
1323
+
1324
+ data = json.loads(json_path.read_text())
1325
+ timestamps = data["time"]
1326
+ frames = data["set_target_data"]
1327
+
1328
+ # Should have ~2s worth of data (not 10s)
1329
+ assert len(frames) > 100, f"Expected ~200 frames for 2s, got {len(frames)}"
1330
+ assert len(frames) < 500, f"Expected <5s of data, got {len(frames)} frames"
1331
+ assert len(timestamps) == len(frames)
1332
+
1333
+ # Verify frames have valid structure
1334
+ for frame in frames[:5]:
1335
+ assert "head" in frame
1336
+ assert "antennas" in frame
1337
+
1338
+ print(f"\nPartial recording: {len(frames)} frames in {timestamps[-1]:.2f}s")
1339
+
1340
+ # Cleanup
1341
+ httpx.delete(f"{base_url}/api/moves/{move_id}", timeout=5)
1342
+
1343
+
1344
+ class TestMultipleRecordPlayCycles:
1345
+ """Back-to-back record/play cycles — the real-world usage pattern.
1346
+
1347
+ Users record several takes in a row, playing them back between takes.
1348
+ Motor state drift and resource leaks show up after multiple cycles.
1349
+ """
1350
+
1351
+ def test_five_record_play_cycles(self, base_url: str, hw_marionette):
1352
+ """Run 5 record→play cycles, verify each completes and robot stays healthy."""
1353
+ import httpx
1354
+
1355
+ move_ids = []
1356
+ for i in range(5):
1357
+ _ensure_idle(base_url)
1358
+
1359
+ # Record 1.5s
1360
+ resp = httpx.post(
1361
+ f"{base_url}/api/record",
1362
+ json={"duration": 1.5, "record_audio": False, "label": f"cycle-{i}"},
1363
+ timeout=5,
1364
+ )
1365
+ assert resp.status_code == 200, f"Cycle {i}: record failed"
1366
+ move_id = resp.json()["move_id"]
1367
+ move_ids.append(move_id)
1368
+ _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + 1.5 + 10)
1369
+
1370
+ # Verify recording exists
1371
+ state = httpx.get(f"{base_url}/api/state", timeout=5).json()
1372
+ assert any(m["id"] == move_id for m in state["moves"]), (
1373
+ f"Cycle {i}: move {move_id} not in list"
1374
+ )
1375
+
1376
+ # Play it back
1377
+ resp = httpx.post(
1378
+ f"{base_url}/api/play",
1379
+ json={"move_id": move_id},
1380
+ timeout=5,
1381
+ )
1382
+ assert resp.status_code == 200, f"Cycle {i}: play failed"
1383
+ _wait_for_mode(base_url, "idle", timeout=1.5 + 15)
1384
+
1385
+ print(f" Cycle {i+1}/5 complete")
1386
+
1387
+ # Verify all 5 moves still exist after all cycles
1388
+ state = httpx.get(f"{base_url}/api/state", timeout=5).json()
1389
+ for mid in move_ids:
1390
+ assert any(m["id"] == mid for m in state["moves"]), (
1391
+ f"Move {mid} disappeared after cycles"
1392
+ )
1393
+
1394
+ # Cleanup
1395
+ for mid in move_ids:
1396
+ httpx.delete(f"{base_url}/api/moves/{mid}", timeout=5)
1397
+
1398
+
1399
+ class TestPlaybackWithCorruptFile:
1400
+ """Playback when the JSON file is missing or corrupt.
1401
+
1402
+ Users may delete files from the dataset folder while the app is running.
1403
+ """
1404
+
1405
+ def test_playback_deleted_file_returns_error(self, base_url: str, hw_marionette):
1406
+ """Delete a move's JSON mid-session, try to play — should fail gracefully."""
1407
+ import httpx
1408
+
1409
+ _ensure_idle(base_url)
1410
+
1411
+ # Record a move
1412
+ resp = httpx.post(
1413
+ f"{base_url}/api/record",
1414
+ json={"duration": 1.5, "record_audio": False, "label": "corrupt-test"},
1415
+ timeout=5,
1416
+ )
1417
+ assert resp.status_code == 200
1418
+ move_id = resp.json()["move_id"]
1419
+ _wait_for_mode(base_url, "idle", timeout=COUNTDOWN_SECONDS + 1.5 + 10)
1420
+
1421
+ # Delete the JSON file behind the app's back
1422
+ json_path = hw_marionette._dataset_dir / f"{move_id}.json"
1423
+ assert json_path.exists()
1424
+ json_path.unlink()
1425
+
1426
+ # Refresh so the app picks up the deletion
1427
+ hw_marionette._refresh_recordings()
1428
+
1429
+ # Try to play — should get 404 (move no longer exists)
1430
+ resp = httpx.post(
1431
+ f"{base_url}/api/play",
1432
+ json={"move_id": move_id},
1433
+ timeout=5,
1434
+ )
1435
+ assert resp.status_code in (404, 409), (
1436
+ f"Expected 404/409 for deleted move, got {resp.status_code}"
1437
+ )
1438
+
1439
+ state = httpx.get(f"{base_url}/api/state", timeout=5).json()
1440
+ assert state["mode"] in ("idle", "error")
1441
+
1442
+ def test_playback_truncated_json_recovers(self, base_url: str, hw_marionette):
1443
+ """Write a truncated JSON, try to play — should fail without crashing."""
1444
+ import httpx
1445
+
1446
+ _ensure_idle(base_url)
1447
+
1448
+ # Create a corrupt JSON file
1449
+ move_id = "corrupt-truncated"
1450
+ json_path = hw_marionette._dataset_dir / f"{move_id}.json"
1451
+ json_path.write_text('{"time": [0.0, 0.01], "set_target_data": [{"head', encoding="utf-8")
1452
+ hw_marionette._refresh_recordings()
1453
+
1454
+ # Try to play — should fail gracefully (corrupt JSON can't be loaded)
1455
+ resp = httpx.post(
1456
+ f"{base_url}/api/play",
1457
+ json={"move_id": move_id},
1458
+ timeout=5,
1459
+ )
1460
+
1461
+ # The move may not parse during refresh (skipped) or fail during load
1462
+ if resp.status_code == 200:
1463
+ # If it was accepted, it should recover to idle/error
1464
+ time.sleep(3)
1465
+ state = httpx.get(f"{base_url}/api/state", timeout=5).json()
1466
+ assert state["mode"] in ("idle", "error"), (
1467
+ f"Server stuck after corrupt playback: mode={state['mode']}"
1468
+ )
1469
+ else:
1470
+ # 404 is fine — refresh skipped the corrupt file
1471
+ assert resp.status_code in (404, 409, 500)
1472
+
1473
+ _ensure_idle(base_url)
1474
+
1475
+ # Cleanup
1476
+ if json_path.exists():
1477
+ json_path.unlink()
1478
+ hw_marionette._refresh_recordings()
1479
+
1480
+
1481
  class TestHardwareAudio:
1482
  """Audio recording/playback tests — run last.
1483