#!/usr/bin/env python3 """Display the test coverage matrix and test catalog without running tests. Reads test_results.json and static/test_coverage.json, then prints the latest result for each OS + robot + test-suite combination plus a summary of what's tested. Usage: cd marionette python tests/show_matrix.py """ import json import sys from datetime import datetime from pathlib import Path RESULTS_FILE = Path(__file__).parent / "test_results.json" WEB_COVERAGE_FILE = Path(__file__).parent.parent / "marionette" / "static" / "test_coverage.json" OS_LIST = ["Linux", "macOS", "Windows"] ROBOT_LIST = ["None", "Lite", "Wireless"] SUITES = [ ("unit", None, "Unit Tests"), ("e2e", "chromium", "Chromium"), ("e2e", "firefox", "Firefox"), ("e2e", "webkit", "WebKit"), ("hardware", None, "Hardware"), ] def print_matrix(results: list[dict]) -> None: """Print the coverage matrix from a list of result entries.""" # Build lookup: (os, robot, suite, browser) -> latest result latest: dict[tuple, dict] = {} for r in results: key = (r["os"], r["robot"], r["suite"], r.get("browser")) existing = latest.get(key) if existing is None or r["timestamp"] >= existing["timestamp"]: latest[key] = r # Only show columns that have at least one result active_suites = [ s for s in SUITES if any( latest.get((os_name, robot, s[0], s[1])) for os_name in OS_LIST for robot in ROBOT_LIST ) ] if not active_suites: active_suites = SUITES[:4] # Show defaults even if empty col_w = 11 label_w = 13 print("Coverage Matrix (latest result per combination)\n") # Header row header = f"{'':>{label_w}} \u2502 " + " \u2502 ".join( f"{s[2]:^{col_w}}" for s in active_suites ) print(header) # Separator sep = "\u2500" * label_w + "\u2500\u253c\u2500" + "\u2500\u253c\u2500".join( "\u2500" * (col_w + 1) for _ in active_suites ) print(sep) # Data rows for os_name in OS_LIST: for robot in ROBOT_LIST: short = robot[:4] + "." if len(robot) > 4 else robot combo = f"{os_name}/{short}" cells = [] for suite, browser, _ in active_suites: key = (os_name, robot, suite, browser) r = latest.get(key) if r is None: cells.append(f"{'—':^{col_w}}") else: dt = datetime.fromisoformat(r["timestamp"]) date_str = dt.strftime("%b %d") icon = "\u2705" if r["status"] == "pass" else "\u274c" cells.append(f"{icon + ' ' + date_str:^{col_w}}") row = f"{combo:>{label_w}} \u2502 " + " \u2502 ".join(cells) print(row) print() def print_catalog(catalog: list[dict]) -> None: """Print a grouped test catalog summary.""" if not catalog: return print("Test Catalog\n") tier_order = ["unit", "e2e", "hardware", "unknown"] tier_labels = { "unit": "Unit Tests", "e2e": "E2E Browser Tests", "hardware": "Hardware Tests", "unknown": "Other", } tiers: dict[str, list[dict]] = {} for entry in catalog: tiers.setdefault(entry["tier"], []).append(entry) for tier in tier_order: entries = tiers.get(tier) if not entries: continue total = sum(e["count"] for e in entries) label = tier_labels.get(tier, tier) print(f" {label} — {total} tests") for e in entries: pad = max(1, 28 - len(e["class"]) - len(str(e["count"]))) print(f" {e['class']} ({e['count']}){' ' * pad} {e['description']}") print() def main() -> None: if not RESULTS_FILE.exists(): print(f"No results file found at {RESULTS_FILE}") print("Run tests first: python tests/run_tests.py") sys.exit(1) results = json.loads(RESULTS_FILE.read_text()) if not results: print("No test results recorded yet.") print("Run tests first: python tests/run_tests.py") sys.exit(1) print_matrix(results) # Summary stats total_combos = len(OS_LIST) * len(ROBOT_LIST) * len(SUITES) tested = len({ (r["os"], r["robot"], r["suite"], r.get("browser")) for r in results }) print(f"Coverage: {tested}/{total_combos} combinations tested\n") # Show test catalog from web coverage file if available if WEB_COVERAGE_FILE.exists(): try: coverage = json.loads(WEB_COVERAGE_FILE.read_text()) catalog = coverage.get("test_catalog", []) if catalog: print_catalog(catalog) except (json.JSONDecodeError, KeyError): pass else: print("Tip: Run python tests/run_tests.py to generate the test catalog.\n") if __name__ == "__main__": main()