| from __future__ import annotations |
|
|
| import importlib.util |
| from pathlib import Path |
|
|
|
|
| def _load_module(): |
| script_path = Path(__file__).resolve().parents[1] / "scripts" / "compare_runs.py" |
| spec = importlib.util.spec_from_file_location("compare_runs", script_path) |
| module = importlib.util.module_from_spec(spec) |
| assert spec.loader is not None |
| spec.loader.exec_module(module) |
| return module |
|
|
|
|
| def test_compare_summaries_includes_breakdowns_and_markdown() -> None: |
| module = _load_module() |
|
|
| baseline = { |
| "run_id": "base", |
| "experiment_name": "baseline_exp", |
| "aggregate_metrics": { |
| "attack_success_rate": 0.6, |
| "successful_attacks": 6, |
| "prompt_leakage_cases": 1, |
| "refusals": 0, |
| "manual_review_cases": 1, |
| "benign_utility_rate": 0.5, |
| "false_positive_rate": 0.0, |
| "by_attack_category": { |
| "instruction_override": { |
| "successful_attacks": 4, |
| "total_cases": 5, |
| "attack_success_rate": 0.8, |
| } |
| }, |
| "by_attack_surface": { |
| "direct": { |
| "successful_attacks": 6, |
| "total_cases": 10, |
| "attack_success_rate": 0.6, |
| } |
| }, |
| "by_source": { |
| "HackAPrompt": { |
| "successful_attacks": 6, |
| "total_cases": 10, |
| "attack_success_rate": 0.6, |
| } |
| }, |
| }, |
| } |
| defended = { |
| "run_id": "def", |
| "experiment_name": "defended_exp", |
| "aggregate_metrics": { |
| "attack_success_rate": 0.3, |
| "successful_attacks": 3, |
| "prompt_leakage_cases": 0, |
| "refusals": 1, |
| "manual_review_cases": 0, |
| "benign_utility_rate": 0.5, |
| "false_positive_rate": 0.1, |
| "by_attack_category": { |
| "instruction_override": { |
| "successful_attacks": 1, |
| "total_cases": 5, |
| "attack_success_rate": 0.2, |
| } |
| }, |
| "by_attack_surface": { |
| "direct": { |
| "successful_attacks": 3, |
| "total_cases": 10, |
| "attack_success_rate": 0.3, |
| } |
| }, |
| "by_source": { |
| "HackAPrompt": { |
| "successful_attacks": 3, |
| "total_cases": 10, |
| "attack_success_rate": 0.3, |
| } |
| }, |
| }, |
| "mitigation_metrics": {"strategy": "surface_aligned", "transformed_cases": 10}, |
| } |
|
|
| comparison = module.compare_summaries(baseline, defended) |
|
|
| assert comparison["metric_diffs"][0]["metric"] == "attack_success_rate" |
| assert comparison["by_attack_category"][0]["label"] == "instruction_override" |
| assert comparison["by_attack_category"][0]["attack_success_rate_delta"] == -0.6000000000000001 |
|
|
| markdown = module.render_markdown_summary(comparison) |
| assert "# Phase 7 Comparison Summary" in markdown |
| assert "## By Attack Category" in markdown |
| assert "instruction_override" in markdown |
|
|
