from __future__ import annotations

import importlib.util
from pathlib import Path


def _load_module():
    script_path = Path(__file__).resolve().parents[1] / "scripts" / "compare_runs.py"
    spec = importlib.util.spec_from_file_location("compare_runs", script_path)
    module = importlib.util.module_from_spec(spec)
    assert spec.loader is not None
    spec.loader.exec_module(module)
    return module


def test_compare_summaries_includes_breakdowns_and_markdown() -> None:
    module = _load_module()

    baseline = {
        "run_id": "base",
        "experiment_name": "baseline_exp",
        "aggregate_metrics": {
            "attack_success_rate": 0.6,
            "successful_attacks": 6,
            "prompt_leakage_cases": 1,
            "refusals": 0,
            "manual_review_cases": 1,
            "benign_utility_rate": 0.5,
            "false_positive_rate": 0.0,
            "by_attack_category": {
                "instruction_override": {
                    "successful_attacks": 4,
                    "total_cases": 5,
                    "attack_success_rate": 0.8,
                }
            },
            "by_attack_surface": {
                "direct": {
                    "successful_attacks": 6,
                    "total_cases": 10,
                    "attack_success_rate": 0.6,
                }
            },
            "by_source": {
                "HackAPrompt": {
                    "successful_attacks": 6,
                    "total_cases": 10,
                    "attack_success_rate": 0.6,
                }
            },
        },
    }
    defended = {
        "run_id": "def",
        "experiment_name": "defended_exp",
        "aggregate_metrics": {
            "attack_success_rate": 0.3,
            "successful_attacks": 3,
            "prompt_leakage_cases": 0,
            "refusals": 1,
            "manual_review_cases": 0,
            "benign_utility_rate": 0.5,
            "false_positive_rate": 0.1,
            "by_attack_category": {
                "instruction_override": {
                    "successful_attacks": 1,
                    "total_cases": 5,
                    "attack_success_rate": 0.2,
                }
            },
            "by_attack_surface": {
                "direct": {
                    "successful_attacks": 3,
                    "total_cases": 10,
                    "attack_success_rate": 0.3,
                }
            },
            "by_source": {
                "HackAPrompt": {
                    "successful_attacks": 3,
                    "total_cases": 10,
                    "attack_success_rate": 0.3,
                }
            },
        },
        "mitigation_metrics": {"strategy": "surface_aligned", "transformed_cases": 10},
    }

    comparison = module.compare_summaries(baseline, defended)

    assert comparison["metric_diffs"][0]["metric"] == "attack_success_rate"
    assert comparison["by_attack_category"][0]["label"] == "instruction_override"
    assert comparison["by_attack_category"][0]["attack_success_rate_delta"] == -0.6000000000000001

    markdown = module.render_markdown_summary(comparison)
    assert "# Phase 7 Comparison Summary" in markdown
    assert "## By Attack Category" in markdown
    assert "instruction_override" in markdown