"""Regression checks for validation failures that can silently change eval claims."""
import json
from pathlib import Path
import tempfile
import unittest

from analyze_mmlu import ValidationError, compare, exact_mcnemar, load_index, load_scores


class AnalysisTests(unittest.TestCase):
    def setUp(self):
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.root = Path(self.temp.name)
        self.index = {
            0: {"i": 0, "question_id": 10, "category": "a", "n_options": 2, "answer_index": 0},
            1: {"i": 1, "question_id": 20, "category": "b", "n_options": 3, "answer_index": 2},
        }
        self.valid = "0\t0\t0\t-1\t-2\n1\t2\t2\t-3\t-2\t-1\n"

    def read(self, text):
        path = self.root / "scores.tsv"
        path.write_text(text)
        return load_scores(path, self.index)

    def test_valid_and_unsorted_rows(self):
        rows, validation = self.read(self.valid)
        self.assertTrue(validation["passed"])
        reversed_rows, _ = self.read("\n".join(reversed(self.valid.splitlines())) + "\n")
        self.assertEqual(rows, reversed_rows)

    def test_failures_rejected_instead_of_changing_denominator(self):
        malformed = {
            "duplicate": self.valid + self.valid.splitlines()[0] + "\n",
            "missing": self.valid.splitlines()[0] + "\n",
            "extra": self.valid + "2\t0\t0\t-1\t-2\n",
            "wrong_gold": self.valid.replace("0\t0\t0", "0\t0\t1"),
            "wrong_options": self.valid.replace("-3\t-2\t-1", "-3\t-1"),
            "nan": self.valid.replace("-3", "nan"),
            "infinity": self.valid.replace("-3", "-inf"),
            "wrong_argmax": self.valid.replace("0\t0\t0", "0\t1\t0"),
            "invalid_prediction": self.valid.replace("0\t0\t0", "0\t2\t0"),
            "positive_logprob": self.valid.replace("-1", "1"),
            "probability_mass": self.valid.replace("-1\t-2", "0\t0"),
            "blank_row": self.valid + "\n",
            "non_numeric": self.valid.replace("-3", "oops"),
        }
        for name, text in malformed.items():
            with self.subTest(name=name), self.assertRaises(ValidationError):
                self.read(text)

    def test_rounded_tie_is_flagged(self):
        _, validation = self.read(self.valid.replace("-1\t-2", "-1\t-1"))
        self.assertEqual(validation["rounded_argmax_tie_indices"], [0])

    def test_index_matches_subset_order_and_allocation(self):
        index_path, subset_path = self.root / "index.json", self.root / "subset.json"
        index_path.write_text(json.dumps(list(self.index.values())))
        subset = {"n": 2, "allocation": {"a": 1, "b": 1}, "question_ids": [10, 20]}
        subset_path.write_text(json.dumps(subset))
        self.assertEqual(load_index(index_path, subset_path, expected_n=2)[0], self.index)
        for change in ({"question_ids": [20, 10]}, {"allocation": {"a": 2}}, {"n": 3}):
            subset_path.write_text(json.dumps({**subset, **change}))
            with self.assertRaises(ValidationError):
                load_index(index_path, subset_path, expected_n=2)

    def test_exact_p_values(self):
        for a, b, expected in [(0, 0, 1.0), (4, 0, 0.125), (3, 1, 0.625), (2, 2, 1.0)]:
            self.assertEqual(exact_mcnemar(a, b), expected)
            self.assertEqual(exact_mcnemar(b, a), expected)

    def test_delta_sign_and_prediction_not_correctness_agreement(self):
        a, _ = self.read(self.valid.replace("1\t2\t2\t-3\t-2\t-1", "1\t0\t2\t-1\t-2\t-3"))
        b, _ = self.read("0\t1\t0\t-2\t-1\n1\t1\t2\t-2\t-1\t-3\n")
        stats = compare(a, b)
        self.assertEqual(stats["delta_pp"], 50.0)
        self.assertEqual(stats["a_only_correct"], 1)
        self.assertEqual(stats["prediction_agreement_count"], 0)
        self.assertEqual(stats["correctness_agreement_count"], 1)
        self.assertEqual(compare(b, a)["delta_pp"], -50.0)
        with self.assertRaises(ValidationError):
            compare(a, {0: b[0]})


if __name__ == "__main__":
    unittest.main()