source/scripts/summarize_profile.py

"""Create a checked report and standalone plots from completed frozen evaluations.

Usage: python scripts/summarize_profile.py --campaign FLEET --prepared PROTOCOL_DIR
  --profile THREE_METHOD_RUN [--expanded-profile TRAINED_ONLY_RUN] --output NEW_DIR

No predictions are read until fleet selection exists and final evaluation is complete.
This is CPU-only reporting: no models, calibration, selection or inference are run.
"""
import argparse
import csv
from datetime import datetime, timezone
import hashlib
import json
import math
from pathlib import Path
import shutil
import sys
import tempfile

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from scripts.profile_inference import accuracy_summary, distribution, timing_summary

METHODS = ('trained', 'base_verifier', 'base_label')
LABELS = {'trained': 'Selected scorer', 'base_verifier': 'Base yes/no verifier',
          'base_label': 'Base one-token label', 'expanded': 'Expanded scorer'}
FINAL_METRICS = ('n', 'accuracy', 'nll', 'brier_multiclass_sum', 'ece_top_label_10_equal_width_bins')


def sha(path):
    digest = hashlib.sha256()
    with Path(path).open('rb') as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b''):
            digest.update(chunk)
    return digest.hexdigest()


def read_json(path):
    return json.loads(Path(path).read_text())


def json_write(path, value):
    Path(path).write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + '\n')


def close(actual, expected, name):
    if not isinstance(actual, (int, float)) or not math.isfinite(actual) or not math.isclose(actual, expected, rel_tol=1e-8, abs_tol=1e-10):
        raise ValueError('Recorded value disagrees with raw evidence: ' + name)


def identities(rows):
    result = {}
    for row in rows:
        key = row['id']
        if key in result:
            raise ValueError('Duplicate decision ID')
        result[key] = (row['group'], row['family'], row['target'], row['choices'])
    return result


def read_predictions(path, expected):
    # Completion is checked before this function is called; partial lines are errors.
    rows = [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()]
    if len(rows) != len(expected) or identities(rows) != identities(expected):
        raise ValueError('Predictions do not match every frozen decision identity')
    for row in rows:
        values, probabilities, choices = row['logits'], row['probabilities'], row['choices']
        if len(values) != len(choices) or len(probabilities) != len(choices) or len(choices) < 2:
            raise ValueError('Prediction choice vector length mismatch')
        expected_probabilities = distribution(values)
        for actual, expected_probability in zip(probabilities, expected_probabilities):
            close(actual, expected_probability, 'probability')
        index = max(range(len(values)), key=values.__getitem__)
        if (type(row['target']) is not int or not 0 <= row['target'] < len(choices) or
                type(row['predicted_index']) is not int or row['predicted_index'] != index or
                row.get('predicted_choice') != choices[index]):
            raise ValueError('Prediction target or chosen label disagrees with logits')
    by_id = {row['id']: row for row in rows}
    return [by_id[row['id']] for row in expected]


def verify_accuracy(recorded, rows):
    actual = accuracy_summary(rows)
    if set(recorded.get('per_family', {})) != set(actual['per_family']):
        raise ValueError('Accuracy family inventory mismatch')
    for name in ('overall', *actual['per_family']):
        expected = actual['overall'] if name == 'overall' else actual['per_family'][name]
        saved = recorded['overall'] if name == 'overall' else recorded['per_family'][name]
        if saved.get('count') != expected['count'] or saved.get('correct') != expected['correct']:
            raise ValueError('Accuracy decision/correct count mismatch')
        close(saved.get('accuracy'), expected['accuracy'], 'accuracy')
    return actual


def verify_speed(directory, expected_cases, protocol, method):
    records = read_json(directory / 'speed.json')
    by_id = {row['case']: row for row in records}
    if len(records) != len(by_id) or set(by_id) != {row['id'] for row in expected_cases}:
        raise ValueError('Timing cases are missing, duplicated or unexpected')
    samples = {}
    for line in (directory / 'timing_samples.jsonl').read_text().splitlines():
        row = json.loads(line)
        key = (row['case'], row['repeat'])
        if key in samples:
            raise ValueError('Duplicate timing repetition')
        samples[key] = row['seconds']
    expected_keys = {(case['id'], repeat) for case in expected_cases for repeat in range(protocol['repeats'])}
    if set(samples) != expected_keys:
        raise ValueError('Raw timing repetitions are incomplete or unexpected')
    verified = []
    for case in expected_cases:
        saved = by_id[case['id']]
        values = [samples[case['id'], repeat] for repeat in range(protocol['repeats'])]
        recomputed = timing_summary(values)
        if saved['samples_seconds'] != values or saved['sample_count'] != len(values):
            raise ValueError('Timing summary differs from raw samples')
        for key in ('median_seconds', 'p95_seconds'):
            close(saved.get(key), recomputed[key], key)
        for key, value in [('questions', case['questions']), ('choices_per_question', case['choices_per_question']),
                           ('state_tokens', case['state_tokens_actual'])]:
            if saved[key] != value:
                raise ValueError('Timing shape changed from frozen request')
        token_count = (sum(spec['label']['input_tokens'] for spec in case['compiled']) if method == 'base_label'
                       else sum(sum(spec['verifier_branch_tokens']) for spec in case['compiled']))
        if saved['input_tokens_processed'] != token_count:
            raise ValueError('Timing token accounting disagrees with frozen requests')
        verified.append({'case': case['id'], 'state_tokens': case['state_tokens_actual'],
            'questions': case['questions'], 'choices_per_question': case['choices_per_question'],
            'input_tokens_processed': token_count, **recomputed,
            'requests_per_second': 1 / recomputed['median_seconds'],
            'questions_per_second': case['questions'] / recomputed['median_seconds'],
            'choice_probabilities_per_second': case['questions'] * case['choices_per_question'] / recomputed['median_seconds']})
    return verified


def verify_final(campaign):
    selection_path = campaign / 'selection.json'
    if not selection_path.is_file():
        raise ValueError('Checkpoint selection must be frozen before any held-out predictions are read')
    selection = read_json(selection_path)
    selected = selection['selected']
    metrics = read_json(campaign / 'evaluation/metrics.json')
    if not selected.get('eligible') or metrics.get('status') != 'complete':
        raise ValueError('Final evaluation must be complete for the frozen eligible selection')
    manifest = read_json(campaign / 'evaluation/manifest.json')
    if (manifest['checkpoint_sha256'] != selected['checkpoint_sha256'] or
            manifest['data_signature'] != selected['data_signature'] or
            metrics['selected_step'] != selected['step'] or manifest['selected_step'] != selected['step']):
        raise ValueError('Final evaluation provenance differs from frozen selection')
    if sha(campaign / 'evaluation/model.pt') != metrics['model_sha256']:
        raise ValueError('Final calibrated model checksum mismatch')
    for split in ('test', 'holdout'):
        block = metrics[split]
        counts = []
        for method in ('trained', 'calibrated', 'base', 'base_calibrated'):
            overall = block[method]['overall']
            if type(overall['n']) is not int or overall['n'] <= 0:
                raise ValueError('Final evaluation decision count is invalid')
            counts.append(overall['n'])
            for key in ('accuracy', 'nll', 'brier_multiclass_sum', 'ece_top_label_10_equal_width_bins'):
                value = overall[key]
                if not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
                    raise ValueError('Final evaluation metric is invalid')
        if len(set(counts)) != 1:
            raise ValueError('Final methods have unmatched decision counts')
        difference = block['calibrated_difference_95pct']
        for metric, key in [('accuracy', 'accuracy'), ('nll', 'nll'), ('brier', 'brier_multiclass_sum')]:
            point = block['calibrated']['overall'][key] - block['base_calibrated']['overall'][key]
            close(difference['point_delta'][metric], point, 'calibrated trained-minus-base delta')
            interval = difference[metric]
            if len(interval) != 2 or any(not math.isfinite(value) for value in interval) or interval[0] > interval[1]:
                raise ValueError('Final bootstrap interval is malformed')
    return selection, metrics, manifest


def load_profile(path, protocol, requests, expected_protocol_hash, expected_requests_hash,
                 methods=METHODS, speed_required=True):
    result = {}
    for method in methods:
        directory = path / method
        summary, manifest = read_json(directory / 'summary.json'), read_json(directory / 'manifest.json')
        if summary.get('status') != 'complete' or summary.get('method') != method or manifest.get('method') != method:
            raise ValueError('Profiling method did not complete')
        if (manifest['protocol_sha256'] != expected_protocol_hash or manifest['requests_sha256'] != expected_requests_hash or
                manifest['model_provenance'] != protocol['model_provenance'] or manifest.get('precision') != 'float32'):
            raise ValueError('Profiling methods do not use the same frozen inputs/model/precision')
        if manifest.get('artifact_temperature_applied') is not False:
            raise ValueError('Profile comparison must preserve declared raw probabilities')
        accuracy = read_json(directory / 'accuracy.json')
        rows, checked_accuracy = {}, {}
        for split in ('heldout', 'diagnostics'):
            rows[split] = read_predictions(directory / (split + '_predictions.jsonl'), requests['accuracy'][split])
            checked_accuracy[split] = verify_accuracy(accuracy[split], rows[split])
            verify_accuracy(summary['accuracy'][split], rows[split])
        timing = verify_speed(directory, requests['timing'], protocol, method) if speed_required else None
        if speed_required and summary['completed_timing_cases'] != len(timing):
            raise ValueError('Completed timing-case count mismatch')
        result[method] = {'manifest': manifest, 'accuracy': checked_accuracy, 'predictions': rows, 'speed': timing}
    return result


def paired_difference(selected, expanded):
    if identities(selected) != identities(expanded):
        raise ValueError('Expanded comparison does not have matched identities')
    b = {row['id']: row for row in expanded}
    result = {'count': len(selected), 'selected_only_correct': 0, 'expanded_only_correct': 0,
              'both_correct': 0, 'both_wrong': 0}
    for row in selected:
        first = row['predicted_index'] == row['target']
        second = b[row['id']]['predicted_index'] == row['target']
        key = 'both_correct' if first and second else 'both_wrong' if not first and not second else 'selected_only_correct' if first else 'expanded_only_correct'
        result[key] += 1
    result['expanded_minus_selected_accuracy_pp'] = 100 * (result['expanded_only_correct'] - result['selected_only_correct']) / len(selected)
    return result


def collect(campaign, prepared, profile, expanded_profile=None, lineage_proof=None):
    campaign, prepared, profile = Path(campaign), Path(prepared), Path(profile)
    selection, final, evaluation_manifest = verify_final(campaign)  # Must precede every profile/prediction read.
    protocol = read_json(prepared / 'protocol.json')
    preparation = read_json(prepared / 'preparation.json')
    protocol_hash, requests_hash = sha(prepared / 'protocol.json'), sha(prepared / 'requests.json')
    if ((prepared / 'protocol.sha256').read_text().strip() != protocol_hash or
            preparation['protocol_sha256'] != protocol_hash or preparation['requests_sha256'] != requests_hash):
        raise ValueError('Frozen profiling protocol or requests changed')
    requests = read_json(prepared / 'requests.json')
    if len(requests['timing']) != 12 or len(requests['accuracy']['heldout']) != 320 or len(requests['accuracy']['diagnostics']) != 383:
        raise ValueError('Expected the frozen 12 timing cases, 320 held-out decisions and 383 diagnostics')
    if protocol['repeats'] < 10 or protocol['warmups'] < 2 or protocol['accuracy_count'] != 320:
        raise ValueError('Profiling protocol is weaker than the declared comparison')
    methods = load_profile(profile, protocol, requests, protocol_hash, requests_hash)
    selected = selection['selected']
    selected_summary_path = Path(selected['checkpoint']).parent / 'summary.json'
    final_validation = read_json(selected_summary_path) if selected_summary_path.is_file() else None
    if final_validation and (final_validation.get('status') != 'complete' or
            final_validation.get('selected_step') != selected['step'] or
            final_validation.get('best_sha256') != selected['checkpoint_sha256']):
        raise ValueError('Final-validation summary does not describe the frozen selected checkpoint')
    tied = [row['name'] for row in selection.get('candidates', []) if row.get('eligible') and
            row.get('metrics', {}).get('selection_score') == selected.get('metrics', {}).get('selection_score')]
    lineage = None
    if lineage_proof is not None:
        proof = read_json(lineage_proof)
        if (proof.get('all_parent_weights_exact') is not True or proof.get('step') != 0 or
                proof.get('data_signature') != selected['data_signature'] or selected['step'] != 0):
            raise ValueError('Warm-start lineage proof does not describe the selected step-zero model')
        lineage = {'proof_sha256': sha(lineage_proof), 'all_parent_weights_exact': True,
            'parent_step': proof['initialization']['parent_step'],
            'parent_checkpoint_sha256': proof['initialization']['parent_checkpoint_sha256'],
            'trainable_tensors': proof['trainable_tensors']}
    primary_hash = methods['trained']['manifest']['checkpoint_sha256']
    if primary_hash not in {selected['checkpoint_sha256'], final['model_sha256']}:
        raise ValueError('Primary profile is not the frozen selected model')
    expanded = None
    if expanded_profile is not None:
        expanded_path = Path(expanded_profile)
        expanded = load_profile(expanded_path, protocol, requests, protocol_hash, requests_hash,
                                methods=('trained',), speed_required=False)['trained']
        declaration = read_json(expanded_path / 'comparison.json')
        if (declaration['checkpoint_sha256'] != expanded['manifest']['checkpoint_sha256'] or
                declaration['protocol_sha256'] != protocol_hash):
            raise ValueError('Expanded checkpoint differs from its comparison declaration')
        if final_validation and (expanded['manifest']['checkpoint_step'] != final_validation['latest_step'] or
                expanded['manifest']['checkpoint_sha256'] != final_validation['latest_sha256']):
            raise ValueError('Expanded profile is not the selected branch latest evaluated checkpoint')
    speed = []
    for index, case in enumerate(requests['timing']):
        row = {key: methods['trained']['speed'][index][key] for key in ('case', 'state_tokens', 'questions', 'choices_per_question')}
        row['methods'] = {method: methods[method]['speed'][index] for method in METHODS}
        for baseline in ('base_verifier', 'base_label'):
            row[baseline + '_over_trained'] = row['methods'][baseline]['median_seconds'] / row['methods']['trained']['median_seconds']
        speed.append(row)
    accuracy = {method: value['accuracy'] for method, value in methods.items()}
    comparisons = {}
    if expanded:
        accuracy['expanded'] = expanded['accuracy']
        for split in ('heldout', 'diagnostics'):
            first, second = methods['trained']['predictions'][split], expanded['predictions'][split]
            comparisons[split] = {'overall': paired_difference(first, second), 'per_family': {family: paired_difference(
                [row for row in first if row['family'] == family], [row for row in second if row['family'] == family])
                for family in sorted({row['family'] for row in first})}}
    # Preserve useful provenance without duplicating hundreds of fold decision IDs.
    selected = {key: value for key, value in selected.items() if key != 'metrics'} | {
        'metrics': {key: value for key, value in selected['metrics'].items() if key != 'selection'}}
    compact_final = {key: value for key, value in final.items() if key not in ('test', 'holdout')}
    for split in ('test', 'holdout'):
        compact_final[split] = {'calibrated_difference_95pct': final[split]['calibrated_difference_95pct']}
        for method in ('trained', 'calibrated', 'base', 'base_calibrated'):
            block = final[split][method]
            compact_final[split][method] = {
                'overall': {key: block['overall'][key] for key in FINAL_METRICS},
                'per_family': {family: {key: row[key] for key in FINAL_METRICS}
                               for family, row in block.get('per_family', {}).items()}}
    return {'created_utc': datetime.now(timezone.utc).isoformat(), 'selected': selected,
        'selected_final_validation': final_validation, 'warm_start_lineage': lineage,
        'selection_tied_candidate_names': sorted(tied),
        'final_evaluation': compact_final, 'evaluation_manifest': evaluation_manifest, 'profile_protocol': protocol,
        'profile_protocol_sha256': protocol_hash, 'profile_requests_sha256': requests_hash,
        'profile_manifests': {**{method: value['manifest'] for method, value in methods.items()},
                             **({'expanded': expanded['manifest']} if expanded else {})},
        'profile_accuracy': accuracy, 'expanded_comparison': comparisons, 'speed': speed,
        'evidence_sha256': {'selection.json': sha(campaign / 'selection.json'),
                            'evaluation/metrics.json': sha(campaign / 'evaluation/metrics.json')},
        'limitations': ['Profile accuracy uses fixed matched samples; point differences have no significance claim.',
            'Base labels jointly condition on all options; verifier paths score each option independently.',
            'Profiles use raw probabilities without applying an artifact temperature.',
            'p95 is an exploratory nearest-rank statistic from the recorded small repeat count.',
            'Warm local timings exclude model loading and network latency; no shared-prefix optimization is used.',
            'Throughput is derived from serial warm median latency; it makes no concurrent-serving capacity claim.',
            'Expanded comparisons are post-selection diagnostics and cannot change the frozen winner.']}


def csv_write(path, rows):
    with Path(path).open('w', newline='') as handle:
        writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)


def report_text(result):
    selected, final = result['selected'], result['final_evaluation']
    lines = ['# Accuracy and inference profile', '',
        f"Frozen selection: **{selected['name']}**, checkpoint step **{selected['step']}**. "
        'Selection used validation only; the following evaluation does not change the winner.', '',
        '## Complete final evaluation', '',
        '| Split | Decisions | Calibrated selected accuracy | Calibrated base accuracy | Selected NLL | Base NLL |',
        '| --- | ---: | ---: | ---: | ---: | ---: |']
    notes = []
    if result['warm_start_lineage']:
        lineage = result['warm_start_lineage']
        notes.append(f"The selected branch's step 0 retains the exact warm-start parent weights from step {lineage['parent_step']}; "
                     'it is not an untrained base model. The saved CPU proof verifies equality of all trainable tensors.')
    if len(result['selection_tied_candidate_names']) > 1:
        names = result['selection_tied_candidate_names']
        if selected['name'] == min(names):
            notes.append('Validation-score tie: ' + ', '.join('`' + name + '`' for name in names) +
                         '. The coordinator selected the first name in deterministic alphabetical order.')
    completed = result['selected_final_validation']
    if completed and completed.get('promoted_latest') is False:
        notes.append(f"The branch's latest checkpoint, step {completed['latest_step']}, was evaluated but not promoted by the fixed validation rule. "
                     'Its expansion-diagnostic results are post-selection comparisons.')
    if notes:
        lines[4:4] = [' '.join(notes), '']
    for split in ('test', 'holdout'):
        a, b = final[split]['calibrated']['overall'], final[split]['base_calibrated']['overall']
        lines.append(f"| {split} | {a['n']} | {a['accuracy']:.2%} | {b['accuracy']:.2%} | {a['nll']:.4f} | {b['nll']:.4f} |")
    lines += ['', 'Paired 95% source-group bootstrap intervals are **calibrated selected minus calibrated base**. '
              'Positive accuracy differences favor the selected model; negative NLL/Brier differences favor it.', '']
    for split in ('test', 'holdout'):
        ci = final[split]['calibrated_difference_95pct']
        lines.append(f"- {split}: accuracy {100*ci['point_delta']['accuracy']:+.2f} pp "
            f"[{100*ci['accuracy'][0]:+.2f}, {100*ci['accuracy'][1]:+.2f}]; "
            f"NLL {ci['point_delta']['nll']:+.4f} [{ci['nll'][0]:+.4f}, {ci['nll'][1]:+.4f}]; "
            f"Brier {ci['point_delta']['brier']:+.4f} [{ci['brier'][0]:+.4f}, {ci['brier'][1]:+.4f}].")
    methods = list(result['profile_accuracy'])
    lines += ['', '## Matched profiling accuracy', '',
        '320 held-out decisions and 383 expansion diagnostics are identical across methods. '
        'Each family row reports its exact sample count. Differences below are descriptive point estimates.', '',
        '| Sample / family | N | ' + ' | '.join(LABELS[m] for m in methods) + ' |',
        '| --- | ---: | ' + ' | '.join('---:' for _ in methods) + ' |']
    for split in ('heldout', 'diagnostics'):
        first = result['profile_accuracy']['trained'][split]
        for family in ('overall', *first['per_family']):
            def entry(method):
                block = result['profile_accuracy'][method][split]
                return block['overall'] if family == 'overall' else block['per_family'][family]
            lines.append(f"| {split} / {family} | {entry('trained')['count']} | " +
                         ' | '.join(f"{entry(m)['accuracy']:.2%}" for m in methods) + ' |')
    if result['expanded_comparison']:
        step = result['profile_manifests']['expanded']['checkpoint_step']
        lines += ['', f'Expanded scorer checkpoint step: **{step}**. Paired expanded-minus-selected accuracy:', '']
        for split, block in result['expanded_comparison'].items():
            for family, row in [('overall', block['overall']), *block['per_family'].items()]:
                lines.append(f"- {split}/{family}: {row['expanded_minus_selected_accuracy_pp']:+.2f} pp; "
                    f"expanded alone correct {row['expanded_only_correct']}, selected alone correct {row['selected_only_correct']} (N={row['count']}).")
    lines += ['', 'The base label method computes one constrained next-token label from a prompt containing all options. '
        'The two verifier methods score each candidate independently. No free-text reasoning or JSON generation is timed.', '',
        '## Warm local speed', '',
        f"Each cell uses {result['profile_protocol']['warmups']} warmups and {result['profile_protocol']['repeats']} measured repeats. "
        'Times are median / exploratory p95 in seconds. Ratios are **base median ÷ selected median**: '
        '**above 1 means the selected scorer is faster; below 1 means the baseline is faster**. '
        '`summary.json` and `speed.csv` also report requests, questions and choice probabilities per second, '
        'derived from serial warm median latency; these do not measure concurrent serving.', '',
        '| State tokens | Questions × choices | Selected median / p95 | Base verifier median / p95 | Base label median / p95 | Verifier / selected | Label / selected |',
        '| ---: | ---: | ---: | ---: | ---: | ---: | ---: |']
    for row in result['speed']:
        cells = [f"{row['methods'][m]['median_seconds']:.4f} / {row['methods'][m]['p95_seconds']:.4f}" for m in METHODS]
        lines.append(f"| {row['state_tokens']} | {row['questions']} × {row['choices_per_question']} | " +
            ' | '.join(cells) + f" | {row['base_verifier_over_trained']:.2f}× | {row['base_label_over_trained']:.2f}× |")
    lines += ['', '![Warm latency comparison](latency.png)', '', '![Matched sample accuracy](accuracy.png)', '',
              '## Scope and evidence', '']
    lines += ['- ' + item for item in result['limitations']]
    lines += ['', '`summary.json` contains metrics, intervals and provenance. `final_evaluation.csv`, `accuracy.csv` and `speed.csv` contain table/chart data.',
              f"Frozen protocol SHA-256: `{result['profile_protocol_sha256']}`.", '']
    return '\n'.join(lines)


def plots(result, directory):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    from matplotlib.ticker import PercentFormatter
    colors = {'trained': '#2367a2', 'base_verifier': '#787878', 'base_label': '#de8524', 'expanded': '#479367'}
    fig, ax = plt.subplots(figsize=(14, 6), layout='constrained')
    width = .24
    for index, method in enumerate(METHODS):
        xs = [i + (index-1)*width for i in range(len(result['speed']))]
        medians = [row['methods'][method]['median_seconds'] for row in result['speed']]
        p95 = [row['methods'][method]['p95_seconds'] for row in result['speed']]
        ax.bar(xs, medians, width=width, label=LABELS[method], color=colors[method])
        ax.plot(xs, p95, linestyle='none', marker='_', markersize=9, color='#202020', label='Exploratory p95' if index == 0 else None)
    ax.set_yscale('log')
    ax.set_ylabel('Seconds per request (log scale)')
    ax.set_xticks(range(len(result['speed'])), [f"{r['state_tokens']} tokens\n{r['questions']}q × {r['choices_per_question']}c" for r in result['speed']])
    ax.set_title('Warm local latency: bars are medians; p95 markers are not confidence intervals')
    ax.grid(axis='y', which='both', alpha=.18)
    ax.legend(loc='upper left', ncols=2)
    fig.savefig(directory / 'latency.png', dpi=180)
    plt.close(fig)
    methods = list(result['profile_accuracy'])
    groups = [(split, family) for split in ('heldout', 'diagnostics')
              for family in result['profile_accuracy']['trained'][split]['per_family']]
    fig, ax = plt.subplots(figsize=(13, 6), layout='constrained')
    width = .8 / len(methods)
    for index, method in enumerate(methods):
        xs = [i + (index-(len(methods)-1)/2)*width for i in range(len(groups))]
        values = [result['profile_accuracy'][method][split]['per_family'][family]['accuracy'] for split, family in groups]
        ax.bar(xs, values, width=width, label=LABELS[method], color=colors[method])
    ax.set_xticks(range(len(groups)), [f"{split}\n{family}\nn={result['profile_accuracy']['trained'][split]['per_family'][family]['count']}"
                                     for split, family in groups])
    ax.set_ylim(0, 1.03)
    ax.yaxis.set_major_formatter(PercentFormatter(1))
    ax.set_ylabel('Accuracy on matched decisions')
    ax.set_title('Fixed profiling samples: descriptive accuracy, without significance claims')
    ax.grid(axis='y', alpha=.18)
    ax.legend(loc='lower right')
    fig.savefig(directory / 'accuracy.png', dpi=180)
    plt.close(fig)


def write_report(result, output):
    output = Path(output).resolve()
    if output.exists() or output.is_relative_to(ROOT):
        raise ValueError('Report output must be a new directory outside the source checkout')
    output.parent.mkdir(parents=True, exist_ok=True)
    temporary = Path(tempfile.mkdtemp(prefix=output.name + '.tmp-', dir=output.parent))
    try:
        json_write(temporary / 'summary.json', result)
        (temporary / 'report.md').write_text(report_text(result))
        accuracy_rows = []
        for method, splits in result['profile_accuracy'].items():
            for split, block in splits.items():
                for family, row in [('overall', block['overall']), *block['per_family'].items()]:
                    accuracy_rows.append({'scope': 'profile', 'split': split, 'method': method, 'family': family,
                        'count': row['count'], 'correct': row['correct'], 'accuracy': row['accuracy']})
        csv_write(temporary / 'accuracy.csv', accuracy_rows)
        final_rows = []
        for split in ('test', 'holdout'):
            for method in ('trained', 'calibrated', 'base', 'base_calibrated'):
                block = result['final_evaluation'][split][method]
                for family, row in [('overall', block['overall']), *block['per_family'].items()]:
                    final_rows.append({'split': split, 'method': method, 'family': family, **row})
        csv_write(temporary / 'final_evaluation.csv', final_rows)
        speed_rows = []
        for row in result['speed']:
            for method, values in row['methods'].items():
                speed_rows.append({'case': row['case'], 'method': method, 'state_tokens': row['state_tokens'],
                    'questions': row['questions'], 'choices_per_question': row['choices_per_question'],
                    'input_tokens_processed': values['input_tokens_processed'], 'sample_count': values['sample_count'],
                    'median_seconds': values['median_seconds'], 'p95_seconds': values['p95_seconds'],
                    'requests_per_second': values['requests_per_second'],
                    'questions_per_second': values['questions_per_second'],
                    'choice_probabilities_per_second': values['choice_probabilities_per_second'],
                    'method_median_over_selected_median': values['median_seconds'] / row['methods']['trained']['median_seconds']})
        csv_write(temporary / 'speed.csv', speed_rows)
        plots(result, temporary)
        temporary.rename(output)
    except BaseException:
        shutil.rmtree(temporary)
        raise


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--campaign', required=True)
    parser.add_argument('--prepared', required=True)
    parser.add_argument('--profile', required=True)
    parser.add_argument('--expanded-profile')
    parser.add_argument('--lineage-proof', help='Optional saved CPU proof that selected step-zero weights equal their warm-start parent')
    parser.add_argument('--output', required=True)
    args = parser.parse_args()
    result = collect(args.campaign, args.prepared, args.profile, args.expanded_profile, args.lineage_proof)
    write_report(result, args.output)
    print(json.dumps({'status': 'complete', 'output': args.output, 'timing_cases': len(result['speed']),
                      'heldout_decisions': 320, 'diagnostic_decisions': 383}))


if __name__ == '__main__':
    main()