#!/usr/bin/env python3 """Compare Claude session mode simulations across two git refs.""" from __future__ import annotations import argparse import json import os import shutil import subprocess import sys import tempfile from dataclasses import asdict, dataclass from pathlib import Path from typing import Any if __package__ in {None, ""}: sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from benchmarks.claude_session_mode_benchmark import ( IMPACT_DIRECTION, OUTPUT_JSON, PROXY_MODE_CACHE, PROXY_MODE_TOKEN, format_currency, ) DEFAULT_OUTPUT_DIR = Path("benchmark_results") / "branch_compare" @dataclass class BranchResult: ref: str label: str commit: str summary: str dataset: dict[str, Any] observed: dict[str, Any] summaries: dict[str, dict[str, Any]] winners: dict[str, str] output_dir: str def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--left-ref", default="upstream/main") parser.add_argument("--right-ref", default="HEAD") parser.add_argument("--left-label", default="main") parser.add_argument("--right-label", default="pr") parser.add_argument("--root", type=Path, default=Path.home() / ".claude" / "projects") parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) parser.add_argument("--max-sessions", type=int, default=None) parser.add_argument("--recent-turns-per-session", type=int, default=None) parser.add_argument("--cache-ttl-minutes", type=int, default=5) parser.add_argument("--cache-write-multiplier", type=float, default=1.25) parser.add_argument("--workers", type=int, default=1) parser.add_argument( "--python", default=sys.executable, help="Python executable to use inside each worktree.", ) parser.add_argument( "--keep-worktrees", action="store_true", help="Do not remove temporary worktrees after the comparison run.", ) return parser.parse_args() def _run_git(args: list[str], cwd: Path) -> str: completed = subprocess.run( ["git", *args], cwd=cwd, check=True, capture_output=True, text=True, ) return completed.stdout.strip() def _ref_slug(ref: str) -> str: return "".join(ch if ch.isalnum() else "-" for ch in ref).strip("-").lower() or "ref" def _branch_output_dir(base: Path, label: str) -> Path: return base / _ref_slug(label) def _comparison_paths(base: Path) -> tuple[Path, Path, Path]: return ( base / "claude_session_branch_compare.md", base / "claude_session_branch_compare.json", base / "claude_session_branch_compare.html", ) def _mode_metric(branch: BranchResult, mode: str, field: str) -> float: summary = branch.summaries[mode] if field == "no_cache_total_cost_usd": if "no_cache_total_cost_usd" in summary: value = summary["no_cache_total_cost_usd"] else: value = ( float(summary["paid_input_cost_usd"]) + (float(summary["cache_read_cost_usd"]) * 10.0) + float(summary["paid_output_cost_usd"]) ) elif field == "prompt_window_with_cache": value = float(summary["forwarded_input_tokens"]) elif field == "prompt_window_without_cache_reads": value = float(summary["forwarded_input_tokens"]) - float(summary["cache_read_tokens"]) else: value = summary[field] if isinstance(value, bool): return float(value) return float(value) def _delta(left: float, right: float) -> float: return right - left def _classify_delta(field: str, delta: float) -> str: direction = IMPACT_DIRECTION.get(field, "same") tolerance = 1e-9 if abs(delta) <= tolerance: return "no_change" if direction == "lower": return "assist" if delta < 0 else "harm" if direction == "higher": return "assist" if delta > 0 else "harm" return "harm" def _build_benchmark_command( python_executable: str, script_path: Path, root: Path, output_dir: Path, max_sessions: int | None, recent_turns_per_session: int | None, cache_ttl_minutes: int, cache_write_multiplier: float, workers: int, ) -> list[str]: command = [ python_executable, str(script_path), "--root", str(root), "--output-dir", str(output_dir), "--cache-ttl-minutes", str(cache_ttl_minutes), "--cache-write-multiplier", str(cache_write_multiplier), "--workers", str(workers), ] if max_sessions is not None: command.extend(["--max-sessions", str(max_sessions)]) if recent_turns_per_session is not None: command.extend(["--recent-turns-per-session", str(recent_turns_per_session)]) return command def _load_branch_result( repo_root: Path, ref: str, label: str, branch_output_dir: Path, ) -> BranchResult: payload = json.loads((branch_output_dir / OUTPUT_JSON).read_text(encoding="utf-8")) commit = _run_git(["rev-parse", ref], repo_root) summary = _run_git(["show", "-s", "--format=%s", ref], repo_root) return BranchResult( ref=ref, label=label, commit=commit, summary=summary, dataset=payload["dataset"], observed=payload["observed"], summaries=payload["summaries"], winners=payload["winners"], output_dir=str(branch_output_dir), ) def _run_branch_benchmark( repo_root: Path, ref: str, label: str, args: argparse.Namespace, worktree_root: Path, ) -> BranchResult: worktree_dir = worktree_root / _ref_slug(label) branch_output_dir = _branch_output_dir(args.output_dir, label) branch_output_dir.mkdir(parents=True, exist_ok=True) if worktree_dir.exists(): shutil.rmtree(worktree_dir) _run_git(["worktree", "add", "--detach", str(worktree_dir), ref], repo_root) try: command = _build_benchmark_command( python_executable=args.python, script_path=worktree_dir / "benchmarks" / "claude_session_mode_benchmark.py", root=args.root, output_dir=branch_output_dir, max_sessions=args.max_sessions, recent_turns_per_session=args.recent_turns_per_session, cache_ttl_minutes=args.cache_ttl_minutes, cache_write_multiplier=args.cache_write_multiplier, workers=args.workers, ) env = os.environ.copy() env["PYTHONPATH"] = os.pathsep.join([str(worktree_dir), env.get("PYTHONPATH", "")]).rstrip( os.pathsep ) subprocess.run(command, cwd=worktree_dir, check=True, env=env) return _load_branch_result(repo_root, ref, label, branch_output_dir) finally: if not args.keep_worktrees: subprocess.run( ["git", "worktree", "remove", "--force", str(worktree_dir)], cwd=repo_root, check=True, ) def _winner_line(metric: str, left: BranchResult, right: BranchResult) -> str: left_winner = left.winners[metric] right_winner = right.winners[metric] if left_winner == right_winner: return f"- {metric}: both pick `{left_winner}`" return ( f"- {metric}: `{left.label}` picks `{left_winner}`, `{right.label}` picks `{right_winner}`" ) def _build_six_way_rows( left: BranchResult, right: BranchResult ) -> list[dict[str, str | float | int]]: rows: list[dict[str, str | float | int]] = [] for branch in (left, right): for mode in ("baseline", PROXY_MODE_TOKEN, PROXY_MODE_CACHE): summary = branch.summaries[mode] cost_delta = _mode_metric(branch, mode, "total_cost_usd") - _mode_metric( branch, "baseline", "total_cost_usd" ) window_delta = int( _mode_metric(branch, mode, "prompt_window_with_cache") - _mode_metric(branch, "baseline", "prompt_window_with_cache") ) read_delta = int( _mode_metric(branch, mode, "cache_read_tokens") - _mode_metric(branch, "baseline", "cache_read_tokens") ) write_delta = int( _mode_metric(branch, mode, "cache_write_tokens") - _mode_metric(branch, "baseline", "cache_write_tokens") ) paid_input_delta = int( _mode_metric(branch, mode, "regular_input_tokens") - _mode_metric(branch, "baseline", "regular_input_tokens") ) rows.append( { "branch": branch.label, "mode": mode, "forwarded_input_tokens": int(summary["forwarded_input_tokens"]), "cache_read_tokens": int(summary["cache_read_tokens"]), "cache_write_tokens": int(summary["cache_write_tokens"]), "regular_input_tokens": int(summary["regular_input_tokens"]), "output_tokens": int(summary["output_tokens"]), "total_cost_usd": float(summary["total_cost_usd"]), "cost_delta_vs_branch_baseline": cost_delta, "window_delta_vs_branch_baseline": window_delta, "cache_read_delta_vs_branch_baseline": read_delta, "cache_write_delta_vs_branch_baseline": write_delta, "paid_input_delta_vs_branch_baseline": paid_input_delta, "is_branch_winner": "yes" if branch.winners["total_cost"] == mode else "no", } ) return rows def build_compare_markdown(left: BranchResult, right: BranchResult) -> str: six_way_rows = _build_six_way_rows(left, right) lines = [ "# Claude Session Branch Comparison", "", "## Branches", "", f"- {left.label}: `{left.ref}` @ `{left.commit[:12]}` - {left.summary}", f"- {right.label}: `{right.ref}` @ `{right.commit[:12]}` - {right.summary}", "", "## Dataset", "", f"- Projects: {right.dataset['projects']}", f"- Sessions: {right.dataset['sessions']}", f"- Requests: {right.dataset['requests']}", f"- Sampled requests: {right.dataset.get('sampled_requests', 0)}", f"- Sampling: {right.dataset.get('sampling_note', 'Full sessions')}", "", "## Winner Comparison", "", _winner_line("total_cost", left, right), _winner_line("no_cache_total_cost", left, right), _winner_line("window_with_cache", left, right), _winner_line("window_without_cache_reads", left, right), "", "## Six-Way Mode Matrix", "", "| Branch | Mode | Forwarded Input | Cache Read | Cache Write | Paid Input | Paid Output | Total Cost | Cost Δ vs Branch Baseline | Window Δ vs Branch Baseline | Winner |", "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", *[ "| " + " | ".join( [ str(row["branch"]), str(row["mode"]), f"{int(row['forwarded_input_tokens']):,}", f"{int(row['cache_read_tokens']):,}", f"{int(row['cache_write_tokens']):,}", f"{int(row['regular_input_tokens']):,}", f"{int(row['output_tokens']):,}", format_currency(float(row["total_cost_usd"])), format_currency(float(row["cost_delta_vs_branch_baseline"])), f"{int(row['window_delta_vs_branch_baseline']):,}", str(row["is_branch_winner"]), ] ) + " |" for row in six_way_rows ], "", "## Mode Deltas", "", f"| Mode | Metric | {left.label} | {right.label} | Delta ({right.label} - {left.label}) | Classification |", "| --- | --- | ---: | ---: | ---: | --- |", ] metrics = [ ("total_cost_usd", "Total Cost", format_currency), ("no_cache_total_cost_usd", "No-Cache Total Cost", format_currency), ("forwarded_input_tokens", "Forwarded Input Tokens", lambda v: f"{int(v):,}"), ("cache_read_tokens", "Cache Read Tokens", lambda v: f"{int(v):,}"), ("cache_write_tokens", "Cache Write Tokens", lambda v: f"{int(v):,}"), ("cache_bust_turns", "Cache Bust Turns", lambda v: f"{int(v):,}"), ("ttl_expiry_turns", "TTL Expiry Turns", lambda v: f"{int(v):,}"), ("prompt_window_with_cache", "Window With Cache", lambda v: f"{int(v):,}"), ( "prompt_window_without_cache_reads", "Window Without Cache Reads", lambda v: f"{int(v):,}", ), ] for mode in ("baseline", PROXY_MODE_TOKEN, PROXY_MODE_CACHE): for field, label, formatter in metrics: left_value = _mode_metric(left, mode, field) right_value = _mode_metric(right, mode, field) delta = _delta(left_value, right_value) delta_text = format_currency(delta) if "cost" in field else f"{int(delta):,}" classification = _classify_delta(field, delta) lines.append( f"| {mode} | {label} | {formatter(left_value)} | {formatter(right_value)} | {delta_text} | {classification} |" ) return "\n".join(lines) def build_compare_html(left: BranchResult, right: BranchResult) -> str: six_way_rows = [] for row in _build_six_way_rows(left, right): six_way_rows.append( "
{branch.commit[:12]}
{branch.summary}
" "Same local Claude transcript corpus. Same simulation knobs. Two git refs. This report isolates code-level behavior changes between the branches.
| Branch | Mode | Forwarded Input | Cache Read | Cache Write | Paid Input | Paid Output | Total Cost | Cost Δ vs Branch Baseline | Window Δ vs Branch Baseline | Winner |
|---|
| Mode | Metric | {left.label} | {right.label} | Delta | Classification |
|---|