#!/usr/bin/env python3 """ Real before/after test - NO MARKETING, JUST FACTS. This script makes actual API calls to demonstrate Headroom compression. """ import json import os import httpx # API Key from environment ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY") if not ANTHROPIC_API_KEY: raise ValueError("ANTHROPIC_API_KEY environment variable required") # Realistic tool output: 100 search results from a code search MOCK_TOOL_OUTPUT = json.dumps( [ { "file": f"src/components/{['Button', 'Modal', 'Form', 'Table', 'Card'][i % 5]}.tsx", "line": 10 + (i * 3), "content": f"export function {['Button', 'Modal', 'Form', 'Table', 'Card'][i % 5]}Component{i}(props: Props) {{", "language": "typescript", "repository": "frontend-app", "branch": "main", "last_modified": "2024-12-15T10:00:00Z", "author": f"dev{i % 10}@company.com", "match_score": 0.95 - (i * 0.005), "context": { "before": ["import React from 'react';", "import { useCallback } from 'react';"], "after": [" return
...
;", "}"], }, "metadata": { "size_bytes": 1500 + (i * 10), "encoding": "utf-8", "mime_type": "text/typescript", }, } for i in range(100) ] ) # The conversation we'll send def create_messages(tool_content: str) -> list: return [ {"role": "user", "content": "Find all React components that use forms"}, { "role": "assistant", "content": [ {"type": "text", "text": "I'll search for React form components."}, { "type": "tool_use", "id": "search_1", "name": "code_search", "input": {"query": "React form component", "limit": 100}, }, ], }, { "role": "user", "content": [ {"type": "tool_result", "tool_use_id": "search_1", "content": tool_content} ], }, ] def count_tokens_anthropic(text: str) -> int: """Rough token estimate (actual would use anthropic tokenizer)""" # Claude's tokenizer is roughly 4 chars per token for JSON return len(text) // 4 def make_api_call(base_url: str, messages: list, label: str) -> dict: """Make actual API call and return usage stats.""" headers = { "x-api-key": ANTHROPIC_API_KEY, "anthropic-version": "2023-06-01", "content-type": "application/json", } payload = { "model": "claude-sonnet-4-20250514", "max_tokens": 500, "messages": messages, "tools": [ { "name": "code_search", "description": "Search for code in the repository", "input_schema": { "type": "object", "properties": {"query": {"type": "string"}, "limit": {"type": "integer"}}, "required": ["query"], }, } ], } print(f"\n{'=' * 60}") print(f"{label}") print(f"{'=' * 60}") print(f"Endpoint: {base_url}") try: with httpx.Client(timeout=60.0) as client: response = client.post(f"{base_url}/v1/messages", headers=headers, json=payload) if response.status_code != 200: print(f"Error: {response.status_code}") print(response.text[:500]) return {"error": response.text} data = response.json() usage = data.get("usage", {}) result = { "input_tokens": usage.get("input_tokens", 0), "output_tokens": usage.get("output_tokens", 0), "response_preview": str(data.get("content", [{}])[0].get("text", ""))[:200], } print(f"Input tokens: {result['input_tokens']:,}") print(f"Output tokens: {result['output_tokens']:,}") print(f"Response: {result['response_preview']}...") return result except Exception as e: print(f"Exception: {e}") return {"error": str(e)} def main(): print("\n" + "=" * 70) print("HEADROOM REAL BEFORE/AFTER TEST") print("NO MARKETING - JUST ACTUAL API RESULTS") print("=" * 70) # Show what we're testing print(f"\nTest data: {len(json.loads(MOCK_TOOL_OUTPUT))} code search results") print(f"Raw JSON size: {len(MOCK_TOOL_OUTPUT):,} characters") print(f"Estimated tokens: ~{count_tokens_anthropic(MOCK_TOOL_OUTPUT):,}") messages = create_messages(MOCK_TOOL_OUTPUT) # Test 1: Direct to Anthropic API (baseline) baseline = make_api_call( "https://api.anthropic.com", messages, "BASELINE: Direct to Anthropic API" ) # Test 2: Through Headroom proxy optimized = make_api_call( "http://localhost:8787", messages, "OPTIMIZED: Through Headroom Proxy" ) # Results print("\n" + "=" * 70) print("RESULTS") print("=" * 70) if "error" not in baseline and "error" not in optimized: baseline_input = baseline["input_tokens"] optimized_input = optimized["input_tokens"] saved = baseline_input - optimized_input percent = (saved / baseline_input * 100) if baseline_input > 0 else 0 # Cost calculation (Claude Sonnet: $3/1M input, $15/1M output) cost_baseline = (baseline_input * 3 + baseline["output_tokens"] * 15) / 1_000_000 cost_optimized = (optimized_input * 3 + optimized["output_tokens"] * 15) / 1_000_000 cost_saved = cost_baseline - cost_optimized print(f""" Input Tokens: Baseline: {baseline_input:,} Optimized: {optimized_input:,} Saved: {saved:,} ({percent:.1f}%) Cost per request (Claude Sonnet pricing): Baseline: ${cost_baseline:.6f} Optimized: ${cost_optimized:.6f} Saved: ${cost_saved:.6f} At 10,000 requests/day: Daily savings: ${cost_saved * 10000:.2f} Monthly savings: ${cost_saved * 10000 * 30:.2f} """) # Return data for README return { "baseline_tokens": baseline_input, "optimized_tokens": optimized_input, "tokens_saved": saved, "percent_saved": percent, "tool_output_size": len(MOCK_TOOL_OUTPUT), "num_items": len(json.loads(MOCK_TOOL_OUTPUT)), } else: print("Test failed - check errors above") return None if __name__ == "__main__": result = main() if result: print("\n" + "=" * 70) print("JSON FOR README:") print("=" * 70) print(json.dumps(result, indent=2))