#!/usr/bin/env python3 """Headroom CLI - The Context Optimization Layer for LLM Applications. Usage: headroom proxy [OPTIONS] Start the optimization proxy server headroom memory-eval [OPTIONS] Run LoCoMo memory evaluation headroom --version Show version headroom --help Show this help message Examples: # Start proxy on default port (8787) headroom proxy # Start proxy on custom port headroom proxy --port 8080 # Start with optimization disabled (passthrough mode) headroom proxy --no-optimize # Use with Claude Code ANTHROPIC_BASE_URL=http://localhost:8787 claude # Run memory evaluation on 3 conversations headroom memory-eval -n 3 # Run memory evaluation with LLM answering and judging headroom memory-eval --answer-model gpt-4o --llm-judge """ from __future__ import annotations import argparse import sys from collections.abc import Callable def get_version() -> str: """Get the current version.""" try: from headroom import __version__ return __version__ except ImportError: return "unknown" def cmd_proxy(args: argparse.Namespace) -> int: """Start the proxy server.""" try: from headroom.proxy.server import ProxyConfig, run_server except ImportError as e: print("Error: Proxy dependencies not installed. Run: pip install headroom[proxy]") print(f"Details: {e}") return 1 config = ProxyConfig( host=args.host, port=args.port, optimize=not args.no_optimize, cache_enabled=not args.no_cache, rate_limit_enabled=not args.no_rate_limit, log_file=args.log_file, budget_limit_usd=args.budget, # LLMLingua: ON by default (use --no-llmlingua to disable) llmlingua_enabled=not args.no_llmlingua, llmlingua_device=args.llmlingua_device, llmlingua_target_rate=args.llmlingua_rate, # Code-aware: ON by default (use --no-code-aware to disable) code_aware_enabled=not args.no_code_aware, # Memory System memory_enabled=args.memory, memory_backend=args.memory_backend, memory_db_path=args.memory_db_path, memory_inject_tools=not args.no_memory_tools, memory_inject_context=not args.no_memory_context, memory_top_k=args.memory_top_k, ) memory_status = "DISABLED" if config.memory_enabled: memory_status = f"ENABLED ({config.memory_backend})" print(f""" ╔═══════════════════════════════════════════════════════════════════════╗ ║ HEADROOM PROXY ║ ║ The Context Optimization Layer for LLM Applications ║ ╚═══════════════════════════════════════════════════════════════════════╝ Starting proxy server... URL: http://{config.host}:{config.port} Optimization: {"ENABLED" if config.optimize else "DISABLED"} Caching: {"ENABLED" if config.cache_enabled else "DISABLED"} Rate Limit: {"ENABLED" if config.rate_limit_enabled else "DISABLED"} Memory: {memory_status} Usage with Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude Usage with OpenAI-compatible clients: OPENAI_BASE_URL=http://{config.host}:{config.port}/v1 your-app { "" if not config.memory_enabled else ''' Memory Requirements: Set x-headroom-user-id header to identify the user for memory scoping. ''' } Endpoints: GET /health Health check GET /stats Detailed statistics GET /metrics Prometheus metrics POST /v1/messages Anthropic API POST /v1/chat/completions OpenAI API Press Ctrl+C to stop. """) try: run_server(config) except KeyboardInterrupt: print("\nShutting down...") return 0 return 0 def cmd_version(args: argparse.Namespace) -> int: """Print version information.""" print(f"headroom {get_version()}") return 0 def cmd_memory_eval(args: argparse.Namespace) -> int: """Run LoCoMo memory evaluation.""" # Suppress noisy pydantic warnings from litellm import warnings warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*") warnings.filterwarnings("ignore", category=UserWarning, module="pydantic") try: from headroom.evals.memory import ( LoCoMoEvaluator, MemoryEvalConfig, create_anthropic_judge, create_litellm_judge, create_openai_judge, simple_judge, ) from headroom.memory import MemoryConfig except ImportError as e: print("Error: Memory eval dependencies not installed.") print("Run: pip install headroom[memory,evals]") print(f"Details: {e}") return 1 import asyncio # Build configuration categories = None if args.categories: categories = [int(c) for c in args.categories.split(",")] memory_config = MemoryConfig() eval_config = MemoryEvalConfig( n_conversations=args.n_conversations, categories=categories, skip_adversarial=not args.include_adversarial, top_k_memories=args.top_k, llm_judge_enabled=args.llm_judge, llm_judge_model=args.judge_model, memory_config=memory_config, f1_threshold=args.f1_threshold, extract_memories=not args.no_extract, extraction_model=args.extraction_model, pass_all_memories=args.pass_all, parallel_workers=args.parallel, debug=args.debug, ) # Create answer function based on provider answer_fn = None if args.answer_model: try: import litellm def answer_fn(question: str, memories: list[str]) -> str: if not memories: return "I don't have information about that." # Format memories - use all if pass_all, else top 10 context = "\n".join(f"- {m}" for m in memories) prompt = f"""You are answering questions about a conversation between two people based on extracted memories/facts. ## Memories from the conversation: {context} ## Question: {question} ## Instructions: 1. Find the specific fact(s) in the memories that answer this question 2. Answer with JUST the key information requested - be concise 3. For "when" questions: give the specific date if mentioned (e.g., "7 May 2023", "2022") 4. For "what" questions: give the specific thing/action 5. For "who" questions: give the name 6. If the exact answer is in the memories, use those exact words/dates 7. If you cannot find the answer, say "Information not found" ## Answer (be concise - just the facts):""" response = litellm.completion( model=args.answer_model, messages=[{"role": "user", "content": prompt}], temperature=0.0, max_tokens=150, ) return response.choices[0].message.content or "" except ImportError: print("Error: litellm required for --answer-model. Run: pip install litellm") return 1 # Create LLM judge if enabled llm_judge_fn: Callable[[str, str, str], tuple[float, str]] | None = None if args.llm_judge: # Use answer model for judge if not explicitly set judge_model = args.judge_model if args.answer_model and args.judge_model == "gpt-4o": judge_model = args.answer_model # Match the answer model if args.judge_provider == "simple": llm_judge_fn = simple_judge elif args.judge_provider == "openai": llm_judge_fn = create_openai_judge(model=judge_model) elif args.judge_provider == "anthropic": llm_judge_fn = create_anthropic_judge(model=judge_model) else: llm_judge_fn = create_litellm_judge(model=judge_model) # Determine judge info for display judge_info = "DISABLED" if args.llm_judge: if args.judge_provider == "simple": judge_info = "ENABLED (rule-based F1)" else: jm = args.judge_model if args.answer_model and args.judge_model == "gpt-4o": jm = args.answer_model judge_info = f"ENABLED ({args.judge_provider}: {jm})" extract_info = ( f"ENABLED ({args.extraction_model})" if not args.no_extract else "DISABLED (raw dialogue)" ) retrieval_info = "ALL memories (Path A)" if args.pass_all else f"Top-{args.top_k} retrieval" print(f""" ╔═══════════════════════════════════════════════════════════════════════╗ ║ HEADROOM MEMORY EVALUATION ║ ║ LoCoMo Benchmark ║ ╚═══════════════════════════════════════════════════════════════════════╝ Configuration: Conversations: {args.n_conversations or "all"} Categories: {categories or "[1,2,3,4]"} Retrieval: {retrieval_info} Memory Extract: {extract_info} Answer Model: {args.answer_model or "default (retrieval)"} LLM Judge: {judge_info} Parallelism: {args.parallel} workers Debug: {"ENABLED" if args.debug else "DISABLED"} Running evaluation... """) # Run evaluation evaluator = LoCoMoEvaluator( answer_fn=answer_fn, llm_judge_fn=llm_judge_fn, config=eval_config, ) try: result = asyncio.run(evaluator.run()) except KeyboardInterrupt: print("\nEvaluation interrupted.") return 1 # Print results print(result.summary()) # Save results if output path specified if args.output: result.save(args.output) print(f"\nResults saved to: {args.output}") return 0 def cmd_memory_eval_v2(args: argparse.Namespace) -> int: """Run LoCoMo V2 memory evaluation (LLM-controlled tools).""" # Suppress noisy pydantic warnings from litellm import warnings warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*") warnings.filterwarnings("ignore", category=UserWarning, module="pydantic") try: from headroom.evals.memory import ( LoCoMoEvaluatorV2, MemoryEvalConfigV2, ) except ImportError as e: print("Error: Memory eval V2 dependencies not installed.") print("Run: pip install headroom[memory,evals]") print(f"Details: {e}") return 1 import asyncio # Build configuration categories = None if args.categories: categories = [int(c) for c in args.categories.split(",")] eval_config = MemoryEvalConfigV2( n_conversations=args.n_conversations, categories=categories, skip_adversarial=not args.include_adversarial, llm_judge_enabled=args.llm_judge, llm_judge_model=args.judge_model, f1_threshold=args.f1_threshold, parallel_workers=args.parallel, debug=args.debug, save_model=args.save_model, answer_model=args.answer_model, max_search_results=args.max_results, include_graph_expansion=not args.no_graph, ) print(f""" ╔═══════════════════════════════════════════════════════════════════════╗ ║ HEADROOM MEMORY EVALUATION V2 ║ ║ LLM-Controlled Memory Architecture ║ ╚═══════════════════════════════════════════════════════════════════════╝ Configuration: Conversations: {args.n_conversations or "all"} Categories: {categories or "[1,2,3,4]"} Save Model: {args.save_model} Answer Model: {args.answer_model} Max Results: {args.max_results} Graph Expansion: {"DISABLED" if args.no_graph else "ENABLED"} LLM Judge: {"ENABLED" if args.llm_judge else "DISABLED"} Parallelism: {args.parallel} workers Debug: {"ENABLED" if args.debug else "DISABLED"} Key Differences from V1: - LLM decides WHAT to save (memory_save tool) - LLM decides HOW to search (memory_search tool) - Graph expansion enables multi-hop reasoning Running evaluation... """) # Run evaluation evaluator = LoCoMoEvaluatorV2( answer_model=args.answer_model, config=eval_config, ) try: result = asyncio.run(evaluator.run()) except KeyboardInterrupt: print("\nEvaluation interrupted.") return 1 # Print results print(result.summary()) # Save results if output path specified if args.output: result.save(args.output) print(f"\nResults saved to: {args.output}") return 0 def main(argv: list[str] | None = None) -> int: """Main CLI entry point.""" parser = argparse.ArgumentParser( prog="headroom", description="The Context Optimization Layer for LLM Applications", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: headroom proxy Start proxy on port 8787 headroom proxy --port 8080 Start proxy on port 8080 headroom proxy --no-optimize Passthrough mode (no optimization) Environment Variables: ANTHROPIC_API_KEY Your Anthropic API key (for proxying) OPENAI_API_KEY Your OpenAI API key (for proxying) Documentation: https://github.com/headroom-sdk/headroom """, ) parser.add_argument( "--version", "-V", action="store_true", help="Show version and exit", ) subparsers = parser.add_subparsers(dest="command", help="Commands") # Proxy command proxy_parser = subparsers.add_parser( "proxy", help="Start the optimization proxy server", formatter_class=argparse.RawDescriptionHelpFormatter, ) proxy_parser.add_argument( "--host", default="127.0.0.1", help="Host to bind to (default: 127.0.0.1)", ) proxy_parser.add_argument( "--port", "-p", type=int, default=8787, help="Port to bind to (default: 8787)", ) proxy_parser.add_argument( "--no-optimize", action="store_true", help="Disable optimization (passthrough mode)", ) proxy_parser.add_argument( "--no-cache", action="store_true", help="Disable semantic caching", ) proxy_parser.add_argument( "--no-rate-limit", action="store_true", help="Disable rate limiting", ) proxy_parser.add_argument( "--log-file", help="Path to JSONL log file", ) proxy_parser.add_argument( "--budget", type=float, help="Daily budget limit in USD", ) # LLMLingua ML-based compression (ON by default if installed) proxy_parser.add_argument( "--no-llmlingua", action="store_true", help="Disable LLMLingua-2 ML-based compression", ) proxy_parser.add_argument( "--llmlingua-device", choices=["auto", "cuda", "cpu", "mps"], default="auto", help="Device for LLMLingua model (default: auto)", ) proxy_parser.add_argument( "--llmlingua-rate", type=float, default=0.3, help="LLMLingua compression rate 0.0-1.0 (default: 0.3 = keep 30%%)", ) # Code-aware compression (ON by default if installed) proxy_parser.add_argument( "--no-code-aware", action="store_true", help="Disable AST-based code compression", ) # Memory System proxy_parser.add_argument( "--memory", action="store_true", help="Enable persistent user memory (requires x-headroom-user-id header)", ) proxy_parser.add_argument( "--memory-backend", choices=["local", "qdrant-neo4j"], default="local", help="Memory storage backend: local (SQLite+HNSW) or qdrant-neo4j (default: local)", ) proxy_parser.add_argument( "--memory-db-path", default="headroom_memory.db", help="Path to memory database file for local backend (default: headroom_memory.db)", ) proxy_parser.add_argument( "--no-memory-tools", action="store_true", help="Disable automatic memory tool injection", ) proxy_parser.add_argument( "--no-memory-context", action="store_true", help="Disable automatic memory context injection", ) proxy_parser.add_argument( "--memory-top-k", type=int, default=10, help="Number of memories to inject as context (default: 10)", ) proxy_parser.set_defaults(func=cmd_proxy) # Memory eval command eval_parser = subparsers.add_parser( "memory-eval", help="Run LoCoMo memory evaluation benchmark", formatter_class=argparse.RawDescriptionHelpFormatter, description=""" Run the LoCoMo memory benchmark to evaluate the Headroom memory system. LoCoMo (Long-term Conversational Memory) tests memory across: - Single-hop questions (simple fact recall) - Temporal questions (time-based) - Multi-hop questions (reasoning across memories) - Open-domain questions (interpretation required) Example: headroom memory-eval --n-conversations 3 headroom memory-eval --answer-model gpt-4o --llm-judge """, ) eval_parser.add_argument( "--n-conversations", "-n", type=int, help="Number of conversations to evaluate (default: all 10)", ) eval_parser.add_argument( "--categories", help="Comma-separated list of categories 1-5 (default: 1,2,3,4)", ) eval_parser.add_argument( "--include-adversarial", action="store_true", help="Include category 5 (unanswerable questions)", ) eval_parser.add_argument( "--top-k", type=int, default=10, help="Number of memories to retrieve per question (default: 10)", ) eval_parser.add_argument( "--f1-threshold", type=float, default=0.5, help="F1 score threshold for 'correct' (default: 0.5)", ) eval_parser.add_argument( "--answer-model", help="LLM model for generating answers (e.g., gpt-4o, claude-sonnet-4-20250514)", ) eval_parser.add_argument( "--llm-judge", action="store_true", help="Use LLM-as-judge scoring", ) eval_parser.add_argument( "--judge-provider", choices=["openai", "anthropic", "litellm", "simple"], default="litellm", help="LLM judge provider (default: litellm - uses same model as answer-model)", ) eval_parser.add_argument( "--judge-model", default="gpt-4o", help="Model for LLM judge (default: gpt-4o)", ) eval_parser.add_argument( "--output", "-o", help="Path to save JSON results", ) eval_parser.add_argument( "--no-extract", action="store_true", help="Disable LLM memory extraction (store raw dialogue instead)", ) eval_parser.add_argument( "--extraction-model", default="gpt-4o-mini", help="Model for memory extraction (default: gpt-4o-mini)", ) eval_parser.add_argument( "--pass-all", action="store_true", help="Pass ALL memories to LLM (Path A: no retrieval bottleneck)", ) eval_parser.add_argument( "--parallel", type=int, default=10, help="Number of parallel workers for LLM calls (default: 10)", ) eval_parser.add_argument( "--debug", action="store_true", help="Enable debug logging (saved to results JSON)", ) eval_parser.set_defaults(func=cmd_memory_eval) # Memory eval V2 command (LLM-controlled tools) eval_v2_parser = subparsers.add_parser( "memory-eval-v2", help="Run LoCoMo V2 evaluation with LLM-controlled memory tools", formatter_class=argparse.RawDescriptionHelpFormatter, description=""" Run the LoCoMo V2 memory benchmark with LLM-controlled memory architecture. This evaluator tests the new architecture where: - LLM decides what to save (memory_save tool) - LLM decides when to search (memory_search tool) - Graph relationships enable multi-hop reasoning Example: headroom memory-eval-v2 --n-conversations 3 headroom memory-eval-v2 --answer-model gpt-4o --save-model gpt-4o-mini """, ) eval_v2_parser.add_argument( "--n-conversations", "-n", type=int, help="Number of conversations to evaluate (default: all 10)", ) eval_v2_parser.add_argument( "--categories", help="Comma-separated list of categories 1-5 (default: 1,2,3,4)", ) eval_v2_parser.add_argument( "--include-adversarial", action="store_true", help="Include category 5 (unanswerable questions)", ) eval_v2_parser.add_argument( "--f1-threshold", type=float, default=0.5, help="F1 score threshold for 'correct' (default: 0.5)", ) eval_v2_parser.add_argument( "--save-model", default="gpt-4o-mini", help="LLM model for deciding what to save (default: gpt-4o-mini)", ) eval_v2_parser.add_argument( "--answer-model", default="gpt-4o", help="LLM model for answering questions (default: gpt-4o)", ) eval_v2_parser.add_argument( "--max-results", type=int, default=10, help="Maximum memories to retrieve per search (default: 10)", ) eval_v2_parser.add_argument( "--no-graph", action="store_true", help="Disable graph expansion in search", ) eval_v2_parser.add_argument( "--llm-judge", action="store_true", help="Use LLM-as-judge scoring", ) eval_v2_parser.add_argument( "--judge-model", default="gpt-4o", help="Model for LLM judge (default: gpt-4o)", ) eval_v2_parser.add_argument( "--output", "-o", help="Path to save JSON results", ) eval_v2_parser.add_argument( "--parallel", type=int, default=5, help="Number of parallel workers for LLM calls (default: 5)", ) eval_v2_parser.add_argument( "--debug", action="store_true", help="Enable debug logging (saved to results JSON)", ) eval_v2_parser.set_defaults(func=cmd_memory_eval_v2) args = parser.parse_args(argv) if args.version: return cmd_version(args) if args.command is None: parser.print_help() return 0 result = args.func(args) return int(result) if result is not None else 0 if __name__ == "__main__": sys.exit(main())