chopratejas commited on
Commit
93b0ca8
·
1 Parent(s): b78eec8

Plugin architecture for headroom learn + live traffic flush

Browse files

Refactor headroom learn into a plugin architecture where each coding
agent (Claude Code, Codex, Gemini CLI) is a self-contained plugin
with scanner, writer, and detection logic. External plugins can
register via the headroom.learn_plugin entry point.

- Add LearnPlugin ABC (base.py) and plugin registry (registry.py)
- Move scanners from monolithic scanner.py into plugins/ directory
- Extract shared error classification and tool name map (_shared.py)
- Add GeminiScanner for Google Gemini CLI session parsing
- CLI uses dynamic agent detection via registry (no hardcoded choices)
- All existing imports preserved via backwards-compat re-exports
- Wire agent_type through wrap → proxy → TrafficLearner
- Flush learned patterns to correct .md file at proxy shutdown
- Fix shutdown queue drain bug (patterns were lost on exit)
- 97 tests pass (84 existing + 13 new registry/plugin tests)

headroom/cli/learn.py CHANGED
@@ -8,59 +8,53 @@ from typing import TYPE_CHECKING
8
  import click
9
 
10
  if TYPE_CHECKING:
11
- from ..learn.scanner import ConversationScanner
12
- from ..learn.writer import ContextWriter
13
 
14
  from .main import main
15
 
16
- _AGENT_HELP = """Which coding agent to analyze. Auto-detects by default.
17
-
18
- \b
19
- Supported agents:
20
- claude Claude Code (~/.claude/)
21
- codex OpenAI Codex CLI (~/.codex/)
22
- gemini Google Gemini CLI (~/.gemini/)
23
- auto Auto-detect (check all, default)
24
- """
25
-
26
 
27
- def _get_scanner_writer(agent: str) -> tuple[ConversationScanner, ContextWriter]:
28
- """Get the appropriate scanner and writer for an agent type."""
29
- from ..learn.scanner import ClaudeCodeScanner, CodexScanner
30
- from ..learn.writer import ClaudeCodeWriter, CodexWriter
31
 
32
- scanners = {
33
- "claude": (ClaudeCodeScanner, ClaudeCodeWriter),
34
- "codex": (CodexScanner, CodexWriter),
35
- # Gemini scanner not yet implemented (protobuf sessions)
36
- # Cursor scanner not yet implemented (SQLite blobs)
37
- }
38
 
39
- if agent in scanners:
40
- scanner_cls, writer_cls = scanners[agent]
41
- return scanner_cls(), writer_cls()
42
 
43
- raise click.BadParameter(f"Unknown agent: {agent}. Supported: claude, codex")
 
 
 
 
 
 
 
 
44
 
 
 
 
 
 
45
 
46
- def _auto_detect_agents() -> list[tuple[str, ConversationScanner, ContextWriter]]:
47
- """Auto-detect which agents have data on this machine."""
48
- from ..learn.scanner import ClaudeCodeScanner, CodexScanner
49
- from ..learn.writer import ClaudeCodeWriter, CodexWriter
 
 
 
50
 
51
- agents: list[tuple[str, ConversationScanner, ContextWriter]] = []
 
52
 
53
- # Claude Code
54
- claude_dir = Path.home() / ".claude" / "projects"
55
- if claude_dir.exists() and any(claude_dir.iterdir()):
56
- agents.append(("claude", ClaudeCodeScanner(), ClaudeCodeWriter()))
57
 
58
- # Codex
59
- codex_dir = Path.home() / ".codex" / "sessions"
60
- if codex_dir.exists() and any(codex_dir.glob("*.json")):
61
- agents.append(("codex", CodexScanner(), CodexWriter()))
62
 
63
- return agents
 
 
 
64
 
65
 
66
  @main.command()
@@ -85,7 +79,7 @@ def _auto_detect_agents() -> list[tuple[str, ConversationScanner, ContextWriter]
85
  )
86
  @click.option(
87
  "--agent",
88
- type=click.Choice(["auto", "claude", "codex", "gemini"], case_sensitive=False),
89
  default="auto",
90
  help=_AGENT_HELP,
91
  )
@@ -109,8 +103,9 @@ def learn(
109
  (wrong paths, missing modules, stubborn retries) and generates context
110
  that prevents them from recurring.
111
 
112
- Supports multiple coding agents: Claude Code, Codex, Gemini CLI.
113
- Uses LiteLLM for provider-agnostic LLM access (100+ models).
 
114
 
115
  \b
116
  Examples:
@@ -121,6 +116,7 @@ def learn(
121
  headroom learn --agent codex --all # Analyze all Codex sessions
122
  """
123
  from ..learn.analyzer import SessionAnalyzer, _detect_default_model
 
124
 
125
  # Resolve model early to fail fast with a clear message
126
  try:
@@ -132,15 +128,18 @@ def learn(
132
  analyzer = SessionAnalyzer(model=resolved_model)
133
 
134
  # Determine which agents to scan
 
 
135
  if agent == "auto":
136
- agent_configs = _auto_detect_agents()
137
- if not agent_configs:
138
- click.echo("No coding agent data found. Checked: ~/.claude/, ~/.codex/")
139
  return
140
- click.echo(f"Detected agents: {', '.join(name for name, _, _ in agent_configs)}")
 
141
  else:
142
- scanner, writer = _get_scanner_writer(agent)
143
- agent_configs = [(agent, scanner, writer)]
144
 
145
  total_projects = 0
146
  total_failures = 0
@@ -148,11 +147,12 @@ def learn(
148
  matched_projects = 0
149
  available_projects: list[tuple[str, Path]] = []
150
 
151
- for agent_name, scanner, writer in agent_configs:
152
- all_projects = scanner.discover_projects()
 
153
  if not all_projects:
154
  continue
155
- available_projects.extend((agent_name, p.project_path) for p in all_projects)
156
 
157
  # Filter to target project(s)
158
  if analyze_all:
@@ -174,8 +174,8 @@ def learn(
174
  click.echo(f"No {agent_name} project data found for {cwd}")
175
  click.echo("Try: headroom learn --all or headroom learn --project <path>")
176
  click.echo(f"\nAvailable {agent_name} projects:")
177
- for p in all_projects[:10]:
178
- click.echo(f" {p.name:30s} {p.project_path}")
179
  return
180
 
181
  for proj in targets:
@@ -185,7 +185,7 @@ def learn(
185
  click.echo(f"Path: {proj.project_path}")
186
  click.echo(f"{'=' * 60}")
187
 
188
- sessions = scanner.scan_project(proj)
189
  if not sessions:
190
  click.echo(" No conversation data found.")
191
  continue
 
8
  import click
9
 
10
  if TYPE_CHECKING:
11
+ from ..learn.base import LearnPlugin
 
12
 
13
  from .main import main
14
 
 
 
 
 
 
 
 
 
 
 
15
 
16
+ class _AgentChoice(click.ParamType):
17
+ """Dynamic Click type that validates against the plugin registry."""
 
 
18
 
19
+ name = "agent"
 
 
 
 
 
20
 
21
+ def get_metavar(self, param: click.Parameter, ctx: click.Context | None = None) -> str | None:
22
+ return "[auto|<agent>]"
 
23
 
24
+ def convert(
25
+ self,
26
+ value: str,
27
+ param: click.Parameter | None,
28
+ ctx: click.Context | None,
29
+ ) -> str:
30
+ if value == "auto":
31
+ return value
32
+ from ..learn.registry import get_registry
33
 
34
+ reg = get_registry()
35
+ if value.lower() not in reg:
36
+ available = ", ".join(sorted(reg.keys()))
37
+ self.fail(f"Unknown agent: {value}. Available: auto, {available}", param, ctx)
38
+ return value.lower()
39
 
40
+ def shell_complete(
41
+ self,
42
+ ctx: click.Context,
43
+ param: click.Parameter,
44
+ incomplete: str,
45
+ ) -> list[click.shell_completion.CompletionItem]:
46
+ from ..learn.registry import available_agent_names
47
 
48
+ names = ["auto"] + available_agent_names()
49
+ return [click.shell_completion.CompletionItem(n) for n in names if n.startswith(incomplete)]
50
 
 
 
 
 
51
 
52
+ _AGENT_HELP = """Which coding agent to analyze. Auto-detects by default.
 
 
 
53
 
54
+ \b
55
+ Built-in: claude, codex, gemini.
56
+ External plugins register via 'headroom.learn_plugin' entry point.
57
+ Use 'auto' (default) to scan all detected agents."""
58
 
59
 
60
  @main.command()
 
79
  )
80
  @click.option(
81
  "--agent",
82
+ type=_AgentChoice(),
83
  default="auto",
84
  help=_AGENT_HELP,
85
  )
 
103
  (wrong paths, missing modules, stubborn retries) and generates context
104
  that prevents them from recurring.
105
 
106
+ Supports multiple coding agents via a plugin architecture. Built-in
107
+ support for Claude Code, Codex, and Gemini CLI. External plugins can
108
+ be installed via pip (entry point: headroom.learn_plugin).
109
 
110
  \b
111
  Examples:
 
116
  headroom learn --agent codex --all # Analyze all Codex sessions
117
  """
118
  from ..learn.analyzer import SessionAnalyzer, _detect_default_model
119
+ from ..learn.registry import auto_detect_plugins, get_plugin
120
 
121
  # Resolve model early to fail fast with a clear message
122
  try:
 
128
  analyzer = SessionAnalyzer(model=resolved_model)
129
 
130
  # Determine which agents to scan
131
+ agent_configs: list[tuple[str, LearnPlugin]] = []
132
+
133
  if agent == "auto":
134
+ detected = auto_detect_plugins()
135
+ if not detected:
136
+ click.echo("No coding agent data found.")
137
  return
138
+ click.echo(f"Detected agents: {', '.join(p.display_name for p in detected)}")
139
+ agent_configs = [(p.name, p) for p in detected]
140
  else:
141
+ selected = get_plugin(agent)
142
+ agent_configs = [(selected.name, selected)]
143
 
144
  total_projects = 0
145
  total_failures = 0
 
147
  matched_projects = 0
148
  available_projects: list[tuple[str, Path]] = []
149
 
150
+ for agent_name, plugin in agent_configs:
151
+ writer = plugin.create_writer()
152
+ all_projects = plugin.discover_projects()
153
  if not all_projects:
154
  continue
155
+ available_projects.extend((agent_name, proj.project_path) for proj in all_projects)
156
 
157
  # Filter to target project(s)
158
  if analyze_all:
 
174
  click.echo(f"No {agent_name} project data found for {cwd}")
175
  click.echo("Try: headroom learn --all or headroom learn --project <path>")
176
  click.echo(f"\nAvailable {agent_name} projects:")
177
+ for proj_info in all_projects[:10]:
178
+ click.echo(f" {proj_info.name:30s} {proj_info.project_path}")
179
  return
180
 
181
  for proj in targets:
 
185
  click.echo(f"Path: {proj.project_path}")
186
  click.echo(f"{'=' * 60}")
187
 
188
+ sessions = plugin.scan_project(proj)
189
  if not sessions:
190
  click.echo(" No conversation data found.")
191
  continue
headroom/cli/proxy.py CHANGED
@@ -269,6 +269,7 @@ def proxy(
269
  memory_top_k=memory_top_k,
270
  # Traffic Learning: only with --learn, never with --no-learn
271
  traffic_learning_enabled=learn and not no_learn,
 
272
  # Backend (Anthropic direct, Bedrock, LiteLLM, or any-llm)
273
  backend=backend,
274
  bedrock_region=bedrock_region or region,
 
269
  memory_top_k=memory_top_k,
270
  # Traffic Learning: only with --learn, never with --no-learn
271
  traffic_learning_enabled=learn and not no_learn,
272
+ traffic_learning_agent_type=os.environ.get("HEADROOM_AGENT_TYPE", "unknown"),
273
  # Backend (Anthropic direct, Bedrock, LiteLLM, or any-llm)
274
  backend=backend,
275
  bedrock_region=bedrock_region or region,
headroom/cli/wrap.py CHANGED
@@ -60,6 +60,7 @@ def _start_proxy(
60
  port: int,
61
  *,
62
  learn: bool = False,
 
63
  backend: str | None = None,
64
  anyllm_provider: str | None = None,
65
  region: str | None = None,
@@ -101,6 +102,10 @@ def _start_proxy(
101
  proxy_env = os.environ.copy()
102
  proxy_env["PYTHONIOENCODING"] = "utf-8"
103
 
 
 
 
 
104
  proc = subprocess.Popen(
105
  cmd,
106
  stdout=log_file,
@@ -313,6 +318,7 @@ def _ensure_proxy(
313
  no_proxy: bool,
314
  *,
315
  learn: bool = False,
 
316
  backend: str | None = None,
317
  anyllm_provider: str | None = None,
318
  region: str | None = None,
@@ -328,6 +334,7 @@ def _ensure_proxy(
328
  proc = _start_proxy(
329
  port,
330
  learn=learn,
 
331
  backend=backend,
332
  anyllm_provider=anyllm_provider,
333
  region=region,
@@ -392,6 +399,7 @@ def _launch_tool(
392
  env_vars_display: list[str],
393
  *,
394
  learn: bool = False,
 
395
  backend: str | None = None,
396
  anyllm_provider: str | None = None,
397
  region: str | None = None,
@@ -414,6 +422,7 @@ def _launch_tool(
414
  port,
415
  no_proxy,
416
  learn=learn,
 
417
  backend=backend,
418
  anyllm_provider=anyllm_provider,
419
  region=region,
@@ -713,7 +722,7 @@ def claude(
713
  click.echo(" ╚═══════════════════════════════════════════════╝")
714
  click.echo()
715
 
716
- proxy_holder[0] = _ensure_proxy(port, no_proxy, learn=learn)
717
 
718
  if not no_rtk:
719
  click.echo(" Setting up rtk...")
@@ -832,6 +841,7 @@ def codex(
832
  tool_label="CODEX",
833
  env_vars_display=[f"OPENAI_BASE_URL=http://127.0.0.1:{port}/v1"],
834
  learn=learn,
 
835
  backend=backend,
836
  anyllm_provider=anyllm_provider,
837
  region=region,
@@ -912,6 +922,7 @@ def aider(
912
  f"ANTHROPIC_BASE_URL=http://127.0.0.1:{port}",
913
  ],
914
  learn=learn,
 
915
  backend=backend,
916
  anyllm_provider=anyllm_provider,
917
  region=region,
@@ -961,7 +972,7 @@ def cursor(port: int, no_rtk: bool, no_proxy: bool, learn: bool, verbose: bool)
961
  click.echo(" ╚═══════════════════════════════════════════════╝")
962
  click.echo()
963
 
964
- proxy_holder[0] = _ensure_proxy(port, no_proxy, learn=learn)
965
 
966
  # Setup rtk for Cursor (binary + .cursorrules instructions)
967
  if not no_rtk:
 
60
  port: int,
61
  *,
62
  learn: bool = False,
63
+ agent_type: str = "unknown",
64
  backend: str | None = None,
65
  anyllm_provider: str | None = None,
66
  region: str | None = None,
 
102
  proxy_env = os.environ.copy()
103
  proxy_env["PYTHONIOENCODING"] = "utf-8"
104
 
105
+ # Tell the proxy which agent is being wrapped (for traffic learning output)
106
+ if agent_type != "unknown":
107
+ proxy_env["HEADROOM_AGENT_TYPE"] = agent_type
108
+
109
  proc = subprocess.Popen(
110
  cmd,
111
  stdout=log_file,
 
318
  no_proxy: bool,
319
  *,
320
  learn: bool = False,
321
+ agent_type: str = "unknown",
322
  backend: str | None = None,
323
  anyllm_provider: str | None = None,
324
  region: str | None = None,
 
334
  proc = _start_proxy(
335
  port,
336
  learn=learn,
337
+ agent_type=agent_type,
338
  backend=backend,
339
  anyllm_provider=anyllm_provider,
340
  region=region,
 
399
  env_vars_display: list[str],
400
  *,
401
  learn: bool = False,
402
+ agent_type: str = "unknown",
403
  backend: str | None = None,
404
  anyllm_provider: str | None = None,
405
  region: str | None = None,
 
422
  port,
423
  no_proxy,
424
  learn=learn,
425
+ agent_type=agent_type,
426
  backend=backend,
427
  anyllm_provider=anyllm_provider,
428
  region=region,
 
722
  click.echo(" ╚═══════════════════════════════════════════════╝")
723
  click.echo()
724
 
725
+ proxy_holder[0] = _ensure_proxy(port, no_proxy, learn=learn, agent_type="claude")
726
 
727
  if not no_rtk:
728
  click.echo(" Setting up rtk...")
 
841
  tool_label="CODEX",
842
  env_vars_display=[f"OPENAI_BASE_URL=http://127.0.0.1:{port}/v1"],
843
  learn=learn,
844
+ agent_type="codex",
845
  backend=backend,
846
  anyllm_provider=anyllm_provider,
847
  region=region,
 
922
  f"ANTHROPIC_BASE_URL=http://127.0.0.1:{port}",
923
  ],
924
  learn=learn,
925
+ agent_type="aider",
926
  backend=backend,
927
  anyllm_provider=anyllm_provider,
928
  region=region,
 
972
  click.echo(" ╚═══════════════════════════════════════════════╝")
973
  click.echo()
974
 
975
+ proxy_holder[0] = _ensure_proxy(port, no_proxy, learn=learn, agent_type="cursor")
976
 
977
  # Setup rtk for Cursor (binary + .cursorrules instructions)
978
  if not no_rtk:
headroom/learn/__init__.py CHANGED
@@ -1,12 +1,14 @@
1
  """Headroom Learn — offline session learning for coding agents.
2
 
3
- Analyzes conversation logs using Sonnet 4.6 to extract actionable patterns
4
- and generates context (CLAUDE.md, MEMORY.md, AGENTS.md, etc.) that prevents
5
- future token waste.
6
 
7
- Architecture:
8
- Scanner (adapter) → Analyzer (LLM) → Writer (adapter)
9
- ├── ClaudeCodeScanner SessionAnalyzer ├ ClaudeCodeWriter
10
- └── CodexScanner (Sonnet 4.6) ├─ CodexWriter
11
- └── GeminiWriter
 
 
12
  """
 
1
  """Headroom Learn — offline session learning for coding agents.
2
 
3
+ Analyzes conversation logs using an LLM to extract actionable patterns
4
+ and generates context (CLAUDE.md, AGENTS.md, GEMINI.md, etc.) that
5
+ prevents future token waste.
6
 
7
+ Plugin architecture:
8
+ plugins/claude.py ─┐
9
+ plugins/codex.py┤→ Analyzer (LLM) → Writer (adapter)
10
+ plugins/gemini.py
11
+
12
+ Built-in plugins are auto-discovered from headroom.learn.plugins.*.
13
+ External plugins register via the ``headroom.learn_plugin`` entry point.
14
  """
headroom/learn/_shared.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared utilities for headroom learn plugins.
2
+
3
+ Error classification, tool name normalization, and other helpers
4
+ used across all scanner plugins.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import re
10
+
11
+ from .models import ErrorCategory
12
+
13
+ # =============================================================================
14
+ # Error Classification
15
+ # =============================================================================
16
+
17
+ # Patterns checked in order — first match wins
18
+ _ERROR_PATTERNS: list[tuple[re.Pattern[str], ErrorCategory]] = [
19
+ (
20
+ re.compile(r"No such file or directory|ENOENT|FileNotFoundError|does not exist", re.I),
21
+ ErrorCategory.FILE_NOT_FOUND,
22
+ ),
23
+ (
24
+ re.compile(r"ModuleNotFoundError|ImportError|No module named", re.I),
25
+ ErrorCategory.MODULE_NOT_FOUND,
26
+ ),
27
+ (re.compile(r"command not found", re.I), ErrorCategory.COMMAND_NOT_FOUND),
28
+ (
29
+ re.compile(r"Permission denied|EACCES|EPERM|auto-denied", re.I),
30
+ ErrorCategory.PERMISSION_DENIED,
31
+ ),
32
+ (
33
+ re.compile(r"file is too large|too many lines|exceeds.*limit", re.I),
34
+ ErrorCategory.FILE_TOO_LARGE,
35
+ ),
36
+ (re.compile(r"EISDIR|Is a directory", re.I), ErrorCategory.IS_DIRECTORY),
37
+ (re.compile(r"SyntaxError|IndentationError", re.I), ErrorCategory.SYNTAX_ERROR),
38
+ (re.compile(r"Traceback \(most recent|Exception:|Error:", re.I), ErrorCategory.RUNTIME_ERROR),
39
+ (re.compile(r"timed? ?out|TimeoutError|deadline exceeded", re.I), ErrorCategory.TIMEOUT),
40
+ (re.compile(r"No (?:matches|files|results) found|0 matches", re.I), ErrorCategory.NO_MATCHES),
41
+ (
42
+ re.compile(r"user.*reject|user.*denied|declined|didn't want to proceed", re.I),
43
+ ErrorCategory.USER_REJECTED,
44
+ ),
45
+ (re.compile(r"[Ss]ibling tool call errored", re.I), ErrorCategory.SIBLING_ERROR),
46
+ (re.compile(r"exit code|non-zero|exited with", re.I), ErrorCategory.EXIT_CODE),
47
+ (
48
+ re.compile(r"ConnectionError|ConnectionRefused|ECONNREFUSED|network", re.I),
49
+ ErrorCategory.CONNECTION_ERROR,
50
+ ),
51
+ (
52
+ re.compile(r"BUILD FAILED|compilation error|compile error", re.I),
53
+ ErrorCategory.BUILD_FAILURE,
54
+ ),
55
+ ]
56
+
57
+
58
+ def classify_error(content: str) -> ErrorCategory:
59
+ """Classify an error message into a category."""
60
+ for pattern, category in _ERROR_PATTERNS:
61
+ if pattern.search(content[:2000]): # Only check first 2KB
62
+ return category
63
+ return ErrorCategory.UNKNOWN
64
+
65
+
66
+ def is_error_content(content: str) -> bool:
67
+ """Heuristic: does this tool result look like an error?"""
68
+ if not content or len(content) < 10:
69
+ return False
70
+ # Check for common error indicators in first 1KB
71
+ snippet = content[:1000]
72
+ indicators = [
73
+ "Error:",
74
+ "error:",
75
+ "ENOENT",
76
+ "No such file",
77
+ "command not found",
78
+ "Permission denied",
79
+ "ModuleNotFoundError",
80
+ "Traceback (most recent",
81
+ "FAILED",
82
+ "EISDIR",
83
+ "auto-denied",
84
+ "Sibling tool call errored",
85
+ "timed out",
86
+ "exit code",
87
+ "FileNotFoundError",
88
+ ]
89
+ return any(ind in snippet for ind in indicators)
90
+
91
+
92
+ # =============================================================================
93
+ # Tool Name Normalization
94
+ # =============================================================================
95
+
96
+ # Consolidated mapping from all agent-specific tool names to the cross-agent schema.
97
+ # Plugins can use normalize_tool_name() or extend this map for custom tools.
98
+ _TOOL_NAME_MAP: dict[str, str] = {
99
+ # Shell / command execution
100
+ "shell": "Bash",
101
+ "run_shell_command": "Bash",
102
+ "execute_command": "Bash",
103
+ "exec_command": "Bash",
104
+ "terminal": "Bash",
105
+ "run_command": "Bash",
106
+ "run_terminal_command": "Bash",
107
+ # File reading
108
+ "read_file": "Read",
109
+ "read_many_files": "Read",
110
+ "readfile": "Read",
111
+ "view_file": "Read",
112
+ "cat": "Read",
113
+ # File writing
114
+ "write_file": "Write",
115
+ "write_new_file": "Write",
116
+ "create_file": "Write",
117
+ "writefile": "Write",
118
+ # File editing
119
+ "edit_file": "Edit",
120
+ "replace_in_file": "Edit",
121
+ "editfile": "Edit",
122
+ "apply_diff": "Edit",
123
+ # File search / glob
124
+ "search_files": "Glob",
125
+ "find_files": "Glob",
126
+ "glob": "Glob",
127
+ "list_directory": "Glob",
128
+ "list_dir": "Glob",
129
+ # Text search / grep
130
+ "grep": "Grep",
131
+ "search_text": "Grep",
132
+ "search_code": "Grep",
133
+ "codebase_search": "Grep",
134
+ # Web
135
+ "browser": "WebFetch",
136
+ "web_search": "WebSearch",
137
+ }
138
+
139
+
140
+ def normalize_tool_name(name: str) -> str:
141
+ """Map agent-specific tool names to the cross-agent schema.
142
+
143
+ Looks up the name (case-insensitive) in the shared tool name map.
144
+ Returns the original name if no mapping exists.
145
+ """
146
+ return _TOOL_NAME_MAP.get(name.lower(), _TOOL_NAME_MAP.get(name, name))
headroom/learn/base.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Base class for headroom learn plugins.
2
+
3
+ Each coding agent (Claude Code, Codex, Gemini, Cursor, etc.) implements
4
+ a LearnPlugin that bundles scanning, writing, and detection into one unit.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from abc import ABC, abstractmethod
10
+
11
+ from .models import ProjectInfo, SessionData
12
+ from .writer import ContextWriter
13
+
14
+
15
+ class ConversationScanner(ABC):
16
+ """Base class for scanning conversation logs from any agent system.
17
+
18
+ Subclasses implement log format parsing for specific tools (Claude Code,
19
+ Cursor, Codex, etc.) and produce normalized ToolCall sequences.
20
+ """
21
+
22
+ @abstractmethod
23
+ def discover_projects(self) -> list[ProjectInfo]:
24
+ """Discover all projects with conversation data."""
25
+ ...
26
+
27
+ @abstractmethod
28
+ def scan_project(self, project: ProjectInfo) -> list[SessionData]:
29
+ """Scan all sessions for a project, returning normalized tool calls."""
30
+ ...
31
+
32
+
33
+ class LearnPlugin(ABC):
34
+ """A self-contained learn plugin for a single coding agent.
35
+
36
+ Bundles identity, detection, scanning, and writer creation.
37
+ Plugins are discovered automatically from headroom.learn.plugins.*
38
+ or via ``headroom.learn_plugin`` entry points for external packages.
39
+
40
+ Example::
41
+
42
+ class MyAgentPlugin(LearnPlugin, ConversationScanner):
43
+ @property
44
+ def name(self) -> str:
45
+ return "myagent"
46
+
47
+ @property
48
+ def display_name(self) -> str:
49
+ return "My Agent"
50
+
51
+ def detect(self) -> bool:
52
+ return Path("~/.myagent/sessions").expanduser().exists()
53
+
54
+ def discover_projects(self) -> list[ProjectInfo]: ...
55
+ def scan_project(self, project: ProjectInfo) -> list[SessionData]: ...
56
+
57
+ def create_writer(self) -> ContextWriter:
58
+ from headroom.learn.writer import GeminiWriter
59
+ return GeminiWriter() # or a custom writer
60
+
61
+ # Module-level instance for auto-discovery
62
+ plugin = MyAgentPlugin()
63
+ """
64
+
65
+ # --- Identity ---
66
+
67
+ @property
68
+ @abstractmethod
69
+ def name(self) -> str:
70
+ """Short lowercase identifier used in CLI (e.g., 'claude', 'cursor')."""
71
+ ...
72
+
73
+ @property
74
+ @abstractmethod
75
+ def display_name(self) -> str:
76
+ """Human-readable name (e.g., 'Claude Code', 'Cursor')."""
77
+ ...
78
+
79
+ @property
80
+ def description(self) -> str:
81
+ """One-line description for --help output."""
82
+ return f"{self.display_name} coding agent"
83
+
84
+ # --- Detection ---
85
+
86
+ @abstractmethod
87
+ def detect(self) -> bool:
88
+ """Return True if this agent has data on the current machine.
89
+
90
+ Called during auto-detection. Must be cheap (stat checks only, no I/O).
91
+ """
92
+ ...
93
+
94
+ # --- Scanning ---
95
+
96
+ @abstractmethod
97
+ def discover_projects(self) -> list[ProjectInfo]:
98
+ """Discover all projects with conversation data for this agent."""
99
+ ...
100
+
101
+ @abstractmethod
102
+ def scan_project(self, project: ProjectInfo) -> list[SessionData]:
103
+ """Scan all sessions for a project, returning normalized data."""
104
+ ...
105
+
106
+ # --- Writing ---
107
+
108
+ @abstractmethod
109
+ def create_writer(self) -> ContextWriter:
110
+ """Return the appropriate ContextWriter for this agent."""
111
+ ...
headroom/learn/plugins/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """Built-in learn plugins for headroom.
2
+
3
+ Each module in this package exposes a ``plugin`` attribute (a ``LearnPlugin``
4
+ instance) that is auto-discovered by the plugin registry.
5
+ """
headroom/learn/plugins/claude.py ADDED
@@ -0,0 +1,399 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Claude Code plugin for headroom learn.
2
+
3
+ Reads conversation logs from ~/.claude/projects/ (JSONL format).
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ import logging
10
+ import re
11
+ from pathlib import Path
12
+
13
+ from .._shared import classify_error, is_error_content
14
+ from ..base import ConversationScanner, LearnPlugin
15
+ from ..models import (
16
+ ErrorCategory,
17
+ ProjectInfo,
18
+ SessionData,
19
+ SessionEvent,
20
+ ToolCall,
21
+ )
22
+ from ..writer import ClaudeCodeWriter, ContextWriter
23
+
24
+ logger = logging.getLogger(__name__)
25
+
26
+
27
+ class ClaudeCodePlugin(LearnPlugin, ConversationScanner):
28
+ """Reads Claude Code conversation logs from ~/.claude/projects/.
29
+
30
+ Claude Code stores conversations as JSONL files with these line types:
31
+ - type="assistant": message.content[] has tool_use blocks (name, input, id)
32
+ - type="user": message.content[] has tool_result blocks (tool_use_id, content)
33
+ """
34
+
35
+ def __init__(self, claude_dir: Path | None = None):
36
+ self.claude_dir = claude_dir or Path.home() / ".claude"
37
+ self.projects_dir = self.claude_dir / "projects"
38
+
39
+ # --- LearnPlugin identity ---
40
+
41
+ @property
42
+ def name(self) -> str:
43
+ return "claude"
44
+
45
+ @property
46
+ def display_name(self) -> str:
47
+ return "Claude Code"
48
+
49
+ @property
50
+ def description(self) -> str:
51
+ return "Claude Code (~/.claude/)"
52
+
53
+ def detect(self) -> bool:
54
+ return self.projects_dir.exists() and any(self.projects_dir.iterdir())
55
+
56
+ def create_writer(self) -> ContextWriter:
57
+ return ClaudeCodeWriter()
58
+
59
+ # --- ConversationScanner interface ---
60
+
61
+ def discover_projects(self) -> list[ProjectInfo]:
62
+ """Discover all projects under ~/.claude/projects/."""
63
+ if not self.projects_dir.exists():
64
+ return []
65
+
66
+ projects = []
67
+ for entry in sorted(self.projects_dir.iterdir()):
68
+ if not entry.is_dir() or entry.name.startswith("."):
69
+ continue
70
+
71
+ project_path = _decode_project_path(entry.name)
72
+ if project_path is None:
73
+ fallback_parts = entry.name[1:].split("-")
74
+ if len(fallback_parts[0]) == 1 and fallback_parts[0].isalpha():
75
+ drive = fallback_parts[0].upper()
76
+ project_path = Path(f"{drive}:\\" + "\\".join(fallback_parts[1:]))
77
+ else:
78
+ project_path = Path("/" + entry.name[1:].replace("-", "/"))
79
+
80
+ name = project_path.name if project_path != Path("/") else entry.name
81
+
82
+ context_file = None
83
+ if project_path.exists():
84
+ claude_md = project_path / "CLAUDE.md"
85
+ if claude_md.exists():
86
+ context_file = claude_md
87
+
88
+ memory_dir = entry / "memory"
89
+ memory_file = memory_dir / "MEMORY.md" if memory_dir.exists() else None
90
+ if memory_file and not memory_file.exists():
91
+ memory_file = None
92
+
93
+ jsonl_files = list(entry.glob("*.jsonl"))
94
+ if not jsonl_files:
95
+ continue
96
+
97
+ projects.append(
98
+ ProjectInfo(
99
+ name=name,
100
+ project_path=project_path,
101
+ data_path=entry,
102
+ context_file=context_file,
103
+ memory_file=memory_file,
104
+ )
105
+ )
106
+
107
+ return projects
108
+
109
+ def scan_project(self, project: ProjectInfo) -> list[SessionData]:
110
+ """Scan all conversation JSONL files for a project."""
111
+ sessions = []
112
+ jsonl_files = sorted(project.data_path.glob("*.jsonl"))
113
+
114
+ for jsonl_path in jsonl_files:
115
+ session = self._scan_session(jsonl_path)
116
+ if session and session.tool_calls:
117
+ sessions.append(session)
118
+
119
+ return sessions
120
+
121
+ def _scan_session(self, jsonl_path: Path) -> SessionData | None:
122
+ """Scan a single JSONL conversation file."""
123
+ session_id = jsonl_path.stem
124
+ tool_uses: dict[str, tuple[str, dict]] = {}
125
+ tool_calls: list[ToolCall] = []
126
+ events: list[SessionEvent] = []
127
+ total_input_tokens = 0
128
+ total_output_tokens = 0
129
+ msg_index = 0
130
+
131
+ try:
132
+ with open(jsonl_path) as f:
133
+ for line in f:
134
+ try:
135
+ d = json.loads(line)
136
+ except json.JSONDecodeError:
137
+ continue
138
+
139
+ msg_index += 1
140
+ line_type = d.get("type", "")
141
+ ts = d.get("timestamp", None)
142
+
143
+ if line_type == "assistant":
144
+ self._extract_tool_uses(d, tool_uses)
145
+ usage = d.get("message", {}).get("usage", {})
146
+ total_input_tokens += usage.get("input_tokens", 0)
147
+ total_input_tokens += usage.get("cache_read_input_tokens", 0)
148
+ total_input_tokens += usage.get("cache_creation_input_tokens", 0)
149
+ total_output_tokens += usage.get("output_tokens", 0)
150
+ elif line_type == "user":
151
+ self._extract_tool_results(d, tool_uses, tool_calls, events, msg_index, ts)
152
+ self._extract_user_events(d, events, msg_index, ts)
153
+
154
+ except (OSError, UnicodeDecodeError) as e:
155
+ logger.debug("Failed to read %s: %s", jsonl_path, e)
156
+ return None
157
+
158
+ for tc in tool_calls:
159
+ if not any(e.type == "tool_call" and e.tool_call is tc for e in events):
160
+ events.append(SessionEvent(type="tool_call", msg_index=tc.msg_index, tool_call=tc))
161
+ events.sort(key=lambda e: e.msg_index)
162
+
163
+ return SessionData(
164
+ session_id=session_id,
165
+ tool_calls=tool_calls,
166
+ events=events,
167
+ total_input_tokens=total_input_tokens,
168
+ total_output_tokens=total_output_tokens,
169
+ )
170
+
171
+ def _extract_tool_uses(self, d: dict, tool_uses: dict[str, tuple[str, dict]]) -> None:
172
+ """Extract tool_use blocks from an assistant message."""
173
+ msg = d.get("message", {})
174
+ content = msg.get("content", [])
175
+ if not isinstance(content, list):
176
+ return
177
+
178
+ for block in content:
179
+ if not isinstance(block, dict) or block.get("type") != "tool_use":
180
+ continue
181
+ tc_id = block.get("id", "")
182
+ name = block.get("name", "")
183
+ inp = block.get("input", {})
184
+ if tc_id and name:
185
+ tool_uses[tc_id] = (name, inp if isinstance(inp, dict) else {})
186
+
187
+ def _extract_tool_results(
188
+ self,
189
+ d: dict,
190
+ tool_uses: dict[str, tuple[str, dict]],
191
+ tool_calls: list[ToolCall],
192
+ events: list[SessionEvent],
193
+ msg_index: int,
194
+ timestamp: str | None = None,
195
+ ) -> None:
196
+ """Extract tool_result blocks from a user message and match to tool_uses."""
197
+ msg = d.get("message", {})
198
+ content = msg.get("content", [])
199
+ if not isinstance(content, list):
200
+ return
201
+
202
+ for block in content:
203
+ if not isinstance(block, dict) or block.get("type") != "tool_result":
204
+ continue
205
+
206
+ tc_id = block.get("tool_use_id", "")
207
+ result_content = block.get("content", "")
208
+ if not isinstance(result_content, str):
209
+ result_content = str(result_content)
210
+
211
+ if tc_id not in tool_uses:
212
+ continue
213
+
214
+ name, inp = tool_uses[tc_id]
215
+
216
+ explicit_error = block.get("is_error", False)
217
+ detected_error = is_error_content(result_content)
218
+ is_err = explicit_error or detected_error
219
+
220
+ error_cat = classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
221
+
222
+ tc = ToolCall(
223
+ name=name,
224
+ tool_call_id=tc_id,
225
+ input_data=inp,
226
+ output=result_content,
227
+ is_error=is_err,
228
+ error_category=error_cat,
229
+ msg_index=msg_index,
230
+ output_bytes=len(result_content.encode("utf-8")),
231
+ )
232
+ tool_calls.append(tc)
233
+ events.append(
234
+ SessionEvent(
235
+ type="tool_call", msg_index=msg_index, timestamp=timestamp, tool_call=tc
236
+ )
237
+ )
238
+
239
+ if name in ("Agent", "agent"):
240
+ tool_result_meta = d.get("toolUseResult", {})
241
+ if isinstance(tool_result_meta, dict):
242
+ events.append(
243
+ SessionEvent(
244
+ type="agent_summary",
245
+ msg_index=msg_index,
246
+ timestamp=timestamp,
247
+ agent_id=tool_result_meta.get("agentId", ""),
248
+ agent_tool_count=tool_result_meta.get("totalToolUseCount", 0),
249
+ agent_tokens=tool_result_meta.get("totalTokens", 0),
250
+ agent_duration_ms=tool_result_meta.get("totalDurationMs", 0),
251
+ agent_prompt=tool_result_meta.get("prompt", "")[:200],
252
+ )
253
+ )
254
+
255
+ def _extract_user_events(
256
+ self,
257
+ d: dict,
258
+ events: list[SessionEvent],
259
+ msg_index: int,
260
+ timestamp: str | None = None,
261
+ ) -> None:
262
+ """Extract user text messages and interruptions from a user line."""
263
+ msg = d.get("message", {})
264
+ content = msg.get("content", "")
265
+
266
+ if isinstance(content, str) and content.strip():
267
+ events.append(
268
+ SessionEvent(
269
+ type="user_message",
270
+ msg_index=msg_index,
271
+ timestamp=timestamp,
272
+ text=content[:500],
273
+ )
274
+ )
275
+ return
276
+
277
+ if isinstance(content, list):
278
+ for block in content:
279
+ if not isinstance(block, dict):
280
+ continue
281
+ if block.get("type") == "text":
282
+ text = block.get("text", "")
283
+ if "[Request interrupted by user" in text:
284
+ events.append(
285
+ SessionEvent(
286
+ type="interruption",
287
+ msg_index=msg_index,
288
+ timestamp=timestamp,
289
+ text=text[:200],
290
+ )
291
+ )
292
+
293
+
294
+ # =============================================================================
295
+ # Path Decode Helpers (Claude Code specific)
296
+ # =============================================================================
297
+
298
+
299
+ def _decode_project_path(escaped_name: str) -> Path | None:
300
+ """Decode a Claude Code escaped project path."""
301
+ if not escaped_name.startswith("-"):
302
+ return None
303
+
304
+ parts = escaped_name[1:].split("-")
305
+ if len(parts) < 2:
306
+ return None
307
+
308
+ if len(parts[0]) == 1 and parts[0].isalpha():
309
+ drive = parts[0].upper()
310
+ win_path = Path(f"{drive}:\\" + "\\".join(parts[1:]))
311
+ if win_path.exists():
312
+ return win_path
313
+ win_base = Path(f"{drive}:\\{parts[1]}") if len(parts) > 1 else win_path
314
+ if win_base.exists() and len(parts) > 2:
315
+ result = _greedy_path_decode(win_base, parts[2:])
316
+ if result:
317
+ return result
318
+
319
+ simple = Path("/" + escaped_name[1:].replace("-", "/"))
320
+ if simple.exists():
321
+ return simple
322
+
323
+ if len(parts) < 3:
324
+ return None
325
+
326
+ if parts[0] == "Users" and len(parts) > 2:
327
+ base = Path(f"/{parts[0]}/{parts[1]}")
328
+ remaining = parts[2:]
329
+ return _greedy_path_decode(base, remaining)
330
+
331
+ if parts[0] == "home" and len(parts) > 2:
332
+ base = Path(f"/{parts[0]}/{parts[1]}")
333
+ remaining = parts[2:]
334
+ return _greedy_path_decode(base, remaining)
335
+
336
+ return None
337
+
338
+
339
+ def _greedy_path_decode(base: Path, parts: list[str]) -> Path | None:
340
+ """Greedily decode remaining path parts using real child directories."""
341
+ if not parts:
342
+ return base if base.exists() else None
343
+
344
+ if not base.exists() or not base.is_dir():
345
+ return None
346
+
347
+ try:
348
+ children = sorted(child for child in base.iterdir() if child.is_dir())
349
+ except OSError:
350
+ return None
351
+
352
+ for child in children:
353
+ for tokenization in _component_tokenizations(child.name):
354
+ n_tokens = len(tokenization)
355
+ if parts[:n_tokens] != tokenization:
356
+ continue
357
+
358
+ result = _greedy_path_decode(child, parts[n_tokens:])
359
+ if result:
360
+ return result
361
+
362
+ return None
363
+
364
+
365
+ def _component_tokenizations(component: str) -> list[list[str]]:
366
+ """Return possible escaped token sequences for a real path component."""
367
+ tokenizations: list[list[str]] = []
368
+ seen: set[tuple[str, ...]] = set()
369
+
370
+ def add(tokens: list[str]) -> None:
371
+ key = tuple(tokens)
372
+ if tokens and key not in seen:
373
+ seen.add(key)
374
+ tokenizations.append(tokens)
375
+
376
+ add([component])
377
+
378
+ for separator in ("-", ".", None):
379
+ if separator is None:
380
+ tokens = [token for token in re.split(r"[-.]", component) if token]
381
+ else:
382
+ tokens = [token for token in component.split(separator) if token]
383
+ add(tokens)
384
+
385
+ if component.startswith(".") and len(component) > 1:
386
+ hidden_component = component[1:]
387
+ add(["", hidden_component])
388
+ for separator in ("-", ".", None):
389
+ if separator is None:
390
+ tokens = [token for token in re.split(r"[-.]", hidden_component) if token]
391
+ else:
392
+ tokens = [token for token in hidden_component.split(separator) if token]
393
+ add(["", *tokens])
394
+
395
+ return tokenizations
396
+
397
+
398
+ # Module-level instance for auto-discovery by the plugin registry
399
+ plugin = ClaudeCodePlugin()
headroom/learn/plugins/codex.py ADDED
@@ -0,0 +1,315 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OpenAI Codex CLI plugin for headroom learn.
2
+
3
+ Reads session logs from ~/.codex/sessions/ (JSON and JSONL formats).
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ import logging
10
+ from pathlib import Path
11
+
12
+ from .._shared import classify_error, is_error_content, normalize_tool_name
13
+ from ..base import ConversationScanner, LearnPlugin
14
+ from ..models import (
15
+ ErrorCategory,
16
+ ProjectInfo,
17
+ SessionData,
18
+ ToolCall,
19
+ )
20
+ from ..writer import CodexWriter, ContextWriter
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+
25
+ class CodexPlugin(LearnPlugin, ConversationScanner):
26
+ """Reads OpenAI Codex CLI session logs from ~/.codex/sessions/.
27
+
28
+ Codex stores sessions as JSON files with:
29
+ - session.id, session.timestamp, session.instructions
30
+ - items[]: array of message/function_call/function_call_output/reasoning objects
31
+
32
+ function_call items have: name, call_id, arguments (JSON string)
33
+ function_call_output items have: call_id, output (string or JSON string)
34
+ """
35
+
36
+ def __init__(self, codex_dir: Path | None = None):
37
+ self.codex_dir = codex_dir or Path.home() / ".codex"
38
+ self.sessions_dir = self.codex_dir / "sessions"
39
+
40
+ # --- LearnPlugin identity ---
41
+
42
+ @property
43
+ def name(self) -> str:
44
+ return "codex"
45
+
46
+ @property
47
+ def display_name(self) -> str:
48
+ return "OpenAI Codex CLI"
49
+
50
+ @property
51
+ def description(self) -> str:
52
+ return "OpenAI Codex CLI (~/.codex/)"
53
+
54
+ def detect(self) -> bool:
55
+ if not self.sessions_dir.exists():
56
+ return False
57
+ return bool(
58
+ any(self.sessions_dir.rglob("*.json")) or any(self.sessions_dir.rglob("*.jsonl"))
59
+ )
60
+
61
+ def create_writer(self) -> ContextWriter:
62
+ return CodexWriter()
63
+
64
+ # --- ConversationScanner interface ---
65
+
66
+ def _iter_session_files(self, root: Path | None = None) -> list[Path]:
67
+ """Return all known Codex session files, including nested rollouts."""
68
+ search_root = root or self.sessions_dir
69
+ session_files = list(search_root.rglob("*.json")) + list(search_root.rglob("*.jsonl"))
70
+ return sorted(path for path in session_files if path.is_file())
71
+
72
+ def discover_projects(self) -> list[ProjectInfo]:
73
+ """Codex doesn't organize by project — return a single 'codex' project."""
74
+ if not self.sessions_dir.exists():
75
+ return []
76
+
77
+ session_files = self._iter_session_files()
78
+ if not session_files:
79
+ return []
80
+
81
+ agents_md = self.codex_dir / "AGENTS.md"
82
+ instructions_md = self.codex_dir / "instructions.md"
83
+
84
+ return [
85
+ ProjectInfo(
86
+ name="codex",
87
+ project_path=Path.cwd(),
88
+ data_path=self.sessions_dir,
89
+ context_file=agents_md if agents_md.exists() else None,
90
+ memory_file=instructions_md if instructions_md.exists() else None,
91
+ )
92
+ ]
93
+
94
+ def scan_project(self, project: ProjectInfo) -> list[SessionData]:
95
+ """Scan all Codex session JSON files."""
96
+ sessions = []
97
+ for json_path in self._iter_session_files(project.data_path):
98
+ session = self._scan_session(json_path)
99
+ if session and session.tool_calls:
100
+ sessions.append(session)
101
+ return sessions
102
+
103
+ def _scan_session(self, json_path: Path) -> SessionData | None:
104
+ """Parse a single Codex session file."""
105
+ if json_path.suffix == ".jsonl":
106
+ return self._scan_jsonl_session(json_path)
107
+ return self._scan_json_session(json_path)
108
+
109
+ def _scan_json_session(self, json_path: Path) -> SessionData | None:
110
+ """Parse a single Codex session file."""
111
+ try:
112
+ with open(json_path) as f:
113
+ data = json.load(f)
114
+ except (OSError, json.JSONDecodeError) as e:
115
+ logger.debug("Failed to read Codex session %s: %s", json_path, e)
116
+ return None
117
+
118
+ session_info = data.get("session", {})
119
+ session_id = session_info.get("id", json_path.stem)
120
+ items = data.get("items", [])
121
+
122
+ if not items:
123
+ return None
124
+
125
+ func_calls: dict[str, tuple[str, dict]] = {}
126
+ tool_calls: list[ToolCall] = []
127
+ msg_index = 0
128
+
129
+ for item in items:
130
+ msg_index += 1
131
+ item_type = item.get("type", "")
132
+
133
+ if item_type == "function_call":
134
+ call_id = item.get("call_id", "")
135
+ name = item.get("name", "")
136
+ raw_args = item.get("arguments", "")
137
+ if isinstance(raw_args, str):
138
+ try:
139
+ parsed = json.loads(raw_args)
140
+ except (json.JSONDecodeError, TypeError):
141
+ parsed = {"raw": raw_args}
142
+ elif isinstance(raw_args, dict):
143
+ parsed = raw_args
144
+ else:
145
+ parsed = {"raw": str(raw_args)}
146
+
147
+ # Codex-specific: extract command from shell args
148
+ if name == "shell" and "command" in parsed:
149
+ cmd = parsed["command"]
150
+ if isinstance(cmd, list):
151
+ parsed["command"] = cmd[-1] if cmd else ""
152
+ name = "Bash"
153
+ else:
154
+ name = normalize_tool_name(name)
155
+
156
+ if call_id:
157
+ func_calls[call_id] = (name, parsed)
158
+
159
+ elif item_type == "function_call_output":
160
+ call_id = item.get("call_id", "")
161
+ output_raw = item.get("output", "")
162
+
163
+ if call_id not in func_calls:
164
+ continue
165
+
166
+ name, inp = func_calls[call_id]
167
+ result_content = _parse_codex_output(output_raw)
168
+
169
+ is_err = is_error_content(result_content)
170
+ error_cat = classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
171
+
172
+ tool_calls.append(
173
+ ToolCall(
174
+ name=name,
175
+ tool_call_id=call_id,
176
+ input_data=inp,
177
+ output=result_content,
178
+ is_error=is_err,
179
+ error_category=error_cat,
180
+ msg_index=msg_index,
181
+ output_bytes=len(result_content.encode("utf-8")),
182
+ )
183
+ )
184
+
185
+ return SessionData(session_id=session_id, tool_calls=tool_calls)
186
+
187
+ def _scan_jsonl_session(self, jsonl_path: Path) -> SessionData | None:
188
+ """Parse a modern Codex rollout session stored as JSONL."""
189
+ session_id = jsonl_path.stem
190
+ func_calls: dict[str, tuple[str, dict]] = {}
191
+ tool_calls: list[ToolCall] = []
192
+ msg_index = 0
193
+
194
+ try:
195
+ with open(jsonl_path) as f:
196
+ for line in f:
197
+ try:
198
+ entry = json.loads(line)
199
+ except json.JSONDecodeError:
200
+ continue
201
+
202
+ if entry.get("type") == "session_meta":
203
+ payload = entry.get("payload", {})
204
+ if isinstance(payload, dict):
205
+ session_id = payload.get("id", session_id)
206
+ continue
207
+
208
+ if entry.get("type") != "response_item":
209
+ continue
210
+
211
+ payload = entry.get("payload", {})
212
+ if not isinstance(payload, dict):
213
+ continue
214
+
215
+ msg_index += 1
216
+ item_type = payload.get("type", "")
217
+
218
+ if item_type in ("function_call", "custom_tool_call"):
219
+ call_id = payload.get("call_id", "")
220
+ name = payload.get("name", "")
221
+ parsed = _parse_codex_arguments(payload)
222
+ name, parsed = _normalize_codex_tool(name, parsed)
223
+ if call_id and name:
224
+ func_calls[call_id] = (name, parsed)
225
+ continue
226
+
227
+ if item_type not in ("function_call_output", "custom_tool_call_output"):
228
+ continue
229
+
230
+ call_id = payload.get("call_id", "")
231
+ if call_id not in func_calls:
232
+ continue
233
+
234
+ name, inp = func_calls[call_id]
235
+ result_content = _parse_codex_output(payload.get("output", ""))
236
+ is_err = is_error_content(result_content)
237
+ error_cat = classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
238
+
239
+ tool_calls.append(
240
+ ToolCall(
241
+ name=name,
242
+ tool_call_id=call_id,
243
+ input_data=inp,
244
+ output=result_content,
245
+ is_error=is_err,
246
+ error_category=error_cat,
247
+ msg_index=msg_index,
248
+ output_bytes=len(result_content.encode("utf-8")),
249
+ )
250
+ )
251
+
252
+ except OSError as e:
253
+ logger.debug("Failed to read Codex session %s: %s", jsonl_path, e)
254
+ return None
255
+
256
+ if not tool_calls:
257
+ return None
258
+
259
+ return SessionData(session_id=session_id, tool_calls=tool_calls)
260
+
261
+
262
+ # =============================================================================
263
+ # Codex-specific Helpers
264
+ # =============================================================================
265
+
266
+
267
+ def _parse_codex_arguments(payload: dict) -> dict:
268
+ """Parse arguments for either legacy or rollout Codex tool calls."""
269
+ raw_args = payload.get("arguments", payload.get("input", ""))
270
+ if isinstance(raw_args, str):
271
+ try:
272
+ parsed = json.loads(raw_args)
273
+ return parsed if isinstance(parsed, dict) else {"raw": raw_args}
274
+ except (json.JSONDecodeError, TypeError):
275
+ return {"raw": raw_args}
276
+ if isinstance(raw_args, dict):
277
+ return raw_args
278
+ return {"raw": str(raw_args)}
279
+
280
+
281
+ def _normalize_codex_tool(name: str, parsed: dict) -> tuple[str, dict]:
282
+ """Normalize modern Codex tool names to the cross-agent schema."""
283
+ if name == "shell" and "command" in parsed:
284
+ cmd = parsed["command"]
285
+ if isinstance(cmd, list):
286
+ parsed["command"] = cmd[-1] if cmd else ""
287
+ return "Bash", parsed
288
+
289
+ if name == "exec_command" and "cmd" in parsed:
290
+ parsed = dict(parsed)
291
+ parsed["command"] = parsed.get("cmd", "")
292
+ return "Bash", parsed
293
+
294
+ return normalize_tool_name(name), parsed
295
+
296
+
297
+ def _parse_codex_output(output_raw: object) -> str:
298
+ """Parse tool output from Codex rollout records."""
299
+ if isinstance(output_raw, str):
300
+ try:
301
+ parsed_out = json.loads(output_raw)
302
+ except (json.JSONDecodeError, TypeError):
303
+ return output_raw
304
+
305
+ if isinstance(parsed_out, dict):
306
+ if "output" in parsed_out:
307
+ return str(parsed_out["output"])
308
+ return json.dumps(parsed_out)
309
+ return output_raw
310
+
311
+ return str(output_raw)
312
+
313
+
314
+ # Module-level instance for auto-discovery by the plugin registry
315
+ plugin = CodexPlugin()
headroom/learn/plugins/gemini.py ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Google Gemini CLI plugin for headroom learn.
2
+
3
+ Reads session logs from ~/.gemini/tmp/<project_hash>/chats/ (JSON and JSONL).
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ import logging
10
+ from pathlib import Path
11
+
12
+ from .._shared import classify_error, is_error_content, normalize_tool_name
13
+ from ..base import ConversationScanner, LearnPlugin
14
+ from ..models import (
15
+ ErrorCategory,
16
+ ProjectInfo,
17
+ SessionData,
18
+ SessionEvent,
19
+ ToolCall,
20
+ )
21
+ from ..writer import ContextWriter, GeminiWriter
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ class GeminiPlugin(LearnPlugin, ConversationScanner):
27
+ """Reads Google Gemini CLI session logs from ~/.gemini/tmp/<project>/chats/.
28
+
29
+ Gemini CLI stores sessions as JSON or JSONL files with messages in the
30
+ Gemini API format:
31
+ - role: "user" or "model"
32
+ - parts[]: array containing text, functionCall, or functionResponse objects
33
+
34
+ Tool calls use:
35
+ - functionCall: {name, args} (in model messages)
36
+ - functionResponse: {name, response} (in user messages)
37
+ """
38
+
39
+ def __init__(self, gemini_dir: Path | None = None):
40
+ self.gemini_dir = gemini_dir or Path.home() / ".gemini"
41
+ self.tmp_dir = self.gemini_dir / "tmp"
42
+
43
+ # --- LearnPlugin identity ---
44
+
45
+ @property
46
+ def name(self) -> str:
47
+ return "gemini"
48
+
49
+ @property
50
+ def display_name(self) -> str:
51
+ return "Google Gemini CLI"
52
+
53
+ @property
54
+ def description(self) -> str:
55
+ return "Google Gemini CLI (~/.gemini/)"
56
+
57
+ def detect(self) -> bool:
58
+ if not self.tmp_dir.exists():
59
+ return False
60
+ return bool(
61
+ any(self.tmp_dir.rglob("session-*.json")) or any(self.tmp_dir.rglob("session-*.jsonl"))
62
+ )
63
+
64
+ def create_writer(self) -> ContextWriter:
65
+ return GeminiWriter()
66
+
67
+ # --- ConversationScanner interface ---
68
+
69
+ def discover_projects(self) -> list[ProjectInfo]:
70
+ """Discover all projects with Gemini session data."""
71
+ if not self.tmp_dir.exists():
72
+ return []
73
+
74
+ projects = []
75
+ for project_dir in sorted(self.tmp_dir.iterdir()):
76
+ if not project_dir.is_dir():
77
+ continue
78
+
79
+ chats_dir = project_dir / "chats"
80
+ if not chats_dir.exists():
81
+ continue
82
+
83
+ session_files = list(chats_dir.glob("session-*.json")) + list(
84
+ chats_dir.glob("session-*.jsonl")
85
+ )
86
+ if not session_files:
87
+ continue
88
+
89
+ project_path = self._detect_project_path(session_files[0])
90
+
91
+ gemini_md = None
92
+ if project_path and project_path.exists():
93
+ candidate = project_path / "GEMINI.md"
94
+ if candidate.exists():
95
+ gemini_md = candidate
96
+
97
+ projects.append(
98
+ ProjectInfo(
99
+ name=project_path.name if project_path else project_dir.name,
100
+ project_path=project_path or Path.cwd(),
101
+ data_path=chats_dir,
102
+ context_file=gemini_md,
103
+ memory_file=None,
104
+ )
105
+ )
106
+
107
+ return projects
108
+
109
+ def scan_project(self, project: ProjectInfo) -> list[SessionData]:
110
+ """Scan all Gemini session files for a project."""
111
+ sessions = []
112
+ session_files = sorted(project.data_path.glob("session-*.json")) + sorted(
113
+ project.data_path.glob("session-*.jsonl")
114
+ )
115
+
116
+ for session_path in session_files:
117
+ session = self._scan_session(session_path)
118
+ if session and session.tool_calls:
119
+ sessions.append(session)
120
+
121
+ return sessions
122
+
123
+ def _scan_session(self, session_path: Path) -> SessionData | None:
124
+ """Parse a single Gemini session file (JSON or JSONL)."""
125
+ if session_path.suffix == ".jsonl":
126
+ return self._scan_jsonl_session(session_path)
127
+ return self._scan_json_session(session_path)
128
+
129
+ def _scan_json_session(self, json_path: Path) -> SessionData | None:
130
+ """Parse a Gemini JSON session file."""
131
+ try:
132
+ with open(json_path) as f:
133
+ data = json.load(f)
134
+ except (OSError, json.JSONDecodeError) as e:
135
+ logger.debug("Failed to read Gemini session %s: %s", json_path, e)
136
+ return None
137
+
138
+ session_id = json_path.stem
139
+
140
+ if isinstance(data, list):
141
+ messages = data
142
+ elif isinstance(data, dict):
143
+ messages = data.get("messages", data.get("history", data.get("contents", []))) or []
144
+ session_id = str(data.get("id", data.get("session_id", session_id)))
145
+ else:
146
+ return None
147
+
148
+ return self._parse_messages(session_id, messages)
149
+
150
+ def _scan_jsonl_session(self, jsonl_path: Path) -> SessionData | None:
151
+ """Parse a Gemini JSONL session file."""
152
+ session_id = jsonl_path.stem
153
+ messages: list[dict] = []
154
+
155
+ try:
156
+ with open(jsonl_path) as f:
157
+ for line in f:
158
+ try:
159
+ entry = json.loads(line)
160
+ except json.JSONDecodeError:
161
+ continue
162
+
163
+ entry_type = entry.get("type", "")
164
+
165
+ if entry_type == "session_metadata":
166
+ session_id = entry.get("id", entry.get("session_id", session_id))
167
+ continue
168
+
169
+ role = entry.get("role", "")
170
+ if not role:
171
+ if entry_type in ("user", "gemini", "model"):
172
+ role = "model" if entry_type == "gemini" else entry_type
173
+ else:
174
+ continue
175
+
176
+ parts = entry.get("parts", [])
177
+ if parts:
178
+ messages.append({"role": role, "parts": parts})
179
+
180
+ except (OSError, UnicodeDecodeError) as e:
181
+ logger.debug("Failed to read Gemini session %s: %s", jsonl_path, e)
182
+ return None
183
+
184
+ return self._parse_messages(session_id, messages)
185
+
186
+ def _parse_messages(self, session_id: str, messages: list) -> SessionData | None:
187
+ """Parse Gemini API format messages into normalized SessionData."""
188
+ tool_calls_pending: dict[str, tuple[str, dict, int]] = {}
189
+ tool_calls: list[ToolCall] = []
190
+ events: list[SessionEvent] = []
191
+ msg_index = 0
192
+ total_input_tokens = 0
193
+ total_output_tokens = 0
194
+
195
+ for msg in messages:
196
+ if not isinstance(msg, dict):
197
+ continue
198
+
199
+ msg_index += 1
200
+ role = msg.get("role", "")
201
+ parts = msg.get("parts", [])
202
+
203
+ usage = msg.get("usageMetadata", msg.get("usage", {}))
204
+ if isinstance(usage, dict):
205
+ total_input_tokens += usage.get("promptTokenCount", 0)
206
+ total_input_tokens += usage.get("cachedContentTokenCount", 0)
207
+ total_output_tokens += usage.get("candidatesTokenCount", 0)
208
+ total_output_tokens += (
209
+ usage.get("totalTokenCount", 0) - usage.get("promptTokenCount", 0)
210
+ if usage.get("totalTokenCount")
211
+ else 0
212
+ )
213
+
214
+ if not isinstance(parts, list):
215
+ continue
216
+
217
+ for part in parts:
218
+ if not isinstance(part, dict):
219
+ continue
220
+
221
+ if role == "user" and "text" in part:
222
+ text = part["text"]
223
+ if isinstance(text, str) and text.strip():
224
+ events.append(
225
+ SessionEvent(
226
+ type="user_message",
227
+ msg_index=msg_index,
228
+ text=text[:500],
229
+ )
230
+ )
231
+
232
+ if "functionCall" in part:
233
+ fc = part["functionCall"]
234
+ if isinstance(fc, dict):
235
+ name = fc.get("name", "")
236
+ args = fc.get("args", {})
237
+ if not isinstance(args, dict):
238
+ args = {}
239
+ normalized_name = normalize_tool_name(name)
240
+ if name:
241
+ tool_calls_pending[name] = (normalized_name, args, msg_index)
242
+
243
+ if "functionResponse" in part:
244
+ fr = part["functionResponse"]
245
+ if isinstance(fr, dict):
246
+ name = fr.get("name", "")
247
+ response = fr.get("response", {})
248
+
249
+ if isinstance(response, dict):
250
+ result_content = response.get("output", response.get("result", ""))
251
+ if not isinstance(result_content, str):
252
+ result_content = json.dumps(response)
253
+ elif isinstance(response, str):
254
+ result_content = response
255
+ else:
256
+ result_content = str(response)
257
+
258
+ if name in tool_calls_pending:
259
+ normalized_name, args, call_idx = tool_calls_pending.pop(name)
260
+ else:
261
+ normalized_name = normalize_tool_name(name)
262
+ args = {}
263
+ call_idx = msg_index
264
+
265
+ call_id = f"{session_id}_{call_idx}_{name}"
266
+ is_err = is_error_content(result_content)
267
+ error_cat = (
268
+ classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
269
+ )
270
+
271
+ tc = ToolCall(
272
+ name=normalized_name,
273
+ tool_call_id=call_id,
274
+ input_data=args,
275
+ output=result_content,
276
+ is_error=is_err,
277
+ error_category=error_cat,
278
+ msg_index=msg_index,
279
+ output_bytes=len(result_content.encode("utf-8")),
280
+ )
281
+ tool_calls.append(tc)
282
+ events.append(
283
+ SessionEvent(
284
+ type="tool_call",
285
+ msg_index=msg_index,
286
+ tool_call=tc,
287
+ )
288
+ )
289
+
290
+ events.sort(key=lambda e: e.msg_index)
291
+
292
+ return SessionData(
293
+ session_id=session_id,
294
+ tool_calls=tool_calls,
295
+ events=events,
296
+ total_input_tokens=total_input_tokens,
297
+ total_output_tokens=total_output_tokens,
298
+ )
299
+
300
+ def _detect_project_path(self, session_path: Path) -> Path | None:
301
+ """Try to detect the project path from a session file."""
302
+ try:
303
+ with open(session_path) as f:
304
+ data = json.load(f)
305
+ except (OSError, json.JSONDecodeError):
306
+ return None
307
+
308
+ if isinstance(data, dict):
309
+ project_path = data.get("projectPath", data.get("project_path", ""))
310
+ if project_path and Path(project_path).exists():
311
+ return Path(project_path)
312
+ cwd = data.get("cwd", data.get("workingDirectory", ""))
313
+ if cwd and Path(cwd).exists():
314
+ return Path(cwd)
315
+
316
+ return None
317
+
318
+
319
+ # Module-level instance for auto-discovery by the plugin registry
320
+ plugin = GeminiPlugin()
headroom/learn/registry.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Plugin registry for headroom learn.
2
+
3
+ Discovers built-in plugins from headroom.learn.plugins.* and external
4
+ plugins registered via the ``headroom.learn_plugin`` entry point group.
5
+
6
+ Follows the same pattern as headroom.storage_backend (storage/__init__.py).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import importlib
12
+ import logging
13
+ import pkgutil
14
+
15
+ from .base import LearnPlugin
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ _registry: dict[str, LearnPlugin] | None = None
20
+
21
+
22
+ def _discover() -> dict[str, LearnPlugin]:
23
+ """Discover all built-in and external plugins."""
24
+ plugins: dict[str, LearnPlugin] = {}
25
+
26
+ # 1. Built-in: scan headroom.learn.plugins.* submodules
27
+ from headroom.learn import plugins as plugins_pkg
28
+
29
+ for _, mod_name, _ in pkgutil.iter_modules(plugins_pkg.__path__):
30
+ try:
31
+ mod = importlib.import_module(f"headroom.learn.plugins.{mod_name}")
32
+ if hasattr(mod, "plugin"):
33
+ p = mod.plugin
34
+ if isinstance(p, LearnPlugin):
35
+ plugins[p.name] = p
36
+ logger.debug("Loaded built-in learn plugin: %s", p.name)
37
+ except Exception:
38
+ logger.debug("Failed to load built-in plugin: %s", mod_name, exc_info=True)
39
+
40
+ # 2. External: entry_points(group="headroom.learn_plugin")
41
+ try:
42
+ from importlib.metadata import entry_points
43
+
44
+ for ep in entry_points(group="headroom.learn_plugin"):
45
+ try:
46
+ obj = ep.load()
47
+ # Support both instances and factory callables
48
+ if isinstance(obj, LearnPlugin):
49
+ p = obj
50
+ elif callable(obj):
51
+ p = obj()
52
+ else:
53
+ logger.warning("Learn plugin %s is not a LearnPlugin instance", ep.name)
54
+ continue
55
+
56
+ if isinstance(p, LearnPlugin):
57
+ plugins[p.name] = p # External overrides built-in on name collision
58
+ logger.debug("Loaded external learn plugin: %s (%s)", p.name, ep.name)
59
+ except Exception:
60
+ logger.warning("Failed to load external learn plugin: %s", ep.name, exc_info=True)
61
+ except Exception:
62
+ logger.debug("Entry point discovery failed", exc_info=True)
63
+
64
+ return plugins
65
+
66
+
67
+ def get_registry() -> dict[str, LearnPlugin]:
68
+ """Get the plugin registry, discovering plugins on first call.
69
+
70
+ Returns a name → LearnPlugin mapping of all available plugins.
71
+ """
72
+ global _registry
73
+ if _registry is None:
74
+ _registry = _discover()
75
+ return _registry
76
+
77
+
78
+ def get_plugin(name: str) -> LearnPlugin:
79
+ """Look up a plugin by name.
80
+
81
+ Raises KeyError with a helpful message if not found.
82
+ """
83
+ reg = get_registry()
84
+ if name not in reg:
85
+ available = ", ".join(sorted(reg.keys()))
86
+ raise KeyError(f"Unknown agent: {name!r}. Available: {available}")
87
+ return reg[name]
88
+
89
+
90
+ def auto_detect_plugins() -> list[LearnPlugin]:
91
+ """Return plugins that have data on the current machine.
92
+
93
+ Calls ``detect()`` on each registered plugin and filters to those
94
+ that return True.
95
+ """
96
+ return [p for p in get_registry().values() if p.detect()]
97
+
98
+
99
+ def available_agent_names() -> list[str]:
100
+ """Return sorted list of all registered agent names."""
101
+ return sorted(get_registry().keys())
102
+
103
+
104
+ def reset_registry() -> None:
105
+ """Clear the registry cache. Used in tests."""
106
+ global _registry
107
+ _registry = None
headroom/learn/scanner.py CHANGED
@@ -1,793 +1,26 @@
1
- """Conversation scanners — read tool call logs from different agent systems.
2
 
3
- Scanners normalize conversation data into ToolCall sequences that analyzers
4
- can process regardless of the source system.
 
 
 
5
  """
6
 
7
  from __future__ import annotations
8
 
9
- import json
10
- import logging
11
- import re
12
- from abc import ABC, abstractmethod
13
- from pathlib import Path
14
-
15
- from .models import (
16
- ErrorCategory,
17
- ProjectInfo,
18
- SessionData,
19
- SessionEvent,
20
- ToolCall,
21
- )
22
-
23
- logger = logging.getLogger(__name__)
24
-
25
- # =============================================================================
26
- # Error Classification
27
- # =============================================================================
28
-
29
- # Patterns checked in order — first match wins
30
- _ERROR_PATTERNS: list[tuple[re.Pattern, ErrorCategory]] = [
31
- (
32
- re.compile(r"No such file or directory|ENOENT|FileNotFoundError|does not exist", re.I),
33
- ErrorCategory.FILE_NOT_FOUND,
34
- ),
35
- (
36
- re.compile(r"ModuleNotFoundError|ImportError|No module named", re.I),
37
- ErrorCategory.MODULE_NOT_FOUND,
38
- ),
39
- (re.compile(r"command not found", re.I), ErrorCategory.COMMAND_NOT_FOUND),
40
- (
41
- re.compile(r"Permission denied|EACCES|EPERM|auto-denied", re.I),
42
- ErrorCategory.PERMISSION_DENIED,
43
- ),
44
- (
45
- re.compile(r"file is too large|too many lines|exceeds.*limit", re.I),
46
- ErrorCategory.FILE_TOO_LARGE,
47
- ),
48
- (re.compile(r"EISDIR|Is a directory", re.I), ErrorCategory.IS_DIRECTORY),
49
- (re.compile(r"SyntaxError|IndentationError", re.I), ErrorCategory.SYNTAX_ERROR),
50
- (re.compile(r"Traceback \(most recent|Exception:|Error:", re.I), ErrorCategory.RUNTIME_ERROR),
51
- (re.compile(r"timed? ?out|TimeoutError|deadline exceeded", re.I), ErrorCategory.TIMEOUT),
52
- (re.compile(r"No (?:matches|files|results) found|0 matches", re.I), ErrorCategory.NO_MATCHES),
53
- (
54
- re.compile(r"user.*reject|user.*denied|declined|didn't want to proceed", re.I),
55
- ErrorCategory.USER_REJECTED,
56
- ),
57
- (re.compile(r"[Ss]ibling tool call errored", re.I), ErrorCategory.SIBLING_ERROR),
58
- (re.compile(r"exit code|non-zero|exited with", re.I), ErrorCategory.EXIT_CODE),
59
- (
60
- re.compile(r"ConnectionError|ConnectionRefused|ECONNREFUSED|network", re.I),
61
- ErrorCategory.CONNECTION_ERROR,
62
- ),
63
- (
64
- re.compile(r"BUILD FAILED|compilation error|compile error", re.I),
65
- ErrorCategory.BUILD_FAILURE,
66
- ),
67
- ]
68
-
69
-
70
- def classify_error(content: str) -> ErrorCategory:
71
- """Classify an error message into a category."""
72
- for pattern, category in _ERROR_PATTERNS:
73
- if pattern.search(content[:2000]): # Only check first 2KB
74
- return category
75
- return ErrorCategory.UNKNOWN
76
-
77
-
78
- def is_error_content(content: str) -> bool:
79
- """Heuristic: does this tool result look like an error?"""
80
- if not content or len(content) < 10:
81
- return False
82
- # Check for common error indicators in first 1KB
83
- snippet = content[:1000]
84
- indicators = [
85
- "Error:",
86
- "error:",
87
- "ENOENT",
88
- "No such file",
89
- "command not found",
90
- "Permission denied",
91
- "ModuleNotFoundError",
92
- "Traceback (most recent",
93
- "FAILED",
94
- "EISDIR",
95
- "auto-denied",
96
- "Sibling tool call errored",
97
- "timed out",
98
- "exit code",
99
- "FileNotFoundError",
100
- ]
101
- return any(ind in snippet for ind in indicators)
102
-
103
-
104
- # =============================================================================
105
- # Abstract Scanner
106
- # =============================================================================
107
-
108
-
109
- class ConversationScanner(ABC):
110
- """Base class for scanning conversation logs from any agent system.
111
-
112
- Subclasses implement log format parsing for specific tools (Claude Code,
113
- Cursor, Codex, etc.) and produce normalized ToolCall sequences.
114
- """
115
-
116
- @abstractmethod
117
- def discover_projects(self) -> list[ProjectInfo]:
118
- """Discover all projects with conversation data."""
119
- ...
120
-
121
- @abstractmethod
122
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
123
- """Scan all sessions for a project, returning normalized tool calls."""
124
- ...
125
-
126
-
127
- # =============================================================================
128
- # Claude Code Scanner
129
- # =============================================================================
130
-
131
-
132
- class ClaudeCodeScanner(ConversationScanner):
133
- """Reads Claude Code conversation logs from ~/.claude/projects/.
134
-
135
- Claude Code stores conversations as JSONL files with these line types:
136
- - type="assistant": message.content[] has tool_use blocks (name, input, id)
137
- - type="user": message.content[] has tool_result blocks (tool_use_id, content)
138
- """
139
-
140
- def __init__(self, claude_dir: Path | None = None):
141
- self.claude_dir = claude_dir or Path.home() / ".claude"
142
- self.projects_dir = self.claude_dir / "projects"
143
-
144
- def discover_projects(self) -> list[ProjectInfo]:
145
- """Discover all projects under ~/.claude/projects/."""
146
- if not self.projects_dir.exists():
147
- return []
148
-
149
- projects = []
150
- for entry in sorted(self.projects_dir.iterdir()):
151
- if not entry.is_dir() or entry.name.startswith("."):
152
- continue
153
-
154
- # Decode project path from escaped directory name. Fall back to a
155
- # simple slash replacement for display if we can't recover the real
156
- # on-disk path from the filesystem.
157
- project_path = _decode_project_path(entry.name)
158
- if project_path is None:
159
- # Fallback: detect Windows drive letter pattern (e.g., -C-MQ2-macros)
160
- fallback_parts = entry.name[1:].split("-")
161
- if len(fallback_parts[0]) == 1 and fallback_parts[0].isalpha():
162
- drive = fallback_parts[0].upper()
163
- project_path = Path(f"{drive}:\\" + "\\".join(fallback_parts[1:]))
164
- else:
165
- project_path = Path("/" + entry.name[1:].replace("-", "/"))
166
-
167
- # Derive human-readable name
168
- name = project_path.name if project_path != Path("/") else entry.name
169
-
170
- # Check for CLAUDE.md in actual project directory
171
- context_file = None
172
- if project_path.exists():
173
- claude_md = project_path / "CLAUDE.md"
174
- if claude_md.exists():
175
- context_file = claude_md
176
-
177
- # Check for MEMORY.md
178
- memory_dir = entry / "memory"
179
- memory_file = memory_dir / "MEMORY.md" if memory_dir.exists() else None
180
- if memory_file and not memory_file.exists():
181
- memory_file = None
182
-
183
- # Only include projects with JSONL files
184
- jsonl_files = list(entry.glob("*.jsonl"))
185
- if not jsonl_files:
186
- continue
187
-
188
- projects.append(
189
- ProjectInfo(
190
- name=name,
191
- project_path=project_path,
192
- data_path=entry,
193
- context_file=context_file,
194
- memory_file=memory_file,
195
- )
196
- )
197
-
198
- return projects
199
-
200
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
201
- """Scan all conversation JSONL files for a project."""
202
- sessions = []
203
-
204
- # Find all JSONL files (main conversations, not subagent files)
205
- jsonl_files = sorted(project.data_path.glob("*.jsonl"))
206
-
207
- for jsonl_path in jsonl_files:
208
- session = self._scan_session(jsonl_path)
209
- if session and session.tool_calls:
210
- sessions.append(session)
211
-
212
- return sessions
213
-
214
- def _scan_session(self, jsonl_path: Path) -> SessionData | None:
215
- """Scan a single JSONL conversation file."""
216
- session_id = jsonl_path.stem
217
- tool_uses: dict[str, tuple[str, dict]] = {} # tc_id → (tool_name, input)
218
- tool_calls: list[ToolCall] = []
219
- events: list[SessionEvent] = []
220
- total_input_tokens = 0
221
- total_output_tokens = 0
222
- msg_index = 0
223
-
224
- try:
225
- with open(jsonl_path) as f:
226
- for line in f:
227
- try:
228
- d = json.loads(line)
229
- except json.JSONDecodeError:
230
- continue
231
-
232
- msg_index += 1
233
- line_type = d.get("type", "")
234
- ts = d.get("timestamp", None)
235
-
236
- if line_type == "assistant":
237
- self._extract_tool_uses(d, tool_uses)
238
- # Extract token usage
239
- usage = d.get("message", {}).get("usage", {})
240
- total_input_tokens += usage.get("input_tokens", 0)
241
- total_input_tokens += usage.get("cache_read_input_tokens", 0)
242
- total_input_tokens += usage.get("cache_creation_input_tokens", 0)
243
- total_output_tokens += usage.get("output_tokens", 0)
244
- elif line_type == "user":
245
- self._extract_tool_results(d, tool_uses, tool_calls, events, msg_index, ts)
246
- self._extract_user_events(d, events, msg_index, ts)
247
-
248
- except (OSError, UnicodeDecodeError) as e:
249
- logger.debug("Failed to read %s: %s", jsonl_path, e)
250
- return None
251
-
252
- # Also wrap tool_calls as events for unified access
253
- for tc in tool_calls:
254
- if not any(e.type == "tool_call" and e.tool_call is tc for e in events):
255
- events.append(SessionEvent(type="tool_call", msg_index=tc.msg_index, tool_call=tc))
256
- events.sort(key=lambda e: e.msg_index)
257
-
258
- return SessionData(
259
- session_id=session_id,
260
- tool_calls=tool_calls,
261
- events=events,
262
- total_input_tokens=total_input_tokens,
263
- total_output_tokens=total_output_tokens,
264
- )
265
-
266
- def _extract_tool_uses(self, d: dict, tool_uses: dict[str, tuple[str, dict]]) -> None:
267
- """Extract tool_use blocks from an assistant message."""
268
- msg = d.get("message", {})
269
- content = msg.get("content", [])
270
- if not isinstance(content, list):
271
- return
272
-
273
- for block in content:
274
- if not isinstance(block, dict) or block.get("type") != "tool_use":
275
- continue
276
- tc_id = block.get("id", "")
277
- name = block.get("name", "")
278
- inp = block.get("input", {})
279
- if tc_id and name:
280
- tool_uses[tc_id] = (name, inp if isinstance(inp, dict) else {})
281
-
282
- def _extract_tool_results(
283
- self,
284
- d: dict,
285
- tool_uses: dict[str, tuple[str, dict]],
286
- tool_calls: list[ToolCall],
287
- events: list[SessionEvent],
288
- msg_index: int,
289
- timestamp: str | None = None,
290
- ) -> None:
291
- """Extract tool_result blocks from a user message and match to tool_uses."""
292
- msg = d.get("message", {})
293
- content = msg.get("content", [])
294
- if not isinstance(content, list):
295
- return
296
-
297
- for block in content:
298
- if not isinstance(block, dict) or block.get("type") != "tool_result":
299
- continue
300
-
301
- tc_id = block.get("tool_use_id", "")
302
- result_content = block.get("content", "")
303
- if not isinstance(result_content, str):
304
- result_content = str(result_content)
305
-
306
- # Match to tool_use
307
- if tc_id not in tool_uses:
308
- continue
309
-
310
- name, inp = tool_uses[tc_id]
311
-
312
- # Determine if error
313
- explicit_error = block.get("is_error", False)
314
- detected_error = is_error_content(result_content)
315
- is_err = explicit_error or detected_error
316
-
317
- error_cat = classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
318
-
319
- tc = ToolCall(
320
- name=name,
321
- tool_call_id=tc_id,
322
- input_data=inp,
323
- output=result_content,
324
- is_error=is_err,
325
- error_category=error_cat,
326
- msg_index=msg_index,
327
- output_bytes=len(result_content.encode("utf-8")),
328
- )
329
- tool_calls.append(tc)
330
- events.append(
331
- SessionEvent(
332
- type="tool_call", msg_index=msg_index, timestamp=timestamp, tool_call=tc
333
- )
334
- )
335
-
336
- # Extract subagent summary from toolUseResult metadata
337
- if name in ("Agent", "agent"):
338
- tool_result_meta = d.get("toolUseResult", {})
339
- if isinstance(tool_result_meta, dict):
340
- events.append(
341
- SessionEvent(
342
- type="agent_summary",
343
- msg_index=msg_index,
344
- timestamp=timestamp,
345
- agent_id=tool_result_meta.get("agentId", ""),
346
- agent_tool_count=tool_result_meta.get("totalToolUseCount", 0),
347
- agent_tokens=tool_result_meta.get("totalTokens", 0),
348
- agent_duration_ms=tool_result_meta.get("totalDurationMs", 0),
349
- agent_prompt=tool_result_meta.get("prompt", "")[:200],
350
- )
351
- )
352
-
353
- def _extract_user_events(
354
- self,
355
- d: dict,
356
- events: list[SessionEvent],
357
- msg_index: int,
358
- timestamp: str | None = None,
359
- ) -> None:
360
- """Extract user text messages and interruptions from a user line."""
361
- msg = d.get("message", {})
362
- content = msg.get("content", "")
363
-
364
- # Human text messages have content as a string, not a list
365
- if isinstance(content, str) and content.strip():
366
- events.append(
367
- SessionEvent(
368
- type="user_message",
369
- msg_index=msg_index,
370
- timestamp=timestamp,
371
- text=content[:500],
372
- )
373
- )
374
- return
375
-
376
- # Check for interruptions in list-format content
377
- if isinstance(content, list):
378
- for block in content:
379
- if not isinstance(block, dict):
380
- continue
381
- if block.get("type") == "text":
382
- text = block.get("text", "")
383
- if "[Request interrupted by user" in text:
384
- events.append(
385
- SessionEvent(
386
- type="interruption",
387
- msg_index=msg_index,
388
- timestamp=timestamp,
389
- text=text[:200],
390
- )
391
- )
392
-
393
-
394
- def _decode_project_path(escaped_name: str) -> Path | None:
395
- """Decode a Claude Code escaped project path.
396
-
397
- Claude Code escapes paths by replacing / with -.
398
- e.g., "-Users-tchopra-claude-projects-headroom"
399
- → "/Users/tchopra/claude-projects/headroom"
400
-
401
- Since - is ambiguous (path separator vs literal hyphen), we try
402
- progressively and check which decoded path actually exists.
403
- """
404
- if not escaped_name.startswith("-"):
405
- return None
406
-
407
- parts = escaped_name[1:].split("-")
408
- if len(parts) < 2:
409
- return None
410
-
411
- # Windows drive letter detection: -C-Users-foo or -D-MQ2-macros
412
- # First part is a single letter → treat as drive letter (C:\...)
413
- if len(parts[0]) == 1 and parts[0].isalpha():
414
- drive = parts[0].upper()
415
- win_path = Path(f"{drive}:\\" + "\\".join(parts[1:]))
416
- if win_path.exists():
417
- return win_path
418
- # Try greedy decode for Windows paths with hyphens in dir names
419
- win_base = Path(f"{drive}:\\{parts[1]}") if len(parts) > 1 else win_path
420
- if win_base.exists() and len(parts) > 2:
421
- result = _greedy_path_decode(win_base, parts[2:])
422
- if result:
423
- return result
424
-
425
- # Unix: simple approach — replace all - with / and check if path exists
426
- simple = Path("/" + escaped_name[1:].replace("-", "/"))
427
- if simple.exists():
428
- return simple
429
-
430
- # Try common Unix patterns: /Users/username/...
431
- if len(parts) < 3:
432
- return None
433
-
434
- # Build path greedily: try joining with / and check existence
435
- if parts[0] == "Users" and len(parts) > 2:
436
- base = Path(f"/{parts[0]}/{parts[1]}")
437
- remaining = parts[2:]
438
- return _greedy_path_decode(base, remaining)
439
-
440
- # Try /home/username/... (Linux)
441
- if parts[0] == "home" and len(parts) > 2:
442
- base = Path(f"/{parts[0]}/{parts[1]}")
443
- remaining = parts[2:]
444
- return _greedy_path_decode(base, remaining)
445
 
446
- return None
 
447
 
448
-
449
- def _greedy_path_decode(base: Path, parts: list[str]) -> Path | None:
450
- """Greedily decode remaining path parts using real child directories.
451
-
452
- Claude's project directory escaping is ambiguous: both ``/`` and literal
453
- punctuation such as ``-`` or ``.`` may show up as ``-`` in the escaped
454
- directory name. Instead of guessing every separator combination, walk the
455
- actual filesystem and match each child directory against the remaining
456
- escaped tokens.
457
- """
458
- if not parts:
459
- return base if base.exists() else None
460
-
461
- if not base.exists() or not base.is_dir():
462
- return None
463
-
464
- try:
465
- children = sorted(child for child in base.iterdir() if child.is_dir())
466
- except OSError:
467
- return None
468
-
469
- for child in children:
470
- for tokenization in _component_tokenizations(child.name):
471
- n_tokens = len(tokenization)
472
- if parts[:n_tokens] != tokenization:
473
- continue
474
-
475
- result = _greedy_path_decode(child, parts[n_tokens:])
476
- if result:
477
- return result
478
-
479
- return None
480
-
481
-
482
- def _component_tokenizations(component: str) -> list[list[str]]:
483
- """Return possible escaped token sequences for a real path component."""
484
- tokenizations: list[list[str]] = []
485
- seen: set[tuple[str, ...]] = set()
486
-
487
- def add(tokens: list[str]) -> None:
488
- key = tuple(tokens)
489
- if tokens and key not in seen:
490
- seen.add(key)
491
- tokenizations.append(tokens)
492
-
493
- add([component])
494
-
495
- for separator in ("-", ".", None):
496
- if separator is None:
497
- tokens = [token for token in re.split(r"[-.]", component) if token]
498
- else:
499
- tokens = [token for token in component.split(separator) if token]
500
- add(tokens)
501
-
502
- # Claude's flattened encoding can turn a leading "." in hidden directory
503
- # names into an empty token followed by the remaining component tokens.
504
- if component.startswith(".") and len(component) > 1:
505
- hidden_component = component[1:]
506
- add(["", hidden_component])
507
- for separator in ("-", ".", None):
508
- if separator is None:
509
- tokens = [token for token in re.split(r"[-.]", hidden_component) if token]
510
- else:
511
- tokens = [token for token in hidden_component.split(separator) if token]
512
- add(["", *tokens])
513
-
514
- return tokenizations
515
-
516
-
517
- # =============================================================================
518
- # Codex Scanner (OpenAI Codex CLI)
519
- # =============================================================================
520
-
521
-
522
- class CodexScanner(ConversationScanner):
523
- """Reads OpenAI Codex CLI session logs from ~/.codex/sessions/.
524
-
525
- Codex stores sessions as JSON files with:
526
- - session.id, session.timestamp, session.instructions
527
- - items[]: array of message/function_call/function_call_output/reasoning objects
528
-
529
- function_call items have: name, call_id, arguments (JSON string)
530
- function_call_output items have: call_id, output (string or JSON string)
531
- """
532
-
533
- def __init__(self, codex_dir: Path | None = None):
534
- self.codex_dir = codex_dir or Path.home() / ".codex"
535
- self.sessions_dir = self.codex_dir / "sessions"
536
-
537
- def _iter_session_files(self, root: Path | None = None) -> list[Path]:
538
- """Return all known Codex session files, including nested rollouts."""
539
- search_root = root or self.sessions_dir
540
- session_files = list(search_root.rglob("*.json")) + list(search_root.rglob("*.jsonl"))
541
- return sorted(path for path in session_files if path.is_file())
542
-
543
- def discover_projects(self) -> list[ProjectInfo]:
544
- """Codex doesn't organize by project — return a single 'codex' project.
545
-
546
- Codex sessions aren't scoped to projects. We treat all sessions as one
547
- project and use the cwd from session data (if available) to group later.
548
- """
549
- if not self.sessions_dir.exists():
550
- return []
551
-
552
- session_files = self._iter_session_files()
553
- if not session_files:
554
- return []
555
-
556
- # Check for global AGENTS.md
557
- agents_md = self.codex_dir / "AGENTS.md"
558
- instructions_md = self.codex_dir / "instructions.md"
559
-
560
- return [
561
- ProjectInfo(
562
- name="codex",
563
- project_path=Path.cwd(), # Codex doesn't track project paths
564
- data_path=self.sessions_dir,
565
- context_file=agents_md if agents_md.exists() else None,
566
- memory_file=instructions_md if instructions_md.exists() else None,
567
- )
568
- ]
569
-
570
- def scan_project(self, project: ProjectInfo) -> list[SessionData]:
571
- """Scan all Codex session JSON files."""
572
- sessions = []
573
- for json_path in self._iter_session_files(project.data_path):
574
- session = self._scan_session(json_path)
575
- if session and session.tool_calls:
576
- sessions.append(session)
577
- return sessions
578
-
579
- def _scan_session(self, json_path: Path) -> SessionData | None:
580
- """Parse a single Codex session file."""
581
- if json_path.suffix == ".jsonl":
582
- return self._scan_jsonl_session(json_path)
583
- return self._scan_json_session(json_path)
584
-
585
- def _scan_json_session(self, json_path: Path) -> SessionData | None:
586
- """Parse a single Codex session file."""
587
- try:
588
- with open(json_path) as f:
589
- data = json.load(f)
590
- except (OSError, json.JSONDecodeError) as e:
591
- logger.debug("Failed to read Codex session %s: %s", json_path, e)
592
- return None
593
-
594
- session_info = data.get("session", {})
595
- session_id = session_info.get("id", json_path.stem)
596
- items = data.get("items", [])
597
-
598
- if not items:
599
- return None
600
-
601
- # Build call_id → (name, input) map from function_call items
602
- func_calls: dict[str, tuple[str, dict]] = {}
603
- tool_calls: list[ToolCall] = []
604
- msg_index = 0
605
-
606
- for item in items:
607
- msg_index += 1
608
- item_type = item.get("type", "")
609
-
610
- if item_type == "function_call":
611
- call_id = item.get("call_id", "")
612
- name = item.get("name", "")
613
- # Parse arguments (JSON string or list)
614
- raw_args = item.get("arguments", "")
615
- if isinstance(raw_args, str):
616
- try:
617
- parsed = json.loads(raw_args)
618
- except (json.JSONDecodeError, TypeError):
619
- parsed = {"raw": raw_args}
620
- elif isinstance(raw_args, dict):
621
- parsed = raw_args
622
- else:
623
- parsed = {"raw": str(raw_args)}
624
-
625
- # Codex uses "shell" as the tool name with command in args
626
- # Normalize: extract command for consistency with other scanners
627
- if name == "shell" and "command" in parsed:
628
- cmd = parsed["command"]
629
- if isinstance(cmd, list):
630
- # Codex passes commands as ["bash", "-lc", "actual command"]
631
- parsed["command"] = cmd[-1] if cmd else ""
632
- name = "Bash" # Normalize to match Claude Code tool names
633
-
634
- if call_id:
635
- func_calls[call_id] = (name, parsed)
636
-
637
- elif item_type == "function_call_output":
638
- call_id = item.get("call_id", "")
639
- output_raw = item.get("output", "")
640
-
641
- if call_id not in func_calls:
642
- continue
643
-
644
- name, inp = func_calls[call_id]
645
-
646
- # Output may be JSON string with "output" field
647
- if isinstance(output_raw, str):
648
- try:
649
- parsed_out = json.loads(output_raw)
650
- if isinstance(parsed_out, dict) and "output" in parsed_out:
651
- result_content = str(parsed_out["output"])
652
- else:
653
- result_content = output_raw
654
- except (json.JSONDecodeError, TypeError):
655
- result_content = output_raw
656
- else:
657
- result_content = str(output_raw)
658
-
659
- is_err = is_error_content(result_content)
660
- error_cat = classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
661
-
662
- tool_calls.append(
663
- ToolCall(
664
- name=name,
665
- tool_call_id=call_id,
666
- input_data=inp,
667
- output=result_content,
668
- is_error=is_err,
669
- error_category=error_cat,
670
- msg_index=msg_index,
671
- output_bytes=len(result_content.encode("utf-8")),
672
- )
673
- )
674
-
675
- return SessionData(session_id=session_id, tool_calls=tool_calls)
676
-
677
- def _scan_jsonl_session(self, jsonl_path: Path) -> SessionData | None:
678
- """Parse a modern Codex rollout session stored as JSONL."""
679
- session_id = jsonl_path.stem
680
- func_calls: dict[str, tuple[str, dict]] = {}
681
- tool_calls: list[ToolCall] = []
682
- msg_index = 0
683
-
684
- try:
685
- with open(jsonl_path) as f:
686
- for line in f:
687
- try:
688
- entry = json.loads(line)
689
- except json.JSONDecodeError:
690
- continue
691
-
692
- if entry.get("type") == "session_meta":
693
- payload = entry.get("payload", {})
694
- if isinstance(payload, dict):
695
- session_id = payload.get("id", session_id)
696
- continue
697
-
698
- if entry.get("type") != "response_item":
699
- continue
700
-
701
- payload = entry.get("payload", {})
702
- if not isinstance(payload, dict):
703
- continue
704
-
705
- msg_index += 1
706
- item_type = payload.get("type", "")
707
-
708
- if item_type in ("function_call", "custom_tool_call"):
709
- call_id = payload.get("call_id", "")
710
- name = payload.get("name", "")
711
- parsed = self._parse_codex_arguments(payload)
712
- name, parsed = self._normalize_codex_tool(name, parsed)
713
- if call_id and name:
714
- func_calls[call_id] = (name, parsed)
715
- continue
716
-
717
- if item_type not in ("function_call_output", "custom_tool_call_output"):
718
- continue
719
-
720
- call_id = payload.get("call_id", "")
721
- if call_id not in func_calls:
722
- continue
723
-
724
- name, inp = func_calls[call_id]
725
- result_content = self._parse_codex_output(payload.get("output", ""))
726
- is_err = is_error_content(result_content)
727
- error_cat = classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
728
-
729
- tool_calls.append(
730
- ToolCall(
731
- name=name,
732
- tool_call_id=call_id,
733
- input_data=inp,
734
- output=result_content,
735
- is_error=is_err,
736
- error_category=error_cat,
737
- msg_index=msg_index,
738
- output_bytes=len(result_content.encode("utf-8")),
739
- )
740
- )
741
-
742
- except OSError as e:
743
- logger.debug("Failed to read Codex session %s: %s", jsonl_path, e)
744
- return None
745
-
746
- if not tool_calls:
747
- return None
748
-
749
- return SessionData(session_id=session_id, tool_calls=tool_calls)
750
-
751
- def _parse_codex_arguments(self, payload: dict) -> dict:
752
- """Parse arguments for either legacy or rollout Codex tool calls."""
753
- raw_args = payload.get("arguments", payload.get("input", ""))
754
- if isinstance(raw_args, str):
755
- try:
756
- parsed = json.loads(raw_args)
757
- return parsed if isinstance(parsed, dict) else {"raw": raw_args}
758
- except (json.JSONDecodeError, TypeError):
759
- return {"raw": raw_args}
760
- if isinstance(raw_args, dict):
761
- return raw_args
762
- return {"raw": str(raw_args)}
763
-
764
- def _normalize_codex_tool(self, name: str, parsed: dict) -> tuple[str, dict]:
765
- """Normalize modern Codex tool names to the cross-agent schema."""
766
- if name == "shell" and "command" in parsed:
767
- cmd = parsed["command"]
768
- if isinstance(cmd, list):
769
- parsed["command"] = cmd[-1] if cmd else ""
770
- return "Bash", parsed
771
-
772
- if name == "exec_command" and "cmd" in parsed:
773
- parsed = dict(parsed)
774
- parsed["command"] = parsed.get("cmd", "")
775
- return "Bash", parsed
776
-
777
- return name, parsed
778
-
779
- def _parse_codex_output(self, output_raw: object) -> str:
780
- """Parse tool output from Codex rollout records."""
781
- if isinstance(output_raw, str):
782
- try:
783
- parsed_out = json.loads(output_raw)
784
- except (json.JSONDecodeError, TypeError):
785
- return output_raw
786
-
787
- if isinstance(parsed_out, dict):
788
- if "output" in parsed_out:
789
- return str(parsed_out["output"])
790
- return json.dumps(parsed_out)
791
- return output_raw
792
-
793
- return str(output_raw)
 
1
+ """Conversation scanners — backwards-compatible re-exports.
2
 
3
+ Concrete scanner implementations have moved to headroom.learn.plugins.*.
4
+ This module re-exports them so existing imports continue to work:
5
+
6
+ from headroom.learn.scanner import ClaudeCodeScanner # still works
7
+ from headroom.learn.scanner import is_error_content # still works
8
  """
9
 
10
  from __future__ import annotations
11
 
12
+ # Shared helpers (moved to _shared.py)
13
+ from ._shared import classify_error, is_error_content # noqa: F401
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
 
15
+ # ConversationScanner ABC (canonical home is base.py)
16
+ from .base import ConversationScanner # noqa: F401
17
 
18
+ # Concrete scanners (moved to plugins/*, aliased to old names)
19
+ from .plugins.claude import ClaudeCodePlugin as ClaudeCodeScanner # noqa: F401
20
+ from .plugins.claude import ( # noqa: F401
21
+ _component_tokenizations,
22
+ _decode_project_path,
23
+ _greedy_path_decode,
24
+ )
25
+ from .plugins.codex import CodexPlugin as CodexScanner # noqa: F401
26
+ from .plugins.gemini import GeminiPlugin as GeminiScanner # noqa: F401
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
headroom/memory/traffic_learner.py CHANGED
@@ -149,6 +149,7 @@ class TrafficLearner:
149
  self,
150
  backend: LocalBackend | None = None,
151
  user_id: str = "default",
 
152
  max_history: int = 20,
153
  dedup_window: int = 100,
154
  min_evidence: int = 2,
@@ -159,12 +160,15 @@ class TrafficLearner:
159
  backend: Memory backend to save patterns to. If None, patterns
160
  are accumulated but not persisted until a backend is set.
161
  user_id: Default user ID for saved memories.
 
 
162
  max_history: Number of recent tool calls to keep for pattern matching.
163
  dedup_window: Number of recent pattern hashes to track for dedup.
164
  min_evidence: Minimum times a pattern must be seen before saving.
165
  """
166
  self._backend = backend
167
  self._user_id = user_id
 
168
  self._max_history = max_history
169
  self._min_evidence = min_evidence
170
 
@@ -186,6 +190,7 @@ class TrafficLearner:
186
  # Background save queue
187
  self._save_queue: asyncio.Queue[ExtractedPattern] = asyncio.Queue(maxsize=100)
188
  self._save_task: asyncio.Task[None] | None = None
 
189
 
190
  # =========================================================================
191
  # Public API
@@ -201,14 +206,111 @@ class TrafficLearner:
201
  self._save_task = asyncio.create_task(self._save_worker())
202
 
203
  async def stop(self) -> None:
204
- """Stop the background save worker."""
 
205
  if self._save_task and not self._save_task.done():
 
 
206
  self._save_task.cancel()
207
  try:
208
  await self._save_task
209
  except asyncio.CancelledError:
210
  pass
211
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
212
  async def on_tool_result(
213
  self,
214
  tool_name: str,
 
149
  self,
150
  backend: LocalBackend | None = None,
151
  user_id: str = "default",
152
+ agent_type: str = "unknown",
153
  max_history: int = 20,
154
  dedup_window: int = 100,
155
  min_evidence: int = 2,
 
160
  backend: Memory backend to save patterns to. If None, patterns
161
  are accumulated but not persisted until a backend is set.
162
  user_id: Default user ID for saved memories.
163
+ agent_type: Which coding agent is being wrapped (claude, codex, gemini, etc.).
164
+ Used to determine the correct output file for flushing patterns.
165
  max_history: Number of recent tool calls to keep for pattern matching.
166
  dedup_window: Number of recent pattern hashes to track for dedup.
167
  min_evidence: Minimum times a pattern must be seen before saving.
168
  """
169
  self._backend = backend
170
  self._user_id = user_id
171
+ self.agent_type = agent_type
172
  self._max_history = max_history
173
  self._min_evidence = min_evidence
174
 
 
190
  # Background save queue
191
  self._save_queue: asyncio.Queue[ExtractedPattern] = asyncio.Queue(maxsize=100)
192
  self._save_task: asyncio.Task[None] | None = None
193
+ self._stopping = False
194
 
195
  # =========================================================================
196
  # Public API
 
206
  self._save_task = asyncio.create_task(self._save_worker())
207
 
208
  async def stop(self) -> None:
209
+ """Stop the background save worker, draining the queue first."""
210
+ # Drain any remaining patterns in the queue before cancelling
211
  if self._save_task and not self._save_task.done():
212
+ # Signal the worker to stop by putting a sentinel
213
+ self._stopping = True
214
  self._save_task.cancel()
215
  try:
216
  await self._save_task
217
  except asyncio.CancelledError:
218
  pass
219
 
220
+ # Drain any patterns left in the queue (worker may have been cancelled mid-flight)
221
+ while not self._save_queue.empty():
222
+ try:
223
+ pattern = self._save_queue.get_nowait()
224
+ if self._backend is not None:
225
+ await self._backend.save_memory(
226
+ content=pattern.content,
227
+ user_id=self._user_id,
228
+ metadata={
229
+ "source": "traffic_learner",
230
+ "category": pattern.category.value,
231
+ "evidence_count": pattern.evidence_count,
232
+ **pattern.metadata,
233
+ },
234
+ )
235
+ self._patterns_saved += 1
236
+ except Exception:
237
+ break
238
+
239
+ # Flush learned patterns to the agent-native .md file
240
+ await self.flush_to_file()
241
+
242
+ async def flush_to_file(self) -> None:
243
+ """Flush accumulated patterns to the agent-native context file.
244
+
245
+ Uses the learn plugin registry to find the correct writer for the
246
+ current agent_type (e.g., MEMORY.md for Claude, AGENTS.md for Codex).
247
+ """
248
+ patterns = self.get_learned_patterns()
249
+ if not patterns or self.agent_type == "unknown":
250
+ return
251
+
252
+ try:
253
+ from headroom.learn.models import ProjectInfo, Recommendation, RecommendationTarget
254
+ from headroom.learn.registry import get_plugin
255
+
256
+ plugin = get_plugin(self.agent_type)
257
+ writer = plugin.create_writer()
258
+
259
+ # Convert patterns to Recommendations
260
+ recommendations = []
261
+ for p in patterns:
262
+ recommendations.append(
263
+ Recommendation(
264
+ target=RecommendationTarget.CONTEXT_FILE,
265
+ section="Learned Patterns (Live Traffic)",
266
+ content=f"- {p.content}",
267
+ confidence=p.importance,
268
+ evidence_count=p.evidence_count,
269
+ )
270
+ )
271
+
272
+ if not recommendations:
273
+ return
274
+
275
+ # Use cwd as project path — the proxy runs in the project directory
276
+ from pathlib import Path
277
+
278
+ project = ProjectInfo(
279
+ name=Path.cwd().name,
280
+ project_path=Path.cwd(),
281
+ data_path=Path.cwd(),
282
+ )
283
+
284
+ result = writer.write(recommendations, project, dry_run=False)
285
+ if result.files_written:
286
+ logger.info(
287
+ "Traffic learner flushed %d patterns to %s",
288
+ len(recommendations),
289
+ ", ".join(str(f) for f in result.files_written),
290
+ )
291
+ except KeyError:
292
+ logger.debug("No learn plugin for agent_type=%s, skipping file flush", self.agent_type)
293
+ except Exception as e:
294
+ logger.warning("Traffic learner flush_to_file failed: %s", e)
295
+
296
+ def get_learned_patterns(self) -> list[ExtractedPattern]:
297
+ """Return all patterns that have been saved or met the evidence threshold.
298
+
299
+ Includes patterns still in the accumulator that haven't hit min_evidence
300
+ but have been seen at least once (for end-of-session flush).
301
+ """
302
+ patterns: list[ExtractedPattern] = []
303
+
304
+ # Patterns that met the threshold and were queued
305
+ # (already saved to DB, but also want them in the .md file)
306
+ # We track what was saved via _saved_hashes, but don't keep the content.
307
+ # So we collect from the accumulator — patterns still accumulating.
308
+ for pattern, count in self._pattern_counts.values():
309
+ if count >= 1: # At shutdown, flush even single-evidence patterns
310
+ patterns.append(pattern)
311
+
312
+ return patterns
313
+
314
  async def on_tool_result(
315
  self,
316
  tool_name: str,
headroom/proxy/models.py CHANGED
@@ -176,6 +176,7 @@ class ProxyConfig:
176
  memory_db_path: str = "headroom_memory.db"
177
  memory_inject_tools: bool = True
178
  traffic_learning_enabled: bool = False
 
179
  memory_use_native_tool: bool = False
180
  memory_inject_context: bool = True
181
  memory_top_k: int = 10
 
176
  memory_db_path: str = "headroom_memory.db"
177
  memory_inject_tools: bool = True
178
  traffic_learning_enabled: bool = False
179
+ traffic_learning_agent_type: str = "unknown" # Which agent is being wrapped
180
  memory_use_native_tool: bool = False
181
  memory_inject_context: bool = True
182
  memory_top_k: int = 10
headroom/proxy/server.py CHANGED
@@ -447,11 +447,13 @@ class HeadroomProxy(
447
  # Traffic Learner (live pattern extraction from proxy traffic)
448
  # Only activates with --learn flag; requires --memory for backend
449
  self.traffic_learner: TrafficLearner | None = None
 
450
  if config.traffic_learning_enabled:
451
  from headroom.memory.traffic_learner import TrafficLearner
452
 
453
  self.traffic_learner = TrafficLearner(
454
  user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),
 
455
  )
456
 
457
  def _get_compression_cache(self, session_id: str) -> CompressionCache:
 
447
  # Traffic Learner (live pattern extraction from proxy traffic)
448
  # Only activates with --learn flag; requires --memory for backend
449
  self.traffic_learner: TrafficLearner | None = None
450
+ self.traffic_learning_agent_type: str = config.traffic_learning_agent_type
451
  if config.traffic_learning_enabled:
452
  from headroom.memory.traffic_learner import TrafficLearner
453
 
454
  self.traffic_learner = TrafficLearner(
455
  user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),
456
+ agent_type=config.traffic_learning_agent_type,
457
  )
458
 
459
  def _get_compression_cache(self, session_id: str) -> CompressionCache:
tests/test_learn/test_gemini_scanner.py ADDED
@@ -0,0 +1,531 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Unit tests for GeminiScanner — Google Gemini CLI session parsing.
2
+
3
+ Tests use synthetic session data in tmp directories, no real Gemini data needed.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ from pathlib import Path
10
+
11
+ from headroom.learn.models import ProjectInfo, Recommendation, RecommendationTarget
12
+ from headroom.learn.scanner import GeminiScanner
13
+ from headroom.learn.writer import GeminiWriter
14
+
15
+ # =============================================================================
16
+ # Helpers
17
+ # =============================================================================
18
+
19
+
20
+ def _make_gemini_session(
21
+ messages: list[dict],
22
+ session_id: str = "session-2026-04-09T10-00-abc123",
23
+ ) -> dict:
24
+ """Wrap messages in a Gemini session JSON structure."""
25
+ return {
26
+ "id": session_id,
27
+ "messages": messages,
28
+ }
29
+
30
+
31
+ def _write_json_session(chats_dir: Path, data: dict, name: str = "session-test.json") -> Path:
32
+ """Write a session JSON file to a chats directory."""
33
+ path = chats_dir / name
34
+ path.write_text(json.dumps(data))
35
+ return path
36
+
37
+
38
+ def _write_jsonl_session(
39
+ chats_dir: Path, records: list[dict], name: str = "session-test.jsonl"
40
+ ) -> Path:
41
+ """Write a session JSONL file to a chats directory."""
42
+ path = chats_dir / name
43
+ path.write_text("\n".join(json.dumps(r) for r in records))
44
+ return path
45
+
46
+
47
+ def _setup_gemini_dir(tmp_path: Path) -> tuple[Path, Path]:
48
+ """Create ~/.gemini/tmp/<project>/chats/ directory structure."""
49
+ gemini_dir = tmp_path / ".gemini"
50
+ project_dir = gemini_dir / "tmp" / "abc123"
51
+ chats_dir = project_dir / "chats"
52
+ chats_dir.mkdir(parents=True)
53
+ return gemini_dir, chats_dir
54
+
55
+
56
+ # =============================================================================
57
+ # Project Discovery
58
+ # =============================================================================
59
+
60
+
61
+ class TestProjectDiscovery:
62
+ def test_no_gemini_dir(self, tmp_path):
63
+ scanner = GeminiScanner(gemini_dir=tmp_path / ".gemini")
64
+ assert scanner.discover_projects() == []
65
+
66
+ def test_empty_tmp_dir(self, tmp_path):
67
+ (tmp_path / ".gemini" / "tmp").mkdir(parents=True)
68
+ scanner = GeminiScanner(gemini_dir=tmp_path / ".gemini")
69
+ assert scanner.discover_projects() == []
70
+
71
+ def test_no_session_files(self, tmp_path):
72
+ chats_dir = tmp_path / ".gemini" / "tmp" / "proj1" / "chats"
73
+ chats_dir.mkdir(parents=True)
74
+ scanner = GeminiScanner(gemini_dir=tmp_path / ".gemini")
75
+ assert scanner.discover_projects() == []
76
+
77
+ def test_discovers_project_with_sessions(self, tmp_path):
78
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
79
+ session = _make_gemini_session(
80
+ [
81
+ {"role": "user", "parts": [{"text": "hello"}]},
82
+ {
83
+ "role": "model",
84
+ "parts": [
85
+ {"functionCall": {"name": "read_file", "args": {"path": "/tmp/test.py"}}},
86
+ ],
87
+ },
88
+ {
89
+ "role": "user",
90
+ "parts": [
91
+ {
92
+ "functionResponse": {
93
+ "name": "read_file",
94
+ "response": {"output": "print('hello')"},
95
+ }
96
+ },
97
+ ],
98
+ },
99
+ ]
100
+ )
101
+ _write_json_session(chats_dir, session)
102
+
103
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
104
+ projects = scanner.discover_projects()
105
+ assert len(projects) == 1
106
+ assert projects[0].data_path == chats_dir
107
+
108
+
109
+ # =============================================================================
110
+ # JSON Session Parsing
111
+ # =============================================================================
112
+
113
+
114
+ class TestJsonSessionParsing:
115
+ def test_basic_tool_call(self, tmp_path):
116
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
117
+ session = _make_gemini_session(
118
+ [
119
+ {"role": "user", "parts": [{"text": "read the config file"}]},
120
+ {
121
+ "role": "model",
122
+ "parts": [
123
+ {
124
+ "functionCall": {
125
+ "name": "read_file",
126
+ "args": {"path": "/app/config.yaml"},
127
+ }
128
+ },
129
+ ],
130
+ },
131
+ {
132
+ "role": "user",
133
+ "parts": [
134
+ {
135
+ "functionResponse": {
136
+ "name": "read_file",
137
+ "response": {"output": "port: 8080\nhost: localhost"},
138
+ }
139
+ },
140
+ ],
141
+ },
142
+ {"role": "model", "parts": [{"text": "The config has port 8080."}]},
143
+ ]
144
+ )
145
+ _write_json_session(chats_dir, session)
146
+
147
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
148
+ projects = scanner.discover_projects()
149
+ sessions = scanner.scan_project(projects[0])
150
+
151
+ assert len(sessions) == 1
152
+ assert len(sessions[0].tool_calls) == 1
153
+ tc = sessions[0].tool_calls[0]
154
+ assert tc.name == "Read" # Normalized from read_file
155
+ assert tc.output == "port: 8080\nhost: localhost"
156
+ assert not tc.is_error
157
+
158
+ def test_multiple_tool_calls(self, tmp_path):
159
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
160
+ session = _make_gemini_session(
161
+ [
162
+ {"role": "user", "parts": [{"text": "find and read test files"}]},
163
+ {
164
+ "role": "model",
165
+ "parts": [
166
+ {
167
+ "functionCall": {
168
+ "name": "search_files",
169
+ "args": {"pattern": "test_*.py"},
170
+ }
171
+ },
172
+ ],
173
+ },
174
+ {
175
+ "role": "user",
176
+ "parts": [
177
+ {
178
+ "functionResponse": {
179
+ "name": "search_files",
180
+ "response": {"output": "tests/test_main.py\ntests/test_utils.py"},
181
+ }
182
+ },
183
+ ],
184
+ },
185
+ {
186
+ "role": "model",
187
+ "parts": [
188
+ {
189
+ "functionCall": {
190
+ "name": "read_file",
191
+ "args": {"path": "tests/test_main.py"},
192
+ }
193
+ },
194
+ ],
195
+ },
196
+ {
197
+ "role": "user",
198
+ "parts": [
199
+ {
200
+ "functionResponse": {
201
+ "name": "read_file",
202
+ "response": {"output": "def test_main(): pass"},
203
+ }
204
+ },
205
+ ],
206
+ },
207
+ ]
208
+ )
209
+ _write_json_session(chats_dir, session)
210
+
211
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
212
+ projects = scanner.discover_projects()
213
+ sessions = scanner.scan_project(projects[0])
214
+
215
+ assert len(sessions[0].tool_calls) == 2
216
+ assert sessions[0].tool_calls[0].name == "Glob" # search_files → Glob
217
+ assert sessions[0].tool_calls[1].name == "Read" # read_file → Read
218
+
219
+ def test_error_detection(self, tmp_path):
220
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
221
+ session = _make_gemini_session(
222
+ [
223
+ {
224
+ "role": "model",
225
+ "parts": [
226
+ {"functionCall": {"name": "read_file", "args": {"path": "/missing.txt"}}},
227
+ ],
228
+ },
229
+ {
230
+ "role": "user",
231
+ "parts": [
232
+ {
233
+ "functionResponse": {
234
+ "name": "read_file",
235
+ "response": {
236
+ "output": "FileNotFoundError: No such file or directory: '/missing.txt'"
237
+ },
238
+ }
239
+ },
240
+ ],
241
+ },
242
+ ]
243
+ )
244
+ _write_json_session(chats_dir, session)
245
+
246
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
247
+ projects = scanner.discover_projects()
248
+ sessions = scanner.scan_project(projects[0])
249
+
250
+ tc = sessions[0].tool_calls[0]
251
+ assert tc.is_error
252
+ assert tc.error_category.value == "file_not_found"
253
+
254
+ def test_shell_command_normalized(self, tmp_path):
255
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
256
+ session = _make_gemini_session(
257
+ [
258
+ {
259
+ "role": "model",
260
+ "parts": [
261
+ {
262
+ "functionCall": {
263
+ "name": "run_shell_command",
264
+ "args": {"command": "ls -la"},
265
+ }
266
+ },
267
+ ],
268
+ },
269
+ {
270
+ "role": "user",
271
+ "parts": [
272
+ {
273
+ "functionResponse": {
274
+ "name": "run_shell_command",
275
+ "response": {
276
+ "output": "total 0\ndrwxr-xr-x 2 user user 64 Apr 9 10:00 ."
277
+ },
278
+ }
279
+ },
280
+ ],
281
+ },
282
+ ]
283
+ )
284
+ _write_json_session(chats_dir, session)
285
+
286
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
287
+ projects = scanner.discover_projects()
288
+ sessions = scanner.scan_project(projects[0])
289
+
290
+ assert sessions[0].tool_calls[0].name == "Bash"
291
+
292
+ def test_user_messages_extracted(self, tmp_path):
293
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
294
+ session = _make_gemini_session(
295
+ [
296
+ {"role": "user", "parts": [{"text": "What files are in this project?"}]},
297
+ {
298
+ "role": "model",
299
+ "parts": [
300
+ {"functionCall": {"name": "search_files", "args": {"pattern": "*"}}},
301
+ ],
302
+ },
303
+ {
304
+ "role": "user",
305
+ "parts": [
306
+ {
307
+ "functionResponse": {
308
+ "name": "search_files",
309
+ "response": {"output": "main.py"},
310
+ }
311
+ },
312
+ ],
313
+ },
314
+ {"role": "user", "parts": [{"text": "Now run the tests"}]},
315
+ ]
316
+ )
317
+ _write_json_session(chats_dir, session)
318
+
319
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
320
+ projects = scanner.discover_projects()
321
+ sessions = scanner.scan_project(projects[0])
322
+
323
+ user_events = [e for e in sessions[0].events if e.type == "user_message"]
324
+ assert len(user_events) == 2
325
+ assert "What files" in user_events[0].text
326
+ assert "run the tests" in user_events[1].text
327
+
328
+ def test_array_format_messages(self, tmp_path):
329
+ """Sessions stored as bare array of messages (no wrapper object)."""
330
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
331
+ messages = [
332
+ {
333
+ "role": "model",
334
+ "parts": [
335
+ {
336
+ "functionCall": {
337
+ "name": "write_file",
338
+ "args": {"path": "test.py", "content": "pass"},
339
+ }
340
+ },
341
+ ],
342
+ },
343
+ {
344
+ "role": "user",
345
+ "parts": [
346
+ {
347
+ "functionResponse": {
348
+ "name": "write_file",
349
+ "response": {"output": "File written"},
350
+ }
351
+ },
352
+ ],
353
+ },
354
+ ]
355
+ _write_json_session(chats_dir, messages) # Write array directly
356
+
357
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
358
+ projects = scanner.discover_projects()
359
+ sessions = scanner.scan_project(projects[0])
360
+
361
+ assert len(sessions) == 1
362
+ assert sessions[0].tool_calls[0].name == "Write"
363
+
364
+ def test_no_tool_calls_returns_empty(self, tmp_path):
365
+ """Session with only text (no tool calls) produces no tool_calls."""
366
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
367
+ session = _make_gemini_session(
368
+ [
369
+ {"role": "user", "parts": [{"text": "What is Python?"}]},
370
+ {"role": "model", "parts": [{"text": "Python is a programming language."}]},
371
+ ]
372
+ )
373
+ _write_json_session(chats_dir, session)
374
+
375
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
376
+ projects = scanner.discover_projects()
377
+ sessions = scanner.scan_project(projects[0])
378
+
379
+ # No tool calls → session filtered out
380
+ assert len(sessions) == 0
381
+
382
+
383
+ # =============================================================================
384
+ # JSONL Session Parsing
385
+ # =============================================================================
386
+
387
+
388
+ class TestJsonlSessionParsing:
389
+ def test_basic_jsonl_session(self, tmp_path):
390
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
391
+ records = [
392
+ {"type": "session_metadata", "id": "ses-001"},
393
+ {"role": "user", "parts": [{"text": "list files"}]},
394
+ {
395
+ "role": "model",
396
+ "parts": [
397
+ {"functionCall": {"name": "run_shell_command", "args": {"command": "ls"}}},
398
+ ],
399
+ },
400
+ {
401
+ "role": "user",
402
+ "parts": [
403
+ {
404
+ "functionResponse": {
405
+ "name": "run_shell_command",
406
+ "response": {"output": "main.py\ntest.py"},
407
+ }
408
+ },
409
+ ],
410
+ },
411
+ ]
412
+ _write_jsonl_session(chats_dir, records)
413
+
414
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
415
+ projects = scanner.discover_projects()
416
+ sessions = scanner.scan_project(projects[0])
417
+
418
+ assert len(sessions) == 1
419
+ assert sessions[0].tool_calls[0].name == "Bash"
420
+ assert "main.py" in sessions[0].tool_calls[0].output
421
+
422
+ def test_jsonl_type_field_roles(self, tmp_path):
423
+ """JSONL records where role is in the 'type' field (user/gemini)."""
424
+ gemini_dir, chats_dir = _setup_gemini_dir(tmp_path)
425
+ records = [
426
+ {
427
+ "type": "gemini",
428
+ "parts": [
429
+ {"functionCall": {"name": "read_file", "args": {"path": "README.md"}}},
430
+ ],
431
+ },
432
+ {
433
+ "type": "user",
434
+ "parts": [
435
+ {"functionResponse": {"name": "read_file", "response": {"output": "# Hello"}}},
436
+ ],
437
+ },
438
+ ]
439
+ _write_jsonl_session(chats_dir, records)
440
+
441
+ scanner = GeminiScanner(gemini_dir=gemini_dir)
442
+ projects = scanner.discover_projects()
443
+ sessions = scanner.scan_project(projects[0])
444
+
445
+ assert len(sessions) == 1
446
+ assert sessions[0].tool_calls[0].name == "Read"
447
+
448
+
449
+ # =============================================================================
450
+ # Tool Name Normalization
451
+ # =============================================================================
452
+
453
+
454
+ class TestToolNameNormalization:
455
+ def test_all_known_names(self):
456
+ from headroom.learn._shared import normalize_tool_name
457
+
458
+ assert normalize_tool_name("run_shell_command") == "Bash"
459
+ assert normalize_tool_name("shell") == "Bash"
460
+ assert normalize_tool_name("execute_command") == "Bash"
461
+ assert normalize_tool_name("read_file") == "Read"
462
+ assert normalize_tool_name("read_many_files") == "Read"
463
+ assert normalize_tool_name("write_file") == "Write"
464
+ assert normalize_tool_name("write_new_file") == "Write"
465
+ assert normalize_tool_name("create_file") == "Write"
466
+ assert normalize_tool_name("edit_file") == "Edit"
467
+ assert normalize_tool_name("replace_in_file") == "Edit"
468
+ assert normalize_tool_name("search_files") == "Glob"
469
+ assert normalize_tool_name("find_files") == "Glob"
470
+ assert normalize_tool_name("grep") == "Grep"
471
+ assert normalize_tool_name("search_text") == "Grep"
472
+ assert normalize_tool_name("list_directory") == "Glob"
473
+
474
+ def test_unknown_name_preserved(self):
475
+ from headroom.learn._shared import normalize_tool_name
476
+
477
+ assert normalize_tool_name("custom_tool") == "custom_tool"
478
+
479
+
480
+ # =============================================================================
481
+ # Writer Integration
482
+ # =============================================================================
483
+
484
+
485
+ class TestGeminiWriter:
486
+ def test_writes_to_gemini_md(self, tmp_path):
487
+ proj = ProjectInfo(name="gemini-test", project_path=tmp_path, data_path=tmp_path)
488
+ recs = [
489
+ Recommendation(
490
+ target=RecommendationTarget.CONTEXT_FILE,
491
+ section="Commands",
492
+ content="- Use `python -m pytest`",
493
+ confidence=0.9,
494
+ evidence_count=5,
495
+ ),
496
+ ]
497
+
498
+ writer = GeminiWriter()
499
+ result = writer.write(recs, proj, dry_run=False)
500
+
501
+ assert len(result.files_written) == 1
502
+ assert result.files_written[0].name == "GEMINI.md"
503
+ content = (tmp_path / "GEMINI.md").read_text()
504
+ assert "python -m pytest" in content
505
+
506
+ def test_empty_recs_no_write(self, tmp_path):
507
+ proj = ProjectInfo(name="clean", project_path=tmp_path, data_path=tmp_path)
508
+ writer = GeminiWriter()
509
+ result = writer.write([], proj, dry_run=False)
510
+ assert result.files_written == []
511
+ assert not (tmp_path / "GEMINI.md").exists()
512
+
513
+ def test_dry_run(self, tmp_path):
514
+ proj = ProjectInfo(name="test", project_path=tmp_path, data_path=tmp_path)
515
+ recs = [
516
+ Recommendation(
517
+ target=RecommendationTarget.CONTEXT_FILE,
518
+ section="Test",
519
+ content="- test",
520
+ confidence=0.8,
521
+ evidence_count=3,
522
+ ),
523
+ ]
524
+
525
+ writer = GeminiWriter()
526
+ result = writer.write(recs, proj, dry_run=True)
527
+
528
+ assert result.dry_run is True
529
+ assert len(result.files_written) == 1
530
+ # Dry run should NOT create the file
531
+ assert not (tmp_path / "GEMINI.md").exists()
tests/test_learn/test_registry.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the learn plugin registry."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from unittest.mock import MagicMock, patch
6
+
7
+ import pytest
8
+
9
+ from headroom.learn.base import LearnPlugin
10
+ from headroom.learn.registry import (
11
+ auto_detect_plugins,
12
+ available_agent_names,
13
+ get_plugin,
14
+ get_registry,
15
+ reset_registry,
16
+ )
17
+ from headroom.learn.scanner import ConversationScanner
18
+
19
+
20
+ @pytest.fixture(autouse=True)
21
+ def _clean_registry():
22
+ """Reset registry before/after each test."""
23
+ reset_registry()
24
+ yield
25
+ reset_registry()
26
+
27
+
28
+ class TestBuiltinDiscovery:
29
+ def test_discovers_three_builtin_plugins(self):
30
+ reg = get_registry()
31
+ assert "claude" in reg
32
+ assert "codex" in reg
33
+ assert "gemini" in reg
34
+ assert len(reg) >= 3
35
+
36
+ def test_all_plugins_are_learn_plugins(self):
37
+ for name, plugin in get_registry().items():
38
+ assert isinstance(plugin, LearnPlugin), f"{name} is not a LearnPlugin"
39
+
40
+ def test_all_plugins_are_conversation_scanners(self):
41
+ """Backwards compat: plugins must also be ConversationScanners."""
42
+ for name, plugin in get_registry().items():
43
+ assert isinstance(plugin, ConversationScanner), f"{name} is not a ConversationScanner"
44
+
45
+ def test_plugins_have_identity(self):
46
+ for name, plugin in get_registry().items():
47
+ assert plugin.name == name
48
+ assert plugin.display_name # non-empty
49
+ assert plugin.description # non-empty
50
+
51
+
52
+ class TestGetPlugin:
53
+ def test_get_existing_plugin(self):
54
+ plugin = get_plugin("claude")
55
+ assert plugin.name == "claude"
56
+ assert plugin.display_name == "Claude Code"
57
+
58
+ def test_get_unknown_raises_keyerror(self):
59
+ with pytest.raises(KeyError, match="Unknown agent.*cursor"):
60
+ get_plugin("cursor")
61
+
62
+ def test_error_message_lists_available(self):
63
+ with pytest.raises(KeyError, match="claude"):
64
+ get_plugin("nonexistent")
65
+
66
+
67
+ class TestAutoDetect:
68
+ def test_filters_to_detected_only(self):
69
+ """Only plugins where detect() returns True are included."""
70
+ detected = auto_detect_plugins()
71
+ for plugin in detected:
72
+ assert plugin.detect()
73
+
74
+ def test_returns_empty_when_nothing_detected(self):
75
+ """All plugins returning False → empty list."""
76
+ with (
77
+ patch.object(get_registry()["claude"], "detect", return_value=False),
78
+ patch.object(get_registry()["codex"], "detect", return_value=False),
79
+ patch.object(get_registry()["gemini"], "detect", return_value=False),
80
+ ):
81
+ assert auto_detect_plugins() == []
82
+
83
+
84
+ class TestAvailableNames:
85
+ def test_returns_sorted_list(self):
86
+ names = available_agent_names()
87
+ assert names == sorted(names)
88
+ assert "claude" in names
89
+ assert "codex" in names
90
+ assert "gemini" in names
91
+
92
+
93
+ class TestExternalPlugin:
94
+ def test_external_plugin_via_entry_point(self):
95
+ """Mock an external plugin registered via entry_points."""
96
+ mock_plugin = MagicMock(spec=LearnPlugin)
97
+ mock_plugin.name = "cursor"
98
+ mock_plugin.display_name = "Cursor"
99
+ mock_plugin.description = "Cursor IDE (~/.cursor/)"
100
+
101
+ mock_ep = MagicMock()
102
+ mock_ep.load.return_value = mock_plugin
103
+ mock_ep.name = "cursor"
104
+
105
+ with patch("importlib.metadata.entry_points", return_value=[mock_ep]):
106
+ reset_registry()
107
+ reg = get_registry()
108
+ assert "cursor" in reg
109
+ assert reg["cursor"].name == "cursor"
110
+
111
+
112
+ class TestResetRegistry:
113
+ def test_reset_clears_cache(self):
114
+ reg1 = get_registry()
115
+ assert reg1 is get_registry() # Same object (cached)
116
+ reset_registry()
117
+ reg2 = get_registry()
118
+ assert reg2 is not reg1 # New object (cache cleared)
119
+
120
+
121
+ class TestPluginCreateWriter:
122
+ def test_all_plugins_create_valid_writers(self):
123
+ from headroom.learn.writer import ContextWriter
124
+
125
+ for name, plugin in get_registry().items():
126
+ writer = plugin.create_writer()
127
+ assert isinstance(writer, ContextWriter), f"{name} writer is not a ContextWriter"