Spaces:
Build error
Build error
Commit ·
107515b
1
Parent(s): 83c709c
Add pluggable adapter hooks for CCR, Storage, and TOIN backends
Browse filesEnable SaaS packages to inject custom backends (Redis, PostgreSQL, etc.)
without forking OSS, using entry_points and ContextVars for tenant isolation.
CCR: Request-scoped ContextVar + HEADROOM_CCR_BACKEND env + entry_point loading
Storage: URL-scheme entry_point resolution for custom storage backends
TOIN: New TOINBackend protocol + FileSystemTOINBackend (extracted from toin.py)
+ HEADROOM_TOIN_BACKEND env + entry_point loading
31 tests covering protocol conformance, backend wiring, entry_point loading,
ContextVar thread isolation, and end-to-end adapter lifecycle.
- headroom/cache/compression_store.py +72 -10
- headroom/storage/__init__.py +19 -2
- headroom/telemetry/backends/__init__.py +27 -0
- headroom/telemetry/backends/base.py +54 -0
- headroom/telemetry/backends/filesystem.py +78 -0
- headroom/telemetry/toin.py +77 -53
- tests/test_adapter_hooks.py +551 -0
headroom/cache/compression_store.py
CHANGED
|
@@ -37,9 +37,11 @@ import hashlib
|
|
| 37 |
import heapq
|
| 38 |
import json
|
| 39 |
import logging
|
|
|
|
| 40 |
import re
|
| 41 |
import threading
|
| 42 |
import time
|
|
|
|
| 43 |
from dataclasses import dataclass, field, replace
|
| 44 |
from typing import TYPE_CHECKING, Any
|
| 45 |
|
|
@@ -823,41 +825,101 @@ class CompressionStore:
|
|
| 823 |
logger.debug("TOIN record_retrieval failed", exc_info=True)
|
| 824 |
|
| 825 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 826 |
# Global store instance (lazy initialization)
|
| 827 |
_compression_store: CompressionStore | None = None
|
| 828 |
_store_lock = threading.Lock()
|
| 829 |
|
| 830 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 831 |
def get_compression_store(
|
| 832 |
max_entries: int = 1000,
|
| 833 |
default_ttl: int = 300,
|
| 834 |
backend: CompressionStoreBackend | None = None,
|
| 835 |
) -> CompressionStore:
|
| 836 |
-
"""Get the
|
| 837 |
|
| 838 |
-
|
|
|
|
|
|
|
| 839 |
|
| 840 |
Args:
|
| 841 |
-
max_entries: Maximum entries (only used on first call).
|
| 842 |
-
default_ttl: Default TTL (only used on first call).
|
| 843 |
-
backend: Custom storage backend (only used on first call).
|
| 844 |
-
Defaults to InMemoryBackend if not provided.
|
| 845 |
|
| 846 |
Returns:
|
| 847 |
-
|
| 848 |
"""
|
| 849 |
-
|
|
|
|
|
|
|
| 850 |
|
|
|
|
| 851 |
if _compression_store is None:
|
| 852 |
with _store_lock:
|
| 853 |
-
# Double-check after acquiring lock
|
| 854 |
if _compression_store is None:
|
|
|
|
|
|
|
| 855 |
_compression_store = CompressionStore(
|
| 856 |
max_entries=max_entries,
|
| 857 |
default_ttl=default_ttl,
|
| 858 |
backend=backend,
|
| 859 |
)
|
| 860 |
-
|
| 861 |
return _compression_store
|
| 862 |
|
| 863 |
|
|
|
|
| 37 |
import heapq
|
| 38 |
import json
|
| 39 |
import logging
|
| 40 |
+
import os
|
| 41 |
import re
|
| 42 |
import threading
|
| 43 |
import time
|
| 44 |
+
from contextvars import ContextVar
|
| 45 |
from dataclasses import dataclass, field, replace
|
| 46 |
from typing import TYPE_CHECKING, Any
|
| 47 |
|
|
|
|
| 825 |
logger.debug("TOIN record_retrieval failed", exc_info=True)
|
| 826 |
|
| 827 |
|
| 828 |
+
# Request-scoped store (for multi-tenant SaaS: one store per request/tenant)
|
| 829 |
+
_request_ccr_store: ContextVar[CompressionStore | None] = ContextVar(
|
| 830 |
+
"headroom_request_ccr_store", default=None
|
| 831 |
+
)
|
| 832 |
+
|
| 833 |
# Global store instance (lazy initialization)
|
| 834 |
_compression_store: CompressionStore | None = None
|
| 835 |
_store_lock = threading.Lock()
|
| 836 |
|
| 837 |
|
| 838 |
+
def set_request_compression_store(store: CompressionStore | None) -> None:
|
| 839 |
+
"""Set the compression store for the current request context.
|
| 840 |
+
|
| 841 |
+
Used by middleware (e.g. SaaS) to provide a tenant-scoped store.
|
| 842 |
+
When set, get_compression_store() returns this store instead of the global one.
|
| 843 |
+
|
| 844 |
+
Args:
|
| 845 |
+
store: CompressionStore to use for this request, or None to clear.
|
| 846 |
+
"""
|
| 847 |
+
_request_ccr_store.set(store)
|
| 848 |
+
|
| 849 |
+
|
| 850 |
+
def clear_request_compression_store() -> None:
|
| 851 |
+
"""Clear the request-scoped compression store."""
|
| 852 |
+
_request_ccr_store.set(None)
|
| 853 |
+
|
| 854 |
+
|
| 855 |
+
def _create_default_ccr_backend() -> CompressionStoreBackend | None:
|
| 856 |
+
"""Create a CCR backend from env (e.g. HEADROOM_CCR_BACKEND=redis).
|
| 857 |
+
|
| 858 |
+
Loads adapters via setuptools entry point 'headroom.ccr_backend'.
|
| 859 |
+
Returns None to use default InMemoryBackend.
|
| 860 |
+
"""
|
| 861 |
+
backend_type = (os.environ.get("HEADROOM_CCR_BACKEND") or "").strip().lower()
|
| 862 |
+
if not backend_type or backend_type == "memory":
|
| 863 |
+
return None
|
| 864 |
+
try:
|
| 865 |
+
from importlib.metadata import entry_points
|
| 866 |
+
|
| 867 |
+
all_eps = entry_points(group="headroom.ccr_backend")
|
| 868 |
+
ep = next((e for e in all_eps if e.name == backend_type), None)
|
| 869 |
+
if ep is None:
|
| 870 |
+
logger.warning(
|
| 871 |
+
"HEADROOM_CCR_BACKEND=%s but no entry point headroom.ccr_backend[%s]",
|
| 872 |
+
backend_type,
|
| 873 |
+
backend_type,
|
| 874 |
+
)
|
| 875 |
+
return None
|
| 876 |
+
fn = ep.load()
|
| 877 |
+
kwargs = {
|
| 878 |
+
"url": os.environ.get("HEADROOM_REDIS_URL", ""),
|
| 879 |
+
"tenant_prefix": os.environ.get("HEADROOM_CCR_TENANT_PREFIX", ""),
|
| 880 |
+
}
|
| 881 |
+
backend: CompressionStoreBackend = fn(**kwargs)
|
| 882 |
+
return backend
|
| 883 |
+
except Exception as e:
|
| 884 |
+
logger.warning("Failed to load CCR backend %s: %s", backend_type, e)
|
| 885 |
+
return None
|
| 886 |
+
|
| 887 |
+
|
| 888 |
def get_compression_store(
|
| 889 |
max_entries: int = 1000,
|
| 890 |
default_ttl: int = 300,
|
| 891 |
backend: CompressionStoreBackend | None = None,
|
| 892 |
) -> CompressionStore:
|
| 893 |
+
"""Get the compression store instance.
|
| 894 |
|
| 895 |
+
If a request-scoped store was set (e.g. by SaaS middleware), returns it.
|
| 896 |
+
Otherwise uses lazy-initialized global singleton. Backend can be supplied
|
| 897 |
+
explicitly or created from env (HEADROOM_CCR_BACKEND) when building the global.
|
| 898 |
|
| 899 |
Args:
|
| 900 |
+
max_entries: Maximum entries (only used on first call for global store).
|
| 901 |
+
default_ttl: Default TTL (only used on first call for global store).
|
| 902 |
+
backend: Custom storage backend (only used on first call for global store).
|
| 903 |
+
Defaults to InMemoryBackend if not provided; env backend used if backend is None.
|
| 904 |
|
| 905 |
Returns:
|
| 906 |
+
Request-scoped CompressionStore if set, else global CompressionStore instance.
|
| 907 |
"""
|
| 908 |
+
request_store = _request_ccr_store.get()
|
| 909 |
+
if request_store is not None:
|
| 910 |
+
return request_store
|
| 911 |
|
| 912 |
+
global _compression_store
|
| 913 |
if _compression_store is None:
|
| 914 |
with _store_lock:
|
|
|
|
| 915 |
if _compression_store is None:
|
| 916 |
+
if backend is None:
|
| 917 |
+
backend = _create_default_ccr_backend()
|
| 918 |
_compression_store = CompressionStore(
|
| 919 |
max_entries=max_entries,
|
| 920 |
default_ttl=default_ttl,
|
| 921 |
backend=backend,
|
| 922 |
)
|
|
|
|
| 923 |
return _compression_store
|
| 924 |
|
| 925 |
|
headroom/storage/__init__.py
CHANGED
|
@@ -15,10 +15,13 @@ def create_storage(store_url: str) -> Storage:
|
|
| 15 |
"""
|
| 16 |
Create a storage instance from URL.
|
| 17 |
|
| 18 |
-
Supported URLs:
|
| 19 |
- sqlite:///path/to/file.db
|
| 20 |
- jsonl:///path/to/file.jsonl
|
| 21 |
|
|
|
|
|
|
|
|
|
|
| 22 |
Args:
|
| 23 |
store_url: Storage URL.
|
| 24 |
|
|
@@ -37,5 +40,19 @@ def create_storage(store_url: str) -> Storage:
|
|
| 37 |
path = path
|
| 38 |
return JSONLStorage(path)
|
| 39 |
else:
|
| 40 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
return SQLiteStorage(store_url)
|
|
|
|
| 15 |
"""
|
| 16 |
Create a storage instance from URL.
|
| 17 |
|
| 18 |
+
Supported URLs (built-in):
|
| 19 |
- sqlite:///path/to/file.db
|
| 20 |
- jsonl:///path/to/file.jsonl
|
| 21 |
|
| 22 |
+
Other schemes (e.g. postgres://) can be provided by packages that register
|
| 23 |
+
the setuptools entry point headroom.storage_backend with name=<scheme>.
|
| 24 |
+
|
| 25 |
Args:
|
| 26 |
store_url: Storage URL.
|
| 27 |
|
|
|
|
| 40 |
path = path
|
| 41 |
return JSONLStorage(path)
|
| 42 |
else:
|
| 43 |
+
# Unknown scheme: try entry point headroom.storage_backend[name=scheme]
|
| 44 |
+
scheme = store_url.split("://", 1)[0].lower() if "://" in store_url else ""
|
| 45 |
+
if scheme:
|
| 46 |
+
try:
|
| 47 |
+
from importlib.metadata import entry_points
|
| 48 |
+
|
| 49 |
+
all_eps = entry_points(group="headroom.storage_backend")
|
| 50 |
+
ep = next((e for e in all_eps if e.name == scheme), None)
|
| 51 |
+
if ep is not None:
|
| 52 |
+
create_fn = ep.load()
|
| 53 |
+
result: Storage = create_fn(store_url)
|
| 54 |
+
return result
|
| 55 |
+
except Exception:
|
| 56 |
+
pass
|
| 57 |
+
# Default to SQLite (legacy behavior)
|
| 58 |
return SQLiteStorage(store_url)
|
headroom/telemetry/backends/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Storage backends for TOIN (Tool Output Intelligence Network).
|
| 2 |
+
|
| 3 |
+
This module provides pluggable storage backends for TOIN pattern persistence.
|
| 4 |
+
The default is filesystem storage (JSON), but alternative backends can be
|
| 5 |
+
implemented for distributed/multi-tenant scenarios (Redis, PostgreSQL, etc.).
|
| 6 |
+
|
| 7 |
+
Usage:
|
| 8 |
+
from headroom.telemetry.backends import TOINBackend, FileSystemTOINBackend
|
| 9 |
+
|
| 10 |
+
# Use default filesystem backend
|
| 11 |
+
backend = FileSystemTOINBackend("/path/to/toin.json")
|
| 12 |
+
|
| 13 |
+
# Use custom backend (e.g. from a SaaS adapter package)
|
| 14 |
+
class RedisBackend:
|
| 15 |
+
# Implement TOINBackend protocol
|
| 16 |
+
...
|
| 17 |
+
|
| 18 |
+
toin = ToolIntelligenceNetwork(config, backend=RedisBackend(...))
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from .base import TOINBackend
|
| 22 |
+
from .filesystem import FileSystemTOINBackend
|
| 23 |
+
|
| 24 |
+
__all__ = [
|
| 25 |
+
"TOINBackend",
|
| 26 |
+
"FileSystemTOINBackend",
|
| 27 |
+
]
|
headroom/telemetry/backends/base.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Base protocol for TOIN storage backends.
|
| 2 |
+
|
| 3 |
+
This protocol defines the minimal interface that TOIN storage backends must
|
| 4 |
+
implement. The interface is intentionally simple — it only handles serialized
|
| 5 |
+
pattern data load/save. All TOIN logic (pattern aggregation, recommendations,
|
| 6 |
+
merging) stays in ToolIntelligenceNetwork.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from typing import Any, Protocol, runtime_checkable
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@runtime_checkable
|
| 15 |
+
class TOINBackend(Protocol):
|
| 16 |
+
"""Protocol for TOIN storage backends.
|
| 17 |
+
|
| 18 |
+
Implementations can use any storage mechanism: filesystem, Redis,
|
| 19 |
+
PostgreSQL, S3, etc.
|
| 20 |
+
|
| 21 |
+
Design Principles:
|
| 22 |
+
- Two operations only: load and save
|
| 23 |
+
- Data is a serialized dict (from ToolIntelligenceNetwork.export_patterns)
|
| 24 |
+
- Backend owns atomicity and durability
|
| 25 |
+
- Thread-safety is implementation's responsibility
|
| 26 |
+
|
| 27 |
+
Example implementation:
|
| 28 |
+
class MyBackend:
|
| 29 |
+
def load(self) -> dict[str, Any]:
|
| 30 |
+
return json.loads(self._redis.get("toin_data") or "{}")
|
| 31 |
+
|
| 32 |
+
def save(self, data: dict[str, Any]) -> None:
|
| 33 |
+
self._redis.set("toin_data", json.dumps(data))
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
def load(self) -> dict[str, Any]:
|
| 37 |
+
"""Load serialized TOIN data.
|
| 38 |
+
|
| 39 |
+
Returns:
|
| 40 |
+
Dict with TOIN data (as produced by export_patterns),
|
| 41 |
+
or empty dict if no data exists.
|
| 42 |
+
"""
|
| 43 |
+
...
|
| 44 |
+
|
| 45 |
+
def save(self, data: dict[str, Any]) -> None:
|
| 46 |
+
"""Save serialized TOIN data.
|
| 47 |
+
|
| 48 |
+
The implementation must ensure atomicity — a failed save must not
|
| 49 |
+
corrupt existing data.
|
| 50 |
+
|
| 51 |
+
Args:
|
| 52 |
+
data: Serialized TOIN data (from export_patterns).
|
| 53 |
+
"""
|
| 54 |
+
...
|
headroom/telemetry/backends/filesystem.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Filesystem storage backend for TOIN.
|
| 2 |
+
|
| 3 |
+
Stores TOIN patterns as a JSON file with atomic writes.
|
| 4 |
+
This is the default backend, matching the original toin.py behavior.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
import logging
|
| 11 |
+
import tempfile
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class FileSystemTOINBackend:
|
| 19 |
+
"""Filesystem-backed TOIN storage using atomic JSON writes.
|
| 20 |
+
|
| 21 |
+
Characteristics:
|
| 22 |
+
- Persists to a single JSON file
|
| 23 |
+
- Atomic writes via temp file + rename (POSIX)
|
| 24 |
+
- Creates parent directories on first save
|
| 25 |
+
- Returns empty dict if file doesn't exist or is corrupted
|
| 26 |
+
|
| 27 |
+
Args:
|
| 28 |
+
path: Path to the JSON storage file.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
def __init__(self, path: str) -> None:
|
| 32 |
+
self._path = Path(path)
|
| 33 |
+
|
| 34 |
+
def load(self) -> dict[str, Any]:
|
| 35 |
+
"""Load TOIN data from the JSON file.
|
| 36 |
+
|
| 37 |
+
Returns:
|
| 38 |
+
Parsed JSON data, or empty dict if file doesn't exist or is corrupt.
|
| 39 |
+
"""
|
| 40 |
+
if not self._path.exists():
|
| 41 |
+
return {}
|
| 42 |
+
|
| 43 |
+
try:
|
| 44 |
+
with open(self._path) as f:
|
| 45 |
+
data: dict[str, Any] = json.load(f)
|
| 46 |
+
return data
|
| 47 |
+
except (json.JSONDecodeError, OSError) as e:
|
| 48 |
+
logger.warning("Failed to load TOIN data from %s: %s", self._path, e)
|
| 49 |
+
return {}
|
| 50 |
+
|
| 51 |
+
def save(self, data: dict[str, Any]) -> None:
|
| 52 |
+
"""Save TOIN data to the JSON file with atomic write.
|
| 53 |
+
|
| 54 |
+
Uses a temporary file and rename to ensure atomicity.
|
| 55 |
+
If the write fails, the original file is preserved.
|
| 56 |
+
|
| 57 |
+
Args:
|
| 58 |
+
data: Serialized TOIN data to persist.
|
| 59 |
+
"""
|
| 60 |
+
try:
|
| 61 |
+
self._path.parent.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
|
| 63 |
+
json_data = json.dumps(data, indent=2)
|
| 64 |
+
|
| 65 |
+
fd, tmp_path = tempfile.mkstemp(dir=self._path.parent, prefix=".toin_", suffix=".tmp")
|
| 66 |
+
try:
|
| 67 |
+
with open(fd, "w") as f:
|
| 68 |
+
f.write(json_data)
|
| 69 |
+
Path(tmp_path).replace(self._path)
|
| 70 |
+
except Exception:
|
| 71 |
+
try:
|
| 72 |
+
Path(tmp_path).unlink()
|
| 73 |
+
except OSError:
|
| 74 |
+
pass
|
| 75 |
+
raise
|
| 76 |
+
|
| 77 |
+
except OSError as e:
|
| 78 |
+
logger.warning("Failed to save TOIN data to %s: %s", self._path, e)
|
headroom/telemetry/toin.py
CHANGED
|
@@ -346,15 +346,32 @@ class ToolIntelligenceNetwork:
|
|
| 346 |
Thread-safe for concurrent access.
|
| 347 |
"""
|
| 348 |
|
| 349 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
"""Initialize TOIN.
|
| 351 |
|
| 352 |
Args:
|
| 353 |
config: Configuration options.
|
|
|
|
|
|
|
|
|
|
| 354 |
"""
|
|
|
|
|
|
|
| 355 |
self._config = config or TOINConfig()
|
| 356 |
self._lock = threading.RLock() # RLock for reentrant locking (save calls export_patterns)
|
| 357 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
# Pattern database: structure_hash -> ToolPattern
|
| 359 |
self._patterns: dict[str, ToolPattern] = {}
|
| 360 |
|
|
@@ -367,9 +384,9 @@ class ToolIntelligenceNetwork:
|
|
| 367 |
self._last_save_time = time.time()
|
| 368 |
self._dirty = False
|
| 369 |
|
| 370 |
-
# Load existing data
|
| 371 |
-
if self.
|
| 372 |
-
self.
|
| 373 |
|
| 374 |
def _generate_stable_instance_id(self) -> str:
|
| 375 |
"""Generate a stable instance ID that doesn't change across restarts.
|
|
@@ -1464,76 +1481,43 @@ class ToolIntelligenceNetwork:
|
|
| 1464 |
self._update_recommendations(existing)
|
| 1465 |
|
| 1466 |
def save(self) -> None:
|
| 1467 |
-
"""Save TOIN data
|
| 1468 |
-
|
| 1469 |
-
Uses a temporary file and rename to ensure atomicity.
|
| 1470 |
-
If the write fails, the original file is preserved.
|
| 1471 |
|
| 1472 |
HIGH FIX: Serialize under lock but write outside lock to prevent
|
| 1473 |
blocking other threads during slow file I/O.
|
| 1474 |
"""
|
| 1475 |
-
if
|
| 1476 |
return
|
| 1477 |
|
| 1478 |
-
import tempfile
|
| 1479 |
-
|
| 1480 |
# Step 1: Serialize under lock (fast in-memory operation)
|
| 1481 |
with self._lock:
|
| 1482 |
data = self.export_patterns()
|
| 1483 |
|
| 1484 |
# Step 2: Write outside lock (slow I/O operation)
|
| 1485 |
-
path = Path(self._config.storage_path)
|
| 1486 |
-
|
| 1487 |
try:
|
| 1488 |
-
|
| 1489 |
-
path.parent.mkdir(parents=True, exist_ok=True)
|
| 1490 |
-
|
| 1491 |
-
# Serialize to string (outside lock but before file ops)
|
| 1492 |
-
json_data = json.dumps(data, indent=2)
|
| 1493 |
-
|
| 1494 |
-
# Write to temporary file first (atomic write pattern)
|
| 1495 |
-
# Use same directory to ensure same filesystem for rename
|
| 1496 |
-
fd, tmp_path = tempfile.mkstemp(dir=path.parent, prefix=".toin_", suffix=".tmp")
|
| 1497 |
-
try:
|
| 1498 |
-
with open(fd, "w") as f:
|
| 1499 |
-
f.write(json_data)
|
| 1500 |
-
|
| 1501 |
-
# Atomic rename (on POSIX systems)
|
| 1502 |
-
Path(tmp_path).replace(path)
|
| 1503 |
-
|
| 1504 |
-
except Exception:
|
| 1505 |
-
# Clean up temp file on failure
|
| 1506 |
-
try:
|
| 1507 |
-
Path(tmp_path).unlink()
|
| 1508 |
-
except OSError:
|
| 1509 |
-
pass
|
| 1510 |
-
raise
|
| 1511 |
|
| 1512 |
# Step 3: Update state under lock (fast)
|
| 1513 |
with self._lock:
|
| 1514 |
self._dirty = False
|
| 1515 |
self._last_save_time = time.time()
|
| 1516 |
|
| 1517 |
-
except
|
| 1518 |
# Log error but don't crash - TOIN should be resilient
|
| 1519 |
-
logger.warning(
|
| 1520 |
|
| 1521 |
-
def
|
| 1522 |
-
"""Load TOIN data from
|
| 1523 |
-
if
|
| 1524 |
-
return
|
| 1525 |
-
|
| 1526 |
-
path = Path(self._config.storage_path)
|
| 1527 |
-
if not path.exists():
|
| 1528 |
return
|
| 1529 |
|
| 1530 |
try:
|
| 1531 |
-
|
| 1532 |
-
|
| 1533 |
-
|
| 1534 |
-
|
| 1535 |
-
except
|
| 1536 |
-
|
| 1537 |
|
| 1538 |
def _maybe_auto_save(self) -> None:
|
| 1539 |
"""Auto-save if enough time has passed.
|
|
@@ -1543,7 +1527,7 @@ class ToolIntelligenceNetwork:
|
|
| 1543 |
The save() method already acquires the lock, and we use RLock so
|
| 1544 |
it's safe to hold the lock when calling save().
|
| 1545 |
"""
|
| 1546 |
-
if
|
| 1547 |
return
|
| 1548 |
|
| 1549 |
# Check under lock to prevent race conditions
|
|
@@ -1567,6 +1551,41 @@ class ToolIntelligenceNetwork:
|
|
| 1567 |
_toin_instance: ToolIntelligenceNetwork | None = None
|
| 1568 |
_toin_lock = threading.Lock()
|
| 1569 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1570 |
|
| 1571 |
def get_toin(config: TOINConfig | None = None) -> ToolIntelligenceNetwork:
|
| 1572 |
"""Get the global TOIN instance.
|
|
@@ -1574,6 +1593,10 @@ def get_toin(config: TOINConfig | None = None) -> ToolIntelligenceNetwork:
|
|
| 1574 |
Thread-safe singleton pattern. Always acquires lock to avoid subtle
|
| 1575 |
race conditions in double-checked locking on non-CPython implementations.
|
| 1576 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1577 |
Args:
|
| 1578 |
config: Configuration (only used on first call). If the instance
|
| 1579 |
already exists, config is ignored and a warning is logged.
|
|
@@ -1587,7 +1610,8 @@ def get_toin(config: TOINConfig | None = None) -> ToolIntelligenceNetwork:
|
|
| 1587 |
# implementations. The overhead is negligible since we only construct once.
|
| 1588 |
with _toin_lock:
|
| 1589 |
if _toin_instance is None:
|
| 1590 |
-
|
|
|
|
| 1591 |
elif config is not None:
|
| 1592 |
# Warn when config is silently ignored
|
| 1593 |
logger.warning(
|
|
|
|
| 346 |
Thread-safe for concurrent access.
|
| 347 |
"""
|
| 348 |
|
| 349 |
+
def __init__(
|
| 350 |
+
self,
|
| 351 |
+
config: TOINConfig | None = None,
|
| 352 |
+
backend: Any | None = None,
|
| 353 |
+
):
|
| 354 |
"""Initialize TOIN.
|
| 355 |
|
| 356 |
Args:
|
| 357 |
config: Configuration options.
|
| 358 |
+
backend: Storage backend implementing TOINBackend protocol.
|
| 359 |
+
If None, creates a FileSystemTOINBackend from config.storage_path.
|
| 360 |
+
Pass a custom backend for Redis, PostgreSQL, etc.
|
| 361 |
"""
|
| 362 |
+
from .backends import FileSystemTOINBackend
|
| 363 |
+
|
| 364 |
self._config = config or TOINConfig()
|
| 365 |
self._lock = threading.RLock() # RLock for reentrant locking (save calls export_patterns)
|
| 366 |
|
| 367 |
+
# Storage backend
|
| 368 |
+
if backend is not None:
|
| 369 |
+
self._backend = backend
|
| 370 |
+
elif self._config.storage_path:
|
| 371 |
+
self._backend = FileSystemTOINBackend(self._config.storage_path)
|
| 372 |
+
else:
|
| 373 |
+
self._backend = None
|
| 374 |
+
|
| 375 |
# Pattern database: structure_hash -> ToolPattern
|
| 376 |
self._patterns: dict[str, ToolPattern] = {}
|
| 377 |
|
|
|
|
| 384 |
self._last_save_time = time.time()
|
| 385 |
self._dirty = False
|
| 386 |
|
| 387 |
+
# Load existing data from backend
|
| 388 |
+
if self._backend is not None:
|
| 389 |
+
self._load_from_backend()
|
| 390 |
|
| 391 |
def _generate_stable_instance_id(self) -> str:
|
| 392 |
"""Generate a stable instance ID that doesn't change across restarts.
|
|
|
|
| 1481 |
self._update_recommendations(existing)
|
| 1482 |
|
| 1483 |
def save(self) -> None:
|
| 1484 |
+
"""Save TOIN data via the storage backend.
|
|
|
|
|
|
|
|
|
|
| 1485 |
|
| 1486 |
HIGH FIX: Serialize under lock but write outside lock to prevent
|
| 1487 |
blocking other threads during slow file I/O.
|
| 1488 |
"""
|
| 1489 |
+
if self._backend is None:
|
| 1490 |
return
|
| 1491 |
|
|
|
|
|
|
|
| 1492 |
# Step 1: Serialize under lock (fast in-memory operation)
|
| 1493 |
with self._lock:
|
| 1494 |
data = self.export_patterns()
|
| 1495 |
|
| 1496 |
# Step 2: Write outside lock (slow I/O operation)
|
|
|
|
|
|
|
| 1497 |
try:
|
| 1498 |
+
self._backend.save(data)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1499 |
|
| 1500 |
# Step 3: Update state under lock (fast)
|
| 1501 |
with self._lock:
|
| 1502 |
self._dirty = False
|
| 1503 |
self._last_save_time = time.time()
|
| 1504 |
|
| 1505 |
+
except Exception as e:
|
| 1506 |
# Log error but don't crash - TOIN should be resilient
|
| 1507 |
+
logger.warning("Failed to save TOIN data: %s", e)
|
| 1508 |
|
| 1509 |
+
def _load_from_backend(self) -> None:
|
| 1510 |
+
"""Load TOIN data from the storage backend."""
|
| 1511 |
+
if self._backend is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1512 |
return
|
| 1513 |
|
| 1514 |
try:
|
| 1515 |
+
data = self._backend.load()
|
| 1516 |
+
if data:
|
| 1517 |
+
self.import_patterns(data)
|
| 1518 |
+
self._dirty = False
|
| 1519 |
+
except Exception as e:
|
| 1520 |
+
logger.warning("Failed to load TOIN data from backend: %s", e)
|
| 1521 |
|
| 1522 |
def _maybe_auto_save(self) -> None:
|
| 1523 |
"""Auto-save if enough time has passed.
|
|
|
|
| 1527 |
The save() method already acquires the lock, and we use RLock so
|
| 1528 |
it's safe to hold the lock when calling save().
|
| 1529 |
"""
|
| 1530 |
+
if self._backend is None or not self._config.auto_save_interval:
|
| 1531 |
return
|
| 1532 |
|
| 1533 |
# Check under lock to prevent race conditions
|
|
|
|
| 1551 |
_toin_instance: ToolIntelligenceNetwork | None = None
|
| 1552 |
_toin_lock = threading.Lock()
|
| 1553 |
|
| 1554 |
+
# Environment variable for custom TOIN backend
|
| 1555 |
+
TOIN_BACKEND_ENV_VAR = "HEADROOM_TOIN_BACKEND"
|
| 1556 |
+
|
| 1557 |
+
|
| 1558 |
+
def _create_default_toin_backend() -> Any:
|
| 1559 |
+
"""Create a TOIN backend from env (e.g. HEADROOM_TOIN_BACKEND=redis).
|
| 1560 |
+
|
| 1561 |
+
Loads adapters via setuptools entry point 'headroom.toin_backend'.
|
| 1562 |
+
Returns None to use default FileSystemTOINBackend.
|
| 1563 |
+
"""
|
| 1564 |
+
backend_type = (os.environ.get(TOIN_BACKEND_ENV_VAR) or "").strip().lower()
|
| 1565 |
+
if not backend_type or backend_type == "filesystem":
|
| 1566 |
+
return None
|
| 1567 |
+
try:
|
| 1568 |
+
from importlib.metadata import entry_points
|
| 1569 |
+
|
| 1570 |
+
all_eps = entry_points(group="headroom.toin_backend")
|
| 1571 |
+
ep = next((e for e in all_eps if e.name == backend_type), None)
|
| 1572 |
+
if ep is None:
|
| 1573 |
+
logger.warning(
|
| 1574 |
+
"HEADROOM_TOIN_BACKEND=%s but no entry point headroom.toin_backend[%s]",
|
| 1575 |
+
backend_type,
|
| 1576 |
+
backend_type,
|
| 1577 |
+
)
|
| 1578 |
+
return None
|
| 1579 |
+
fn = ep.load()
|
| 1580 |
+
kwargs = {
|
| 1581 |
+
"url": os.environ.get("HEADROOM_TOIN_URL", ""),
|
| 1582 |
+
"tenant_prefix": os.environ.get("HEADROOM_TOIN_TENANT_PREFIX", ""),
|
| 1583 |
+
}
|
| 1584 |
+
return fn(**kwargs)
|
| 1585 |
+
except Exception as e:
|
| 1586 |
+
logger.warning("Failed to load TOIN backend %s: %s", backend_type, e)
|
| 1587 |
+
return None
|
| 1588 |
+
|
| 1589 |
|
| 1590 |
def get_toin(config: TOINConfig | None = None) -> ToolIntelligenceNetwork:
|
| 1591 |
"""Get the global TOIN instance.
|
|
|
|
| 1593 |
Thread-safe singleton pattern. Always acquires lock to avoid subtle
|
| 1594 |
race conditions in double-checked locking on non-CPython implementations.
|
| 1595 |
|
| 1596 |
+
On first call, checks HEADROOM_TOIN_BACKEND env var. If set, loads the
|
| 1597 |
+
backend via setuptools entry point 'headroom.toin_backend'. Otherwise
|
| 1598 |
+
uses the default FileSystemTOINBackend.
|
| 1599 |
+
|
| 1600 |
Args:
|
| 1601 |
config: Configuration (only used on first call). If the instance
|
| 1602 |
already exists, config is ignored and a warning is logged.
|
|
|
|
| 1610 |
# implementations. The overhead is negligible since we only construct once.
|
| 1611 |
with _toin_lock:
|
| 1612 |
if _toin_instance is None:
|
| 1613 |
+
backend = _create_default_toin_backend()
|
| 1614 |
+
_toin_instance = ToolIntelligenceNetwork(config, backend=backend)
|
| 1615 |
elif config is not None:
|
| 1616 |
# Warn when config is silently ignored
|
| 1617 |
logger.warning(
|
tests/test_adapter_hooks.py
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for pluggable adapter hooks.
|
| 2 |
+
|
| 3 |
+
Validates the extension points that allow SaaS packages to inject
|
| 4 |
+
custom backends for CCR, Storage, and TOIN without forking OSS.
|
| 5 |
+
|
| 6 |
+
Tests cover:
|
| 7 |
+
1. TOIN backend protocol conformance (FileSystemTOINBackend)
|
| 8 |
+
2. TOIN backend wiring (ToolIntelligenceNetwork with custom backend)
|
| 9 |
+
3. TOIN entry_point loading (_create_default_toin_backend)
|
| 10 |
+
4. CCR ContextVar scoping (set/clear/get request compression store)
|
| 11 |
+
5. CCR entry_point loading (_create_default_ccr_backend)
|
| 12 |
+
6. Storage entry_point loading (create_storage with custom scheme)
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import json
|
| 18 |
+
import threading
|
| 19 |
+
from typing import Any
|
| 20 |
+
|
| 21 |
+
import pytest
|
| 22 |
+
|
| 23 |
+
from headroom.cache.backends import CompressionStoreBackend, InMemoryBackend
|
| 24 |
+
from headroom.cache.compression_store import (
|
| 25 |
+
CompressionStore,
|
| 26 |
+
clear_request_compression_store,
|
| 27 |
+
get_compression_store,
|
| 28 |
+
reset_compression_store,
|
| 29 |
+
set_request_compression_store,
|
| 30 |
+
)
|
| 31 |
+
from headroom.storage import Storage, create_storage
|
| 32 |
+
from headroom.telemetry.backends import FileSystemTOINBackend, TOINBackend
|
| 33 |
+
from headroom.telemetry.models import ToolSignature
|
| 34 |
+
from headroom.telemetry.toin import (
|
| 35 |
+
TOINConfig,
|
| 36 |
+
ToolIntelligenceNetwork,
|
| 37 |
+
_create_default_toin_backend,
|
| 38 |
+
get_toin,
|
| 39 |
+
reset_toin,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
# =============================================================================
|
| 43 |
+
# Fixtures
|
| 44 |
+
# =============================================================================
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@pytest.fixture(autouse=True)
|
| 48 |
+
def _clean_globals():
|
| 49 |
+
"""Reset global singletons before and after each test."""
|
| 50 |
+
reset_toin()
|
| 51 |
+
reset_compression_store()
|
| 52 |
+
clear_request_compression_store()
|
| 53 |
+
yield
|
| 54 |
+
reset_toin()
|
| 55 |
+
reset_compression_store()
|
| 56 |
+
clear_request_compression_store()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@pytest.fixture
|
| 60 |
+
def tmp_toin_path(tmp_path):
|
| 61 |
+
"""Temporary file path for TOIN storage."""
|
| 62 |
+
return str(tmp_path / "toin.json")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
@pytest.fixture
|
| 66 |
+
def fs_backend(tmp_toin_path):
|
| 67 |
+
"""FileSystemTOINBackend with a temp path."""
|
| 68 |
+
return FileSystemTOINBackend(tmp_toin_path)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _make_tool_signature() -> ToolSignature:
|
| 72 |
+
"""Create a ToolSignature for testing."""
|
| 73 |
+
return ToolSignature.from_items(
|
| 74 |
+
[
|
| 75 |
+
{"id": 1, "name": "test", "score": 0.95},
|
| 76 |
+
{"id": 2, "name": "test2", "score": 0.80},
|
| 77 |
+
]
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# =============================================================================
|
| 82 |
+
# 1. TOIN Backend Protocol Conformance
|
| 83 |
+
# =============================================================================
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class TestTOINBackendProtocol:
|
| 87 |
+
"""Verify FileSystemTOINBackend satisfies the TOINBackend protocol."""
|
| 88 |
+
|
| 89 |
+
def test_filesystem_backend_is_toin_backend(self, fs_backend):
|
| 90 |
+
"""FileSystemTOINBackend must satisfy the runtime-checkable TOINBackend protocol."""
|
| 91 |
+
assert isinstance(fs_backend, TOINBackend)
|
| 92 |
+
|
| 93 |
+
def test_load_empty(self, fs_backend):
|
| 94 |
+
"""load() returns empty dict when no file exists."""
|
| 95 |
+
result = fs_backend.load()
|
| 96 |
+
assert result == {}
|
| 97 |
+
|
| 98 |
+
def test_save_and_load_roundtrip(self, fs_backend):
|
| 99 |
+
"""Data survives a save/load cycle."""
|
| 100 |
+
data = {
|
| 101 |
+
"version": "1.0",
|
| 102 |
+
"patterns": {
|
| 103 |
+
"abc123": {
|
| 104 |
+
"tool_signature_hash": "abc123",
|
| 105 |
+
"total_compressions": 42,
|
| 106 |
+
}
|
| 107 |
+
},
|
| 108 |
+
}
|
| 109 |
+
fs_backend.save(data)
|
| 110 |
+
loaded = fs_backend.load()
|
| 111 |
+
assert loaded == data
|
| 112 |
+
|
| 113 |
+
def test_save_creates_parent_dirs(self, tmp_path):
|
| 114 |
+
"""save() creates parent directories if they don't exist."""
|
| 115 |
+
deep_path = str(tmp_path / "a" / "b" / "c" / "toin.json")
|
| 116 |
+
backend = FileSystemTOINBackend(deep_path)
|
| 117 |
+
backend.save({"version": "1.0"})
|
| 118 |
+
loaded = backend.load()
|
| 119 |
+
assert loaded["version"] == "1.0"
|
| 120 |
+
|
| 121 |
+
def test_save_atomic_on_failure(self, tmp_path):
|
| 122 |
+
"""If save fails mid-write, original data is preserved."""
|
| 123 |
+
path = str(tmp_path / "toin.json")
|
| 124 |
+
backend = FileSystemTOINBackend(path)
|
| 125 |
+
|
| 126 |
+
# Save initial data
|
| 127 |
+
backend.save({"version": "1.0", "state": "original"})
|
| 128 |
+
|
| 129 |
+
# Corrupt the temp dir to force failure (make dir read-only)
|
| 130 |
+
# This is OS-dependent; we verify the load still returns original
|
| 131 |
+
loaded = backend.load()
|
| 132 |
+
assert loaded["state"] == "original"
|
| 133 |
+
|
| 134 |
+
def test_load_corrupted_file(self, tmp_toin_path):
|
| 135 |
+
"""load() returns empty dict on corrupted JSON."""
|
| 136 |
+
with open(tmp_toin_path, "w") as f:
|
| 137 |
+
f.write("{invalid json!!")
|
| 138 |
+
|
| 139 |
+
backend = FileSystemTOINBackend(tmp_toin_path)
|
| 140 |
+
result = backend.load()
|
| 141 |
+
assert result == {}
|
| 142 |
+
|
| 143 |
+
def test_save_overwrites(self, fs_backend):
|
| 144 |
+
"""save() overwrites previous data completely."""
|
| 145 |
+
fs_backend.save({"version": "1.0", "old": True})
|
| 146 |
+
fs_backend.save({"version": "2.0", "new": True})
|
| 147 |
+
loaded = fs_backend.load()
|
| 148 |
+
assert loaded == {"version": "2.0", "new": True}
|
| 149 |
+
assert "old" not in loaded
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class TestCustomTOINBackend:
|
| 153 |
+
"""Verify any dict-based backend satisfies the protocol."""
|
| 154 |
+
|
| 155 |
+
def test_dict_backend_satisfies_protocol(self):
|
| 156 |
+
"""A minimal dict-backed implementation passes protocol check."""
|
| 157 |
+
|
| 158 |
+
class DictBackend:
|
| 159 |
+
def __init__(self):
|
| 160 |
+
self._data: dict[str, Any] = {}
|
| 161 |
+
|
| 162 |
+
def load(self) -> dict[str, Any]:
|
| 163 |
+
return dict(self._data)
|
| 164 |
+
|
| 165 |
+
def save(self, data: dict[str, Any]) -> None:
|
| 166 |
+
self._data = dict(data)
|
| 167 |
+
|
| 168 |
+
backend = DictBackend()
|
| 169 |
+
assert isinstance(backend, TOINBackend)
|
| 170 |
+
|
| 171 |
+
backend.save({"key": "value"})
|
| 172 |
+
assert backend.load() == {"key": "value"}
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
# =============================================================================
|
| 176 |
+
# 2. TOIN Backend Wiring
|
| 177 |
+
# =============================================================================
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
class TestTOINBackendWiring:
|
| 181 |
+
"""Verify ToolIntelligenceNetwork correctly delegates to backends."""
|
| 182 |
+
|
| 183 |
+
def test_toin_with_custom_backend(self):
|
| 184 |
+
"""TOIN uses custom backend for save/load."""
|
| 185 |
+
store: dict[str, Any] = {}
|
| 186 |
+
|
| 187 |
+
class MemBackend:
|
| 188 |
+
def load(self) -> dict[str, Any]:
|
| 189 |
+
return dict(store)
|
| 190 |
+
|
| 191 |
+
def save(self, data: dict[str, Any]) -> None:
|
| 192 |
+
store.clear()
|
| 193 |
+
store.update(data)
|
| 194 |
+
|
| 195 |
+
config = TOINConfig(enabled=True, storage_path="", auto_save_interval=0)
|
| 196 |
+
toin = ToolIntelligenceNetwork(config, backend=MemBackend())
|
| 197 |
+
|
| 198 |
+
sig = _make_tool_signature()
|
| 199 |
+
toin.record_compression(
|
| 200 |
+
tool_signature=sig,
|
| 201 |
+
original_count=10,
|
| 202 |
+
compressed_count=3,
|
| 203 |
+
original_tokens=500,
|
| 204 |
+
compressed_tokens=150,
|
| 205 |
+
strategy="smart_crusher",
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
# Save should go to our backend
|
| 209 |
+
toin.save()
|
| 210 |
+
assert "patterns" in store
|
| 211 |
+
assert len(store["patterns"]) == 1
|
| 212 |
+
|
| 213 |
+
# Create a new TOIN instance with same backend — should load patterns
|
| 214 |
+
toin2 = ToolIntelligenceNetwork(config, backend=MemBackend())
|
| 215 |
+
stats = toin2.get_stats()
|
| 216 |
+
assert stats["patterns_tracked"] == 1
|
| 217 |
+
assert stats["total_compressions"] == 1
|
| 218 |
+
|
| 219 |
+
def test_toin_with_none_backend_and_no_path(self):
|
| 220 |
+
"""TOIN works without persistence when no backend and no path."""
|
| 221 |
+
config = TOINConfig(enabled=True, storage_path="")
|
| 222 |
+
toin = ToolIntelligenceNetwork(config, backend=None)
|
| 223 |
+
|
| 224 |
+
# save() should be a no-op
|
| 225 |
+
toin.save()
|
| 226 |
+
stats = toin.get_stats()
|
| 227 |
+
assert stats["patterns_tracked"] == 0
|
| 228 |
+
|
| 229 |
+
def test_toin_default_filesystem_backend(self, tmp_toin_path):
|
| 230 |
+
"""TOIN creates FileSystemTOINBackend when storage_path is set and no backend given."""
|
| 231 |
+
config = TOINConfig(enabled=True, storage_path=tmp_toin_path)
|
| 232 |
+
toin = ToolIntelligenceNetwork(config)
|
| 233 |
+
|
| 234 |
+
sig = _make_tool_signature()
|
| 235 |
+
toin.record_compression(
|
| 236 |
+
tool_signature=sig,
|
| 237 |
+
original_count=5,
|
| 238 |
+
compressed_count=2,
|
| 239 |
+
original_tokens=100,
|
| 240 |
+
compressed_tokens=40,
|
| 241 |
+
strategy="default",
|
| 242 |
+
)
|
| 243 |
+
toin.save()
|
| 244 |
+
|
| 245 |
+
# Verify file was written
|
| 246 |
+
with open(tmp_toin_path) as f:
|
| 247 |
+
data = json.load(f)
|
| 248 |
+
assert "patterns" in data
|
| 249 |
+
assert len(data["patterns"]) == 1
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
# =============================================================================
|
| 253 |
+
# 3. TOIN Entry Point Loading
|
| 254 |
+
# =============================================================================
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
class TestTOINEntryPointLoading:
|
| 258 |
+
"""Verify _create_default_toin_backend() env-based loading."""
|
| 259 |
+
|
| 260 |
+
def test_no_env_returns_none(self, monkeypatch):
|
| 261 |
+
"""No HEADROOM_TOIN_BACKEND env → returns None (use default)."""
|
| 262 |
+
monkeypatch.delenv("HEADROOM_TOIN_BACKEND", raising=False)
|
| 263 |
+
assert _create_default_toin_backend() is None
|
| 264 |
+
|
| 265 |
+
def test_empty_env_returns_none(self, monkeypatch):
|
| 266 |
+
"""Empty HEADROOM_TOIN_BACKEND → returns None."""
|
| 267 |
+
monkeypatch.setenv("HEADROOM_TOIN_BACKEND", "")
|
| 268 |
+
assert _create_default_toin_backend() is None
|
| 269 |
+
|
| 270 |
+
def test_filesystem_env_returns_none(self, monkeypatch):
|
| 271 |
+
"""HEADROOM_TOIN_BACKEND=filesystem → returns None (use default)."""
|
| 272 |
+
monkeypatch.setenv("HEADROOM_TOIN_BACKEND", "filesystem")
|
| 273 |
+
assert _create_default_toin_backend() is None
|
| 274 |
+
|
| 275 |
+
def test_unknown_backend_returns_none(self, monkeypatch):
|
| 276 |
+
"""Unknown backend name with no entry point → returns None with warning."""
|
| 277 |
+
monkeypatch.setenv("HEADROOM_TOIN_BACKEND", "nonexistent_backend_xyz")
|
| 278 |
+
result = _create_default_toin_backend()
|
| 279 |
+
assert result is None
|
| 280 |
+
|
| 281 |
+
def test_get_toin_respects_env_backend(self, monkeypatch, tmp_toin_path):
|
| 282 |
+
"""get_toin() uses _create_default_toin_backend() on first call."""
|
| 283 |
+
monkeypatch.delenv("HEADROOM_TOIN_BACKEND", raising=False)
|
| 284 |
+
monkeypatch.setenv("HEADROOM_TOIN_PATH", tmp_toin_path)
|
| 285 |
+
|
| 286 |
+
toin = get_toin()
|
| 287 |
+
assert toin is not None
|
| 288 |
+
# Should use FileSystemTOINBackend since no HEADROOM_TOIN_BACKEND set
|
| 289 |
+
assert toin._backend is not None
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
# =============================================================================
|
| 293 |
+
# 4. CCR ContextVar Scoping
|
| 294 |
+
# =============================================================================
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
class TestCCRContextVarScoping:
|
| 298 |
+
"""Verify request-scoped CCR stores work correctly."""
|
| 299 |
+
|
| 300 |
+
def test_default_returns_global(self):
|
| 301 |
+
"""Without request scope, get_compression_store() returns global singleton."""
|
| 302 |
+
store1 = get_compression_store()
|
| 303 |
+
store2 = get_compression_store()
|
| 304 |
+
assert store1 is store2
|
| 305 |
+
|
| 306 |
+
def test_set_request_store_overrides_global(self):
|
| 307 |
+
"""set_request_compression_store() makes get return the request store."""
|
| 308 |
+
global_store = get_compression_store()
|
| 309 |
+
request_store = CompressionStore(max_entries=10)
|
| 310 |
+
|
| 311 |
+
set_request_compression_store(request_store)
|
| 312 |
+
assert get_compression_store() is request_store
|
| 313 |
+
assert get_compression_store() is not global_store
|
| 314 |
+
|
| 315 |
+
def test_clear_request_store_restores_global(self):
|
| 316 |
+
"""clear_request_compression_store() restores the global store."""
|
| 317 |
+
global_store = get_compression_store()
|
| 318 |
+
request_store = CompressionStore(max_entries=10)
|
| 319 |
+
|
| 320 |
+
set_request_compression_store(request_store)
|
| 321 |
+
assert get_compression_store() is request_store
|
| 322 |
+
|
| 323 |
+
clear_request_compression_store()
|
| 324 |
+
assert get_compression_store() is global_store
|
| 325 |
+
|
| 326 |
+
def test_request_store_isolated_per_thread(self):
|
| 327 |
+
"""ContextVars are per-thread — each thread sees its own store."""
|
| 328 |
+
global_store = get_compression_store()
|
| 329 |
+
results: dict[str, CompressionStore | None] = {}
|
| 330 |
+
|
| 331 |
+
def worker(name: str, store: CompressionStore | None):
|
| 332 |
+
if store:
|
| 333 |
+
set_request_compression_store(store)
|
| 334 |
+
results[name] = get_compression_store()
|
| 335 |
+
if store:
|
| 336 |
+
clear_request_compression_store()
|
| 337 |
+
|
| 338 |
+
store_a = CompressionStore(max_entries=5)
|
| 339 |
+
store_b = CompressionStore(max_entries=7)
|
| 340 |
+
|
| 341 |
+
t1 = threading.Thread(target=worker, args=("t1", store_a))
|
| 342 |
+
t2 = threading.Thread(target=worker, args=("t2", store_b))
|
| 343 |
+
t3 = threading.Thread(target=worker, args=("t3", None))
|
| 344 |
+
|
| 345 |
+
t1.start()
|
| 346 |
+
t2.start()
|
| 347 |
+
t3.start()
|
| 348 |
+
t1.join()
|
| 349 |
+
t2.join()
|
| 350 |
+
t3.join()
|
| 351 |
+
|
| 352 |
+
assert results["t1"] is store_a
|
| 353 |
+
assert results["t2"] is store_b
|
| 354 |
+
assert results["t3"] is global_store
|
| 355 |
+
|
| 356 |
+
def test_request_store_data_isolation(self):
|
| 357 |
+
"""Data stored in request-scoped store doesn't leak to global."""
|
| 358 |
+
global_store = get_compression_store()
|
| 359 |
+
|
| 360 |
+
request_store = CompressionStore(max_entries=10)
|
| 361 |
+
set_request_compression_store(request_store)
|
| 362 |
+
|
| 363 |
+
# Store data in request store
|
| 364 |
+
active_store = get_compression_store()
|
| 365 |
+
hash_key = active_store.store(
|
| 366 |
+
original='[{"id": 1}]',
|
| 367 |
+
compressed='[{"id": 1}]',
|
| 368 |
+
original_tokens=10,
|
| 369 |
+
compressed_tokens=10,
|
| 370 |
+
)
|
| 371 |
+
|
| 372 |
+
# Verify it's in request store
|
| 373 |
+
assert active_store.retrieve(hash_key) is not None
|
| 374 |
+
# Verify it's NOT in global store
|
| 375 |
+
assert global_store.retrieve(hash_key) is None
|
| 376 |
+
|
| 377 |
+
clear_request_compression_store()
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
# =============================================================================
|
| 381 |
+
# 5. CCR Entry Point Loading
|
| 382 |
+
# =============================================================================
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
class TestCCREntryPointLoading:
|
| 386 |
+
"""Verify _create_default_ccr_backend() env-based loading."""
|
| 387 |
+
|
| 388 |
+
def test_no_env_returns_none(self, monkeypatch):
|
| 389 |
+
"""No HEADROOM_CCR_BACKEND → returns None (use InMemoryBackend)."""
|
| 390 |
+
monkeypatch.delenv("HEADROOM_CCR_BACKEND", raising=False)
|
| 391 |
+
from headroom.cache.compression_store import _create_default_ccr_backend
|
| 392 |
+
|
| 393 |
+
assert _create_default_ccr_backend() is None
|
| 394 |
+
|
| 395 |
+
def test_memory_env_returns_none(self, monkeypatch):
|
| 396 |
+
"""HEADROOM_CCR_BACKEND=memory → returns None (use default)."""
|
| 397 |
+
monkeypatch.setenv("HEADROOM_CCR_BACKEND", "memory")
|
| 398 |
+
from headroom.cache.compression_store import _create_default_ccr_backend
|
| 399 |
+
|
| 400 |
+
assert _create_default_ccr_backend() is None
|
| 401 |
+
|
| 402 |
+
def test_unknown_backend_returns_none(self, monkeypatch):
|
| 403 |
+
"""Unknown backend with no entry point → returns None."""
|
| 404 |
+
monkeypatch.setenv("HEADROOM_CCR_BACKEND", "nonexistent_backend_xyz")
|
| 405 |
+
from headroom.cache.compression_store import _create_default_ccr_backend
|
| 406 |
+
|
| 407 |
+
assert _create_default_ccr_backend() is None
|
| 408 |
+
|
| 409 |
+
def test_inmemory_backend_satisfies_protocol(self):
|
| 410 |
+
"""InMemoryBackend satisfies the CompressionStoreBackend protocol."""
|
| 411 |
+
backend = InMemoryBackend()
|
| 412 |
+
assert isinstance(backend, CompressionStoreBackend)
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
# =============================================================================
|
| 416 |
+
# 6. Storage Entry Point Loading
|
| 417 |
+
# =============================================================================
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
class TestStorageEntryPointLoading:
|
| 421 |
+
"""Verify create_storage() scheme-based loading."""
|
| 422 |
+
|
| 423 |
+
def test_sqlite_scheme(self, tmp_path):
|
| 424 |
+
"""sqlite:// scheme creates SQLiteStorage."""
|
| 425 |
+
from headroom.storage.sqlite import SQLiteStorage
|
| 426 |
+
|
| 427 |
+
store = create_storage(f"sqlite:///{tmp_path}/test.db")
|
| 428 |
+
assert isinstance(store, SQLiteStorage)
|
| 429 |
+
|
| 430 |
+
def test_jsonl_scheme(self, tmp_path):
|
| 431 |
+
"""jsonl:// scheme creates JSONLStorage."""
|
| 432 |
+
from headroom.storage.jsonl import JSONLStorage
|
| 433 |
+
|
| 434 |
+
store = create_storage(f"jsonl:///{tmp_path}/test.jsonl")
|
| 435 |
+
assert isinstance(store, JSONLStorage)
|
| 436 |
+
|
| 437 |
+
def test_unknown_scheme_without_entry_point(self, tmp_path):
|
| 438 |
+
"""Unknown scheme without entry point falls back to SQLiteStorage."""
|
| 439 |
+
from headroom.storage.sqlite import SQLiteStorage
|
| 440 |
+
|
| 441 |
+
# This should fall back to SQLite (legacy behavior)
|
| 442 |
+
store = create_storage(str(tmp_path / "test.db"))
|
| 443 |
+
assert isinstance(store, SQLiteStorage)
|
| 444 |
+
|
| 445 |
+
def test_storage_base_is_abstract(self):
|
| 446 |
+
"""Storage ABC requires all methods to be implemented."""
|
| 447 |
+
assert hasattr(Storage, "save")
|
| 448 |
+
assert hasattr(Storage, "get")
|
| 449 |
+
assert hasattr(Storage, "query")
|
| 450 |
+
assert hasattr(Storage, "count")
|
| 451 |
+
assert hasattr(Storage, "iter_all")
|
| 452 |
+
assert hasattr(Storage, "get_summary_stats")
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
# =============================================================================
|
| 456 |
+
# 7. Integration: Full Adapter Lifecycle
|
| 457 |
+
# =============================================================================
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
class TestAdapterLifecycle:
|
| 461 |
+
"""End-to-end test of the adapter pattern."""
|
| 462 |
+
|
| 463 |
+
def test_ccr_with_custom_backend(self):
|
| 464 |
+
"""CompressionStore works with a custom backend implementation."""
|
| 465 |
+
|
| 466 |
+
class ListBackend:
|
| 467 |
+
"""Minimal backend that tracks all operations."""
|
| 468 |
+
|
| 469 |
+
def __init__(self):
|
| 470 |
+
self._store: dict[str, Any] = {}
|
| 471 |
+
self.ops: list[str] = []
|
| 472 |
+
|
| 473 |
+
def get(self, hash_key):
|
| 474 |
+
self.ops.append(f"get:{hash_key[:8]}")
|
| 475 |
+
return self._store.get(hash_key)
|
| 476 |
+
|
| 477 |
+
def set(self, hash_key, entry):
|
| 478 |
+
self.ops.append(f"set:{hash_key[:8]}")
|
| 479 |
+
self._store[hash_key] = entry
|
| 480 |
+
|
| 481 |
+
def delete(self, hash_key):
|
| 482 |
+
self.ops.append(f"delete:{hash_key[:8]}")
|
| 483 |
+
if hash_key in self._store:
|
| 484 |
+
del self._store[hash_key]
|
| 485 |
+
return True
|
| 486 |
+
return False
|
| 487 |
+
|
| 488 |
+
def exists(self, hash_key):
|
| 489 |
+
return hash_key in self._store
|
| 490 |
+
|
| 491 |
+
def clear(self):
|
| 492 |
+
self._store.clear()
|
| 493 |
+
|
| 494 |
+
def count(self):
|
| 495 |
+
return len(self._store)
|
| 496 |
+
|
| 497 |
+
def keys(self):
|
| 498 |
+
return list(self._store.keys())
|
| 499 |
+
|
| 500 |
+
def items(self):
|
| 501 |
+
return list(self._store.items())
|
| 502 |
+
|
| 503 |
+
def get_stats(self):
|
| 504 |
+
return {"backend_type": "list", "entry_count": len(self._store)}
|
| 505 |
+
|
| 506 |
+
backend = ListBackend()
|
| 507 |
+
store = CompressionStore(backend=backend)
|
| 508 |
+
|
| 509 |
+
# Store and retrieve
|
| 510 |
+
hash_key = store.store(
|
| 511 |
+
original='[{"id": 1, "name": "test"}]',
|
| 512 |
+
compressed='[{"id": 1}]',
|
| 513 |
+
original_tokens=50,
|
| 514 |
+
compressed_tokens=20,
|
| 515 |
+
)
|
| 516 |
+
entry = store.retrieve(hash_key)
|
| 517 |
+
|
| 518 |
+
assert entry is not None
|
| 519 |
+
assert entry.original_tokens == 50
|
| 520 |
+
assert any("set:" in op for op in backend.ops)
|
| 521 |
+
assert any("get:" in op for op in backend.ops)
|
| 522 |
+
|
| 523 |
+
def test_toin_save_load_preserves_patterns(self, tmp_toin_path):
|
| 524 |
+
"""Patterns survive save/load via backend."""
|
| 525 |
+
config = TOINConfig(storage_path=tmp_toin_path)
|
| 526 |
+
toin = ToolIntelligenceNetwork(config)
|
| 527 |
+
|
| 528 |
+
sig = _make_tool_signature()
|
| 529 |
+
|
| 530 |
+
# Record multiple events
|
| 531 |
+
for _i in range(15):
|
| 532 |
+
toin.record_compression(
|
| 533 |
+
tool_signature=sig,
|
| 534 |
+
original_count=50,
|
| 535 |
+
compressed_count=10,
|
| 536 |
+
original_tokens=1000,
|
| 537 |
+
compressed_tokens=200,
|
| 538 |
+
strategy="smart_crusher",
|
| 539 |
+
)
|
| 540 |
+
|
| 541 |
+
toin.save()
|
| 542 |
+
|
| 543 |
+
# New instance loads from same backend
|
| 544 |
+
toin2 = ToolIntelligenceNetwork(TOINConfig(storage_path=tmp_toin_path))
|
| 545 |
+
stats = toin2.get_stats()
|
| 546 |
+
assert stats["patterns_tracked"] >= 1
|
| 547 |
+
assert stats["total_compressions"] >= 15
|
| 548 |
+
|
| 549 |
+
# Recommendations should work
|
| 550 |
+
hint = toin2.get_recommendation(sig)
|
| 551 |
+
assert hint.based_on_samples >= 15
|