Spaces:
Build error
Build error
Commit ·
a71a45d
1
Parent(s): 91b077d
Fix multi-worker beacon spam, cost dashboard, upsert telemetry; bump 0.5.17
Browse files- Fix beacon spam: file lock ensures only one beacon per proxy regardless
of worker count. Workers > 1 caused N beacons firing N rows per cycle.
- Beacon upsert: on_conflict=session_id prevents duplicate rows.
- Beacon stop() guard: skip final report if uptime < 2 minutes.
- Fix dashboard cost: savings_usd now uses model list price (monotonic),
not moving average. Separate breakdown for compression/cache/rtk.
Fixes #83
- headroom/__init__.py +1 -1
- headroom/proxy/server.py +63 -3
- headroom/telemetry/beacon.py +8 -5
- pyproject.toml +1 -1
headroom/__init__.py
CHANGED
|
@@ -153,7 +153,7 @@ from .transforms import (
|
|
| 153 |
TransformPipeline,
|
| 154 |
)
|
| 155 |
|
| 156 |
-
__version__ = "0.5.
|
| 157 |
|
| 158 |
__all__ = [
|
| 159 |
# Main client
|
|
|
|
| 153 |
TransformPipeline,
|
| 154 |
)
|
| 155 |
|
| 156 |
+
__version__ = "0.5.17"
|
| 157 |
|
| 158 |
__all__ = [
|
| 159 |
# Main client
|
headroom/proxy/server.py
CHANGED
|
@@ -7309,7 +7309,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
|
|
| 7309 |
|
| 7310 |
proxy = HeadroomProxy(config)
|
| 7311 |
|
| 7312 |
-
# Telemetry beacon (anonymous aggregate stats)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7313 |
from headroom.telemetry.beacon import TelemetryBeacon
|
| 7314 |
|
| 7315 |
_beacon = TelemetryBeacon(
|
|
@@ -7317,6 +7324,51 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
|
|
| 7317 |
sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
|
| 7318 |
backend=config.backend if hasattr(config, "backend") else "anthropic",
|
| 7319 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7320 |
|
| 7321 |
@asynccontextmanager
|
| 7322 |
async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
|
|
@@ -7327,12 +7379,20 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
|
|
| 7327 |
await proxy.usage_reporter.start(proxy)
|
| 7328 |
if proxy.traffic_learner:
|
| 7329 |
await proxy.traffic_learner.start()
|
| 7330 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7331 |
|
| 7332 |
yield
|
| 7333 |
|
| 7334 |
# Shutdown
|
| 7335 |
-
|
|
|
|
|
|
|
| 7336 |
if proxy.usage_reporter:
|
| 7337 |
await proxy.usage_reporter.stop()
|
| 7338 |
if proxy.traffic_learner:
|
|
|
|
| 7309 |
|
| 7310 |
proxy = HeadroomProxy(config)
|
| 7311 |
|
| 7312 |
+
# Telemetry beacon (anonymous aggregate stats).
|
| 7313 |
+
# With uvicorn workers > 1, each worker runs the lifespan independently.
|
| 7314 |
+
# We must ensure only ONE beacon runs across all workers — otherwise each
|
| 7315 |
+
# worker creates its own beacon, spamming the telemetry table with N rows
|
| 7316 |
+
# per cycle instead of 1 (all reading the same /stats from the same port).
|
| 7317 |
+
#
|
| 7318 |
+
# Strategy: use a file lock to ensure only the first worker starts the
|
| 7319 |
+
# beacon. Other workers see the lock and skip.
|
| 7320 |
from headroom.telemetry.beacon import TelemetryBeacon
|
| 7321 |
|
| 7322 |
_beacon = TelemetryBeacon(
|
|
|
|
| 7324 |
sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
|
| 7325 |
backend=config.backend if hasattr(config, "backend") else "anthropic",
|
| 7326 |
)
|
| 7327 |
+
_beacon_lock_path = Path.home() / ".headroom" / f".beacon_lock_{config.port}"
|
| 7328 |
+
_beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor
|
| 7329 |
+
_beacon_is_owner: list = [False]
|
| 7330 |
+
|
| 7331 |
+
def _try_acquire_beacon_lock() -> bool:
|
| 7332 |
+
"""Try to acquire the beacon file lock (non-blocking).
|
| 7333 |
+
|
| 7334 |
+
Returns True if this process is the beacon owner.
|
| 7335 |
+
"""
|
| 7336 |
+
try:
|
| 7337 |
+
_beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
| 7338 |
+
import fcntl
|
| 7339 |
+
|
| 7340 |
+
fd = open(_beacon_lock_path, "w") # noqa: SIM115
|
| 7341 |
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
| 7342 |
+
fd.write(str(os.getpid()))
|
| 7343 |
+
fd.flush()
|
| 7344 |
+
_beacon_lock_fd[0] = fd
|
| 7345 |
+
return True
|
| 7346 |
+
except (OSError, ImportError):
|
| 7347 |
+
# Lock held by another worker, or fcntl not available (Windows)
|
| 7348 |
+
# On Windows, skip locking — workers are rare on Windows anyway
|
| 7349 |
+
try:
|
| 7350 |
+
import fcntl # noqa: F811
|
| 7351 |
+
|
| 7352 |
+
return False # Lock held by another worker
|
| 7353 |
+
except ImportError:
|
| 7354 |
+
return True # Windows: no fcntl, just allow it
|
| 7355 |
+
|
| 7356 |
+
def _release_beacon_lock() -> None:
|
| 7357 |
+
"""Release the beacon file lock."""
|
| 7358 |
+
fd = _beacon_lock_fd[0]
|
| 7359 |
+
if fd:
|
| 7360 |
+
try:
|
| 7361 |
+
import fcntl
|
| 7362 |
+
|
| 7363 |
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
| 7364 |
+
fd.close()
|
| 7365 |
+
except Exception:
|
| 7366 |
+
pass
|
| 7367 |
+
_beacon_lock_fd[0] = None
|
| 7368 |
+
try:
|
| 7369 |
+
_beacon_lock_path.unlink(missing_ok=True)
|
| 7370 |
+
except Exception:
|
| 7371 |
+
pass
|
| 7372 |
|
| 7373 |
@asynccontextmanager
|
| 7374 |
async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
|
|
|
|
| 7379 |
await proxy.usage_reporter.start(proxy)
|
| 7380 |
if proxy.traffic_learner:
|
| 7381 |
await proxy.traffic_learner.start()
|
| 7382 |
+
|
| 7383 |
+
# Only start beacon if we acquire the lock (first worker wins)
|
| 7384 |
+
_beacon_is_owner[0] = _try_acquire_beacon_lock()
|
| 7385 |
+
if _beacon_is_owner[0]:
|
| 7386 |
+
await _beacon.start()
|
| 7387 |
+
else:
|
| 7388 |
+
logger.debug("Beacon: skipping (another worker owns the lock)")
|
| 7389 |
|
| 7390 |
yield
|
| 7391 |
|
| 7392 |
# Shutdown
|
| 7393 |
+
if _beacon_is_owner[0]:
|
| 7394 |
+
await _beacon.stop()
|
| 7395 |
+
_release_beacon_lock()
|
| 7396 |
if proxy.usage_reporter:
|
| 7397 |
await proxy.usage_reporter.stop()
|
| 7398 |
if proxy.traffic_learner:
|
headroom/telemetry/beacon.py
CHANGED
|
@@ -33,7 +33,7 @@ _SUPABASE_KEY = ".".join(
|
|
| 33 |
]
|
| 34 |
)
|
| 35 |
_TABLE = "proxy_telemetry_v2"
|
| 36 |
-
_ENDPOINT = f"{_SUPABASE_URL}/rest/v1/{_TABLE}"
|
| 37 |
|
| 38 |
# Report every 5 minutes
|
| 39 |
_INTERVAL_SECONDS = 300
|
|
@@ -74,8 +74,11 @@ class TelemetryBeacon:
|
|
| 74 |
if self._task:
|
| 75 |
self._task.cancel()
|
| 76 |
self._task = None
|
| 77 |
-
# Final report
|
| 78 |
-
|
|
|
|
|
|
|
|
|
|
| 79 |
await self._report()
|
| 80 |
|
| 81 |
async def _loop(self) -> None:
|
|
@@ -250,7 +253,7 @@ class TelemetryBeacon:
|
|
| 250 |
except Exception:
|
| 251 |
logger.debug("Beacon: failed to extract waste signals", exc_info=True)
|
| 252 |
|
| 253 |
-
# ---- Send to Supabase (fire-and-forget) ----
|
| 254 |
try:
|
| 255 |
async with httpx.AsyncClient(timeout=10.0) as client:
|
| 256 |
await client.post(
|
|
@@ -260,7 +263,7 @@ class TelemetryBeacon:
|
|
| 260 |
"apikey": _SUPABASE_KEY,
|
| 261 |
"Authorization": f"Bearer {_SUPABASE_KEY}",
|
| 262 |
"Content-Type": "application/json",
|
| 263 |
-
"Prefer": "return=minimal",
|
| 264 |
},
|
| 265 |
)
|
| 266 |
except Exception:
|
|
|
|
| 33 |
]
|
| 34 |
)
|
| 35 |
_TABLE = "proxy_telemetry_v2"
|
| 36 |
+
_ENDPOINT = f"{_SUPABASE_URL}/rest/v1/{_TABLE}?on_conflict=session_id"
|
| 37 |
|
| 38 |
# Report every 5 minutes
|
| 39 |
_INTERVAL_SECONDS = 300
|
|
|
|
| 74 |
if self._task:
|
| 75 |
self._task.cancel()
|
| 76 |
self._task = None
|
| 77 |
+
# Final report — but only if the proxy ran for more than 2 minutes.
|
| 78 |
+
# Short-lived restarts (e.g. crash loops, orchestration churn) would
|
| 79 |
+
# otherwise spam the telemetry table with duplicate cumulative stats.
|
| 80 |
+
uptime_seconds = time.time() - self._start_time
|
| 81 |
+
if is_telemetry_enabled() and uptime_seconds > 120:
|
| 82 |
await self._report()
|
| 83 |
|
| 84 |
async def _loop(self) -> None:
|
|
|
|
| 253 |
except Exception:
|
| 254 |
logger.debug("Beacon: failed to extract waste signals", exc_info=True)
|
| 255 |
|
| 256 |
+
# ---- Send to Supabase (fire-and-forget, upsert on session_id) ----
|
| 257 |
try:
|
| 258 |
async with httpx.AsyncClient(timeout=10.0) as client:
|
| 259 |
await client.post(
|
|
|
|
| 263 |
"apikey": _SUPABASE_KEY,
|
| 264 |
"Authorization": f"Bearer {_SUPABASE_KEY}",
|
| 265 |
"Content-Type": "application/json",
|
| 266 |
+
"Prefer": "resolution=merge-duplicates,return=minimal",
|
| 267 |
},
|
| 268 |
)
|
| 269 |
except Exception:
|
pyproject.toml
CHANGED
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "headroom-ai"
|
| 7 |
-
version = "0.5.
|
| 8 |
description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
|
| 9 |
readme = "README.md"
|
| 10 |
license = "Apache-2.0"
|
|
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "headroom-ai"
|
| 7 |
+
version = "0.5.17"
|
| 8 |
description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
|
| 9 |
readme = "README.md"
|
| 10 |
license = "Apache-2.0"
|