chopratejas commited on
Commit
a71a45d
·
1 Parent(s): 91b077d

Fix multi-worker beacon spam, cost dashboard, upsert telemetry; bump 0.5.17

Browse files

- Fix beacon spam: file lock ensures only one beacon per proxy regardless
of worker count. Workers > 1 caused N beacons firing N rows per cycle.
- Beacon upsert: on_conflict=session_id prevents duplicate rows.
- Beacon stop() guard: skip final report if uptime < 2 minutes.
- Fix dashboard cost: savings_usd now uses model list price (monotonic),
not moving average. Separate breakdown for compression/cache/rtk.

Fixes #83

headroom/__init__.py CHANGED
@@ -153,7 +153,7 @@ from .transforms import (
153
  TransformPipeline,
154
  )
155
 
156
- __version__ = "0.5.16"
157
 
158
  __all__ = [
159
  # Main client
 
153
  TransformPipeline,
154
  )
155
 
156
+ __version__ = "0.5.17"
157
 
158
  __all__ = [
159
  # Main client
headroom/proxy/server.py CHANGED
@@ -7309,7 +7309,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
7309
 
7310
  proxy = HeadroomProxy(config)
7311
 
7312
- # Telemetry beacon (anonymous aggregate stats)
 
 
 
 
 
 
 
7313
  from headroom.telemetry.beacon import TelemetryBeacon
7314
 
7315
  _beacon = TelemetryBeacon(
@@ -7317,6 +7324,51 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
7317
  sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
7318
  backend=config.backend if hasattr(config, "backend") else "anthropic",
7319
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7320
 
7321
  @asynccontextmanager
7322
  async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
@@ -7327,12 +7379,20 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
7327
  await proxy.usage_reporter.start(proxy)
7328
  if proxy.traffic_learner:
7329
  await proxy.traffic_learner.start()
7330
- await _beacon.start()
 
 
 
 
 
 
7331
 
7332
  yield
7333
 
7334
  # Shutdown
7335
- await _beacon.stop()
 
 
7336
  if proxy.usage_reporter:
7337
  await proxy.usage_reporter.stop()
7338
  if proxy.traffic_learner:
 
7309
 
7310
  proxy = HeadroomProxy(config)
7311
 
7312
+ # Telemetry beacon (anonymous aggregate stats).
7313
+ # With uvicorn workers > 1, each worker runs the lifespan independently.
7314
+ # We must ensure only ONE beacon runs across all workers — otherwise each
7315
+ # worker creates its own beacon, spamming the telemetry table with N rows
7316
+ # per cycle instead of 1 (all reading the same /stats from the same port).
7317
+ #
7318
+ # Strategy: use a file lock to ensure only the first worker starts the
7319
+ # beacon. Other workers see the lock and skip.
7320
  from headroom.telemetry.beacon import TelemetryBeacon
7321
 
7322
  _beacon = TelemetryBeacon(
 
7324
  sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
7325
  backend=config.backend if hasattr(config, "backend") else "anthropic",
7326
  )
7327
+ _beacon_lock_path = Path.home() / ".headroom" / f".beacon_lock_{config.port}"
7328
+ _beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor
7329
+ _beacon_is_owner: list = [False]
7330
+
7331
+ def _try_acquire_beacon_lock() -> bool:
7332
+ """Try to acquire the beacon file lock (non-blocking).
7333
+
7334
+ Returns True if this process is the beacon owner.
7335
+ """
7336
+ try:
7337
+ _beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)
7338
+ import fcntl
7339
+
7340
+ fd = open(_beacon_lock_path, "w") # noqa: SIM115
7341
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
7342
+ fd.write(str(os.getpid()))
7343
+ fd.flush()
7344
+ _beacon_lock_fd[0] = fd
7345
+ return True
7346
+ except (OSError, ImportError):
7347
+ # Lock held by another worker, or fcntl not available (Windows)
7348
+ # On Windows, skip locking — workers are rare on Windows anyway
7349
+ try:
7350
+ import fcntl # noqa: F811
7351
+
7352
+ return False # Lock held by another worker
7353
+ except ImportError:
7354
+ return True # Windows: no fcntl, just allow it
7355
+
7356
+ def _release_beacon_lock() -> None:
7357
+ """Release the beacon file lock."""
7358
+ fd = _beacon_lock_fd[0]
7359
+ if fd:
7360
+ try:
7361
+ import fcntl
7362
+
7363
+ fcntl.flock(fd, fcntl.LOCK_UN)
7364
+ fd.close()
7365
+ except Exception:
7366
+ pass
7367
+ _beacon_lock_fd[0] = None
7368
+ try:
7369
+ _beacon_lock_path.unlink(missing_ok=True)
7370
+ except Exception:
7371
+ pass
7372
 
7373
  @asynccontextmanager
7374
  async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
 
7379
  await proxy.usage_reporter.start(proxy)
7380
  if proxy.traffic_learner:
7381
  await proxy.traffic_learner.start()
7382
+
7383
+ # Only start beacon if we acquire the lock (first worker wins)
7384
+ _beacon_is_owner[0] = _try_acquire_beacon_lock()
7385
+ if _beacon_is_owner[0]:
7386
+ await _beacon.start()
7387
+ else:
7388
+ logger.debug("Beacon: skipping (another worker owns the lock)")
7389
 
7390
  yield
7391
 
7392
  # Shutdown
7393
+ if _beacon_is_owner[0]:
7394
+ await _beacon.stop()
7395
+ _release_beacon_lock()
7396
  if proxy.usage_reporter:
7397
  await proxy.usage_reporter.stop()
7398
  if proxy.traffic_learner:
headroom/telemetry/beacon.py CHANGED
@@ -33,7 +33,7 @@ _SUPABASE_KEY = ".".join(
33
  ]
34
  )
35
  _TABLE = "proxy_telemetry_v2"
36
- _ENDPOINT = f"{_SUPABASE_URL}/rest/v1/{_TABLE}"
37
 
38
  # Report every 5 minutes
39
  _INTERVAL_SECONDS = 300
@@ -74,8 +74,11 @@ class TelemetryBeacon:
74
  if self._task:
75
  self._task.cancel()
76
  self._task = None
77
- # Final report
78
- if is_telemetry_enabled():
 
 
 
79
  await self._report()
80
 
81
  async def _loop(self) -> None:
@@ -250,7 +253,7 @@ class TelemetryBeacon:
250
  except Exception:
251
  logger.debug("Beacon: failed to extract waste signals", exc_info=True)
252
 
253
- # ---- Send to Supabase (fire-and-forget) ----
254
  try:
255
  async with httpx.AsyncClient(timeout=10.0) as client:
256
  await client.post(
@@ -260,7 +263,7 @@ class TelemetryBeacon:
260
  "apikey": _SUPABASE_KEY,
261
  "Authorization": f"Bearer {_SUPABASE_KEY}",
262
  "Content-Type": "application/json",
263
- "Prefer": "return=minimal",
264
  },
265
  )
266
  except Exception:
 
33
  ]
34
  )
35
  _TABLE = "proxy_telemetry_v2"
36
+ _ENDPOINT = f"{_SUPABASE_URL}/rest/v1/{_TABLE}?on_conflict=session_id"
37
 
38
  # Report every 5 minutes
39
  _INTERVAL_SECONDS = 300
 
74
  if self._task:
75
  self._task.cancel()
76
  self._task = None
77
+ # Final report — but only if the proxy ran for more than 2 minutes.
78
+ # Short-lived restarts (e.g. crash loops, orchestration churn) would
79
+ # otherwise spam the telemetry table with duplicate cumulative stats.
80
+ uptime_seconds = time.time() - self._start_time
81
+ if is_telemetry_enabled() and uptime_seconds > 120:
82
  await self._report()
83
 
84
  async def _loop(self) -> None:
 
253
  except Exception:
254
  logger.debug("Beacon: failed to extract waste signals", exc_info=True)
255
 
256
+ # ---- Send to Supabase (fire-and-forget, upsert on session_id) ----
257
  try:
258
  async with httpx.AsyncClient(timeout=10.0) as client:
259
  await client.post(
 
263
  "apikey": _SUPABASE_KEY,
264
  "Authorization": f"Bearer {_SUPABASE_KEY}",
265
  "Content-Type": "application/json",
266
+ "Prefer": "resolution=merge-duplicates,return=minimal",
267
  },
268
  )
269
  except Exception:
pyproject.toml CHANGED
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
 
5
  [project]
6
  name = "headroom-ai"
7
- version = "0.5.16"
8
  description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
9
  readme = "README.md"
10
  license = "Apache-2.0"
 
4
 
5
  [project]
6
  name = "headroom-ai"
7
+ version = "0.5.17"
8
  description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
9
  readme = "README.md"
10
  license = "Apache-2.0"