Garm commited on
Commit
d30a2e7
·
1 Parent(s): f9ef7e0

Add canonical display-session metrics for downstream dashboards

Browse files
headroom/cli/proxy.py CHANGED
@@ -324,6 +324,7 @@ Usage:
324
  Endpoints:
325
  GET /health Health check
326
  GET /stats Detailed statistics
 
327
  GET /metrics Prometheus metrics
328
 
329
  Press Ctrl+C to stop.
 
324
  Endpoints:
325
  GET /health Health check
326
  GET /stats Detailed statistics
327
+ GET /stats-history Durable compression history + display session
328
  GET /metrics Prometheus metrics
329
 
330
  Press Ctrl+C to stop.
headroom/proxy/savings_tracker.py CHANGED
@@ -1,8 +1,8 @@
1
- """Durable proxy savings history tracking.
2
 
3
- Persists cumulative proxy compression savings to a local JSON file so
4
- historical charts survive proxy restarts and can be shared by multiple
5
- Headroom frontends.
6
  """
7
 
8
  from __future__ import annotations
@@ -23,9 +23,10 @@ logger = logging.getLogger(__name__)
23
  HEADROOM_SAVINGS_PATH_ENV_VAR = "HEADROOM_SAVINGS_PATH"
24
  DEFAULT_SAVINGS_DIR = ".headroom"
25
  DEFAULT_SAVINGS_FILE = "proxy_savings.json"
26
- SCHEMA_VERSION = 1
27
  DEFAULT_MAX_HISTORY_POINTS = 5000
28
  DEFAULT_MAX_HISTORY_AGE_DAYS = 365
 
29
 
30
  try:
31
  import litellm
@@ -140,6 +141,48 @@ def _estimate_compression_savings_usd(model: str, tokens_saved: int) -> float:
140
  return 0.0
141
 
142
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
144
  """Normalize persisted history entries across schema shapes."""
145
  timestamp: datetime | None = None
@@ -178,6 +221,49 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
178
  }
179
 
180
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
181
  class SavingsTracker:
182
  """Persist bounded proxy compression savings history."""
183
 
@@ -186,10 +272,15 @@ class SavingsTracker:
186
  path: str | None = None,
187
  max_history_points: int = DEFAULT_MAX_HISTORY_POINTS,
188
  max_history_age_days: int = DEFAULT_MAX_HISTORY_AGE_DAYS,
 
189
  ) -> None:
190
  self._path = Path(path or get_default_savings_storage_path())
191
  self._max_history_points = max_history_points
192
  self._max_history_age_days = max_history_age_days
 
 
 
 
193
  self._lock = threading.Lock()
194
  self._state = self._load_state()
195
 
@@ -257,6 +348,126 @@ class SavingsTracker:
257
  self._save_locked()
258
  return True
259
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
260
  def stats_preview(self, recent_points: int = 20) -> dict[str, Any]:
261
  """Return a compact preview for `/stats`."""
262
  snapshot = self.snapshot()
@@ -264,6 +475,8 @@ class SavingsTracker:
264
  "schema_version": snapshot["schema_version"],
265
  "storage_path": snapshot["storage_path"],
266
  "lifetime": snapshot["lifetime"],
 
 
267
  "history_points": len(snapshot["history"]),
268
  "recent_history": snapshot["history"][-recent_points:],
269
  "retention": snapshot["retention"],
@@ -284,6 +497,8 @@ class SavingsTracker:
284
  "generated_at": _to_utc_iso(_utc_now()),
285
  "storage_path": snapshot["storage_path"],
286
  "lifetime": snapshot["lifetime"],
 
 
287
  "history": history,
288
  "series": series,
289
  "exports": {
@@ -339,6 +554,10 @@ class SavingsTracker:
339
  "schema_version": SCHEMA_VERSION,
340
  "storage_path": str(self._path),
341
  "lifetime": dict(self._state["lifetime"]),
 
 
 
 
342
  "history": history,
343
  "retention": {
344
  "max_history_points": self._max_history_points,
@@ -350,11 +569,13 @@ class SavingsTracker:
350
  return {
351
  "schema_version": SCHEMA_VERSION,
352
  "lifetime": {
 
353
  "tokens_saved": 0,
354
  "compression_savings_usd": 0.0,
355
  "total_input_tokens": 0,
356
  "total_input_cost_usd": 0.0,
357
  },
 
358
  "history": [],
359
  }
360
 
@@ -386,11 +607,13 @@ class SavingsTracker:
386
  normalized_history.sort(key=lambda item: item["timestamp"])
387
 
388
  lifetime_raw = raw.get("lifetime", {})
 
389
  lifetime_tokens_saved = 0
390
  lifetime_savings_usd = 0.0
391
  lifetime_input_tokens = 0
392
  lifetime_input_cost_usd = 0.0
393
  if isinstance(lifetime_raw, dict):
 
394
  lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))
395
  lifetime_savings_usd = _coerce_float(lifetime_raw.get("compression_savings_usd"))
396
  lifetime_input_tokens = _coerce_int(lifetime_raw.get("total_input_tokens"))
@@ -415,11 +638,13 @@ class SavingsTracker:
415
  state = {
416
  "schema_version": SCHEMA_VERSION,
417
  "lifetime": {
 
418
  "tokens_saved": lifetime_tokens_saved,
419
  "compression_savings_usd": round(lifetime_savings_usd, 6),
420
  "total_input_tokens": lifetime_input_tokens,
421
  "total_input_cost_usd": round(lifetime_input_cost_usd, 6),
422
  },
 
423
  "history": normalized_history,
424
  }
425
 
@@ -463,6 +688,7 @@ class SavingsTracker:
463
  payload = {
464
  "schema_version": SCHEMA_VERSION,
465
  "lifetime": self._state["lifetime"],
 
466
  "history": self._state["history"],
467
  }
468
  json_data = json.dumps(payload, indent=2)
@@ -487,6 +713,47 @@ class SavingsTracker:
487
  except OSError as e:
488
  logger.warning("Failed to save savings history to %s: %s", self._path, e)
489
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
490
  def _build_rollup(self, history: list[dict[str, Any]], bucket: str) -> list[dict[str, Any]]:
491
  if not history:
492
  return []
 
1
+ """Durable proxy savings and display-session tracking.
2
 
3
+ Persists cumulative proxy compression savings plus a canonical display session
4
+ window to a local JSON file so historical charts and dashboard session stats
5
+ survive proxy restarts and can be shared by multiple Headroom frontends.
6
  """
7
 
8
  from __future__ import annotations
 
23
  HEADROOM_SAVINGS_PATH_ENV_VAR = "HEADROOM_SAVINGS_PATH"
24
  DEFAULT_SAVINGS_DIR = ".headroom"
25
  DEFAULT_SAVINGS_FILE = "proxy_savings.json"
26
+ SCHEMA_VERSION = 2
27
  DEFAULT_MAX_HISTORY_POINTS = 5000
28
  DEFAULT_MAX_HISTORY_AGE_DAYS = 365
29
+ DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES = 60
30
 
31
  try:
32
  import litellm
 
141
  return 0.0
142
 
143
 
144
+ def _estimate_input_cost_usd(
145
+ model: str,
146
+ input_tokens: int,
147
+ *,
148
+ cache_read_tokens: int = 0,
149
+ cache_write_tokens: int = 0,
150
+ uncached_input_tokens: int = 0,
151
+ ) -> float:
152
+ """Estimate input spend in USD for a request.
153
+
154
+ Uses provider cache pricing when a complete cache breakdown is available and
155
+ otherwise falls back to list-price input tokens.
156
+ """
157
+ total_input_tokens = _coerce_int(input_tokens)
158
+ if total_input_tokens <= 0 or not LITELLM_AVAILABLE:
159
+ return 0.0
160
+
161
+ cache_read = _coerce_int(cache_read_tokens)
162
+ cache_write = _coerce_int(cache_write_tokens)
163
+ uncached = _coerce_int(uncached_input_tokens)
164
+
165
+ try:
166
+ resolved = _resolve_litellm_model(model)
167
+ info = litellm.model_cost.get(resolved, {})
168
+ input_cost_per_token = info.get("input_cost_per_token")
169
+ if not input_cost_per_token:
170
+ return 0.0
171
+
172
+ if cache_read + cache_write + uncached > 0:
173
+ cache_read_cost = info.get("cache_read_input_token_cost", input_cost_per_token)
174
+ cache_write_cost = info.get("cache_creation_input_token_cost", input_cost_per_token)
175
+ return (
176
+ float(cache_read) * float(cache_read_cost)
177
+ + float(cache_write) * float(cache_write_cost)
178
+ + float(uncached) * float(input_cost_per_token)
179
+ )
180
+
181
+ return float(total_input_tokens) * float(input_cost_per_token)
182
+ except Exception:
183
+ return 0.0
184
+
185
+
186
  def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
187
  """Normalize persisted history entries across schema shapes."""
188
  timestamp: datetime | None = None
 
221
  }
222
 
223
 
224
+ def _empty_display_session() -> dict[str, Any]:
225
+ return {
226
+ "requests": 0,
227
+ "tokens_saved": 0,
228
+ "compression_savings_usd": 0.0,
229
+ "total_input_tokens": 0,
230
+ "total_input_cost_usd": 0.0,
231
+ "savings_percent": 0.0,
232
+ "started_at": None,
233
+ "last_activity_at": None,
234
+ }
235
+
236
+
237
+ def _normalize_display_session(entry: Any) -> dict[str, Any]:
238
+ if not isinstance(entry, dict):
239
+ return _empty_display_session()
240
+
241
+ started_at = _parse_timestamp(entry.get("started_at"))
242
+ last_activity_at = _parse_timestamp(entry.get("last_activity_at"))
243
+
244
+ if started_at is None or last_activity_at is None or last_activity_at < started_at:
245
+ return _empty_display_session()
246
+
247
+ tokens_saved = _coerce_int(entry.get("tokens_saved"))
248
+ total_input_tokens = _coerce_int(entry.get("total_input_tokens"))
249
+ total_before = tokens_saved + total_input_tokens
250
+ savings_percent = round((tokens_saved / total_before * 100) if total_before > 0 else 0.0, 2)
251
+
252
+ return {
253
+ "requests": _coerce_int(entry.get("requests")),
254
+ "tokens_saved": tokens_saved,
255
+ "compression_savings_usd": round(
256
+ _coerce_float(entry.get("compression_savings_usd")),
257
+ 6,
258
+ ),
259
+ "total_input_tokens": total_input_tokens,
260
+ "total_input_cost_usd": round(_coerce_float(entry.get("total_input_cost_usd")), 6),
261
+ "savings_percent": savings_percent,
262
+ "started_at": _to_utc_iso(started_at),
263
+ "last_activity_at": _to_utc_iso(last_activity_at),
264
+ }
265
+
266
+
267
  class SavingsTracker:
268
  """Persist bounded proxy compression savings history."""
269
 
 
272
  path: str | None = None,
273
  max_history_points: int = DEFAULT_MAX_HISTORY_POINTS,
274
  max_history_age_days: int = DEFAULT_MAX_HISTORY_AGE_DAYS,
275
+ display_session_inactivity_minutes: int = DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES,
276
  ) -> None:
277
  self._path = Path(path or get_default_savings_storage_path())
278
  self._max_history_points = max_history_points
279
  self._max_history_age_days = max_history_age_days
280
+ self._display_session_inactivity_minutes = max(
281
+ _coerce_int(display_session_inactivity_minutes, DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES),
282
+ 1,
283
+ )
284
  self._lock = threading.Lock()
285
  self._state = self._load_state()
286
 
 
348
  self._save_locked()
349
  return True
350
 
351
+ def record_request(
352
+ self,
353
+ *,
354
+ model: str,
355
+ input_tokens: int,
356
+ tokens_saved: int,
357
+ cache_read_tokens: int = 0,
358
+ cache_write_tokens: int = 0,
359
+ uncached_input_tokens: int = 0,
360
+ total_input_tokens: int | None = None,
361
+ total_input_cost_usd: float | None = None,
362
+ timestamp: datetime | str | None = None,
363
+ ) -> bool:
364
+ """Persist a canonical display-session update for every request."""
365
+ timestamp_dt = (
366
+ _parse_timestamp(timestamp)
367
+ if isinstance(timestamp, str)
368
+ else timestamp.astimezone(timezone.utc)
369
+ if isinstance(timestamp, datetime)
370
+ else _utc_now()
371
+ )
372
+ if timestamp_dt is None:
373
+ timestamp_dt = _utc_now()
374
+
375
+ delta_tokens_saved = _coerce_int(tokens_saved)
376
+ delta_input_tokens = _coerce_int(input_tokens)
377
+ delta_savings_usd = _estimate_compression_savings_usd(model, delta_tokens_saved)
378
+ delta_input_cost_usd = _estimate_input_cost_usd(
379
+ model,
380
+ delta_input_tokens,
381
+ cache_read_tokens=cache_read_tokens,
382
+ cache_write_tokens=cache_write_tokens,
383
+ uncached_input_tokens=uncached_input_tokens,
384
+ )
385
+
386
+ with self._lock:
387
+ lifetime = self._state["lifetime"]
388
+ previous_total_input_tokens = lifetime["total_input_tokens"]
389
+ previous_total_input_cost_usd = lifetime["total_input_cost_usd"]
390
+
391
+ next_total_input_tokens = max(
392
+ previous_total_input_tokens + delta_input_tokens,
393
+ _coerce_int(
394
+ total_input_tokens,
395
+ default=previous_total_input_tokens + delta_input_tokens,
396
+ ),
397
+ )
398
+ next_total_input_cost_usd = round(
399
+ max(
400
+ previous_total_input_cost_usd + delta_input_cost_usd,
401
+ _coerce_float(
402
+ total_input_cost_usd,
403
+ default=previous_total_input_cost_usd + delta_input_cost_usd,
404
+ ),
405
+ ),
406
+ 6,
407
+ )
408
+ session_input_tokens_delta = max(
409
+ next_total_input_tokens - previous_total_input_tokens,
410
+ 0,
411
+ )
412
+ session_input_cost_delta = round(
413
+ max(next_total_input_cost_usd - previous_total_input_cost_usd, 0.0),
414
+ 6,
415
+ )
416
+
417
+ lifetime["requests"] += 1
418
+ lifetime["tokens_saved"] += delta_tokens_saved
419
+ lifetime["compression_savings_usd"] = round(
420
+ lifetime["compression_savings_usd"] + delta_savings_usd,
421
+ 6,
422
+ )
423
+ lifetime["total_input_tokens"] = next_total_input_tokens
424
+ lifetime["total_input_cost_usd"] = next_total_input_cost_usd
425
+
426
+ session = self._state["display_session"]
427
+ last_activity = _parse_timestamp(session.get("last_activity_at"))
428
+ if last_activity is None or self._is_display_session_expired(
429
+ last_activity,
430
+ reference_time=timestamp_dt,
431
+ ):
432
+ session = _empty_display_session()
433
+ session["started_at"] = _to_utc_iso(timestamp_dt)
434
+ self._state["display_session"] = session
435
+
436
+ session["requests"] += 1
437
+ session["tokens_saved"] += delta_tokens_saved
438
+ session["compression_savings_usd"] = round(
439
+ session["compression_savings_usd"] + delta_savings_usd,
440
+ 6,
441
+ )
442
+ session["total_input_tokens"] += session_input_tokens_delta
443
+ session["total_input_cost_usd"] = round(
444
+ session["total_input_cost_usd"] + session_input_cost_delta,
445
+ 6,
446
+ )
447
+ total_before = session["tokens_saved"] + session["total_input_tokens"]
448
+ session["savings_percent"] = round(
449
+ (session["tokens_saved"] / total_before * 100) if total_before > 0 else 0.0,
450
+ 2,
451
+ )
452
+ session["last_activity_at"] = _to_utc_iso(timestamp_dt)
453
+ if session.get("started_at") is None:
454
+ session["started_at"] = session["last_activity_at"]
455
+
456
+ if delta_tokens_saved > 0:
457
+ self._state["history"].append(
458
+ {
459
+ "timestamp": _to_utc_iso(timestamp_dt),
460
+ "total_tokens_saved": lifetime["tokens_saved"],
461
+ "compression_savings_usd": lifetime["compression_savings_usd"],
462
+ "total_input_tokens": lifetime["total_input_tokens"],
463
+ "total_input_cost_usd": lifetime["total_input_cost_usd"],
464
+ }
465
+ )
466
+ self._trim_history_locked(reference_time=timestamp_dt)
467
+
468
+ self._save_locked()
469
+ return True
470
+
471
  def stats_preview(self, recent_points: int = 20) -> dict[str, Any]:
472
  """Return a compact preview for `/stats`."""
473
  snapshot = self.snapshot()
 
475
  "schema_version": snapshot["schema_version"],
476
  "storage_path": snapshot["storage_path"],
477
  "lifetime": snapshot["lifetime"],
478
+ "display_session": snapshot["display_session"],
479
+ "display_session_policy": snapshot["display_session_policy"],
480
  "history_points": len(snapshot["history"]),
481
  "recent_history": snapshot["history"][-recent_points:],
482
  "retention": snapshot["retention"],
 
497
  "generated_at": _to_utc_iso(_utc_now()),
498
  "storage_path": snapshot["storage_path"],
499
  "lifetime": snapshot["lifetime"],
500
+ "display_session": snapshot["display_session"],
501
+ "display_session_policy": snapshot["display_session_policy"],
502
  "history": history,
503
  "series": series,
504
  "exports": {
 
554
  "schema_version": SCHEMA_VERSION,
555
  "storage_path": str(self._path),
556
  "lifetime": dict(self._state["lifetime"]),
557
+ "display_session": self._display_session_snapshot_locked(),
558
+ "display_session_policy": {
559
+ "rollover_inactivity_minutes": self._display_session_inactivity_minutes,
560
+ },
561
  "history": history,
562
  "retention": {
563
  "max_history_points": self._max_history_points,
 
569
  return {
570
  "schema_version": SCHEMA_VERSION,
571
  "lifetime": {
572
+ "requests": 0,
573
  "tokens_saved": 0,
574
  "compression_savings_usd": 0.0,
575
  "total_input_tokens": 0,
576
  "total_input_cost_usd": 0.0,
577
  },
578
+ "display_session": _empty_display_session(),
579
  "history": [],
580
  }
581
 
 
607
  normalized_history.sort(key=lambda item: item["timestamp"])
608
 
609
  lifetime_raw = raw.get("lifetime", {})
610
+ lifetime_requests = 0
611
  lifetime_tokens_saved = 0
612
  lifetime_savings_usd = 0.0
613
  lifetime_input_tokens = 0
614
  lifetime_input_cost_usd = 0.0
615
  if isinstance(lifetime_raw, dict):
616
+ lifetime_requests = _coerce_int(lifetime_raw.get("requests"))
617
  lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))
618
  lifetime_savings_usd = _coerce_float(lifetime_raw.get("compression_savings_usd"))
619
  lifetime_input_tokens = _coerce_int(lifetime_raw.get("total_input_tokens"))
 
638
  state = {
639
  "schema_version": SCHEMA_VERSION,
640
  "lifetime": {
641
+ "requests": lifetime_requests,
642
  "tokens_saved": lifetime_tokens_saved,
643
  "compression_savings_usd": round(lifetime_savings_usd, 6),
644
  "total_input_tokens": lifetime_input_tokens,
645
  "total_input_cost_usd": round(lifetime_input_cost_usd, 6),
646
  },
647
+ "display_session": _normalize_display_session(raw.get("display_session")),
648
  "history": normalized_history,
649
  }
650
 
 
688
  payload = {
689
  "schema_version": SCHEMA_VERSION,
690
  "lifetime": self._state["lifetime"],
691
+ "display_session": self._state["display_session"],
692
  "history": self._state["history"],
693
  }
694
  json_data = json.dumps(payload, indent=2)
 
713
  except OSError as e:
714
  logger.warning("Failed to save savings history to %s: %s", self._path, e)
715
 
716
+ def _display_session_snapshot_locked(
717
+ self,
718
+ reference_time: datetime | None = None,
719
+ ) -> dict[str, Any]:
720
+ session = dict(self._state["display_session"])
721
+ last_activity = _parse_timestamp(session.get("last_activity_at"))
722
+ if last_activity is None or self._is_display_session_expired(
723
+ last_activity,
724
+ reference_time=reference_time,
725
+ ):
726
+ return _empty_display_session()
727
+
728
+ total_before = _coerce_int(session.get("tokens_saved")) + _coerce_int(
729
+ session.get("total_input_tokens")
730
+ )
731
+ session["savings_percent"] = round(
732
+ (_coerce_int(session.get("tokens_saved")) / total_before * 100)
733
+ if total_before > 0
734
+ else 0.0,
735
+ 2,
736
+ )
737
+ session["compression_savings_usd"] = round(
738
+ _coerce_float(session.get("compression_savings_usd")),
739
+ 6,
740
+ )
741
+ session["total_input_cost_usd"] = round(
742
+ _coerce_float(session.get("total_input_cost_usd")),
743
+ 6,
744
+ )
745
+ return session
746
+
747
+ def _is_display_session_expired(
748
+ self,
749
+ last_activity: datetime,
750
+ *,
751
+ reference_time: datetime | None = None,
752
+ ) -> bool:
753
+ return (reference_time or _utc_now()) - last_activity > timedelta(
754
+ minutes=self._display_session_inactivity_minutes
755
+ )
756
+
757
  def _build_rollup(self, history: list[dict[str, Any]], bucket: str) -> list[dict[str, Any]]:
758
  if not history:
759
  return []
headroom/proxy/server.py CHANGED
@@ -1269,6 +1269,7 @@ class PrometheusMetrics:
1269
  waste_signals: dict[str, int] | None = None,
1270
  cache_read_tokens: int = 0,
1271
  cache_write_tokens: int = 0,
 
1272
  ):
1273
  """Record metrics for a request."""
1274
  async with self._lock:
@@ -1342,14 +1343,17 @@ class PrometheusMetrics:
1342
  if len(self.savings_history) > 500:
1343
  self.savings_history = self.savings_history[-500:]
1344
 
1345
- if tokens_saved > 0:
1346
- total_input_tokens, total_input_cost_usd = self._current_savings_tracker_totals()
1347
- self.savings_tracker.record_compression_savings(
1348
- model=model,
1349
- tokens_saved=tokens_saved,
1350
- total_input_tokens=total_input_tokens,
1351
- total_input_cost_usd=total_input_cost_usd,
1352
- )
 
 
 
1353
 
1354
  async def record_rate_limited(self):
1355
  async with self._lock:
@@ -3082,6 +3086,7 @@ class HeadroomProxy:
3082
  waste_signals=waste_signals_dict,
3083
  cache_read_tokens=cr_tokens,
3084
  cache_write_tokens=cw_tokens,
 
3085
  )
3086
 
3087
  # Log request
@@ -4843,6 +4848,7 @@ class HeadroomProxy:
4843
  pipeline_timing=pipeline_timing,
4844
  cache_read_tokens=cache_read_tokens,
4845
  cache_write_tokens=cache_write_tokens,
 
4846
  )
4847
 
4848
  return StreamingResponse(
@@ -7386,6 +7392,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
7386
  - Request metrics (total, cached, failed, by model/provider)
7387
  - Token usage and savings
7388
  - Cost tracking
 
7389
  - Compression (CCR) statistics
7390
  - Telemetry/TOIN (data flywheel) statistics
7391
  - Cache and rate limiter stats
@@ -7483,6 +7490,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
7483
  cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
7484
  total_tokens_all_layers = compression_tokens + cli_tokens_avoided
7485
  persistent_savings = m.savings_tracker.stats_preview()
 
7486
 
7487
  return {
7488
  "summary": summary,
@@ -7558,6 +7566,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
7558
  else {},
7559
  "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
7560
  "savings_history": m.savings_history[-100:], # Last 100 data points
 
7561
  "persistent_savings": persistent_savings,
7562
  "prefix_cache": prefix_cache_stats,
7563
  "cost": _merge_cost_stats(
@@ -7605,7 +7614,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
7605
  format: Literal["json", "csv"] = "json",
7606
  series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
7607
  ):
7608
- """Get durable proxy compression savings history for frontends."""
7609
  if format == "csv":
7610
  filename = f"headroom-stats-history-{series}.csv"
7611
  return Response(
 
1269
  waste_signals: dict[str, int] | None = None,
1270
  cache_read_tokens: int = 0,
1271
  cache_write_tokens: int = 0,
1272
+ uncached_input_tokens: int = 0,
1273
  ):
1274
  """Record metrics for a request."""
1275
  async with self._lock:
 
1343
  if len(self.savings_history) > 500:
1344
  self.savings_history = self.savings_history[-500:]
1345
 
1346
+ total_input_tokens, total_input_cost_usd = self._current_savings_tracker_totals()
1347
+ self.savings_tracker.record_request(
1348
+ model=model,
1349
+ input_tokens=input_tokens,
1350
+ tokens_saved=tokens_saved,
1351
+ cache_read_tokens=cache_read_tokens,
1352
+ cache_write_tokens=cache_write_tokens,
1353
+ uncached_input_tokens=uncached_input_tokens,
1354
+ total_input_tokens=total_input_tokens,
1355
+ total_input_cost_usd=total_input_cost_usd,
1356
+ )
1357
 
1358
  async def record_rate_limited(self):
1359
  async with self._lock:
 
3086
  waste_signals=waste_signals_dict,
3087
  cache_read_tokens=cr_tokens,
3088
  cache_write_tokens=cw_tokens,
3089
+ uncached_input_tokens=uncached_input_tokens,
3090
  )
3091
 
3092
  # Log request
 
4848
  pipeline_timing=pipeline_timing,
4849
  cache_read_tokens=cache_read_tokens,
4850
  cache_write_tokens=cache_write_tokens,
4851
+ uncached_input_tokens=uncached_input_tokens,
4852
  )
4853
 
4854
  return StreamingResponse(
 
7392
  - Request metrics (total, cached, failed, by model/provider)
7393
  - Token usage and savings
7394
  - Cost tracking
7395
+ - Canonical persisted display_session metrics for downstream dashboards
7396
  - Compression (CCR) statistics
7397
  - Telemetry/TOIN (data flywheel) statistics
7398
  - Cache and rate limiter stats
 
7490
  cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
7491
  total_tokens_all_layers = compression_tokens + cli_tokens_avoided
7492
  persistent_savings = m.savings_tracker.stats_preview()
7493
+ display_session = persistent_savings.get("display_session", {})
7494
 
7495
  return {
7496
  "summary": summary,
 
7566
  else {},
7567
  "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
7568
  "savings_history": m.savings_history[-100:], # Last 100 data points
7569
+ "display_session": display_session,
7570
  "persistent_savings": persistent_savings,
7571
  "prefix_cache": prefix_cache_stats,
7572
  "cost": _merge_cost_stats(
 
7614
  format: Literal["json", "csv"] = "json",
7615
  series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
7616
  ):
7617
+ """Get durable proxy compression history plus display-session state."""
7618
  if format == "csv":
7619
  filename = f"headroom-stats-history-{series}.csv"
7620
  return Response(
tests/test_proxy_savings_history.py CHANGED
@@ -18,15 +18,21 @@ from headroom.proxy.savings_tracker import HEADROOM_SAVINGS_PATH_ENV_VAR, Saving
18
  from headroom.proxy.server import ProxyConfig, create_app
19
 
20
 
21
- def _record_request(client: TestClient, *, model: str, tokens_saved: int) -> None:
 
 
 
 
 
 
22
  proxy = client.app.state.proxy
23
  if proxy.cost_tracker:
24
- proxy.cost_tracker.record_tokens(model, tokens_saved, 120)
25
  asyncio.run(
26
  proxy.metrics.record_request(
27
  provider="openai",
28
  model=model,
29
- input_tokens=120,
30
  output_tokens=24,
31
  tokens_saved=tokens_saved,
32
  latency_ms=15.0,
@@ -99,13 +105,15 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
99
  tracker = SavingsTracker(path=str(path), max_history_points=1, max_history_age_days=2)
100
  snapshot = tracker.snapshot()
101
 
102
- assert snapshot["schema_version"] == 1
103
  assert snapshot["lifetime"] == {
 
104
  "tokens_saved": 30,
105
  "compression_savings_usd": pytest.approx(0.03),
106
  "total_input_tokens": 0,
107
  "total_input_cost_usd": 0.0,
108
  }
 
109
  assert snapshot["history"] == [
110
  {
111
  "timestamp": "2026-03-27T09:00:00Z",
@@ -129,11 +137,13 @@ def test_non_dict_savings_state_resets_to_default(tmp_path):
129
  snapshot = tracker.snapshot()
130
 
131
  assert snapshot["lifetime"] == {
 
132
  "tokens_saved": 0,
133
  "compression_savings_usd": 0.0,
134
  "total_input_tokens": 0,
135
  "total_input_cost_usd": 0.0,
136
  }
 
137
  assert snapshot["history"] == []
138
 
139
 
@@ -223,9 +233,17 @@ def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):
223
  assert savings_tracker_module._estimate_compression_savings_usd(
224
  "claude-sonnet-4-6", 100
225
  ) == pytest.approx(0.2)
 
 
 
 
 
 
 
226
 
227
  fake_litellm.model_cost = {}
228
  assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
 
229
 
230
  monkeypatch.setattr(
231
  fake_litellm,
@@ -237,6 +255,86 @@ def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):
237
 
238
  monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)
239
  assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
240
 
241
 
242
  def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):
@@ -407,11 +505,15 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
407
  history = client.get("/stats-history")
408
  assert history.status_code == 200
409
  history_data = history.json()
410
- assert history_data["schema_version"] == 1
411
  assert history_data["storage_path"] == str(savings_path)
412
  assert history_data["lifetime"]["tokens_saved"] == 40
413
  assert history_data["lifetime"]["total_input_tokens"] == 120
414
  assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)
 
 
 
 
415
  assert list(history_data["series"].keys()) == ["hourly", "daily", "weekly", "monthly"]
416
  assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
417
  assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120
@@ -419,10 +521,14 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
419
  0.24
420
  )
421
 
 
 
 
422
  with TestClient(create_app(config)) as client:
423
  history = client.get("/stats-history")
424
  assert history.status_code == 200
425
  assert history.json()["lifetime"]["tokens_saved"] == 40
 
426
 
427
  _record_request(client, model="gpt-4o", tokens_saved=15)
428
 
@@ -430,7 +536,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
430
  assert updated["lifetime"]["tokens_saved"] == 55
431
  assert updated["lifetime"]["total_input_tokens"] == 240
432
  assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
 
433
  assert len(updated["history"]) == 2
 
 
 
 
434
  assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
435
  assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
436
 
@@ -438,6 +549,7 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
438
  assert persisted["lifetime"]["tokens_saved"] == 55
439
  assert persisted["lifetime"]["total_input_tokens"] == 240
440
  assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
 
441
 
442
 
443
  def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
 
18
  from headroom.proxy.server import ProxyConfig, create_app
19
 
20
 
21
+ def _record_request(
22
+ client: TestClient,
23
+ *,
24
+ model: str,
25
+ tokens_saved: int,
26
+ input_tokens: int = 120,
27
+ ) -> None:
28
  proxy = client.app.state.proxy
29
  if proxy.cost_tracker:
30
+ proxy.cost_tracker.record_tokens(model, tokens_saved, input_tokens)
31
  asyncio.run(
32
  proxy.metrics.record_request(
33
  provider="openai",
34
  model=model,
35
+ input_tokens=input_tokens,
36
  output_tokens=24,
37
  tokens_saved=tokens_saved,
38
  latency_ms=15.0,
 
105
  tracker = SavingsTracker(path=str(path), max_history_points=1, max_history_age_days=2)
106
  snapshot = tracker.snapshot()
107
 
108
+ assert snapshot["schema_version"] == 2
109
  assert snapshot["lifetime"] == {
110
+ "requests": 0,
111
  "tokens_saved": 30,
112
  "compression_savings_usd": pytest.approx(0.03),
113
  "total_input_tokens": 0,
114
  "total_input_cost_usd": 0.0,
115
  }
116
+ assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
117
  assert snapshot["history"] == [
118
  {
119
  "timestamp": "2026-03-27T09:00:00Z",
 
137
  snapshot = tracker.snapshot()
138
 
139
  assert snapshot["lifetime"] == {
140
+ "requests": 0,
141
  "tokens_saved": 0,
142
  "compression_savings_usd": 0.0,
143
  "total_input_tokens": 0,
144
  "total_input_cost_usd": 0.0,
145
  }
146
+ assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
147
  assert snapshot["history"] == []
148
 
149
 
 
233
  assert savings_tracker_module._estimate_compression_savings_usd(
234
  "claude-sonnet-4-6", 100
235
  ) == pytest.approx(0.2)
236
+ assert savings_tracker_module._estimate_input_cost_usd(
237
+ "claude-sonnet-4-6",
238
+ 100,
239
+ cache_read_tokens=10,
240
+ cache_write_tokens=5,
241
+ uncached_input_tokens=85,
242
+ ) == pytest.approx(0.2)
243
 
244
  fake_litellm.model_cost = {}
245
  assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
246
+ assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
247
 
248
  monkeypatch.setattr(
249
  fake_litellm,
 
255
 
256
  monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)
257
  assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
258
+ assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
259
+
260
+
261
+ def test_display_session_rolls_after_inactivity_and_counts_zero_savings_requests(
262
+ tmp_path, monkeypatch
263
+ ):
264
+ path = tmp_path / "proxy_savings.json"
265
+ tracker = SavingsTracker(path=str(path), display_session_inactivity_minutes=30)
266
+ monkeypatch.setattr(
267
+ savings_tracker_module,
268
+ "_estimate_compression_savings_usd",
269
+ lambda model, tokens_saved: tokens_saved / 1000.0,
270
+ )
271
+ monkeypatch.setattr(
272
+ savings_tracker_module,
273
+ "_estimate_input_cost_usd",
274
+ lambda model, input_tokens, **kwargs: input_tokens / 1000.0,
275
+ )
276
+
277
+ tracker.record_request(
278
+ model="gpt-4o",
279
+ input_tokens=120,
280
+ tokens_saved=0,
281
+ timestamp="2026-03-27T09:00:00Z",
282
+ )
283
+ tracker.record_request(
284
+ model="gpt-4o",
285
+ input_tokens=80,
286
+ tokens_saved=20,
287
+ timestamp="2026-03-27T09:10:00Z",
288
+ )
289
+
290
+ monkeypatch.setattr(
291
+ savings_tracker_module,
292
+ "_utc_now",
293
+ lambda: datetime(2026, 3, 27, 9, 15, tzinfo=timezone.utc),
294
+ )
295
+ active_session = tracker.snapshot()["display_session"]
296
+ assert active_session == {
297
+ "requests": 2,
298
+ "tokens_saved": 20,
299
+ "compression_savings_usd": pytest.approx(0.02),
300
+ "total_input_tokens": 200,
301
+ "total_input_cost_usd": pytest.approx(0.2),
302
+ "savings_percent": pytest.approx(9.09),
303
+ "started_at": "2026-03-27T09:00:00Z",
304
+ "last_activity_at": "2026-03-27T09:10:00Z",
305
+ }
306
+
307
+ monkeypatch.setattr(
308
+ savings_tracker_module,
309
+ "_utc_now",
310
+ lambda: datetime(2026, 3, 27, 9, 45, tzinfo=timezone.utc),
311
+ )
312
+ assert tracker.snapshot()["display_session"] == savings_tracker_module._empty_display_session()
313
+
314
+ tracker.record_request(
315
+ model="gpt-4o",
316
+ input_tokens=50,
317
+ tokens_saved=5,
318
+ timestamp="2026-03-27T10:05:00Z",
319
+ )
320
+
321
+ monkeypatch.setattr(
322
+ savings_tracker_module,
323
+ "_utc_now",
324
+ lambda: datetime(2026, 3, 27, 10, 10, tzinfo=timezone.utc),
325
+ )
326
+ rolled = tracker.snapshot()
327
+ assert rolled["lifetime"]["requests"] == 3
328
+ assert rolled["display_session"] == {
329
+ "requests": 1,
330
+ "tokens_saved": 5,
331
+ "compression_savings_usd": pytest.approx(0.005),
332
+ "total_input_tokens": 50,
333
+ "total_input_cost_usd": pytest.approx(0.05),
334
+ "savings_percent": pytest.approx(9.09),
335
+ "started_at": "2026-03-27T10:05:00Z",
336
+ "last_activity_at": "2026-03-27T10:05:00Z",
337
+ }
338
 
339
 
340
  def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):
 
505
  history = client.get("/stats-history")
506
  assert history.status_code == 200
507
  history_data = history.json()
508
+ assert history_data["schema_version"] == 2
509
  assert history_data["storage_path"] == str(savings_path)
510
  assert history_data["lifetime"]["tokens_saved"] == 40
511
  assert history_data["lifetime"]["total_input_tokens"] == 120
512
  assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)
513
+ assert history_data["display_session"]["requests"] == 1
514
+ assert history_data["display_session"]["tokens_saved"] == 40
515
+ assert history_data["display_session"]["total_input_tokens"] == 120
516
+ assert history_data["display_session"]["savings_percent"] == pytest.approx(25.0)
517
  assert list(history_data["series"].keys()) == ["hourly", "daily", "weekly", "monthly"]
518
  assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
519
  assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120
 
521
  0.24
522
  )
523
 
524
+ assert stats_data["display_session"] == history_data["display_session"]
525
+ assert stats_data["persistent_savings"]["display_session"] == history_data["display_session"]
526
+
527
  with TestClient(create_app(config)) as client:
528
  history = client.get("/stats-history")
529
  assert history.status_code == 200
530
  assert history.json()["lifetime"]["tokens_saved"] == 40
531
+ assert history.json()["display_session"]["requests"] == 1
532
 
533
  _record_request(client, model="gpt-4o", tokens_saved=15)
534
 
 
536
  assert updated["lifetime"]["tokens_saved"] == 55
537
  assert updated["lifetime"]["total_input_tokens"] == 240
538
  assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
539
+ assert updated["lifetime"]["requests"] == 2
540
  assert len(updated["history"]) == 2
541
+ assert updated["display_session"]["requests"] == 2
542
+ assert updated["display_session"]["tokens_saved"] == 55
543
+ assert updated["display_session"]["total_input_tokens"] == 240
544
+ assert updated["display_session"]["savings_percent"] == pytest.approx(18.64)
545
  assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
546
  assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
547
 
 
549
  assert persisted["lifetime"]["tokens_saved"] == 55
550
  assert persisted["lifetime"]["total_input_tokens"] == 240
551
  assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
552
+ assert persisted["display_session"]["requests"] == 2
553
 
554
 
555
  def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):