Spaces:
Build error
Build error
Garm commited on
Commit ·
d30a2e7
1
Parent(s): f9ef7e0
Add canonical display-session metrics for downstream dashboards
Browse files- headroom/cli/proxy.py +1 -0
- headroom/proxy/savings_tracker.py +272 -5
- headroom/proxy/server.py +18 -9
- tests/test_proxy_savings_history.py +117 -5
headroom/cli/proxy.py
CHANGED
|
@@ -324,6 +324,7 @@ Usage:
|
|
| 324 |
Endpoints:
|
| 325 |
GET /health Health check
|
| 326 |
GET /stats Detailed statistics
|
|
|
|
| 327 |
GET /metrics Prometheus metrics
|
| 328 |
|
| 329 |
Press Ctrl+C to stop.
|
|
|
|
| 324 |
Endpoints:
|
| 325 |
GET /health Health check
|
| 326 |
GET /stats Detailed statistics
|
| 327 |
+
GET /stats-history Durable compression history + display session
|
| 328 |
GET /metrics Prometheus metrics
|
| 329 |
|
| 330 |
Press Ctrl+C to stop.
|
headroom/proxy/savings_tracker.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
-
"""Durable proxy savings
|
| 2 |
|
| 3 |
-
Persists cumulative proxy compression savings
|
| 4 |
-
|
| 5 |
-
Headroom frontends.
|
| 6 |
"""
|
| 7 |
|
| 8 |
from __future__ import annotations
|
|
@@ -23,9 +23,10 @@ logger = logging.getLogger(__name__)
|
|
| 23 |
HEADROOM_SAVINGS_PATH_ENV_VAR = "HEADROOM_SAVINGS_PATH"
|
| 24 |
DEFAULT_SAVINGS_DIR = ".headroom"
|
| 25 |
DEFAULT_SAVINGS_FILE = "proxy_savings.json"
|
| 26 |
-
SCHEMA_VERSION =
|
| 27 |
DEFAULT_MAX_HISTORY_POINTS = 5000
|
| 28 |
DEFAULT_MAX_HISTORY_AGE_DAYS = 365
|
|
|
|
| 29 |
|
| 30 |
try:
|
| 31 |
import litellm
|
|
@@ -140,6 +141,48 @@ def _estimate_compression_savings_usd(model: str, tokens_saved: int) -> float:
|
|
| 140 |
return 0.0
|
| 141 |
|
| 142 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
|
| 144 |
"""Normalize persisted history entries across schema shapes."""
|
| 145 |
timestamp: datetime | None = None
|
|
@@ -178,6 +221,49 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
|
|
| 178 |
}
|
| 179 |
|
| 180 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
class SavingsTracker:
|
| 182 |
"""Persist bounded proxy compression savings history."""
|
| 183 |
|
|
@@ -186,10 +272,15 @@ class SavingsTracker:
|
|
| 186 |
path: str | None = None,
|
| 187 |
max_history_points: int = DEFAULT_MAX_HISTORY_POINTS,
|
| 188 |
max_history_age_days: int = DEFAULT_MAX_HISTORY_AGE_DAYS,
|
|
|
|
| 189 |
) -> None:
|
| 190 |
self._path = Path(path or get_default_savings_storage_path())
|
| 191 |
self._max_history_points = max_history_points
|
| 192 |
self._max_history_age_days = max_history_age_days
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
self._lock = threading.Lock()
|
| 194 |
self._state = self._load_state()
|
| 195 |
|
|
@@ -257,6 +348,126 @@ class SavingsTracker:
|
|
| 257 |
self._save_locked()
|
| 258 |
return True
|
| 259 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
def stats_preview(self, recent_points: int = 20) -> dict[str, Any]:
|
| 261 |
"""Return a compact preview for `/stats`."""
|
| 262 |
snapshot = self.snapshot()
|
|
@@ -264,6 +475,8 @@ class SavingsTracker:
|
|
| 264 |
"schema_version": snapshot["schema_version"],
|
| 265 |
"storage_path": snapshot["storage_path"],
|
| 266 |
"lifetime": snapshot["lifetime"],
|
|
|
|
|
|
|
| 267 |
"history_points": len(snapshot["history"]),
|
| 268 |
"recent_history": snapshot["history"][-recent_points:],
|
| 269 |
"retention": snapshot["retention"],
|
|
@@ -284,6 +497,8 @@ class SavingsTracker:
|
|
| 284 |
"generated_at": _to_utc_iso(_utc_now()),
|
| 285 |
"storage_path": snapshot["storage_path"],
|
| 286 |
"lifetime": snapshot["lifetime"],
|
|
|
|
|
|
|
| 287 |
"history": history,
|
| 288 |
"series": series,
|
| 289 |
"exports": {
|
|
@@ -339,6 +554,10 @@ class SavingsTracker:
|
|
| 339 |
"schema_version": SCHEMA_VERSION,
|
| 340 |
"storage_path": str(self._path),
|
| 341 |
"lifetime": dict(self._state["lifetime"]),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
"history": history,
|
| 343 |
"retention": {
|
| 344 |
"max_history_points": self._max_history_points,
|
|
@@ -350,11 +569,13 @@ class SavingsTracker:
|
|
| 350 |
return {
|
| 351 |
"schema_version": SCHEMA_VERSION,
|
| 352 |
"lifetime": {
|
|
|
|
| 353 |
"tokens_saved": 0,
|
| 354 |
"compression_savings_usd": 0.0,
|
| 355 |
"total_input_tokens": 0,
|
| 356 |
"total_input_cost_usd": 0.0,
|
| 357 |
},
|
|
|
|
| 358 |
"history": [],
|
| 359 |
}
|
| 360 |
|
|
@@ -386,11 +607,13 @@ class SavingsTracker:
|
|
| 386 |
normalized_history.sort(key=lambda item: item["timestamp"])
|
| 387 |
|
| 388 |
lifetime_raw = raw.get("lifetime", {})
|
|
|
|
| 389 |
lifetime_tokens_saved = 0
|
| 390 |
lifetime_savings_usd = 0.0
|
| 391 |
lifetime_input_tokens = 0
|
| 392 |
lifetime_input_cost_usd = 0.0
|
| 393 |
if isinstance(lifetime_raw, dict):
|
|
|
|
| 394 |
lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))
|
| 395 |
lifetime_savings_usd = _coerce_float(lifetime_raw.get("compression_savings_usd"))
|
| 396 |
lifetime_input_tokens = _coerce_int(lifetime_raw.get("total_input_tokens"))
|
|
@@ -415,11 +638,13 @@ class SavingsTracker:
|
|
| 415 |
state = {
|
| 416 |
"schema_version": SCHEMA_VERSION,
|
| 417 |
"lifetime": {
|
|
|
|
| 418 |
"tokens_saved": lifetime_tokens_saved,
|
| 419 |
"compression_savings_usd": round(lifetime_savings_usd, 6),
|
| 420 |
"total_input_tokens": lifetime_input_tokens,
|
| 421 |
"total_input_cost_usd": round(lifetime_input_cost_usd, 6),
|
| 422 |
},
|
|
|
|
| 423 |
"history": normalized_history,
|
| 424 |
}
|
| 425 |
|
|
@@ -463,6 +688,7 @@ class SavingsTracker:
|
|
| 463 |
payload = {
|
| 464 |
"schema_version": SCHEMA_VERSION,
|
| 465 |
"lifetime": self._state["lifetime"],
|
|
|
|
| 466 |
"history": self._state["history"],
|
| 467 |
}
|
| 468 |
json_data = json.dumps(payload, indent=2)
|
|
@@ -487,6 +713,47 @@ class SavingsTracker:
|
|
| 487 |
except OSError as e:
|
| 488 |
logger.warning("Failed to save savings history to %s: %s", self._path, e)
|
| 489 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
def _build_rollup(self, history: list[dict[str, Any]], bucket: str) -> list[dict[str, Any]]:
|
| 491 |
if not history:
|
| 492 |
return []
|
|
|
|
| 1 |
+
"""Durable proxy savings and display-session tracking.
|
| 2 |
|
| 3 |
+
Persists cumulative proxy compression savings plus a canonical display session
|
| 4 |
+
window to a local JSON file so historical charts and dashboard session stats
|
| 5 |
+
survive proxy restarts and can be shared by multiple Headroom frontends.
|
| 6 |
"""
|
| 7 |
|
| 8 |
from __future__ import annotations
|
|
|
|
| 23 |
HEADROOM_SAVINGS_PATH_ENV_VAR = "HEADROOM_SAVINGS_PATH"
|
| 24 |
DEFAULT_SAVINGS_DIR = ".headroom"
|
| 25 |
DEFAULT_SAVINGS_FILE = "proxy_savings.json"
|
| 26 |
+
SCHEMA_VERSION = 2
|
| 27 |
DEFAULT_MAX_HISTORY_POINTS = 5000
|
| 28 |
DEFAULT_MAX_HISTORY_AGE_DAYS = 365
|
| 29 |
+
DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES = 60
|
| 30 |
|
| 31 |
try:
|
| 32 |
import litellm
|
|
|
|
| 141 |
return 0.0
|
| 142 |
|
| 143 |
|
| 144 |
+
def _estimate_input_cost_usd(
|
| 145 |
+
model: str,
|
| 146 |
+
input_tokens: int,
|
| 147 |
+
*,
|
| 148 |
+
cache_read_tokens: int = 0,
|
| 149 |
+
cache_write_tokens: int = 0,
|
| 150 |
+
uncached_input_tokens: int = 0,
|
| 151 |
+
) -> float:
|
| 152 |
+
"""Estimate input spend in USD for a request.
|
| 153 |
+
|
| 154 |
+
Uses provider cache pricing when a complete cache breakdown is available and
|
| 155 |
+
otherwise falls back to list-price input tokens.
|
| 156 |
+
"""
|
| 157 |
+
total_input_tokens = _coerce_int(input_tokens)
|
| 158 |
+
if total_input_tokens <= 0 or not LITELLM_AVAILABLE:
|
| 159 |
+
return 0.0
|
| 160 |
+
|
| 161 |
+
cache_read = _coerce_int(cache_read_tokens)
|
| 162 |
+
cache_write = _coerce_int(cache_write_tokens)
|
| 163 |
+
uncached = _coerce_int(uncached_input_tokens)
|
| 164 |
+
|
| 165 |
+
try:
|
| 166 |
+
resolved = _resolve_litellm_model(model)
|
| 167 |
+
info = litellm.model_cost.get(resolved, {})
|
| 168 |
+
input_cost_per_token = info.get("input_cost_per_token")
|
| 169 |
+
if not input_cost_per_token:
|
| 170 |
+
return 0.0
|
| 171 |
+
|
| 172 |
+
if cache_read + cache_write + uncached > 0:
|
| 173 |
+
cache_read_cost = info.get("cache_read_input_token_cost", input_cost_per_token)
|
| 174 |
+
cache_write_cost = info.get("cache_creation_input_token_cost", input_cost_per_token)
|
| 175 |
+
return (
|
| 176 |
+
float(cache_read) * float(cache_read_cost)
|
| 177 |
+
+ float(cache_write) * float(cache_write_cost)
|
| 178 |
+
+ float(uncached) * float(input_cost_per_token)
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
return float(total_input_tokens) * float(input_cost_per_token)
|
| 182 |
+
except Exception:
|
| 183 |
+
return 0.0
|
| 184 |
+
|
| 185 |
+
|
| 186 |
def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
|
| 187 |
"""Normalize persisted history entries across schema shapes."""
|
| 188 |
timestamp: datetime | None = None
|
|
|
|
| 221 |
}
|
| 222 |
|
| 223 |
|
| 224 |
+
def _empty_display_session() -> dict[str, Any]:
|
| 225 |
+
return {
|
| 226 |
+
"requests": 0,
|
| 227 |
+
"tokens_saved": 0,
|
| 228 |
+
"compression_savings_usd": 0.0,
|
| 229 |
+
"total_input_tokens": 0,
|
| 230 |
+
"total_input_cost_usd": 0.0,
|
| 231 |
+
"savings_percent": 0.0,
|
| 232 |
+
"started_at": None,
|
| 233 |
+
"last_activity_at": None,
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _normalize_display_session(entry: Any) -> dict[str, Any]:
|
| 238 |
+
if not isinstance(entry, dict):
|
| 239 |
+
return _empty_display_session()
|
| 240 |
+
|
| 241 |
+
started_at = _parse_timestamp(entry.get("started_at"))
|
| 242 |
+
last_activity_at = _parse_timestamp(entry.get("last_activity_at"))
|
| 243 |
+
|
| 244 |
+
if started_at is None or last_activity_at is None or last_activity_at < started_at:
|
| 245 |
+
return _empty_display_session()
|
| 246 |
+
|
| 247 |
+
tokens_saved = _coerce_int(entry.get("tokens_saved"))
|
| 248 |
+
total_input_tokens = _coerce_int(entry.get("total_input_tokens"))
|
| 249 |
+
total_before = tokens_saved + total_input_tokens
|
| 250 |
+
savings_percent = round((tokens_saved / total_before * 100) if total_before > 0 else 0.0, 2)
|
| 251 |
+
|
| 252 |
+
return {
|
| 253 |
+
"requests": _coerce_int(entry.get("requests")),
|
| 254 |
+
"tokens_saved": tokens_saved,
|
| 255 |
+
"compression_savings_usd": round(
|
| 256 |
+
_coerce_float(entry.get("compression_savings_usd")),
|
| 257 |
+
6,
|
| 258 |
+
),
|
| 259 |
+
"total_input_tokens": total_input_tokens,
|
| 260 |
+
"total_input_cost_usd": round(_coerce_float(entry.get("total_input_cost_usd")), 6),
|
| 261 |
+
"savings_percent": savings_percent,
|
| 262 |
+
"started_at": _to_utc_iso(started_at),
|
| 263 |
+
"last_activity_at": _to_utc_iso(last_activity_at),
|
| 264 |
+
}
|
| 265 |
+
|
| 266 |
+
|
| 267 |
class SavingsTracker:
|
| 268 |
"""Persist bounded proxy compression savings history."""
|
| 269 |
|
|
|
|
| 272 |
path: str | None = None,
|
| 273 |
max_history_points: int = DEFAULT_MAX_HISTORY_POINTS,
|
| 274 |
max_history_age_days: int = DEFAULT_MAX_HISTORY_AGE_DAYS,
|
| 275 |
+
display_session_inactivity_minutes: int = DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES,
|
| 276 |
) -> None:
|
| 277 |
self._path = Path(path or get_default_savings_storage_path())
|
| 278 |
self._max_history_points = max_history_points
|
| 279 |
self._max_history_age_days = max_history_age_days
|
| 280 |
+
self._display_session_inactivity_minutes = max(
|
| 281 |
+
_coerce_int(display_session_inactivity_minutes, DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES),
|
| 282 |
+
1,
|
| 283 |
+
)
|
| 284 |
self._lock = threading.Lock()
|
| 285 |
self._state = self._load_state()
|
| 286 |
|
|
|
|
| 348 |
self._save_locked()
|
| 349 |
return True
|
| 350 |
|
| 351 |
+
def record_request(
|
| 352 |
+
self,
|
| 353 |
+
*,
|
| 354 |
+
model: str,
|
| 355 |
+
input_tokens: int,
|
| 356 |
+
tokens_saved: int,
|
| 357 |
+
cache_read_tokens: int = 0,
|
| 358 |
+
cache_write_tokens: int = 0,
|
| 359 |
+
uncached_input_tokens: int = 0,
|
| 360 |
+
total_input_tokens: int | None = None,
|
| 361 |
+
total_input_cost_usd: float | None = None,
|
| 362 |
+
timestamp: datetime | str | None = None,
|
| 363 |
+
) -> bool:
|
| 364 |
+
"""Persist a canonical display-session update for every request."""
|
| 365 |
+
timestamp_dt = (
|
| 366 |
+
_parse_timestamp(timestamp)
|
| 367 |
+
if isinstance(timestamp, str)
|
| 368 |
+
else timestamp.astimezone(timezone.utc)
|
| 369 |
+
if isinstance(timestamp, datetime)
|
| 370 |
+
else _utc_now()
|
| 371 |
+
)
|
| 372 |
+
if timestamp_dt is None:
|
| 373 |
+
timestamp_dt = _utc_now()
|
| 374 |
+
|
| 375 |
+
delta_tokens_saved = _coerce_int(tokens_saved)
|
| 376 |
+
delta_input_tokens = _coerce_int(input_tokens)
|
| 377 |
+
delta_savings_usd = _estimate_compression_savings_usd(model, delta_tokens_saved)
|
| 378 |
+
delta_input_cost_usd = _estimate_input_cost_usd(
|
| 379 |
+
model,
|
| 380 |
+
delta_input_tokens,
|
| 381 |
+
cache_read_tokens=cache_read_tokens,
|
| 382 |
+
cache_write_tokens=cache_write_tokens,
|
| 383 |
+
uncached_input_tokens=uncached_input_tokens,
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
with self._lock:
|
| 387 |
+
lifetime = self._state["lifetime"]
|
| 388 |
+
previous_total_input_tokens = lifetime["total_input_tokens"]
|
| 389 |
+
previous_total_input_cost_usd = lifetime["total_input_cost_usd"]
|
| 390 |
+
|
| 391 |
+
next_total_input_tokens = max(
|
| 392 |
+
previous_total_input_tokens + delta_input_tokens,
|
| 393 |
+
_coerce_int(
|
| 394 |
+
total_input_tokens,
|
| 395 |
+
default=previous_total_input_tokens + delta_input_tokens,
|
| 396 |
+
),
|
| 397 |
+
)
|
| 398 |
+
next_total_input_cost_usd = round(
|
| 399 |
+
max(
|
| 400 |
+
previous_total_input_cost_usd + delta_input_cost_usd,
|
| 401 |
+
_coerce_float(
|
| 402 |
+
total_input_cost_usd,
|
| 403 |
+
default=previous_total_input_cost_usd + delta_input_cost_usd,
|
| 404 |
+
),
|
| 405 |
+
),
|
| 406 |
+
6,
|
| 407 |
+
)
|
| 408 |
+
session_input_tokens_delta = max(
|
| 409 |
+
next_total_input_tokens - previous_total_input_tokens,
|
| 410 |
+
0,
|
| 411 |
+
)
|
| 412 |
+
session_input_cost_delta = round(
|
| 413 |
+
max(next_total_input_cost_usd - previous_total_input_cost_usd, 0.0),
|
| 414 |
+
6,
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
lifetime["requests"] += 1
|
| 418 |
+
lifetime["tokens_saved"] += delta_tokens_saved
|
| 419 |
+
lifetime["compression_savings_usd"] = round(
|
| 420 |
+
lifetime["compression_savings_usd"] + delta_savings_usd,
|
| 421 |
+
6,
|
| 422 |
+
)
|
| 423 |
+
lifetime["total_input_tokens"] = next_total_input_tokens
|
| 424 |
+
lifetime["total_input_cost_usd"] = next_total_input_cost_usd
|
| 425 |
+
|
| 426 |
+
session = self._state["display_session"]
|
| 427 |
+
last_activity = _parse_timestamp(session.get("last_activity_at"))
|
| 428 |
+
if last_activity is None or self._is_display_session_expired(
|
| 429 |
+
last_activity,
|
| 430 |
+
reference_time=timestamp_dt,
|
| 431 |
+
):
|
| 432 |
+
session = _empty_display_session()
|
| 433 |
+
session["started_at"] = _to_utc_iso(timestamp_dt)
|
| 434 |
+
self._state["display_session"] = session
|
| 435 |
+
|
| 436 |
+
session["requests"] += 1
|
| 437 |
+
session["tokens_saved"] += delta_tokens_saved
|
| 438 |
+
session["compression_savings_usd"] = round(
|
| 439 |
+
session["compression_savings_usd"] + delta_savings_usd,
|
| 440 |
+
6,
|
| 441 |
+
)
|
| 442 |
+
session["total_input_tokens"] += session_input_tokens_delta
|
| 443 |
+
session["total_input_cost_usd"] = round(
|
| 444 |
+
session["total_input_cost_usd"] + session_input_cost_delta,
|
| 445 |
+
6,
|
| 446 |
+
)
|
| 447 |
+
total_before = session["tokens_saved"] + session["total_input_tokens"]
|
| 448 |
+
session["savings_percent"] = round(
|
| 449 |
+
(session["tokens_saved"] / total_before * 100) if total_before > 0 else 0.0,
|
| 450 |
+
2,
|
| 451 |
+
)
|
| 452 |
+
session["last_activity_at"] = _to_utc_iso(timestamp_dt)
|
| 453 |
+
if session.get("started_at") is None:
|
| 454 |
+
session["started_at"] = session["last_activity_at"]
|
| 455 |
+
|
| 456 |
+
if delta_tokens_saved > 0:
|
| 457 |
+
self._state["history"].append(
|
| 458 |
+
{
|
| 459 |
+
"timestamp": _to_utc_iso(timestamp_dt),
|
| 460 |
+
"total_tokens_saved": lifetime["tokens_saved"],
|
| 461 |
+
"compression_savings_usd": lifetime["compression_savings_usd"],
|
| 462 |
+
"total_input_tokens": lifetime["total_input_tokens"],
|
| 463 |
+
"total_input_cost_usd": lifetime["total_input_cost_usd"],
|
| 464 |
+
}
|
| 465 |
+
)
|
| 466 |
+
self._trim_history_locked(reference_time=timestamp_dt)
|
| 467 |
+
|
| 468 |
+
self._save_locked()
|
| 469 |
+
return True
|
| 470 |
+
|
| 471 |
def stats_preview(self, recent_points: int = 20) -> dict[str, Any]:
|
| 472 |
"""Return a compact preview for `/stats`."""
|
| 473 |
snapshot = self.snapshot()
|
|
|
|
| 475 |
"schema_version": snapshot["schema_version"],
|
| 476 |
"storage_path": snapshot["storage_path"],
|
| 477 |
"lifetime": snapshot["lifetime"],
|
| 478 |
+
"display_session": snapshot["display_session"],
|
| 479 |
+
"display_session_policy": snapshot["display_session_policy"],
|
| 480 |
"history_points": len(snapshot["history"]),
|
| 481 |
"recent_history": snapshot["history"][-recent_points:],
|
| 482 |
"retention": snapshot["retention"],
|
|
|
|
| 497 |
"generated_at": _to_utc_iso(_utc_now()),
|
| 498 |
"storage_path": snapshot["storage_path"],
|
| 499 |
"lifetime": snapshot["lifetime"],
|
| 500 |
+
"display_session": snapshot["display_session"],
|
| 501 |
+
"display_session_policy": snapshot["display_session_policy"],
|
| 502 |
"history": history,
|
| 503 |
"series": series,
|
| 504 |
"exports": {
|
|
|
|
| 554 |
"schema_version": SCHEMA_VERSION,
|
| 555 |
"storage_path": str(self._path),
|
| 556 |
"lifetime": dict(self._state["lifetime"]),
|
| 557 |
+
"display_session": self._display_session_snapshot_locked(),
|
| 558 |
+
"display_session_policy": {
|
| 559 |
+
"rollover_inactivity_minutes": self._display_session_inactivity_minutes,
|
| 560 |
+
},
|
| 561 |
"history": history,
|
| 562 |
"retention": {
|
| 563 |
"max_history_points": self._max_history_points,
|
|
|
|
| 569 |
return {
|
| 570 |
"schema_version": SCHEMA_VERSION,
|
| 571 |
"lifetime": {
|
| 572 |
+
"requests": 0,
|
| 573 |
"tokens_saved": 0,
|
| 574 |
"compression_savings_usd": 0.0,
|
| 575 |
"total_input_tokens": 0,
|
| 576 |
"total_input_cost_usd": 0.0,
|
| 577 |
},
|
| 578 |
+
"display_session": _empty_display_session(),
|
| 579 |
"history": [],
|
| 580 |
}
|
| 581 |
|
|
|
|
| 607 |
normalized_history.sort(key=lambda item: item["timestamp"])
|
| 608 |
|
| 609 |
lifetime_raw = raw.get("lifetime", {})
|
| 610 |
+
lifetime_requests = 0
|
| 611 |
lifetime_tokens_saved = 0
|
| 612 |
lifetime_savings_usd = 0.0
|
| 613 |
lifetime_input_tokens = 0
|
| 614 |
lifetime_input_cost_usd = 0.0
|
| 615 |
if isinstance(lifetime_raw, dict):
|
| 616 |
+
lifetime_requests = _coerce_int(lifetime_raw.get("requests"))
|
| 617 |
lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))
|
| 618 |
lifetime_savings_usd = _coerce_float(lifetime_raw.get("compression_savings_usd"))
|
| 619 |
lifetime_input_tokens = _coerce_int(lifetime_raw.get("total_input_tokens"))
|
|
|
|
| 638 |
state = {
|
| 639 |
"schema_version": SCHEMA_VERSION,
|
| 640 |
"lifetime": {
|
| 641 |
+
"requests": lifetime_requests,
|
| 642 |
"tokens_saved": lifetime_tokens_saved,
|
| 643 |
"compression_savings_usd": round(lifetime_savings_usd, 6),
|
| 644 |
"total_input_tokens": lifetime_input_tokens,
|
| 645 |
"total_input_cost_usd": round(lifetime_input_cost_usd, 6),
|
| 646 |
},
|
| 647 |
+
"display_session": _normalize_display_session(raw.get("display_session")),
|
| 648 |
"history": normalized_history,
|
| 649 |
}
|
| 650 |
|
|
|
|
| 688 |
payload = {
|
| 689 |
"schema_version": SCHEMA_VERSION,
|
| 690 |
"lifetime": self._state["lifetime"],
|
| 691 |
+
"display_session": self._state["display_session"],
|
| 692 |
"history": self._state["history"],
|
| 693 |
}
|
| 694 |
json_data = json.dumps(payload, indent=2)
|
|
|
|
| 713 |
except OSError as e:
|
| 714 |
logger.warning("Failed to save savings history to %s: %s", self._path, e)
|
| 715 |
|
| 716 |
+
def _display_session_snapshot_locked(
|
| 717 |
+
self,
|
| 718 |
+
reference_time: datetime | None = None,
|
| 719 |
+
) -> dict[str, Any]:
|
| 720 |
+
session = dict(self._state["display_session"])
|
| 721 |
+
last_activity = _parse_timestamp(session.get("last_activity_at"))
|
| 722 |
+
if last_activity is None or self._is_display_session_expired(
|
| 723 |
+
last_activity,
|
| 724 |
+
reference_time=reference_time,
|
| 725 |
+
):
|
| 726 |
+
return _empty_display_session()
|
| 727 |
+
|
| 728 |
+
total_before = _coerce_int(session.get("tokens_saved")) + _coerce_int(
|
| 729 |
+
session.get("total_input_tokens")
|
| 730 |
+
)
|
| 731 |
+
session["savings_percent"] = round(
|
| 732 |
+
(_coerce_int(session.get("tokens_saved")) / total_before * 100)
|
| 733 |
+
if total_before > 0
|
| 734 |
+
else 0.0,
|
| 735 |
+
2,
|
| 736 |
+
)
|
| 737 |
+
session["compression_savings_usd"] = round(
|
| 738 |
+
_coerce_float(session.get("compression_savings_usd")),
|
| 739 |
+
6,
|
| 740 |
+
)
|
| 741 |
+
session["total_input_cost_usd"] = round(
|
| 742 |
+
_coerce_float(session.get("total_input_cost_usd")),
|
| 743 |
+
6,
|
| 744 |
+
)
|
| 745 |
+
return session
|
| 746 |
+
|
| 747 |
+
def _is_display_session_expired(
|
| 748 |
+
self,
|
| 749 |
+
last_activity: datetime,
|
| 750 |
+
*,
|
| 751 |
+
reference_time: datetime | None = None,
|
| 752 |
+
) -> bool:
|
| 753 |
+
return (reference_time or _utc_now()) - last_activity > timedelta(
|
| 754 |
+
minutes=self._display_session_inactivity_minutes
|
| 755 |
+
)
|
| 756 |
+
|
| 757 |
def _build_rollup(self, history: list[dict[str, Any]], bucket: str) -> list[dict[str, Any]]:
|
| 758 |
if not history:
|
| 759 |
return []
|
headroom/proxy/server.py
CHANGED
|
@@ -1269,6 +1269,7 @@ class PrometheusMetrics:
|
|
| 1269 |
waste_signals: dict[str, int] | None = None,
|
| 1270 |
cache_read_tokens: int = 0,
|
| 1271 |
cache_write_tokens: int = 0,
|
|
|
|
| 1272 |
):
|
| 1273 |
"""Record metrics for a request."""
|
| 1274 |
async with self._lock:
|
|
@@ -1342,14 +1343,17 @@ class PrometheusMetrics:
|
|
| 1342 |
if len(self.savings_history) > 500:
|
| 1343 |
self.savings_history = self.savings_history[-500:]
|
| 1344 |
|
| 1345 |
-
|
| 1346 |
-
|
| 1347 |
-
|
| 1348 |
-
|
| 1349 |
-
|
| 1350 |
-
|
| 1351 |
-
|
| 1352 |
-
|
|
|
|
|
|
|
|
|
|
| 1353 |
|
| 1354 |
async def record_rate_limited(self):
|
| 1355 |
async with self._lock:
|
|
@@ -3082,6 +3086,7 @@ class HeadroomProxy:
|
|
| 3082 |
waste_signals=waste_signals_dict,
|
| 3083 |
cache_read_tokens=cr_tokens,
|
| 3084 |
cache_write_tokens=cw_tokens,
|
|
|
|
| 3085 |
)
|
| 3086 |
|
| 3087 |
# Log request
|
|
@@ -4843,6 +4848,7 @@ class HeadroomProxy:
|
|
| 4843 |
pipeline_timing=pipeline_timing,
|
| 4844 |
cache_read_tokens=cache_read_tokens,
|
| 4845 |
cache_write_tokens=cache_write_tokens,
|
|
|
|
| 4846 |
)
|
| 4847 |
|
| 4848 |
return StreamingResponse(
|
|
@@ -7386,6 +7392,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
|
|
| 7386 |
- Request metrics (total, cached, failed, by model/provider)
|
| 7387 |
- Token usage and savings
|
| 7388 |
- Cost tracking
|
|
|
|
| 7389 |
- Compression (CCR) statistics
|
| 7390 |
- Telemetry/TOIN (data flywheel) statistics
|
| 7391 |
- Cache and rate limiter stats
|
|
@@ -7483,6 +7490,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
|
|
| 7483 |
cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
|
| 7484 |
total_tokens_all_layers = compression_tokens + cli_tokens_avoided
|
| 7485 |
persistent_savings = m.savings_tracker.stats_preview()
|
|
|
|
| 7486 |
|
| 7487 |
return {
|
| 7488 |
"summary": summary,
|
|
@@ -7558,6 +7566,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
|
|
| 7558 |
else {},
|
| 7559 |
"waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
|
| 7560 |
"savings_history": m.savings_history[-100:], # Last 100 data points
|
|
|
|
| 7561 |
"persistent_savings": persistent_savings,
|
| 7562 |
"prefix_cache": prefix_cache_stats,
|
| 7563 |
"cost": _merge_cost_stats(
|
|
@@ -7605,7 +7614,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
|
|
| 7605 |
format: Literal["json", "csv"] = "json",
|
| 7606 |
series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
|
| 7607 |
):
|
| 7608 |
-
"""Get durable proxy compression
|
| 7609 |
if format == "csv":
|
| 7610 |
filename = f"headroom-stats-history-{series}.csv"
|
| 7611 |
return Response(
|
|
|
|
| 1269 |
waste_signals: dict[str, int] | None = None,
|
| 1270 |
cache_read_tokens: int = 0,
|
| 1271 |
cache_write_tokens: int = 0,
|
| 1272 |
+
uncached_input_tokens: int = 0,
|
| 1273 |
):
|
| 1274 |
"""Record metrics for a request."""
|
| 1275 |
async with self._lock:
|
|
|
|
| 1343 |
if len(self.savings_history) > 500:
|
| 1344 |
self.savings_history = self.savings_history[-500:]
|
| 1345 |
|
| 1346 |
+
total_input_tokens, total_input_cost_usd = self._current_savings_tracker_totals()
|
| 1347 |
+
self.savings_tracker.record_request(
|
| 1348 |
+
model=model,
|
| 1349 |
+
input_tokens=input_tokens,
|
| 1350 |
+
tokens_saved=tokens_saved,
|
| 1351 |
+
cache_read_tokens=cache_read_tokens,
|
| 1352 |
+
cache_write_tokens=cache_write_tokens,
|
| 1353 |
+
uncached_input_tokens=uncached_input_tokens,
|
| 1354 |
+
total_input_tokens=total_input_tokens,
|
| 1355 |
+
total_input_cost_usd=total_input_cost_usd,
|
| 1356 |
+
)
|
| 1357 |
|
| 1358 |
async def record_rate_limited(self):
|
| 1359 |
async with self._lock:
|
|
|
|
| 3086 |
waste_signals=waste_signals_dict,
|
| 3087 |
cache_read_tokens=cr_tokens,
|
| 3088 |
cache_write_tokens=cw_tokens,
|
| 3089 |
+
uncached_input_tokens=uncached_input_tokens,
|
| 3090 |
)
|
| 3091 |
|
| 3092 |
# Log request
|
|
|
|
| 4848 |
pipeline_timing=pipeline_timing,
|
| 4849 |
cache_read_tokens=cache_read_tokens,
|
| 4850 |
cache_write_tokens=cache_write_tokens,
|
| 4851 |
+
uncached_input_tokens=uncached_input_tokens,
|
| 4852 |
)
|
| 4853 |
|
| 4854 |
return StreamingResponse(
|
|
|
|
| 7392 |
- Request metrics (total, cached, failed, by model/provider)
|
| 7393 |
- Token usage and savings
|
| 7394 |
- Cost tracking
|
| 7395 |
+
- Canonical persisted display_session metrics for downstream dashboards
|
| 7396 |
- Compression (CCR) statistics
|
| 7397 |
- Telemetry/TOIN (data flywheel) statistics
|
| 7398 |
- Cache and rate limiter stats
|
|
|
|
| 7490 |
cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
|
| 7491 |
total_tokens_all_layers = compression_tokens + cli_tokens_avoided
|
| 7492 |
persistent_savings = m.savings_tracker.stats_preview()
|
| 7493 |
+
display_session = persistent_savings.get("display_session", {})
|
| 7494 |
|
| 7495 |
return {
|
| 7496 |
"summary": summary,
|
|
|
|
| 7566 |
else {},
|
| 7567 |
"waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
|
| 7568 |
"savings_history": m.savings_history[-100:], # Last 100 data points
|
| 7569 |
+
"display_session": display_session,
|
| 7570 |
"persistent_savings": persistent_savings,
|
| 7571 |
"prefix_cache": prefix_cache_stats,
|
| 7572 |
"cost": _merge_cost_stats(
|
|
|
|
| 7614 |
format: Literal["json", "csv"] = "json",
|
| 7615 |
series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
|
| 7616 |
):
|
| 7617 |
+
"""Get durable proxy compression history plus display-session state."""
|
| 7618 |
if format == "csv":
|
| 7619 |
filename = f"headroom-stats-history-{series}.csv"
|
| 7620 |
return Response(
|
tests/test_proxy_savings_history.py
CHANGED
|
@@ -18,15 +18,21 @@ from headroom.proxy.savings_tracker import HEADROOM_SAVINGS_PATH_ENV_VAR, Saving
|
|
| 18 |
from headroom.proxy.server import ProxyConfig, create_app
|
| 19 |
|
| 20 |
|
| 21 |
-
def _record_request(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
proxy = client.app.state.proxy
|
| 23 |
if proxy.cost_tracker:
|
| 24 |
-
proxy.cost_tracker.record_tokens(model, tokens_saved,
|
| 25 |
asyncio.run(
|
| 26 |
proxy.metrics.record_request(
|
| 27 |
provider="openai",
|
| 28 |
model=model,
|
| 29 |
-
input_tokens=
|
| 30 |
output_tokens=24,
|
| 31 |
tokens_saved=tokens_saved,
|
| 32 |
latency_ms=15.0,
|
|
@@ -99,13 +105,15 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
|
|
| 99 |
tracker = SavingsTracker(path=str(path), max_history_points=1, max_history_age_days=2)
|
| 100 |
snapshot = tracker.snapshot()
|
| 101 |
|
| 102 |
-
assert snapshot["schema_version"] ==
|
| 103 |
assert snapshot["lifetime"] == {
|
|
|
|
| 104 |
"tokens_saved": 30,
|
| 105 |
"compression_savings_usd": pytest.approx(0.03),
|
| 106 |
"total_input_tokens": 0,
|
| 107 |
"total_input_cost_usd": 0.0,
|
| 108 |
}
|
|
|
|
| 109 |
assert snapshot["history"] == [
|
| 110 |
{
|
| 111 |
"timestamp": "2026-03-27T09:00:00Z",
|
|
@@ -129,11 +137,13 @@ def test_non_dict_savings_state_resets_to_default(tmp_path):
|
|
| 129 |
snapshot = tracker.snapshot()
|
| 130 |
|
| 131 |
assert snapshot["lifetime"] == {
|
|
|
|
| 132 |
"tokens_saved": 0,
|
| 133 |
"compression_savings_usd": 0.0,
|
| 134 |
"total_input_tokens": 0,
|
| 135 |
"total_input_cost_usd": 0.0,
|
| 136 |
}
|
|
|
|
| 137 |
assert snapshot["history"] == []
|
| 138 |
|
| 139 |
|
|
@@ -223,9 +233,17 @@ def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):
|
|
| 223 |
assert savings_tracker_module._estimate_compression_savings_usd(
|
| 224 |
"claude-sonnet-4-6", 100
|
| 225 |
) == pytest.approx(0.2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
|
| 227 |
fake_litellm.model_cost = {}
|
| 228 |
assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
|
|
|
|
| 229 |
|
| 230 |
monkeypatch.setattr(
|
| 231 |
fake_litellm,
|
|
@@ -237,6 +255,86 @@ def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):
|
|
| 237 |
|
| 238 |
monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)
|
| 239 |
assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
|
| 241 |
|
| 242 |
def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):
|
|
@@ -407,11 +505,15 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
|
|
| 407 |
history = client.get("/stats-history")
|
| 408 |
assert history.status_code == 200
|
| 409 |
history_data = history.json()
|
| 410 |
-
assert history_data["schema_version"] ==
|
| 411 |
assert history_data["storage_path"] == str(savings_path)
|
| 412 |
assert history_data["lifetime"]["tokens_saved"] == 40
|
| 413 |
assert history_data["lifetime"]["total_input_tokens"] == 120
|
| 414 |
assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
assert list(history_data["series"].keys()) == ["hourly", "daily", "weekly", "monthly"]
|
| 416 |
assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
|
| 417 |
assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120
|
|
@@ -419,10 +521,14 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
|
|
| 419 |
0.24
|
| 420 |
)
|
| 421 |
|
|
|
|
|
|
|
|
|
|
| 422 |
with TestClient(create_app(config)) as client:
|
| 423 |
history = client.get("/stats-history")
|
| 424 |
assert history.status_code == 200
|
| 425 |
assert history.json()["lifetime"]["tokens_saved"] == 40
|
|
|
|
| 426 |
|
| 427 |
_record_request(client, model="gpt-4o", tokens_saved=15)
|
| 428 |
|
|
@@ -430,7 +536,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
|
|
| 430 |
assert updated["lifetime"]["tokens_saved"] == 55
|
| 431 |
assert updated["lifetime"]["total_input_tokens"] == 240
|
| 432 |
assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
|
|
|
|
| 433 |
assert len(updated["history"]) == 2
|
|
|
|
|
|
|
|
|
|
|
|
|
| 434 |
assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
|
| 435 |
assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
|
| 436 |
|
|
@@ -438,6 +549,7 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
|
|
| 438 |
assert persisted["lifetime"]["tokens_saved"] == 55
|
| 439 |
assert persisted["lifetime"]["total_input_tokens"] == 240
|
| 440 |
assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
|
|
|
|
| 441 |
|
| 442 |
|
| 443 |
def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
|
|
|
|
| 18 |
from headroom.proxy.server import ProxyConfig, create_app
|
| 19 |
|
| 20 |
|
| 21 |
+
def _record_request(
|
| 22 |
+
client: TestClient,
|
| 23 |
+
*,
|
| 24 |
+
model: str,
|
| 25 |
+
tokens_saved: int,
|
| 26 |
+
input_tokens: int = 120,
|
| 27 |
+
) -> None:
|
| 28 |
proxy = client.app.state.proxy
|
| 29 |
if proxy.cost_tracker:
|
| 30 |
+
proxy.cost_tracker.record_tokens(model, tokens_saved, input_tokens)
|
| 31 |
asyncio.run(
|
| 32 |
proxy.metrics.record_request(
|
| 33 |
provider="openai",
|
| 34 |
model=model,
|
| 35 |
+
input_tokens=input_tokens,
|
| 36 |
output_tokens=24,
|
| 37 |
tokens_saved=tokens_saved,
|
| 38 |
latency_ms=15.0,
|
|
|
|
| 105 |
tracker = SavingsTracker(path=str(path), max_history_points=1, max_history_age_days=2)
|
| 106 |
snapshot = tracker.snapshot()
|
| 107 |
|
| 108 |
+
assert snapshot["schema_version"] == 2
|
| 109 |
assert snapshot["lifetime"] == {
|
| 110 |
+
"requests": 0,
|
| 111 |
"tokens_saved": 30,
|
| 112 |
"compression_savings_usd": pytest.approx(0.03),
|
| 113 |
"total_input_tokens": 0,
|
| 114 |
"total_input_cost_usd": 0.0,
|
| 115 |
}
|
| 116 |
+
assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
|
| 117 |
assert snapshot["history"] == [
|
| 118 |
{
|
| 119 |
"timestamp": "2026-03-27T09:00:00Z",
|
|
|
|
| 137 |
snapshot = tracker.snapshot()
|
| 138 |
|
| 139 |
assert snapshot["lifetime"] == {
|
| 140 |
+
"requests": 0,
|
| 141 |
"tokens_saved": 0,
|
| 142 |
"compression_savings_usd": 0.0,
|
| 143 |
"total_input_tokens": 0,
|
| 144 |
"total_input_cost_usd": 0.0,
|
| 145 |
}
|
| 146 |
+
assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
|
| 147 |
assert snapshot["history"] == []
|
| 148 |
|
| 149 |
|
|
|
|
| 233 |
assert savings_tracker_module._estimate_compression_savings_usd(
|
| 234 |
"claude-sonnet-4-6", 100
|
| 235 |
) == pytest.approx(0.2)
|
| 236 |
+
assert savings_tracker_module._estimate_input_cost_usd(
|
| 237 |
+
"claude-sonnet-4-6",
|
| 238 |
+
100,
|
| 239 |
+
cache_read_tokens=10,
|
| 240 |
+
cache_write_tokens=5,
|
| 241 |
+
uncached_input_tokens=85,
|
| 242 |
+
) == pytest.approx(0.2)
|
| 243 |
|
| 244 |
fake_litellm.model_cost = {}
|
| 245 |
assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
|
| 246 |
+
assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
|
| 247 |
|
| 248 |
monkeypatch.setattr(
|
| 249 |
fake_litellm,
|
|
|
|
| 255 |
|
| 256 |
monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)
|
| 257 |
assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
|
| 258 |
+
assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def test_display_session_rolls_after_inactivity_and_counts_zero_savings_requests(
|
| 262 |
+
tmp_path, monkeypatch
|
| 263 |
+
):
|
| 264 |
+
path = tmp_path / "proxy_savings.json"
|
| 265 |
+
tracker = SavingsTracker(path=str(path), display_session_inactivity_minutes=30)
|
| 266 |
+
monkeypatch.setattr(
|
| 267 |
+
savings_tracker_module,
|
| 268 |
+
"_estimate_compression_savings_usd",
|
| 269 |
+
lambda model, tokens_saved: tokens_saved / 1000.0,
|
| 270 |
+
)
|
| 271 |
+
monkeypatch.setattr(
|
| 272 |
+
savings_tracker_module,
|
| 273 |
+
"_estimate_input_cost_usd",
|
| 274 |
+
lambda model, input_tokens, **kwargs: input_tokens / 1000.0,
|
| 275 |
+
)
|
| 276 |
+
|
| 277 |
+
tracker.record_request(
|
| 278 |
+
model="gpt-4o",
|
| 279 |
+
input_tokens=120,
|
| 280 |
+
tokens_saved=0,
|
| 281 |
+
timestamp="2026-03-27T09:00:00Z",
|
| 282 |
+
)
|
| 283 |
+
tracker.record_request(
|
| 284 |
+
model="gpt-4o",
|
| 285 |
+
input_tokens=80,
|
| 286 |
+
tokens_saved=20,
|
| 287 |
+
timestamp="2026-03-27T09:10:00Z",
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
monkeypatch.setattr(
|
| 291 |
+
savings_tracker_module,
|
| 292 |
+
"_utc_now",
|
| 293 |
+
lambda: datetime(2026, 3, 27, 9, 15, tzinfo=timezone.utc),
|
| 294 |
+
)
|
| 295 |
+
active_session = tracker.snapshot()["display_session"]
|
| 296 |
+
assert active_session == {
|
| 297 |
+
"requests": 2,
|
| 298 |
+
"tokens_saved": 20,
|
| 299 |
+
"compression_savings_usd": pytest.approx(0.02),
|
| 300 |
+
"total_input_tokens": 200,
|
| 301 |
+
"total_input_cost_usd": pytest.approx(0.2),
|
| 302 |
+
"savings_percent": pytest.approx(9.09),
|
| 303 |
+
"started_at": "2026-03-27T09:00:00Z",
|
| 304 |
+
"last_activity_at": "2026-03-27T09:10:00Z",
|
| 305 |
+
}
|
| 306 |
+
|
| 307 |
+
monkeypatch.setattr(
|
| 308 |
+
savings_tracker_module,
|
| 309 |
+
"_utc_now",
|
| 310 |
+
lambda: datetime(2026, 3, 27, 9, 45, tzinfo=timezone.utc),
|
| 311 |
+
)
|
| 312 |
+
assert tracker.snapshot()["display_session"] == savings_tracker_module._empty_display_session()
|
| 313 |
+
|
| 314 |
+
tracker.record_request(
|
| 315 |
+
model="gpt-4o",
|
| 316 |
+
input_tokens=50,
|
| 317 |
+
tokens_saved=5,
|
| 318 |
+
timestamp="2026-03-27T10:05:00Z",
|
| 319 |
+
)
|
| 320 |
+
|
| 321 |
+
monkeypatch.setattr(
|
| 322 |
+
savings_tracker_module,
|
| 323 |
+
"_utc_now",
|
| 324 |
+
lambda: datetime(2026, 3, 27, 10, 10, tzinfo=timezone.utc),
|
| 325 |
+
)
|
| 326 |
+
rolled = tracker.snapshot()
|
| 327 |
+
assert rolled["lifetime"]["requests"] == 3
|
| 328 |
+
assert rolled["display_session"] == {
|
| 329 |
+
"requests": 1,
|
| 330 |
+
"tokens_saved": 5,
|
| 331 |
+
"compression_savings_usd": pytest.approx(0.005),
|
| 332 |
+
"total_input_tokens": 50,
|
| 333 |
+
"total_input_cost_usd": pytest.approx(0.05),
|
| 334 |
+
"savings_percent": pytest.approx(9.09),
|
| 335 |
+
"started_at": "2026-03-27T10:05:00Z",
|
| 336 |
+
"last_activity_at": "2026-03-27T10:05:00Z",
|
| 337 |
+
}
|
| 338 |
|
| 339 |
|
| 340 |
def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):
|
|
|
|
| 505 |
history = client.get("/stats-history")
|
| 506 |
assert history.status_code == 200
|
| 507 |
history_data = history.json()
|
| 508 |
+
assert history_data["schema_version"] == 2
|
| 509 |
assert history_data["storage_path"] == str(savings_path)
|
| 510 |
assert history_data["lifetime"]["tokens_saved"] == 40
|
| 511 |
assert history_data["lifetime"]["total_input_tokens"] == 120
|
| 512 |
assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)
|
| 513 |
+
assert history_data["display_session"]["requests"] == 1
|
| 514 |
+
assert history_data["display_session"]["tokens_saved"] == 40
|
| 515 |
+
assert history_data["display_session"]["total_input_tokens"] == 120
|
| 516 |
+
assert history_data["display_session"]["savings_percent"] == pytest.approx(25.0)
|
| 517 |
assert list(history_data["series"].keys()) == ["hourly", "daily", "weekly", "monthly"]
|
| 518 |
assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
|
| 519 |
assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120
|
|
|
|
| 521 |
0.24
|
| 522 |
)
|
| 523 |
|
| 524 |
+
assert stats_data["display_session"] == history_data["display_session"]
|
| 525 |
+
assert stats_data["persistent_savings"]["display_session"] == history_data["display_session"]
|
| 526 |
+
|
| 527 |
with TestClient(create_app(config)) as client:
|
| 528 |
history = client.get("/stats-history")
|
| 529 |
assert history.status_code == 200
|
| 530 |
assert history.json()["lifetime"]["tokens_saved"] == 40
|
| 531 |
+
assert history.json()["display_session"]["requests"] == 1
|
| 532 |
|
| 533 |
_record_request(client, model="gpt-4o", tokens_saved=15)
|
| 534 |
|
|
|
|
| 536 |
assert updated["lifetime"]["tokens_saved"] == 55
|
| 537 |
assert updated["lifetime"]["total_input_tokens"] == 240
|
| 538 |
assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
|
| 539 |
+
assert updated["lifetime"]["requests"] == 2
|
| 540 |
assert len(updated["history"]) == 2
|
| 541 |
+
assert updated["display_session"]["requests"] == 2
|
| 542 |
+
assert updated["display_session"]["tokens_saved"] == 55
|
| 543 |
+
assert updated["display_session"]["total_input_tokens"] == 240
|
| 544 |
+
assert updated["display_session"]["savings_percent"] == pytest.approx(18.64)
|
| 545 |
assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
|
| 546 |
assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
|
| 547 |
|
|
|
|
| 549 |
assert persisted["lifetime"]["tokens_saved"] == 55
|
| 550 |
assert persisted["lifetime"]["total_input_tokens"] == 240
|
| 551 |
assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
|
| 552 |
+
assert persisted["display_session"]["requests"] == 2
|
| 553 |
|
| 554 |
|
| 555 |
def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
|