Spaces:
Build error
Build error
Garm commited on
Commit ·
f9ef7e0
1
Parent(s): e5a93f2
feat: add spend and input-token rollups to stats-history
Browse files- headroom/proxy/savings_tracker.py +78 -2
- headroom/proxy/server.py +55 -2
- tests/test_proxy_savings_history.py +84 -2
headroom/proxy/savings_tracker.py
CHANGED
|
@@ -145,16 +145,24 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
|
|
| 145 |
timestamp: datetime | None = None
|
| 146 |
total_tokens_saved = 0
|
| 147 |
compression_savings_usd = 0.0
|
|
|
|
|
|
|
| 148 |
|
| 149 |
if isinstance(entry, dict):
|
| 150 |
timestamp = _parse_timestamp(entry.get("timestamp"))
|
| 151 |
total_tokens_saved = _coerce_int(entry.get("total_tokens_saved"))
|
| 152 |
compression_savings_usd = _coerce_float(entry.get("compression_savings_usd"))
|
|
|
|
|
|
|
| 153 |
elif isinstance(entry, (list, tuple)) and len(entry) >= 2:
|
| 154 |
timestamp = _parse_timestamp(entry[0])
|
| 155 |
total_tokens_saved = _coerce_int(entry[1])
|
| 156 |
if len(entry) >= 3:
|
| 157 |
compression_savings_usd = _coerce_float(entry[2])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
else:
|
| 159 |
return None
|
| 160 |
|
|
@@ -165,6 +173,8 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
|
|
| 165 |
"timestamp": _to_utc_iso(timestamp),
|
| 166 |
"total_tokens_saved": total_tokens_saved,
|
| 167 |
"compression_savings_usd": round(compression_savings_usd, 6),
|
|
|
|
|
|
|
| 168 |
}
|
| 169 |
|
| 170 |
|
|
@@ -192,6 +202,8 @@ class SavingsTracker:
|
|
| 192 |
*,
|
| 193 |
model: str,
|
| 194 |
tokens_saved: int,
|
|
|
|
|
|
|
| 195 |
timestamp: datetime | str | None = None,
|
| 196 |
) -> bool:
|
| 197 |
"""Persist a cumulative savings checkpoint when compression changed totals."""
|
|
@@ -217,12 +229,28 @@ class SavingsTracker:
|
|
| 217 |
lifetime["compression_savings_usd"] = round(
|
| 218 |
lifetime["compression_savings_usd"] + delta_usd, 6
|
| 219 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
|
| 221 |
self._state["history"].append(
|
| 222 |
{
|
| 223 |
"timestamp": _to_utc_iso(timestamp_dt),
|
| 224 |
"total_tokens_saved": lifetime["tokens_saved"],
|
| 225 |
"compression_savings_usd": lifetime["compression_savings_usd"],
|
|
|
|
|
|
|
| 226 |
}
|
| 227 |
)
|
| 228 |
self._trim_history_locked(reference_time=timestamp_dt)
|
|
@@ -277,7 +305,13 @@ class SavingsTracker:
|
|
| 277 |
"""Export history or rollup series as CSV."""
|
| 278 |
rows = self.export_rows(series=series)
|
| 279 |
if series == "history":
|
| 280 |
-
fieldnames = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
else:
|
| 282 |
fieldnames = [
|
| 283 |
"timestamp",
|
|
@@ -285,6 +319,10 @@ class SavingsTracker:
|
|
| 285 |
"compression_savings_usd_delta",
|
| 286 |
"total_tokens_saved",
|
| 287 |
"compression_savings_usd",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
]
|
| 289 |
|
| 290 |
buffer = StringIO()
|
|
@@ -311,7 +349,12 @@ class SavingsTracker:
|
|
| 311 |
def _default_state(self) -> dict[str, Any]:
|
| 312 |
return {
|
| 313 |
"schema_version": SCHEMA_VERSION,
|
| 314 |
-
"lifetime": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
"history": [],
|
| 316 |
}
|
| 317 |
|
|
@@ -345,9 +388,13 @@ class SavingsTracker:
|
|
| 345 |
lifetime_raw = raw.get("lifetime", {})
|
| 346 |
lifetime_tokens_saved = 0
|
| 347 |
lifetime_savings_usd = 0.0
|
|
|
|
|
|
|
| 348 |
if isinstance(lifetime_raw, dict):
|
| 349 |
lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))
|
| 350 |
lifetime_savings_usd = _coerce_float(lifetime_raw.get("compression_savings_usd"))
|
|
|
|
|
|
|
| 351 |
|
| 352 |
if normalized_history:
|
| 353 |
last = normalized_history[-1]
|
|
@@ -356,12 +403,22 @@ class SavingsTracker:
|
|
| 356 |
lifetime_savings_usd,
|
| 357 |
_coerce_float(last["compression_savings_usd"]),
|
| 358 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
|
| 360 |
state = {
|
| 361 |
"schema_version": SCHEMA_VERSION,
|
| 362 |
"lifetime": {
|
| 363 |
"tokens_saved": lifetime_tokens_saved,
|
| 364 |
"compression_savings_usd": round(lifetime_savings_usd, 6),
|
|
|
|
|
|
|
| 365 |
},
|
| 366 |
"history": normalized_history,
|
| 367 |
}
|
|
@@ -437,6 +494,8 @@ class SavingsTracker:
|
|
| 437 |
aggregated: dict[str, dict[str, Any]] = {}
|
| 438 |
prev_total_tokens = 0
|
| 439 |
prev_total_usd = 0.0
|
|
|
|
|
|
|
| 440 |
|
| 441 |
for point in history:
|
| 442 |
timestamp = _parse_timestamp(point["timestamp"])
|
|
@@ -448,11 +507,17 @@ class SavingsTracker:
|
|
| 448 |
bucket_key = _to_utc_iso(bucket_start)
|
| 449 |
total_tokens_saved = _coerce_int(point.get("total_tokens_saved"))
|
| 450 |
total_usd = _coerce_float(point.get("compression_savings_usd"))
|
|
|
|
|
|
|
| 451 |
delta_tokens = max(total_tokens_saved - prev_total_tokens, 0)
|
| 452 |
delta_usd = max(total_usd - prev_total_usd, 0.0)
|
|
|
|
|
|
|
| 453 |
|
| 454 |
prev_total_tokens = total_tokens_saved
|
| 455 |
prev_total_usd = total_usd
|
|
|
|
|
|
|
| 456 |
|
| 457 |
entry = aggregated.setdefault(
|
| 458 |
bucket_key,
|
|
@@ -462,6 +527,10 @@ class SavingsTracker:
|
|
| 462 |
"compression_savings_usd_delta": 0.0,
|
| 463 |
"total_tokens_saved": total_tokens_saved,
|
| 464 |
"compression_savings_usd": total_usd,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 465 |
},
|
| 466 |
)
|
| 467 |
entry["tokens_saved"] += delta_tokens
|
|
@@ -469,7 +538,14 @@ class SavingsTracker:
|
|
| 469 |
entry["compression_savings_usd_delta"] + delta_usd,
|
| 470 |
6,
|
| 471 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
entry["total_tokens_saved"] = total_tokens_saved
|
| 473 |
entry["compression_savings_usd"] = round(total_usd, 6)
|
|
|
|
|
|
|
| 474 |
|
| 475 |
return list(aggregated.values())
|
|
|
|
| 145 |
timestamp: datetime | None = None
|
| 146 |
total_tokens_saved = 0
|
| 147 |
compression_savings_usd = 0.0
|
| 148 |
+
total_input_tokens = 0
|
| 149 |
+
total_input_cost_usd = 0.0
|
| 150 |
|
| 151 |
if isinstance(entry, dict):
|
| 152 |
timestamp = _parse_timestamp(entry.get("timestamp"))
|
| 153 |
total_tokens_saved = _coerce_int(entry.get("total_tokens_saved"))
|
| 154 |
compression_savings_usd = _coerce_float(entry.get("compression_savings_usd"))
|
| 155 |
+
total_input_tokens = _coerce_int(entry.get("total_input_tokens"))
|
| 156 |
+
total_input_cost_usd = _coerce_float(entry.get("total_input_cost_usd"))
|
| 157 |
elif isinstance(entry, (list, tuple)) and len(entry) >= 2:
|
| 158 |
timestamp = _parse_timestamp(entry[0])
|
| 159 |
total_tokens_saved = _coerce_int(entry[1])
|
| 160 |
if len(entry) >= 3:
|
| 161 |
compression_savings_usd = _coerce_float(entry[2])
|
| 162 |
+
if len(entry) >= 4:
|
| 163 |
+
total_input_tokens = _coerce_int(entry[3])
|
| 164 |
+
if len(entry) >= 5:
|
| 165 |
+
total_input_cost_usd = _coerce_float(entry[4])
|
| 166 |
else:
|
| 167 |
return None
|
| 168 |
|
|
|
|
| 173 |
"timestamp": _to_utc_iso(timestamp),
|
| 174 |
"total_tokens_saved": total_tokens_saved,
|
| 175 |
"compression_savings_usd": round(compression_savings_usd, 6),
|
| 176 |
+
"total_input_tokens": total_input_tokens,
|
| 177 |
+
"total_input_cost_usd": round(total_input_cost_usd, 6),
|
| 178 |
}
|
| 179 |
|
| 180 |
|
|
|
|
| 202 |
*,
|
| 203 |
model: str,
|
| 204 |
tokens_saved: int,
|
| 205 |
+
total_input_tokens: int | None = None,
|
| 206 |
+
total_input_cost_usd: float | None = None,
|
| 207 |
timestamp: datetime | str | None = None,
|
| 208 |
) -> bool:
|
| 209 |
"""Persist a cumulative savings checkpoint when compression changed totals."""
|
|
|
|
| 229 |
lifetime["compression_savings_usd"] = round(
|
| 230 |
lifetime["compression_savings_usd"] + delta_usd, 6
|
| 231 |
)
|
| 232 |
+
lifetime["total_input_tokens"] = max(
|
| 233 |
+
lifetime["total_input_tokens"],
|
| 234 |
+
_coerce_int(total_input_tokens, default=lifetime["total_input_tokens"]),
|
| 235 |
+
)
|
| 236 |
+
lifetime["total_input_cost_usd"] = round(
|
| 237 |
+
max(
|
| 238 |
+
lifetime["total_input_cost_usd"],
|
| 239 |
+
_coerce_float(
|
| 240 |
+
total_input_cost_usd,
|
| 241 |
+
default=lifetime["total_input_cost_usd"],
|
| 242 |
+
),
|
| 243 |
+
),
|
| 244 |
+
6,
|
| 245 |
+
)
|
| 246 |
|
| 247 |
self._state["history"].append(
|
| 248 |
{
|
| 249 |
"timestamp": _to_utc_iso(timestamp_dt),
|
| 250 |
"total_tokens_saved": lifetime["tokens_saved"],
|
| 251 |
"compression_savings_usd": lifetime["compression_savings_usd"],
|
| 252 |
+
"total_input_tokens": lifetime["total_input_tokens"],
|
| 253 |
+
"total_input_cost_usd": lifetime["total_input_cost_usd"],
|
| 254 |
}
|
| 255 |
)
|
| 256 |
self._trim_history_locked(reference_time=timestamp_dt)
|
|
|
|
| 305 |
"""Export history or rollup series as CSV."""
|
| 306 |
rows = self.export_rows(series=series)
|
| 307 |
if series == "history":
|
| 308 |
+
fieldnames = [
|
| 309 |
+
"timestamp",
|
| 310 |
+
"total_tokens_saved",
|
| 311 |
+
"compression_savings_usd",
|
| 312 |
+
"total_input_tokens",
|
| 313 |
+
"total_input_cost_usd",
|
| 314 |
+
]
|
| 315 |
else:
|
| 316 |
fieldnames = [
|
| 317 |
"timestamp",
|
|
|
|
| 319 |
"compression_savings_usd_delta",
|
| 320 |
"total_tokens_saved",
|
| 321 |
"compression_savings_usd",
|
| 322 |
+
"total_input_tokens_delta",
|
| 323 |
+
"total_input_tokens",
|
| 324 |
+
"total_input_cost_usd_delta",
|
| 325 |
+
"total_input_cost_usd",
|
| 326 |
]
|
| 327 |
|
| 328 |
buffer = StringIO()
|
|
|
|
| 349 |
def _default_state(self) -> dict[str, Any]:
|
| 350 |
return {
|
| 351 |
"schema_version": SCHEMA_VERSION,
|
| 352 |
+
"lifetime": {
|
| 353 |
+
"tokens_saved": 0,
|
| 354 |
+
"compression_savings_usd": 0.0,
|
| 355 |
+
"total_input_tokens": 0,
|
| 356 |
+
"total_input_cost_usd": 0.0,
|
| 357 |
+
},
|
| 358 |
"history": [],
|
| 359 |
}
|
| 360 |
|
|
|
|
| 388 |
lifetime_raw = raw.get("lifetime", {})
|
| 389 |
lifetime_tokens_saved = 0
|
| 390 |
lifetime_savings_usd = 0.0
|
| 391 |
+
lifetime_input_tokens = 0
|
| 392 |
+
lifetime_input_cost_usd = 0.0
|
| 393 |
if isinstance(lifetime_raw, dict):
|
| 394 |
lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))
|
| 395 |
lifetime_savings_usd = _coerce_float(lifetime_raw.get("compression_savings_usd"))
|
| 396 |
+
lifetime_input_tokens = _coerce_int(lifetime_raw.get("total_input_tokens"))
|
| 397 |
+
lifetime_input_cost_usd = _coerce_float(lifetime_raw.get("total_input_cost_usd"))
|
| 398 |
|
| 399 |
if normalized_history:
|
| 400 |
last = normalized_history[-1]
|
|
|
|
| 403 |
lifetime_savings_usd,
|
| 404 |
_coerce_float(last["compression_savings_usd"]),
|
| 405 |
)
|
| 406 |
+
lifetime_input_tokens = max(
|
| 407 |
+
lifetime_input_tokens,
|
| 408 |
+
_coerce_int(last.get("total_input_tokens")),
|
| 409 |
+
)
|
| 410 |
+
lifetime_input_cost_usd = max(
|
| 411 |
+
lifetime_input_cost_usd,
|
| 412 |
+
_coerce_float(last.get("total_input_cost_usd")),
|
| 413 |
+
)
|
| 414 |
|
| 415 |
state = {
|
| 416 |
"schema_version": SCHEMA_VERSION,
|
| 417 |
"lifetime": {
|
| 418 |
"tokens_saved": lifetime_tokens_saved,
|
| 419 |
"compression_savings_usd": round(lifetime_savings_usd, 6),
|
| 420 |
+
"total_input_tokens": lifetime_input_tokens,
|
| 421 |
+
"total_input_cost_usd": round(lifetime_input_cost_usd, 6),
|
| 422 |
},
|
| 423 |
"history": normalized_history,
|
| 424 |
}
|
|
|
|
| 494 |
aggregated: dict[str, dict[str, Any]] = {}
|
| 495 |
prev_total_tokens = 0
|
| 496 |
prev_total_usd = 0.0
|
| 497 |
+
prev_total_input_tokens = 0
|
| 498 |
+
prev_total_input_cost_usd = 0.0
|
| 499 |
|
| 500 |
for point in history:
|
| 501 |
timestamp = _parse_timestamp(point["timestamp"])
|
|
|
|
| 507 |
bucket_key = _to_utc_iso(bucket_start)
|
| 508 |
total_tokens_saved = _coerce_int(point.get("total_tokens_saved"))
|
| 509 |
total_usd = _coerce_float(point.get("compression_savings_usd"))
|
| 510 |
+
total_input_tokens = _coerce_int(point.get("total_input_tokens"))
|
| 511 |
+
total_input_cost_usd = _coerce_float(point.get("total_input_cost_usd"))
|
| 512 |
delta_tokens = max(total_tokens_saved - prev_total_tokens, 0)
|
| 513 |
delta_usd = max(total_usd - prev_total_usd, 0.0)
|
| 514 |
+
delta_input_tokens = max(total_input_tokens - prev_total_input_tokens, 0)
|
| 515 |
+
delta_input_cost_usd = max(total_input_cost_usd - prev_total_input_cost_usd, 0.0)
|
| 516 |
|
| 517 |
prev_total_tokens = total_tokens_saved
|
| 518 |
prev_total_usd = total_usd
|
| 519 |
+
prev_total_input_tokens = total_input_tokens
|
| 520 |
+
prev_total_input_cost_usd = total_input_cost_usd
|
| 521 |
|
| 522 |
entry = aggregated.setdefault(
|
| 523 |
bucket_key,
|
|
|
|
| 527 |
"compression_savings_usd_delta": 0.0,
|
| 528 |
"total_tokens_saved": total_tokens_saved,
|
| 529 |
"compression_savings_usd": total_usd,
|
| 530 |
+
"total_input_tokens_delta": 0,
|
| 531 |
+
"total_input_tokens": total_input_tokens,
|
| 532 |
+
"total_input_cost_usd_delta": 0.0,
|
| 533 |
+
"total_input_cost_usd": total_input_cost_usd,
|
| 534 |
},
|
| 535 |
)
|
| 536 |
entry["tokens_saved"] += delta_tokens
|
|
|
|
| 538 |
entry["compression_savings_usd_delta"] + delta_usd,
|
| 539 |
6,
|
| 540 |
)
|
| 541 |
+
entry["total_input_tokens_delta"] += delta_input_tokens
|
| 542 |
+
entry["total_input_cost_usd_delta"] = round(
|
| 543 |
+
entry["total_input_cost_usd_delta"] + delta_input_cost_usd,
|
| 544 |
+
6,
|
| 545 |
+
)
|
| 546 |
entry["total_tokens_saved"] = total_tokens_saved
|
| 547 |
entry["compression_savings_usd"] = round(total_usd, 6)
|
| 548 |
+
entry["total_input_tokens"] = total_input_tokens
|
| 549 |
+
entry["total_input_cost_usd"] = round(total_input_cost_usd, 6)
|
| 550 |
|
| 551 |
return list(aggregated.values())
|
headroom/proxy/server.py
CHANGED
|
@@ -1137,7 +1137,11 @@ class CostTracker:
|
|
| 1137 |
class PrometheusMetrics:
|
| 1138 |
"""Prometheus-compatible metrics."""
|
| 1139 |
|
| 1140 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1141 |
self.requests_total = 0
|
| 1142 |
self.requests_by_provider: dict[str, int] = defaultdict(int)
|
| 1143 |
self.requests_by_model: dict[str, int] = defaultdict(int)
|
|
@@ -1201,9 +1205,55 @@ class PrometheusMetrics:
|
|
| 1201 |
# Cumulative savings history (timestamp → cumulative tokens saved)
|
| 1202 |
self.savings_history: list[tuple[str, int]] = []
|
| 1203 |
self.savings_tracker = savings_tracker or SavingsTracker()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1204 |
|
| 1205 |
self._lock = asyncio.Lock()
|
| 1206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1207 |
async def record_request(
|
| 1208 |
self,
|
| 1209 |
provider: str,
|
|
@@ -1293,9 +1343,12 @@ class PrometheusMetrics:
|
|
| 1293 |
self.savings_history = self.savings_history[-500:]
|
| 1294 |
|
| 1295 |
if tokens_saved > 0:
|
|
|
|
| 1296 |
self.savings_tracker.record_compression_savings(
|
| 1297 |
model=model,
|
| 1298 |
tokens_saved=tokens_saved,
|
|
|
|
|
|
|
| 1299 |
)
|
| 1300 |
|
| 1301 |
async def record_rate_limited(self):
|
|
@@ -1603,7 +1656,7 @@ class HeadroomProxy:
|
|
| 1603 |
else None
|
| 1604 |
)
|
| 1605 |
|
| 1606 |
-
self.metrics = PrometheusMetrics()
|
| 1607 |
|
| 1608 |
# Prefix cache tracking: freeze already-cached messages to avoid
|
| 1609 |
# invalidating the provider's prefix cache with our transforms
|
|
|
|
| 1137 |
class PrometheusMetrics:
|
| 1138 |
"""Prometheus-compatible metrics."""
|
| 1139 |
|
| 1140 |
+
def __init__(
|
| 1141 |
+
self,
|
| 1142 |
+
savings_tracker: SavingsTracker | None = None,
|
| 1143 |
+
cost_tracker: CostTracker | None = None,
|
| 1144 |
+
):
|
| 1145 |
self.requests_total = 0
|
| 1146 |
self.requests_by_provider: dict[str, int] = defaultdict(int)
|
| 1147 |
self.requests_by_model: dict[str, int] = defaultdict(int)
|
|
|
|
| 1205 |
# Cumulative savings history (timestamp → cumulative tokens saved)
|
| 1206 |
self.savings_history: list[tuple[str, int]] = []
|
| 1207 |
self.savings_tracker = savings_tracker or SavingsTracker()
|
| 1208 |
+
self.cost_tracker = cost_tracker
|
| 1209 |
+
tracker_lifetime = self.savings_tracker.snapshot()["lifetime"]
|
| 1210 |
+
self._savings_tracker_input_tokens_offset = max(
|
| 1211 |
+
int(tracker_lifetime.get("total_input_tokens", 0) or 0),
|
| 1212 |
+
0,
|
| 1213 |
+
)
|
| 1214 |
+
self._savings_tracker_input_cost_usd_offset = max(
|
| 1215 |
+
float(tracker_lifetime.get("total_input_cost_usd", 0.0) or 0.0),
|
| 1216 |
+
0.0,
|
| 1217 |
+
)
|
| 1218 |
|
| 1219 |
self._lock = asyncio.Lock()
|
| 1220 |
|
| 1221 |
+
def _current_savings_tracker_totals(self) -> tuple[int, float]:
|
| 1222 |
+
total_input_tokens = self._savings_tracker_input_tokens_offset + self.tokens_input_total
|
| 1223 |
+
total_input_cost_usd = self._savings_tracker_input_cost_usd_offset
|
| 1224 |
+
|
| 1225 |
+
if self.cost_tracker is None:
|
| 1226 |
+
return total_input_tokens, total_input_cost_usd
|
| 1227 |
+
|
| 1228 |
+
try:
|
| 1229 |
+
cost_stats = self.cost_tracker.stats()
|
| 1230 |
+
except Exception:
|
| 1231 |
+
logger.debug("Failed to read cost tracker totals for savings history", exc_info=True)
|
| 1232 |
+
return total_input_tokens, total_input_cost_usd
|
| 1233 |
+
|
| 1234 |
+
tracked_input_tokens = cost_stats.get("total_input_tokens")
|
| 1235 |
+
tracked_input_cost_usd = cost_stats.get("total_input_cost_usd")
|
| 1236 |
+
|
| 1237 |
+
if tracked_input_tokens is not None:
|
| 1238 |
+
try:
|
| 1239 |
+
total_input_tokens = self._savings_tracker_input_tokens_offset + max(
|
| 1240 |
+
int(tracked_input_tokens),
|
| 1241 |
+
0,
|
| 1242 |
+
)
|
| 1243 |
+
except (TypeError, ValueError):
|
| 1244 |
+
pass
|
| 1245 |
+
|
| 1246 |
+
if tracked_input_cost_usd is not None:
|
| 1247 |
+
try:
|
| 1248 |
+
total_input_cost_usd = self._savings_tracker_input_cost_usd_offset + max(
|
| 1249 |
+
float(tracked_input_cost_usd),
|
| 1250 |
+
0.0,
|
| 1251 |
+
)
|
| 1252 |
+
except (TypeError, ValueError):
|
| 1253 |
+
pass
|
| 1254 |
+
|
| 1255 |
+
return total_input_tokens, total_input_cost_usd
|
| 1256 |
+
|
| 1257 |
async def record_request(
|
| 1258 |
self,
|
| 1259 |
provider: str,
|
|
|
|
| 1343 |
self.savings_history = self.savings_history[-500:]
|
| 1344 |
|
| 1345 |
if tokens_saved > 0:
|
| 1346 |
+
total_input_tokens, total_input_cost_usd = self._current_savings_tracker_totals()
|
| 1347 |
self.savings_tracker.record_compression_savings(
|
| 1348 |
model=model,
|
| 1349 |
tokens_saved=tokens_saved,
|
| 1350 |
+
total_input_tokens=total_input_tokens,
|
| 1351 |
+
total_input_cost_usd=total_input_cost_usd,
|
| 1352 |
)
|
| 1353 |
|
| 1354 |
async def record_rate_limited(self):
|
|
|
|
| 1656 |
else None
|
| 1657 |
)
|
| 1658 |
|
| 1659 |
+
self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)
|
| 1660 |
|
| 1661 |
# Prefix cache tracking: freeze already-cached messages to avoid
|
| 1662 |
# invalidating the provider's prefix cache with our transforms
|
tests/test_proxy_savings_history.py
CHANGED
|
@@ -20,6 +20,8 @@ from headroom.proxy.server import ProxyConfig, create_app
|
|
| 20 |
|
| 21 |
def _record_request(client: TestClient, *, model: str, tokens_saved: int) -> None:
|
| 22 |
proxy = client.app.state.proxy
|
|
|
|
|
|
|
| 23 |
asyncio.run(
|
| 24 |
proxy.metrics.record_request(
|
| 25 |
provider="openai",
|
|
@@ -58,6 +60,8 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc
|
|
| 58 |
"timestamp": "2026-03-27T09:00:00Z",
|
| 59 |
"total_tokens_saved": 12,
|
| 60 |
"compression_savings_usd": 0.5,
|
|
|
|
|
|
|
| 61 |
}
|
| 62 |
assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is None
|
| 63 |
assert savings_tracker_module._normalize_history_entry(object()) is None
|
|
@@ -99,12 +103,16 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
|
|
| 99 |
assert snapshot["lifetime"] == {
|
| 100 |
"tokens_saved": 30,
|
| 101 |
"compression_savings_usd": pytest.approx(0.03),
|
|
|
|
|
|
|
| 102 |
}
|
| 103 |
assert snapshot["history"] == [
|
| 104 |
{
|
| 105 |
"timestamp": "2026-03-27T09:00:00Z",
|
| 106 |
"total_tokens_saved": 30,
|
| 107 |
"compression_savings_usd": 0.03,
|
|
|
|
|
|
|
| 108 |
}
|
| 109 |
]
|
| 110 |
assert snapshot["retention"] == {
|
|
@@ -123,6 +131,8 @@ def test_non_dict_savings_state_resets_to_default(tmp_path):
|
|
| 123 |
assert snapshot["lifetime"] == {
|
| 124 |
"tokens_saved": 0,
|
| 125 |
"compression_savings_usd": 0.0,
|
|
|
|
|
|
|
| 126 |
}
|
| 127 |
assert snapshot["history"] == []
|
| 128 |
|
|
@@ -145,6 +155,8 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp
|
|
| 145 |
assert tracker.record_compression_savings(
|
| 146 |
model="gpt-4o",
|
| 147 |
tokens_saved=10,
|
|
|
|
|
|
|
| 148 |
timestamp=local_time,
|
| 149 |
)
|
| 150 |
|
|
@@ -153,6 +165,8 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp
|
|
| 153 |
assert tracker.record_compression_savings(
|
| 154 |
model="gpt-4o",
|
| 155 |
tokens_saved=5,
|
|
|
|
|
|
|
| 156 |
timestamp="not-a-timestamp",
|
| 157 |
)
|
| 158 |
|
|
@@ -162,16 +176,22 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp
|
|
| 162 |
"timestamp": "2026-03-27T08:00:00Z",
|
| 163 |
"total_tokens_saved": 10,
|
| 164 |
"compression_savings_usd": 0.01,
|
|
|
|
|
|
|
| 165 |
},
|
| 166 |
{
|
| 167 |
"timestamp": "2026-03-27T12:34:00Z",
|
| 168 |
"total_tokens_saved": 15,
|
| 169 |
"compression_savings_usd": 0.015,
|
|
|
|
|
|
|
| 170 |
},
|
| 171 |
]
|
| 172 |
|
| 173 |
persisted = json.loads(path.read_text(encoding="utf-8"))
|
| 174 |
assert persisted["lifetime"]["tokens_saved"] == 15
|
|
|
|
|
|
|
| 175 |
assert persisted["history"][-1]["timestamp"] == "2026-03-27T12:34:00Z"
|
| 176 |
|
| 177 |
|
|
@@ -219,7 +239,7 @@ def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):
|
|
| 219 |
assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
|
| 220 |
|
| 221 |
|
| 222 |
-
def
|
| 223 |
path = tmp_path / "proxy_savings.json"
|
| 224 |
tracker = SavingsTracker(path=str(path), max_history_points=100, max_history_age_days=30)
|
| 225 |
monkeypatch.setattr(
|
|
@@ -230,26 +250,36 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):
|
|
| 230 |
tracker.record_compression_savings(
|
| 231 |
model="gpt-4o",
|
| 232 |
tokens_saved=100,
|
|
|
|
|
|
|
| 233 |
timestamp="2026-03-27T09:10:00Z",
|
| 234 |
)
|
| 235 |
tracker.record_compression_savings(
|
| 236 |
model="gpt-4o",
|
| 237 |
tokens_saved=50,
|
|
|
|
|
|
|
| 238 |
timestamp="2026-03-27T09:40:00Z",
|
| 239 |
)
|
| 240 |
tracker.record_compression_savings(
|
| 241 |
model="gpt-4o",
|
| 242 |
tokens_saved=25,
|
|
|
|
|
|
|
| 243 |
timestamp="2026-03-27T10:05:00Z",
|
| 244 |
)
|
| 245 |
tracker.record_compression_savings(
|
| 246 |
model="gpt-4o",
|
| 247 |
tokens_saved=10,
|
|
|
|
|
|
|
| 248 |
timestamp="2026-03-28T08:00:00Z",
|
| 249 |
)
|
| 250 |
tracker.record_compression_savings(
|
| 251 |
model="gpt-4o",
|
| 252 |
tokens_saved=20,
|
|
|
|
|
|
|
| 253 |
timestamp="2026-04-02T14:00:00Z",
|
| 254 |
)
|
| 255 |
|
|
@@ -257,6 +287,8 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):
|
|
| 257 |
|
| 258 |
assert response["lifetime"]["tokens_saved"] == 205
|
| 259 |
assert response["lifetime"]["compression_savings_usd"] == pytest.approx(0.205)
|
|
|
|
|
|
|
| 260 |
assert len(response["history"]) == 5
|
| 261 |
|
| 262 |
hourly = response["series"]["hourly"]
|
|
@@ -268,12 +300,28 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):
|
|
| 268 |
]
|
| 269 |
assert hourly[0]["tokens_saved"] == 150
|
| 270 |
assert hourly[0]["total_tokens_saved"] == 150
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
assert hourly[1]["tokens_saved"] == 25
|
| 272 |
assert hourly[1]["total_tokens_saved"] == 175
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
assert hourly[2]["tokens_saved"] == 10
|
| 274 |
assert hourly[2]["total_tokens_saved"] == 185
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
assert hourly[3]["tokens_saved"] == 20
|
| 276 |
assert hourly[3]["total_tokens_saved"] == 205
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
|
| 278 |
daily = response["series"]["daily"]
|
| 279 |
assert [point["timestamp"] for point in daily] == [
|
|
@@ -283,10 +331,22 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):
|
|
| 283 |
]
|
| 284 |
assert daily[0]["tokens_saved"] == 175
|
| 285 |
assert daily[0]["total_tokens_saved"] == 175
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
assert daily[1]["tokens_saved"] == 10
|
| 287 |
assert daily[1]["total_tokens_saved"] == 185
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
assert daily[2]["tokens_saved"] == 20
|
| 289 |
assert daily[2]["total_tokens_saved"] == 205
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
|
| 291 |
weekly = response["series"]["weekly"]
|
| 292 |
assert [point["timestamp"] for point in weekly] == [
|
|
@@ -321,6 +381,10 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):
|
|
| 321 |
def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):
|
| 322 |
savings_path = tmp_path / "proxy_savings.json"
|
| 323 |
monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
|
| 325 |
config = ProxyConfig(
|
| 326 |
cache_enabled=False,
|
|
@@ -346,8 +410,14 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
|
|
| 346 |
assert history_data["schema_version"] == 1
|
| 347 |
assert history_data["storage_path"] == str(savings_path)
|
| 348 |
assert history_data["lifetime"]["tokens_saved"] == 40
|
|
|
|
|
|
|
| 349 |
assert list(history_data["series"].keys()) == ["hourly", "daily", "weekly", "monthly"]
|
| 350 |
assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
|
| 352 |
with TestClient(create_app(config)) as client:
|
| 353 |
history = client.get("/stats-history")
|
|
@@ -358,15 +428,25 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
|
|
| 358 |
|
| 359 |
updated = client.get("/stats-history").json()
|
| 360 |
assert updated["lifetime"]["tokens_saved"] == 55
|
|
|
|
|
|
|
| 361 |
assert len(updated["history"]) == 2
|
|
|
|
|
|
|
| 362 |
|
| 363 |
persisted = json.loads(savings_path.read_text())
|
| 364 |
assert persisted["lifetime"]["tokens_saved"] == 55
|
|
|
|
|
|
|
| 365 |
|
| 366 |
|
| 367 |
def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
|
| 368 |
savings_path = tmp_path / "proxy_savings.json"
|
| 369 |
monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 370 |
|
| 371 |
config = ProxyConfig(
|
| 372 |
cache_enabled=False,
|
|
@@ -388,10 +468,12 @@ def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
|
|
| 388 |
lines = response.text.strip().splitlines()
|
| 389 |
assert lines[0] == (
|
| 390 |
"timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"
|
| 391 |
-
"compression_savings_usd"
|
|
|
|
| 392 |
)
|
| 393 |
assert len(lines) >= 2
|
| 394 |
assert "total_tokens_saved" in lines[0]
|
|
|
|
| 395 |
|
| 396 |
|
| 397 |
def test_malformed_savings_state_is_ignored_safely(tmp_path, monkeypatch):
|
|
|
|
| 20 |
|
| 21 |
def _record_request(client: TestClient, *, model: str, tokens_saved: int) -> None:
|
| 22 |
proxy = client.app.state.proxy
|
| 23 |
+
if proxy.cost_tracker:
|
| 24 |
+
proxy.cost_tracker.record_tokens(model, tokens_saved, 120)
|
| 25 |
asyncio.run(
|
| 26 |
proxy.metrics.record_request(
|
| 27 |
provider="openai",
|
|
|
|
| 60 |
"timestamp": "2026-03-27T09:00:00Z",
|
| 61 |
"total_tokens_saved": 12,
|
| 62 |
"compression_savings_usd": 0.5,
|
| 63 |
+
"total_input_tokens": 0,
|
| 64 |
+
"total_input_cost_usd": 0.0,
|
| 65 |
}
|
| 66 |
assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is None
|
| 67 |
assert savings_tracker_module._normalize_history_entry(object()) is None
|
|
|
|
| 103 |
assert snapshot["lifetime"] == {
|
| 104 |
"tokens_saved": 30,
|
| 105 |
"compression_savings_usd": pytest.approx(0.03),
|
| 106 |
+
"total_input_tokens": 0,
|
| 107 |
+
"total_input_cost_usd": 0.0,
|
| 108 |
}
|
| 109 |
assert snapshot["history"] == [
|
| 110 |
{
|
| 111 |
"timestamp": "2026-03-27T09:00:00Z",
|
| 112 |
"total_tokens_saved": 30,
|
| 113 |
"compression_savings_usd": 0.03,
|
| 114 |
+
"total_input_tokens": 0,
|
| 115 |
+
"total_input_cost_usd": 0.0,
|
| 116 |
}
|
| 117 |
]
|
| 118 |
assert snapshot["retention"] == {
|
|
|
|
| 131 |
assert snapshot["lifetime"] == {
|
| 132 |
"tokens_saved": 0,
|
| 133 |
"compression_savings_usd": 0.0,
|
| 134 |
+
"total_input_tokens": 0,
|
| 135 |
+
"total_input_cost_usd": 0.0,
|
| 136 |
}
|
| 137 |
assert snapshot["history"] == []
|
| 138 |
|
|
|
|
| 155 |
assert tracker.record_compression_savings(
|
| 156 |
model="gpt-4o",
|
| 157 |
tokens_saved=10,
|
| 158 |
+
total_input_tokens=120,
|
| 159 |
+
total_input_cost_usd=0.24,
|
| 160 |
timestamp=local_time,
|
| 161 |
)
|
| 162 |
|
|
|
|
| 165 |
assert tracker.record_compression_savings(
|
| 166 |
model="gpt-4o",
|
| 167 |
tokens_saved=5,
|
| 168 |
+
total_input_tokens=180,
|
| 169 |
+
total_input_cost_usd=0.36,
|
| 170 |
timestamp="not-a-timestamp",
|
| 171 |
)
|
| 172 |
|
|
|
|
| 176 |
"timestamp": "2026-03-27T08:00:00Z",
|
| 177 |
"total_tokens_saved": 10,
|
| 178 |
"compression_savings_usd": 0.01,
|
| 179 |
+
"total_input_tokens": 120,
|
| 180 |
+
"total_input_cost_usd": 0.24,
|
| 181 |
},
|
| 182 |
{
|
| 183 |
"timestamp": "2026-03-27T12:34:00Z",
|
| 184 |
"total_tokens_saved": 15,
|
| 185 |
"compression_savings_usd": 0.015,
|
| 186 |
+
"total_input_tokens": 180,
|
| 187 |
+
"total_input_cost_usd": 0.36,
|
| 188 |
},
|
| 189 |
]
|
| 190 |
|
| 191 |
persisted = json.loads(path.read_text(encoding="utf-8"))
|
| 192 |
assert persisted["lifetime"]["tokens_saved"] == 15
|
| 193 |
+
assert persisted["lifetime"]["total_input_tokens"] == 180
|
| 194 |
+
assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.36)
|
| 195 |
assert persisted["history"][-1]["timestamp"] == "2026-03-27T12:34:00Z"
|
| 196 |
|
| 197 |
|
|
|
|
| 239 |
assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
|
| 240 |
|
| 241 |
|
| 242 |
+
def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):
|
| 243 |
path = tmp_path / "proxy_savings.json"
|
| 244 |
tracker = SavingsTracker(path=str(path), max_history_points=100, max_history_age_days=30)
|
| 245 |
monkeypatch.setattr(
|
|
|
|
| 250 |
tracker.record_compression_savings(
|
| 251 |
model="gpt-4o",
|
| 252 |
tokens_saved=100,
|
| 253 |
+
total_input_tokens=120,
|
| 254 |
+
total_input_cost_usd=0.24,
|
| 255 |
timestamp="2026-03-27T09:10:00Z",
|
| 256 |
)
|
| 257 |
tracker.record_compression_savings(
|
| 258 |
model="gpt-4o",
|
| 259 |
tokens_saved=50,
|
| 260 |
+
total_input_tokens=210,
|
| 261 |
+
total_input_cost_usd=0.42,
|
| 262 |
timestamp="2026-03-27T09:40:00Z",
|
| 263 |
)
|
| 264 |
tracker.record_compression_savings(
|
| 265 |
model="gpt-4o",
|
| 266 |
tokens_saved=25,
|
| 267 |
+
total_input_tokens=300,
|
| 268 |
+
total_input_cost_usd=0.63,
|
| 269 |
timestamp="2026-03-27T10:05:00Z",
|
| 270 |
)
|
| 271 |
tracker.record_compression_savings(
|
| 272 |
model="gpt-4o",
|
| 273 |
tokens_saved=10,
|
| 274 |
+
total_input_tokens=360,
|
| 275 |
+
total_input_cost_usd=0.75,
|
| 276 |
timestamp="2026-03-28T08:00:00Z",
|
| 277 |
)
|
| 278 |
tracker.record_compression_savings(
|
| 279 |
model="gpt-4o",
|
| 280 |
tokens_saved=20,
|
| 281 |
+
total_input_tokens=450,
|
| 282 |
+
total_input_cost_usd=0.93,
|
| 283 |
timestamp="2026-04-02T14:00:00Z",
|
| 284 |
)
|
| 285 |
|
|
|
|
| 287 |
|
| 288 |
assert response["lifetime"]["tokens_saved"] == 205
|
| 289 |
assert response["lifetime"]["compression_savings_usd"] == pytest.approx(0.205)
|
| 290 |
+
assert response["lifetime"]["total_input_tokens"] == 450
|
| 291 |
+
assert response["lifetime"]["total_input_cost_usd"] == pytest.approx(0.93)
|
| 292 |
assert len(response["history"]) == 5
|
| 293 |
|
| 294 |
hourly = response["series"]["hourly"]
|
|
|
|
| 300 |
]
|
| 301 |
assert hourly[0]["tokens_saved"] == 150
|
| 302 |
assert hourly[0]["total_tokens_saved"] == 150
|
| 303 |
+
assert hourly[0]["total_input_tokens_delta"] == 210
|
| 304 |
+
assert hourly[0]["total_input_tokens"] == 210
|
| 305 |
+
assert hourly[0]["total_input_cost_usd_delta"] == pytest.approx(0.42)
|
| 306 |
+
assert hourly[0]["total_input_cost_usd"] == pytest.approx(0.42)
|
| 307 |
assert hourly[1]["tokens_saved"] == 25
|
| 308 |
assert hourly[1]["total_tokens_saved"] == 175
|
| 309 |
+
assert hourly[1]["total_input_tokens_delta"] == 90
|
| 310 |
+
assert hourly[1]["total_input_tokens"] == 300
|
| 311 |
+
assert hourly[1]["total_input_cost_usd_delta"] == pytest.approx(0.21)
|
| 312 |
+
assert hourly[1]["total_input_cost_usd"] == pytest.approx(0.63)
|
| 313 |
assert hourly[2]["tokens_saved"] == 10
|
| 314 |
assert hourly[2]["total_tokens_saved"] == 185
|
| 315 |
+
assert hourly[2]["total_input_tokens_delta"] == 60
|
| 316 |
+
assert hourly[2]["total_input_tokens"] == 360
|
| 317 |
+
assert hourly[2]["total_input_cost_usd_delta"] == pytest.approx(0.12)
|
| 318 |
+
assert hourly[2]["total_input_cost_usd"] == pytest.approx(0.75)
|
| 319 |
assert hourly[3]["tokens_saved"] == 20
|
| 320 |
assert hourly[3]["total_tokens_saved"] == 205
|
| 321 |
+
assert hourly[3]["total_input_tokens_delta"] == 90
|
| 322 |
+
assert hourly[3]["total_input_tokens"] == 450
|
| 323 |
+
assert hourly[3]["total_input_cost_usd_delta"] == pytest.approx(0.18)
|
| 324 |
+
assert hourly[3]["total_input_cost_usd"] == pytest.approx(0.93)
|
| 325 |
|
| 326 |
daily = response["series"]["daily"]
|
| 327 |
assert [point["timestamp"] for point in daily] == [
|
|
|
|
| 331 |
]
|
| 332 |
assert daily[0]["tokens_saved"] == 175
|
| 333 |
assert daily[0]["total_tokens_saved"] == 175
|
| 334 |
+
assert daily[0]["total_input_tokens_delta"] == 300
|
| 335 |
+
assert daily[0]["total_input_tokens"] == 300
|
| 336 |
+
assert daily[0]["total_input_cost_usd_delta"] == pytest.approx(0.63)
|
| 337 |
+
assert daily[0]["total_input_cost_usd"] == pytest.approx(0.63)
|
| 338 |
assert daily[1]["tokens_saved"] == 10
|
| 339 |
assert daily[1]["total_tokens_saved"] == 185
|
| 340 |
+
assert daily[1]["total_input_tokens_delta"] == 60
|
| 341 |
+
assert daily[1]["total_input_tokens"] == 360
|
| 342 |
+
assert daily[1]["total_input_cost_usd_delta"] == pytest.approx(0.12)
|
| 343 |
+
assert daily[1]["total_input_cost_usd"] == pytest.approx(0.75)
|
| 344 |
assert daily[2]["tokens_saved"] == 20
|
| 345 |
assert daily[2]["total_tokens_saved"] == 205
|
| 346 |
+
assert daily[2]["total_input_tokens_delta"] == 90
|
| 347 |
+
assert daily[2]["total_input_tokens"] == 450
|
| 348 |
+
assert daily[2]["total_input_cost_usd_delta"] == pytest.approx(0.18)
|
| 349 |
+
assert daily[2]["total_input_cost_usd"] == pytest.approx(0.93)
|
| 350 |
|
| 351 |
weekly = response["series"]["weekly"]
|
| 352 |
assert [point["timestamp"] for point in weekly] == [
|
|
|
|
| 381 |
def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):
|
| 382 |
savings_path = tmp_path / "proxy_savings.json"
|
| 383 |
monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
|
| 384 |
+
monkeypatch.setattr(
|
| 385 |
+
"headroom.proxy.server.CostTracker._get_cache_prices",
|
| 386 |
+
lambda self, model: (0.001, 0.0015, 0.002),
|
| 387 |
+
)
|
| 388 |
|
| 389 |
config = ProxyConfig(
|
| 390 |
cache_enabled=False,
|
|
|
|
| 410 |
assert history_data["schema_version"] == 1
|
| 411 |
assert history_data["storage_path"] == str(savings_path)
|
| 412 |
assert history_data["lifetime"]["tokens_saved"] == 40
|
| 413 |
+
assert history_data["lifetime"]["total_input_tokens"] == 120
|
| 414 |
+
assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)
|
| 415 |
assert list(history_data["series"].keys()) == ["hourly", "daily", "weekly", "monthly"]
|
| 416 |
assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
|
| 417 |
+
assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120
|
| 418 |
+
assert history_data["series"]["hourly"][0]["total_input_cost_usd_delta"] == pytest.approx(
|
| 419 |
+
0.24
|
| 420 |
+
)
|
| 421 |
|
| 422 |
with TestClient(create_app(config)) as client:
|
| 423 |
history = client.get("/stats-history")
|
|
|
|
| 428 |
|
| 429 |
updated = client.get("/stats-history").json()
|
| 430 |
assert updated["lifetime"]["tokens_saved"] == 55
|
| 431 |
+
assert updated["lifetime"]["total_input_tokens"] == 240
|
| 432 |
+
assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
|
| 433 |
assert len(updated["history"]) == 2
|
| 434 |
+
assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
|
| 435 |
+
assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
|
| 436 |
|
| 437 |
persisted = json.loads(savings_path.read_text())
|
| 438 |
assert persisted["lifetime"]["tokens_saved"] == 55
|
| 439 |
+
assert persisted["lifetime"]["total_input_tokens"] == 240
|
| 440 |
+
assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
|
| 441 |
|
| 442 |
|
| 443 |
def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
|
| 444 |
savings_path = tmp_path / "proxy_savings.json"
|
| 445 |
monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
|
| 446 |
+
monkeypatch.setattr(
|
| 447 |
+
"headroom.proxy.server.CostTracker._get_cache_prices",
|
| 448 |
+
lambda self, model: (0.001, 0.0015, 0.002),
|
| 449 |
+
)
|
| 450 |
|
| 451 |
config = ProxyConfig(
|
| 452 |
cache_enabled=False,
|
|
|
|
| 468 |
lines = response.text.strip().splitlines()
|
| 469 |
assert lines[0] == (
|
| 470 |
"timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"
|
| 471 |
+
"compression_savings_usd,total_input_tokens_delta,total_input_tokens,"
|
| 472 |
+
"total_input_cost_usd_delta,total_input_cost_usd"
|
| 473 |
)
|
| 474 |
assert len(lines) >= 2
|
| 475 |
assert "total_tokens_saved" in lines[0]
|
| 476 |
+
assert "total_input_cost_usd" in lines[0]
|
| 477 |
|
| 478 |
|
| 479 |
def test_malformed_savings_state_is_ignored_safely(tmp_path, monkeypatch):
|