Spaces:
Build error
Build error
Commit ·
b4d2977
1
Parent(s): 52bd5e8
Add observed cache TTL metrics and dashboard coverage
Browse files- docs/metrics.md +35 -0
- docs/proxy.md +10 -0
- headroom/dashboard/templates/dashboard.html +78 -1
- headroom/proxy/cost.py +81 -1
- headroom/proxy/handlers/anthropic.py +25 -0
- headroom/proxy/handlers/streaming.py +42 -1
- headroom/proxy/prometheus_metrics.py +12 -0
- tests/test_dashboard_cache_ttl_playwright.py +211 -0
- tests/test_proxy_cache_ttl_metrics.py +147 -0
docs/metrics.md
CHANGED
|
@@ -53,6 +53,41 @@ for backward compatibility. The new `persistent_savings` block is durable local
|
|
| 53 |
proxy compression history stored by default at `~/.headroom/proxy_savings.json`.
|
| 54 |
Use `HEADROOM_SAVINGS_PATH` to override the file location.
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
### Historical Savings Endpoint
|
| 57 |
|
| 58 |
```bash
|
|
|
|
| 53 |
proxy compression history stored by default at `~/.headroom/proxy_savings.json`.
|
| 54 |
Use `HEADROOM_SAVINGS_PATH` to override the file location.
|
| 55 |
|
| 56 |
+
For Anthropic-style providers that return cache-write TTL buckets, `/stats`
|
| 57 |
+
also surfaces observed cache TTL usage under `prefix_cache`:
|
| 58 |
+
|
| 59 |
+
```json
|
| 60 |
+
{
|
| 61 |
+
"prefix_cache": {
|
| 62 |
+
"by_provider": {
|
| 63 |
+
"anthropic": {
|
| 64 |
+
"observed_ttl_buckets": {
|
| 65 |
+
"5m": {"tokens": 20000, "requests": 8},
|
| 66 |
+
"1h": {"tokens": 50000, "requests": 12}
|
| 67 |
+
},
|
| 68 |
+
"observed_ttl_mix": {
|
| 69 |
+
"5m_pct": 28.6,
|
| 70 |
+
"1h_pct": 71.4,
|
| 71 |
+
"active_buckets": ["5m", "1h"]
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"totals": {
|
| 76 |
+
"observed_ttl_buckets": {
|
| 77 |
+
"5m": {"tokens": 20000, "requests": 8},
|
| 78 |
+
"1h": {"tokens": 50000, "requests": 12}
|
| 79 |
+
}
|
| 80 |
+
}
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
These fields are observational only:
|
| 86 |
+
|
| 87 |
+
- they reflect provider-reported cache write buckets
|
| 88 |
+
- they do not configure TTL
|
| 89 |
+
- they do not represent remaining expiration time
|
| 90 |
+
|
| 91 |
### Historical Savings Endpoint
|
| 92 |
|
| 93 |
```bash
|
docs/proxy.md
CHANGED
|
@@ -133,6 +133,16 @@ curl http://localhost:8787/stats
|
|
| 133 |
small recent preview. The existing `savings_history` field is still present and
|
| 134 |
remains session-scoped for backward compatibility.
|
| 135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
### Historical Savings
|
| 137 |
|
| 138 |
```bash
|
|
|
|
| 133 |
small recent preview. The existing `savings_history` field is still present and
|
| 134 |
remains session-scoped for backward compatibility.
|
| 135 |
|
| 136 |
+
For providers that return cache-write TTL bucket usage, `/stats` also includes
|
| 137 |
+
observed TTL breakdowns under `prefix_cache`:
|
| 138 |
+
|
| 139 |
+
- `observed_ttl_buckets.5m.tokens`
|
| 140 |
+
- `observed_ttl_buckets.1h.tokens`
|
| 141 |
+
- `observed_ttl_mix`
|
| 142 |
+
|
| 143 |
+
These are provider-reported observations, not configured TTL and not remaining
|
| 144 |
+
expiration time.
|
| 145 |
+
|
| 146 |
### Historical Savings
|
| 147 |
|
| 148 |
```bash
|
headroom/dashboard/templates/dashboard.html
CHANGED
|
@@ -208,6 +208,60 @@
|
|
| 208 |
<span class="flex items-center gap-1"><span class="w-2 h-2 rounded-full bg-border inline-block"></span> Uncached</span>
|
| 209 |
</div>
|
| 210 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
<!-- Per-provider breakdown -->
|
| 212 |
<template x-if="Object.keys(stats.prefix_cache?.by_provider || {}).length > 0">
|
| 213 |
<div class="mt-4 border-t border-border pt-3">
|
|
@@ -219,11 +273,14 @@
|
|
| 219 |
<span class="px-2 py-0.5 bg-border rounded text-xs font-mono" x-text="prov"></span>
|
| 220 |
<span class="text-xs text-gray-500" x-text="pc.label"></span>
|
| 221 |
</div>
|
| 222 |
-
<div class="flex items-center gap-4 text-xs font-mono">
|
| 223 |
<span class="text-emerald-400" x-text="formatNumber(pc.cache_read_tokens) + ' reads (' + pc.read_discount + ' off)'"></span>
|
| 224 |
<template x-if="pc.write_premium !== 'none'">
|
| 225 |
<span class="text-amber-400" x-text="formatNumber(pc.cache_write_tokens) + ' writes (+' + pc.write_premium + ')'"></span>
|
| 226 |
</template>
|
|
|
|
|
|
|
|
|
|
| 227 |
<span :class="pc.bust_count > 0 ? 'text-red-400' : 'text-gray-500'" x-text="pc.bust_count + ' busts'"></span>
|
| 228 |
<span class="text-emerald-400" x-text="'$' + formatCurrency(pc.net_savings_usd)"></span>
|
| 229 |
</div>
|
|
@@ -1054,6 +1111,26 @@
|
|
| 1054 |
return Math.round((t.cache_write_tokens || 0) / total * 100);
|
| 1055 |
},
|
| 1056 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1057 |
// --- Waste Signals ---
|
| 1058 |
|
| 1059 |
wasteSignalLabel(signal) {
|
|
|
|
| 208 |
<span class="flex items-center gap-1"><span class="w-2 h-2 rounded-full bg-border inline-block"></span> Uncached</span>
|
| 209 |
</div>
|
| 210 |
</div>
|
| 211 |
+
<template x-if="hasObservedTtlBuckets">
|
| 212 |
+
<div class="mt-5 border-t border-border pt-4">
|
| 213 |
+
<div class="flex items-center justify-between mb-3">
|
| 214 |
+
<div>
|
| 215 |
+
<div class="text-xs text-gray-500 uppercase tracking-[0.22em]">Observed TTL Buckets</div>
|
| 216 |
+
<div class="text-sm text-gray-300 mt-1">Provider-reported cache write mix</div>
|
| 217 |
+
</div>
|
| 218 |
+
<div class="text-xs text-gray-500 font-mono" x-text="observedTtlWindowLabel"></div>
|
| 219 |
+
</div>
|
| 220 |
+
<div class="grid grid-cols-1 xl:grid-cols-[1.3fr_0.9fr] gap-4">
|
| 221 |
+
<div class="rounded-2xl border border-cyan-500/20 bg-[linear-gradient(180deg,rgba(34,211,238,0.10),rgba(12,16,22,0.88))] p-4 shadow-[inset_0_1px_0_rgba(255,255,255,0.03)]">
|
| 222 |
+
<div class="flex items-start justify-between gap-3">
|
| 223 |
+
<div>
|
| 224 |
+
<div class="text-[11px] uppercase tracking-[0.22em] text-cyan-300/75">Bucket Mix</div>
|
| 225 |
+
<div class="mt-1 text-2xl font-light text-cyan-50">
|
| 226 |
+
<span data-testid="ttl-bucket-mix-1h-total" x-text="formatNumber(stats.prefix_cache?.totals?.cache_write_1h_tokens || 0)"></span>
|
| 227 |
+
<span class="text-sm text-cyan-300/70">1h</span>
|
| 228 |
+
<span class="mx-2 text-cyan-500/60">/</span>
|
| 229 |
+
<span data-testid="ttl-bucket-mix-5m-total" x-text="formatNumber(stats.prefix_cache?.totals?.cache_write_5m_tokens || 0)"></span>
|
| 230 |
+
<span class="text-sm text-cyan-300/70">5m</span>
|
| 231 |
+
</div>
|
| 232 |
+
</div>
|
| 233 |
+
<div data-testid="ttl-bucket-headline" class="rounded-full border border-cyan-400/20 bg-black/20 px-3 py-1 text-[11px] uppercase tracking-[0.18em] text-cyan-200/80" x-text="observedTtlHeadline"></div>
|
| 234 |
+
</div>
|
| 235 |
+
<div class="mt-4">
|
| 236 |
+
<div class="flex justify-between text-xs text-gray-500 mb-1">
|
| 237 |
+
<span>Observed write-token split</span>
|
| 238 |
+
<span x-text="(stats.prefix_cache?.totals?.observed_ttl_mix?.active_buckets || []).join(' + ')"></span>
|
| 239 |
+
</div>
|
| 240 |
+
<div class="h-3 overflow-hidden rounded-full bg-black/30 ring-1 ring-white/5 flex">
|
| 241 |
+
<div class="h-full bg-cyan-400 transition-all duration-500" :style="'width:' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['1h_pct'] || 0) + '%'"></div>
|
| 242 |
+
<div class="h-full bg-violet-400 transition-all duration-500" :style="'width:' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['5m_pct'] || 0) + '%'"></div>
|
| 243 |
+
</div>
|
| 244 |
+
<div class="mt-2 flex flex-wrap gap-3 text-xs text-gray-400">
|
| 245 |
+
<span class="inline-flex items-center gap-1.5"><span class="inline-block h-2 w-2 rounded-full bg-cyan-400"></span><span data-testid="ttl-bucket-mix-1h-pct" x-text="'1h ' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['1h_pct'] || 0).toFixed(1) + '%'"></span></span>
|
| 246 |
+
<span class="inline-flex items-center gap-1.5"><span class="inline-block h-2 w-2 rounded-full bg-violet-400"></span><span data-testid="ttl-bucket-mix-5m-pct" x-text="'5m ' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['5m_pct'] || 0).toFixed(1) + '%'"></span></span>
|
| 247 |
+
</div>
|
| 248 |
+
</div>
|
| 249 |
+
</div>
|
| 250 |
+
<div class="grid grid-cols-1 sm:grid-cols-2 xl:grid-cols-1 gap-3">
|
| 251 |
+
<div class="rounded-2xl border border-white/8 bg-black/20 p-4">
|
| 252 |
+
<div class="text-[11px] uppercase tracking-[0.18em] text-cyan-300/75">1h Cache Writes</div>
|
| 253 |
+
<div data-testid="ttl-bucket-1h-value" class="mt-2 text-3xl font-light text-cyan-50" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['1h']?.tokens || 0)"></div>
|
| 254 |
+
<div class="mt-1 text-xs text-gray-500" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['1h']?.requests || 0) + ' requests observed'"></div>
|
| 255 |
+
</div>
|
| 256 |
+
<div class="rounded-2xl border border-white/8 bg-black/20 p-4">
|
| 257 |
+
<div class="text-[11px] uppercase tracking-[0.18em] text-violet-300/75">5m Cache Writes</div>
|
| 258 |
+
<div data-testid="ttl-bucket-5m-value" class="mt-2 text-3xl font-light text-violet-50" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['5m']?.tokens || 0)"></div>
|
| 259 |
+
<div class="mt-1 text-xs text-gray-500" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['5m']?.requests || 0) + ' requests observed'"></div>
|
| 260 |
+
</div>
|
| 261 |
+
</div>
|
| 262 |
+
</div>
|
| 263 |
+
</div>
|
| 264 |
+
</template>
|
| 265 |
<!-- Per-provider breakdown -->
|
| 266 |
<template x-if="Object.keys(stats.prefix_cache?.by_provider || {}).length > 0">
|
| 267 |
<div class="mt-4 border-t border-border pt-3">
|
|
|
|
| 273 |
<span class="px-2 py-0.5 bg-border rounded text-xs font-mono" x-text="prov"></span>
|
| 274 |
<span class="text-xs text-gray-500" x-text="pc.label"></span>
|
| 275 |
</div>
|
| 276 |
+
<div class="flex flex-wrap items-center justify-end gap-4 text-xs font-mono">
|
| 277 |
<span class="text-emerald-400" x-text="formatNumber(pc.cache_read_tokens) + ' reads (' + pc.read_discount + ' off)'"></span>
|
| 278 |
<template x-if="pc.write_premium !== 'none'">
|
| 279 |
<span class="text-amber-400" x-text="formatNumber(pc.cache_write_tokens) + ' writes (+' + pc.write_premium + ')'"></span>
|
| 280 |
</template>
|
| 281 |
+
<template x-if="pc.observed_ttl_mix">
|
| 282 |
+
<span class="text-cyan-300" x-text="'TTL 1h ' + (pc.observed_ttl_mix['1h_pct'] || 0).toFixed(1) + '% / 5m ' + (pc.observed_ttl_mix['5m_pct'] || 0).toFixed(1) + '%'"></span>
|
| 283 |
+
</template>
|
| 284 |
<span :class="pc.bust_count > 0 ? 'text-red-400' : 'text-gray-500'" x-text="pc.bust_count + ' busts'"></span>
|
| 285 |
<span class="text-emerald-400" x-text="'$' + formatCurrency(pc.net_savings_usd)"></span>
|
| 286 |
</div>
|
|
|
|
| 1111 |
return Math.round((t.cache_write_tokens || 0) / total * 100);
|
| 1112 |
},
|
| 1113 |
|
| 1114 |
+
get hasObservedTtlBuckets() {
|
| 1115 |
+
const buckets = this.stats.prefix_cache?.totals?.observed_ttl_buckets || {};
|
| 1116 |
+
return ((buckets['5m']?.tokens || 0) + (buckets['1h']?.tokens || 0)) > 0;
|
| 1117 |
+
},
|
| 1118 |
+
|
| 1119 |
+
get observedTtlHeadline() {
|
| 1120 |
+
const mix = this.stats.prefix_cache?.totals?.observed_ttl_mix || {};
|
| 1121 |
+
const oneHour = mix['1h_pct'] || 0;
|
| 1122 |
+
const fiveMinute = mix['5m_pct'] || 0;
|
| 1123 |
+
if (oneHour === fiveMinute) return 'Balanced';
|
| 1124 |
+
return oneHour > fiveMinute ? '1h leaning' : '5m leaning';
|
| 1125 |
+
},
|
| 1126 |
+
|
| 1127 |
+
get observedTtlWindowLabel() {
|
| 1128 |
+
const mix = this.stats.prefix_cache?.totals?.observed_ttl_mix || {};
|
| 1129 |
+
const active = mix.active_buckets || [];
|
| 1130 |
+
if (!active.length) return 'No TTL bucket data';
|
| 1131 |
+
return active.length === 1 ? active[0] + ' only' : active.join(' / ');
|
| 1132 |
+
},
|
| 1133 |
+
|
| 1134 |
// --- Waste Signals ---
|
| 1135 |
|
| 1136 |
wasteSignalLabel(signal) {
|
headroom/proxy/cost.py
CHANGED
|
@@ -78,6 +78,10 @@ def build_prefix_cache_stats(
|
|
| 78 |
totals = {
|
| 79 |
"cache_read_tokens": 0,
|
| 80 |
"cache_write_tokens": 0,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
"requests": 0,
|
| 82 |
"hit_requests": 0,
|
| 83 |
"bust_count": 0,
|
|
@@ -119,6 +123,10 @@ def build_prefix_cache_stats(
|
|
| 119 |
# for observability but don't penalise our savings number.
|
| 120 |
read_tokens: int = pc["cache_read_tokens"] # type: ignore[assignment]
|
| 121 |
write_tokens: int = pc["cache_write_tokens"] # type: ignore[assignment]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
savings_usd = 0.0
|
| 123 |
write_premium_usd = 0.0
|
| 124 |
|
|
@@ -134,6 +142,10 @@ def build_prefix_cache_stats(
|
|
| 134 |
provider_stats = {
|
| 135 |
"cache_read_tokens": read_tokens,
|
| 136 |
"cache_write_tokens": write_tokens,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
"requests": pc["requests"],
|
| 138 |
"hit_requests": pc["hit_requests"],
|
| 139 |
"hit_rate": hit_rate,
|
|
@@ -145,12 +157,37 @@ def build_prefix_cache_stats(
|
|
| 145 |
"write_premium_usd": round(write_premium_usd, 4),
|
| 146 |
"net_savings_usd": round(savings_usd, 4),
|
| 147 |
"label": str(econ["label"]),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
by_provider[provider] = provider_stats
|
| 150 |
|
| 151 |
# Accumulate totals
|
| 152 |
totals["cache_read_tokens"] += read_tokens
|
| 153 |
totals["cache_write_tokens"] += write_tokens
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
totals["requests"] += pc["requests"]
|
| 155 |
totals["hit_requests"] += pc["hit_requests"]
|
| 156 |
totals["bust_count"] += pc["bust_count"]
|
|
@@ -164,6 +201,33 @@ def build_prefix_cache_stats(
|
|
| 164 |
totals["hit_rate"] = (
|
| 165 |
round(totals["hit_requests"] / totals["requests"] * 100, 1) if totals["requests"] > 0 else 0
|
| 166 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
return {
|
| 169 |
"by_provider": by_provider,
|
|
@@ -180,7 +244,9 @@ def build_prefix_cache_stats(
|
|
| 180 |
"Prefix caching is performed by the LLM provider (Anthropic, OpenAI). "
|
| 181 |
"Headroom reports cache stats as observed from API responses. "
|
| 182 |
"CacheAligner and prefix freeze improve cache hit rates by stabilizing "
|
| 183 |
-
"the message prefix, but baseline caching happens without Headroom."
|
|
|
|
|
|
|
| 184 |
),
|
| 185 |
}
|
| 186 |
|
|
@@ -356,6 +422,8 @@ class CostTracker:
|
|
| 356 |
# API-reported cache breakdown per model (for accurate cost calculation)
|
| 357 |
self._api_cache_read_by_model: dict[str, int] = {}
|
| 358 |
self._api_cache_write_by_model: dict[str, int] = {}
|
|
|
|
|
|
|
| 359 |
self._api_uncached_by_model: dict[str, int] = {}
|
| 360 |
|
| 361 |
# Cache resolved model names to avoid repeated litellm lookups.
|
|
@@ -480,6 +548,8 @@ class CostTracker:
|
|
| 480 |
tokens_sent: int,
|
| 481 |
cache_read_tokens: int = 0,
|
| 482 |
cache_write_tokens: int = 0,
|
|
|
|
|
|
|
| 483 |
uncached_tokens: int = 0,
|
| 484 |
):
|
| 485 |
"""Record token counts per model.
|
|
@@ -503,6 +573,12 @@ class CostTracker:
|
|
| 503 |
self._api_cache_write_by_model[model] = (
|
| 504 |
self._api_cache_write_by_model.get(model, 0) + cache_write_tokens
|
| 505 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 506 |
self._api_uncached_by_model[model] = (
|
| 507 |
self._api_uncached_by_model.get(model, 0) + uncached_tokens
|
| 508 |
)
|
|
@@ -574,6 +650,8 @@ class CostTracker:
|
|
| 574 |
"requests": reqs,
|
| 575 |
"tokens_saved": saved,
|
| 576 |
"tokens_sent": sent,
|
|
|
|
|
|
|
| 577 |
"reduction_pct": round(saved / (saved + sent) * 100, 1)
|
| 578 |
if (saved + sent) > 0
|
| 579 |
else 0,
|
|
@@ -625,6 +703,8 @@ class CostTracker:
|
|
| 625 |
"total_tokens_saved": total_saved,
|
| 626 |
"total_input_tokens": total_input_tokens,
|
| 627 |
"total_input_cost_usd": round(cost_with_headroom, 4),
|
|
|
|
|
|
|
| 628 |
"per_model": per_model,
|
| 629 |
"cost_with_headroom_usd": round(cost_with_headroom, 4),
|
| 630 |
"savings_usd": round(savings_usd, 4),
|
|
|
|
| 78 |
totals = {
|
| 79 |
"cache_read_tokens": 0,
|
| 80 |
"cache_write_tokens": 0,
|
| 81 |
+
"cache_write_5m_tokens": 0,
|
| 82 |
+
"cache_write_1h_tokens": 0,
|
| 83 |
+
"cache_write_5m_requests": 0,
|
| 84 |
+
"cache_write_1h_requests": 0,
|
| 85 |
"requests": 0,
|
| 86 |
"hit_requests": 0,
|
| 87 |
"bust_count": 0,
|
|
|
|
| 123 |
# for observability but don't penalise our savings number.
|
| 124 |
read_tokens: int = pc["cache_read_tokens"] # type: ignore[assignment]
|
| 125 |
write_tokens: int = pc["cache_write_tokens"] # type: ignore[assignment]
|
| 126 |
+
write_5m_tokens: int = pc["cache_write_5m_tokens"] # type: ignore[assignment]
|
| 127 |
+
write_1h_tokens: int = pc["cache_write_1h_tokens"] # type: ignore[assignment]
|
| 128 |
+
write_5m_requests: int = pc["cache_write_5m_requests"] # type: ignore[assignment]
|
| 129 |
+
write_1h_requests: int = pc["cache_write_1h_requests"] # type: ignore[assignment]
|
| 130 |
savings_usd = 0.0
|
| 131 |
write_premium_usd = 0.0
|
| 132 |
|
|
|
|
| 142 |
provider_stats = {
|
| 143 |
"cache_read_tokens": read_tokens,
|
| 144 |
"cache_write_tokens": write_tokens,
|
| 145 |
+
"cache_write_5m_tokens": write_5m_tokens,
|
| 146 |
+
"cache_write_1h_tokens": write_1h_tokens,
|
| 147 |
+
"cache_write_5m_requests": write_5m_requests,
|
| 148 |
+
"cache_write_1h_requests": write_1h_requests,
|
| 149 |
"requests": pc["requests"],
|
| 150 |
"hit_requests": pc["hit_requests"],
|
| 151 |
"hit_rate": hit_rate,
|
|
|
|
| 157 |
"write_premium_usd": round(write_premium_usd, 4),
|
| 158 |
"net_savings_usd": round(savings_usd, 4),
|
| 159 |
"label": str(econ["label"]),
|
| 160 |
+
"observed_ttl_buckets": {
|
| 161 |
+
"5m": {
|
| 162 |
+
"tokens": write_5m_tokens,
|
| 163 |
+
"requests": write_5m_requests,
|
| 164 |
+
},
|
| 165 |
+
"1h": {
|
| 166 |
+
"tokens": write_1h_tokens,
|
| 167 |
+
"requests": write_1h_requests,
|
| 168 |
+
},
|
| 169 |
+
},
|
| 170 |
}
|
| 171 |
+
total_observed_ttl_tokens = write_5m_tokens + write_1h_tokens
|
| 172 |
+
if total_observed_ttl_tokens > 0:
|
| 173 |
+
provider_stats["observed_ttl_mix"] = {
|
| 174 |
+
"5m_pct": round(write_5m_tokens / total_observed_ttl_tokens * 100, 1),
|
| 175 |
+
"1h_pct": round(write_1h_tokens / total_observed_ttl_tokens * 100, 1),
|
| 176 |
+
"active_buckets": [
|
| 177 |
+
bucket
|
| 178 |
+
for bucket, tokens in (("5m", write_5m_tokens), ("1h", write_1h_tokens))
|
| 179 |
+
if tokens > 0
|
| 180 |
+
],
|
| 181 |
+
}
|
| 182 |
by_provider[provider] = provider_stats
|
| 183 |
|
| 184 |
# Accumulate totals
|
| 185 |
totals["cache_read_tokens"] += read_tokens
|
| 186 |
totals["cache_write_tokens"] += write_tokens
|
| 187 |
+
totals["cache_write_5m_tokens"] += write_5m_tokens
|
| 188 |
+
totals["cache_write_1h_tokens"] += write_1h_tokens
|
| 189 |
+
totals["cache_write_5m_requests"] += write_5m_requests
|
| 190 |
+
totals["cache_write_1h_requests"] += write_1h_requests
|
| 191 |
totals["requests"] += pc["requests"]
|
| 192 |
totals["hit_requests"] += pc["hit_requests"]
|
| 193 |
totals["bust_count"] += pc["bust_count"]
|
|
|
|
| 201 |
totals["hit_rate"] = (
|
| 202 |
round(totals["hit_requests"] / totals["requests"] * 100, 1) if totals["requests"] > 0 else 0
|
| 203 |
)
|
| 204 |
+
total_observed_ttl_tokens = totals["cache_write_5m_tokens"] + totals["cache_write_1h_tokens"]
|
| 205 |
+
totals["observed_ttl_buckets"] = {
|
| 206 |
+
"5m": {
|
| 207 |
+
"tokens": totals["cache_write_5m_tokens"],
|
| 208 |
+
"requests": totals["cache_write_5m_requests"],
|
| 209 |
+
},
|
| 210 |
+
"1h": {
|
| 211 |
+
"tokens": totals["cache_write_1h_tokens"],
|
| 212 |
+
"requests": totals["cache_write_1h_requests"],
|
| 213 |
+
},
|
| 214 |
+
}
|
| 215 |
+
totals["observed_ttl_mix"] = {
|
| 216 |
+
"5m_pct": round(totals["cache_write_5m_tokens"] / total_observed_ttl_tokens * 100, 1)
|
| 217 |
+
if total_observed_ttl_tokens > 0
|
| 218 |
+
else 0.0,
|
| 219 |
+
"1h_pct": round(totals["cache_write_1h_tokens"] / total_observed_ttl_tokens * 100, 1)
|
| 220 |
+
if total_observed_ttl_tokens > 0
|
| 221 |
+
else 0.0,
|
| 222 |
+
"active_buckets": [
|
| 223 |
+
bucket
|
| 224 |
+
for bucket, tokens in (
|
| 225 |
+
("5m", totals["cache_write_5m_tokens"]),
|
| 226 |
+
("1h", totals["cache_write_1h_tokens"]),
|
| 227 |
+
)
|
| 228 |
+
if tokens > 0
|
| 229 |
+
],
|
| 230 |
+
}
|
| 231 |
|
| 232 |
return {
|
| 233 |
"by_provider": by_provider,
|
|
|
|
| 244 |
"Prefix caching is performed by the LLM provider (Anthropic, OpenAI). "
|
| 245 |
"Headroom reports cache stats as observed from API responses. "
|
| 246 |
"CacheAligner and prefix freeze improve cache hit rates by stabilizing "
|
| 247 |
+
"the message prefix, but baseline caching happens without Headroom. "
|
| 248 |
+
"Observed TTL bucket metrics reflect provider-reported cache write usage "
|
| 249 |
+
"(for example Anthropic 5m vs 1h), not configured or remaining TTL."
|
| 250 |
),
|
| 251 |
}
|
| 252 |
|
|
|
|
| 422 |
# API-reported cache breakdown per model (for accurate cost calculation)
|
| 423 |
self._api_cache_read_by_model: dict[str, int] = {}
|
| 424 |
self._api_cache_write_by_model: dict[str, int] = {}
|
| 425 |
+
self._api_cache_write_5m_by_model: dict[str, int] = {}
|
| 426 |
+
self._api_cache_write_1h_by_model: dict[str, int] = {}
|
| 427 |
self._api_uncached_by_model: dict[str, int] = {}
|
| 428 |
|
| 429 |
# Cache resolved model names to avoid repeated litellm lookups.
|
|
|
|
| 548 |
tokens_sent: int,
|
| 549 |
cache_read_tokens: int = 0,
|
| 550 |
cache_write_tokens: int = 0,
|
| 551 |
+
cache_write_5m_tokens: int = 0,
|
| 552 |
+
cache_write_1h_tokens: int = 0,
|
| 553 |
uncached_tokens: int = 0,
|
| 554 |
):
|
| 555 |
"""Record token counts per model.
|
|
|
|
| 573 |
self._api_cache_write_by_model[model] = (
|
| 574 |
self._api_cache_write_by_model.get(model, 0) + cache_write_tokens
|
| 575 |
)
|
| 576 |
+
self._api_cache_write_5m_by_model[model] = (
|
| 577 |
+
self._api_cache_write_5m_by_model.get(model, 0) + cache_write_5m_tokens
|
| 578 |
+
)
|
| 579 |
+
self._api_cache_write_1h_by_model[model] = (
|
| 580 |
+
self._api_cache_write_1h_by_model.get(model, 0) + cache_write_1h_tokens
|
| 581 |
+
)
|
| 582 |
self._api_uncached_by_model[model] = (
|
| 583 |
self._api_uncached_by_model.get(model, 0) + uncached_tokens
|
| 584 |
)
|
|
|
|
| 650 |
"requests": reqs,
|
| 651 |
"tokens_saved": saved,
|
| 652 |
"tokens_sent": sent,
|
| 653 |
+
"cache_write_5m_tokens": self._api_cache_write_5m_by_model.get(model, 0),
|
| 654 |
+
"cache_write_1h_tokens": self._api_cache_write_1h_by_model.get(model, 0),
|
| 655 |
"reduction_pct": round(saved / (saved + sent) * 100, 1)
|
| 656 |
if (saved + sent) > 0
|
| 657 |
else 0,
|
|
|
|
| 703 |
"total_tokens_saved": total_saved,
|
| 704 |
"total_input_tokens": total_input_tokens,
|
| 705 |
"total_input_cost_usd": round(cost_with_headroom, 4),
|
| 706 |
+
"cache_write_5m_tokens": sum(self._api_cache_write_5m_by_model.values()),
|
| 707 |
+
"cache_write_1h_tokens": sum(self._api_cache_write_1h_by_model.values()),
|
| 708 |
"per_model": per_model,
|
| 709 |
"cost_with_headroom_usd": round(cost_with_headroom, 4),
|
| 710 |
"savings_usd": round(savings_usd, 4),
|
headroom/proxy/handlers/anthropic.py
CHANGED
|
@@ -40,6 +40,24 @@ class AnthropicHandlerMixin:
|
|
| 40 |
canonical = str(tool)
|
| 41 |
return (name, canonical)
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
@classmethod
|
| 44 |
def _sort_tools_deterministically(
|
| 45 |
cls, tools: list[dict[str, Any]] | None
|
|
@@ -1196,12 +1214,15 @@ class AnthropicHandlerMixin:
|
|
| 1196 |
output_tokens = 0
|
| 1197 |
cr_tokens = 0
|
| 1198 |
cw_tokens = 0
|
|
|
|
|
|
|
| 1199 |
uncached_input_tokens = 0
|
| 1200 |
if resp_json:
|
| 1201 |
usage = resp_json.get("usage", {})
|
| 1202 |
output_tokens = usage.get("output_tokens", 0)
|
| 1203 |
cr_tokens = usage.get("cache_read_input_tokens", 0)
|
| 1204 |
cw_tokens = usage.get("cache_creation_input_tokens", 0)
|
|
|
|
| 1205 |
uncached_input_tokens = usage.get("input_tokens", 0)
|
| 1206 |
|
| 1207 |
# Update prefix cache tracker for next turn
|
|
@@ -1225,6 +1246,8 @@ class AnthropicHandlerMixin:
|
|
| 1225 |
optimized_tokens,
|
| 1226 |
cache_read_tokens=cr_tokens,
|
| 1227 |
cache_write_tokens=cw_tokens,
|
|
|
|
|
|
|
| 1228 |
uncached_tokens=uncached_input_tokens,
|
| 1229 |
)
|
| 1230 |
|
|
@@ -1252,6 +1275,8 @@ class AnthropicHandlerMixin:
|
|
| 1252 |
waste_signals=waste_signals_dict,
|
| 1253 |
cache_read_tokens=cr_tokens,
|
| 1254 |
cache_write_tokens=cw_tokens,
|
|
|
|
|
|
|
| 1255 |
uncached_input_tokens=uncached_input_tokens,
|
| 1256 |
)
|
| 1257 |
|
|
|
|
| 40 |
canonical = str(tool)
|
| 41 |
return (name, canonical)
|
| 42 |
|
| 43 |
+
@staticmethod
|
| 44 |
+
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
|
| 45 |
+
"""Extract observed Anthropic cache-write TTL bucket usage.
|
| 46 |
+
|
| 47 |
+
Returns (cache_write_5m_tokens, cache_write_1h_tokens) from provider usage.
|
| 48 |
+
These are observational metrics only; they do not imply configured or
|
| 49 |
+
remaining TTL.
|
| 50 |
+
"""
|
| 51 |
+
if not isinstance(usage, dict):
|
| 52 |
+
return (0, 0)
|
| 53 |
+
cache_creation = usage.get("cache_creation")
|
| 54 |
+
if not isinstance(cache_creation, dict):
|
| 55 |
+
return (0, 0)
|
| 56 |
+
return (
|
| 57 |
+
int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
|
| 58 |
+
int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
@classmethod
|
| 62 |
def _sort_tools_deterministically(
|
| 63 |
cls, tools: list[dict[str, Any]] | None
|
|
|
|
| 1214 |
output_tokens = 0
|
| 1215 |
cr_tokens = 0
|
| 1216 |
cw_tokens = 0
|
| 1217 |
+
cw_5m_tokens = 0
|
| 1218 |
+
cw_1h_tokens = 0
|
| 1219 |
uncached_input_tokens = 0
|
| 1220 |
if resp_json:
|
| 1221 |
usage = resp_json.get("usage", {})
|
| 1222 |
output_tokens = usage.get("output_tokens", 0)
|
| 1223 |
cr_tokens = usage.get("cache_read_input_tokens", 0)
|
| 1224 |
cw_tokens = usage.get("cache_creation_input_tokens", 0)
|
| 1225 |
+
cw_5m_tokens, cw_1h_tokens = self._extract_anthropic_cache_ttl_metrics(usage)
|
| 1226 |
uncached_input_tokens = usage.get("input_tokens", 0)
|
| 1227 |
|
| 1228 |
# Update prefix cache tracker for next turn
|
|
|
|
| 1246 |
optimized_tokens,
|
| 1247 |
cache_read_tokens=cr_tokens,
|
| 1248 |
cache_write_tokens=cw_tokens,
|
| 1249 |
+
cache_write_5m_tokens=cw_5m_tokens,
|
| 1250 |
+
cache_write_1h_tokens=cw_1h_tokens,
|
| 1251 |
uncached_tokens=uncached_input_tokens,
|
| 1252 |
)
|
| 1253 |
|
|
|
|
| 1275 |
waste_signals=waste_signals_dict,
|
| 1276 |
cache_read_tokens=cr_tokens,
|
| 1277 |
cache_write_tokens=cw_tokens,
|
| 1278 |
+
cache_write_5m_tokens=cw_5m_tokens,
|
| 1279 |
+
cache_write_1h_tokens=cw_1h_tokens,
|
| 1280 |
uncached_input_tokens=uncached_input_tokens,
|
| 1281 |
)
|
| 1282 |
|
headroom/proxy/handlers/streaming.py
CHANGED
|
@@ -24,6 +24,19 @@ logger = logging.getLogger("headroom.proxy")
|
|
| 24 |
class StreamingMixin:
|
| 25 |
"""Mixin providing streaming response methods for HeadroomProxy."""
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:
|
| 28 |
"""Parse usage information from SSE chunk.
|
| 29 |
|
|
@@ -31,7 +44,9 @@ class StreamingMixin:
|
|
| 31 |
For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)
|
| 32 |
For Gemini: Looks for usageMetadata in each chunk
|
| 33 |
|
| 34 |
-
Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens,
|
|
|
|
|
|
|
| 35 |
Returns None if no usage found in this chunk.
|
| 36 |
"""
|
| 37 |
try:
|
|
@@ -67,6 +82,11 @@ class StreamingMixin:
|
|
| 67 |
usage["cache_creation_input_tokens"] = msg_usage.get(
|
| 68 |
"cache_creation_input_tokens", 0
|
| 69 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
|
| 71 |
elif event_type == "message_delta":
|
| 72 |
delta_usage = data.get("usage", {})
|
|
@@ -148,6 +168,11 @@ class StreamingMixin:
|
|
| 148 |
usage_found["cache_creation_input_tokens"] = msg_usage.get(
|
| 149 |
"cache_creation_input_tokens", 0
|
| 150 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
# INFO logging for cache token tracking (temporary for debugging)
|
| 152 |
logger.info(
|
| 153 |
f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "
|
|
@@ -450,6 +475,8 @@ class StreamingMixin:
|
|
| 450 |
"output_tokens": None,
|
| 451 |
"cache_read_input_tokens": 0,
|
| 452 |
"cache_creation_input_tokens": 0,
|
|
|
|
|
|
|
| 453 |
"total_bytes": 0,
|
| 454 |
"sse_buffer": "", # Buffer for incomplete SSE events
|
| 455 |
"ttfb_ms": None, # Time to first byte from upstream
|
|
@@ -550,6 +577,14 @@ class StreamingMixin:
|
|
| 550 |
stream_state["cache_creation_input_tokens"] = usage[
|
| 551 |
"cache_creation_input_tokens"
|
| 552 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 553 |
|
| 554 |
# Memory tool handling after stream completes
|
| 555 |
# Chunks were already yielded in real-time above, so we only
|
|
@@ -637,6 +672,8 @@ class StreamingMixin:
|
|
| 637 |
# misleading for aggregation (often just 1 with prompt caching).
|
| 638 |
cache_read_tokens = stream_state["cache_read_input_tokens"]
|
| 639 |
cache_write_tokens = stream_state["cache_creation_input_tokens"]
|
|
|
|
|
|
|
| 640 |
uncached_input_tokens = stream_state.get("input_tokens") or 0
|
| 641 |
|
| 642 |
# Structured perf log line for `headroom perf` analysis
|
|
@@ -672,6 +709,8 @@ class StreamingMixin:
|
|
| 672 |
optimized_tokens,
|
| 673 |
cache_read_tokens=cache_read_tokens,
|
| 674 |
cache_write_tokens=cache_write_tokens,
|
|
|
|
|
|
|
| 675 |
uncached_tokens=uncached_input_tokens,
|
| 676 |
)
|
| 677 |
|
|
@@ -687,6 +726,8 @@ class StreamingMixin:
|
|
| 687 |
pipeline_timing=pipeline_timing,
|
| 688 |
cache_read_tokens=cache_read_tokens,
|
| 689 |
cache_write_tokens=cache_write_tokens,
|
|
|
|
|
|
|
| 690 |
uncached_input_tokens=uncached_input_tokens,
|
| 691 |
)
|
| 692 |
|
|
|
|
| 24 |
class StreamingMixin:
|
| 25 |
"""Mixin providing streaming response methods for HeadroomProxy."""
|
| 26 |
|
| 27 |
+
@staticmethod
|
| 28 |
+
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
|
| 29 |
+
"""Extract observed Anthropic cache-write TTL bucket usage."""
|
| 30 |
+
if not isinstance(usage, dict):
|
| 31 |
+
return (0, 0)
|
| 32 |
+
cache_creation = usage.get("cache_creation")
|
| 33 |
+
if not isinstance(cache_creation, dict):
|
| 34 |
+
return (0, 0)
|
| 35 |
+
return (
|
| 36 |
+
int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
|
| 37 |
+
int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:
|
| 41 |
"""Parse usage information from SSE chunk.
|
| 42 |
|
|
|
|
| 44 |
For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)
|
| 45 |
For Gemini: Looks for usageMetadata in each chunk
|
| 46 |
|
| 47 |
+
Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens,
|
| 48 |
+
cache_creation_input_tokens, cache_creation_ephemeral_5m_input_tokens,
|
| 49 |
+
cache_creation_ephemeral_1h_input_tokens
|
| 50 |
Returns None if no usage found in this chunk.
|
| 51 |
"""
|
| 52 |
try:
|
|
|
|
| 82 |
usage["cache_creation_input_tokens"] = msg_usage.get(
|
| 83 |
"cache_creation_input_tokens", 0
|
| 84 |
)
|
| 85 |
+
cache_write_5m, cache_write_1h = (
|
| 86 |
+
self._extract_anthropic_cache_ttl_metrics(msg_usage)
|
| 87 |
+
)
|
| 88 |
+
usage["cache_creation_ephemeral_5m_input_tokens"] = cache_write_5m
|
| 89 |
+
usage["cache_creation_ephemeral_1h_input_tokens"] = cache_write_1h
|
| 90 |
|
| 91 |
elif event_type == "message_delta":
|
| 92 |
delta_usage = data.get("usage", {})
|
|
|
|
| 168 |
usage_found["cache_creation_input_tokens"] = msg_usage.get(
|
| 169 |
"cache_creation_input_tokens", 0
|
| 170 |
)
|
| 171 |
+
cache_write_5m, cache_write_1h = (
|
| 172 |
+
self._extract_anthropic_cache_ttl_metrics(msg_usage)
|
| 173 |
+
)
|
| 174 |
+
usage_found["cache_creation_ephemeral_5m_input_tokens"] = cache_write_5m
|
| 175 |
+
usage_found["cache_creation_ephemeral_1h_input_tokens"] = cache_write_1h
|
| 176 |
# INFO logging for cache token tracking (temporary for debugging)
|
| 177 |
logger.info(
|
| 178 |
f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "
|
|
|
|
| 475 |
"output_tokens": None,
|
| 476 |
"cache_read_input_tokens": 0,
|
| 477 |
"cache_creation_input_tokens": 0,
|
| 478 |
+
"cache_creation_ephemeral_5m_input_tokens": 0,
|
| 479 |
+
"cache_creation_ephemeral_1h_input_tokens": 0,
|
| 480 |
"total_bytes": 0,
|
| 481 |
"sse_buffer": "", # Buffer for incomplete SSE events
|
| 482 |
"ttfb_ms": None, # Time to first byte from upstream
|
|
|
|
| 577 |
stream_state["cache_creation_input_tokens"] = usage[
|
| 578 |
"cache_creation_input_tokens"
|
| 579 |
]
|
| 580 |
+
if "cache_creation_ephemeral_5m_input_tokens" in usage:
|
| 581 |
+
stream_state["cache_creation_ephemeral_5m_input_tokens"] = usage[
|
| 582 |
+
"cache_creation_ephemeral_5m_input_tokens"
|
| 583 |
+
]
|
| 584 |
+
if "cache_creation_ephemeral_1h_input_tokens" in usage:
|
| 585 |
+
stream_state["cache_creation_ephemeral_1h_input_tokens"] = usage[
|
| 586 |
+
"cache_creation_ephemeral_1h_input_tokens"
|
| 587 |
+
]
|
| 588 |
|
| 589 |
# Memory tool handling after stream completes
|
| 590 |
# Chunks were already yielded in real-time above, so we only
|
|
|
|
| 672 |
# misleading for aggregation (often just 1 with prompt caching).
|
| 673 |
cache_read_tokens = stream_state["cache_read_input_tokens"]
|
| 674 |
cache_write_tokens = stream_state["cache_creation_input_tokens"]
|
| 675 |
+
cache_write_5m_tokens = stream_state["cache_creation_ephemeral_5m_input_tokens"]
|
| 676 |
+
cache_write_1h_tokens = stream_state["cache_creation_ephemeral_1h_input_tokens"]
|
| 677 |
uncached_input_tokens = stream_state.get("input_tokens") or 0
|
| 678 |
|
| 679 |
# Structured perf log line for `headroom perf` analysis
|
|
|
|
| 709 |
optimized_tokens,
|
| 710 |
cache_read_tokens=cache_read_tokens,
|
| 711 |
cache_write_tokens=cache_write_tokens,
|
| 712 |
+
cache_write_5m_tokens=cache_write_5m_tokens,
|
| 713 |
+
cache_write_1h_tokens=cache_write_1h_tokens,
|
| 714 |
uncached_tokens=uncached_input_tokens,
|
| 715 |
)
|
| 716 |
|
|
|
|
| 726 |
pipeline_timing=pipeline_timing,
|
| 727 |
cache_read_tokens=cache_read_tokens,
|
| 728 |
cache_write_tokens=cache_write_tokens,
|
| 729 |
+
cache_write_5m_tokens=cache_write_5m_tokens,
|
| 730 |
+
cache_write_1h_tokens=cache_write_1h_tokens,
|
| 731 |
uncached_input_tokens=uncached_input_tokens,
|
| 732 |
)
|
| 733 |
|
headroom/proxy/prometheus_metrics.py
CHANGED
|
@@ -77,6 +77,10 @@ class PrometheusMetrics:
|
|
| 77 |
lambda: {
|
| 78 |
"cache_read_tokens": 0,
|
| 79 |
"cache_write_tokens": 0,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
"requests": 0,
|
| 81 |
"hit_requests": 0, # requests with cache_read > 0
|
| 82 |
"bust_count": 0,
|
|
@@ -158,6 +162,8 @@ class PrometheusMetrics:
|
|
| 158 |
waste_signals: dict[str, int] | None = None,
|
| 159 |
cache_read_tokens: int = 0,
|
| 160 |
cache_write_tokens: int = 0,
|
|
|
|
|
|
|
| 161 |
uncached_input_tokens: int = 0,
|
| 162 |
):
|
| 163 |
"""Record metrics for a request."""
|
|
@@ -178,6 +184,12 @@ class PrometheusMetrics:
|
|
| 178 |
pc = self.cache_by_provider[provider]
|
| 179 |
pc["cache_read_tokens"] += cache_read_tokens
|
| 180 |
pc["cache_write_tokens"] += cache_write_tokens
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
pc["requests"] += 1
|
| 182 |
if cache_read_tokens > 0:
|
| 183 |
pc["hit_requests"] += 1
|
|
|
|
| 77 |
lambda: {
|
| 78 |
"cache_read_tokens": 0,
|
| 79 |
"cache_write_tokens": 0,
|
| 80 |
+
"cache_write_5m_tokens": 0,
|
| 81 |
+
"cache_write_1h_tokens": 0,
|
| 82 |
+
"cache_write_5m_requests": 0,
|
| 83 |
+
"cache_write_1h_requests": 0,
|
| 84 |
"requests": 0,
|
| 85 |
"hit_requests": 0, # requests with cache_read > 0
|
| 86 |
"bust_count": 0,
|
|
|
|
| 162 |
waste_signals: dict[str, int] | None = None,
|
| 163 |
cache_read_tokens: int = 0,
|
| 164 |
cache_write_tokens: int = 0,
|
| 165 |
+
cache_write_5m_tokens: int = 0,
|
| 166 |
+
cache_write_1h_tokens: int = 0,
|
| 167 |
uncached_input_tokens: int = 0,
|
| 168 |
):
|
| 169 |
"""Record metrics for a request."""
|
|
|
|
| 184 |
pc = self.cache_by_provider[provider]
|
| 185 |
pc["cache_read_tokens"] += cache_read_tokens
|
| 186 |
pc["cache_write_tokens"] += cache_write_tokens
|
| 187 |
+
pc["cache_write_5m_tokens"] += cache_write_5m_tokens
|
| 188 |
+
pc["cache_write_1h_tokens"] += cache_write_1h_tokens
|
| 189 |
+
if cache_write_5m_tokens > 0:
|
| 190 |
+
pc["cache_write_5m_requests"] += 1
|
| 191 |
+
if cache_write_1h_tokens > 0:
|
| 192 |
+
pc["cache_write_1h_requests"] += 1
|
| 193 |
pc["requests"] += 1
|
| 194 |
if cache_read_tokens > 0:
|
| 195 |
pc["hit_requests"] += 1
|
tests/test_dashboard_cache_ttl_playwright.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Behavior-driven Playwright validation for dashboard TTL cache metrics."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import pytest
|
| 10 |
+
|
| 11 |
+
from headroom.dashboard import get_dashboard_html
|
| 12 |
+
|
| 13 |
+
playwright = pytest.importorskip("playwright.sync_api")
|
| 14 |
+
Page = playwright.Page
|
| 15 |
+
expect = playwright.expect
|
| 16 |
+
sync_playwright = playwright.sync_playwright
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _sample_stats() -> dict:
|
| 20 |
+
return {
|
| 21 |
+
"cost": {
|
| 22 |
+
"savings_usd": 12.34,
|
| 23 |
+
"compression_savings_usd": 12.34,
|
| 24 |
+
"cache_savings_usd": 5.67,
|
| 25 |
+
"cli_tokens_avoided": 0,
|
| 26 |
+
},
|
| 27 |
+
"requests": {
|
| 28 |
+
"total": 128,
|
| 29 |
+
"cached": 96,
|
| 30 |
+
"rate_limited": 0,
|
| 31 |
+
"failed": 0,
|
| 32 |
+
"by_provider": {"anthropic": 128},
|
| 33 |
+
"by_model": {"claude-opus-4-6": 128},
|
| 34 |
+
},
|
| 35 |
+
"tokens": {
|
| 36 |
+
"input": 245_000,
|
| 37 |
+
"output": 88_000,
|
| 38 |
+
"saved": 143_000,
|
| 39 |
+
"cli_tokens_avoided": 0,
|
| 40 |
+
"total_before_compression": 388_000,
|
| 41 |
+
"savings_percent": 36.86,
|
| 42 |
+
},
|
| 43 |
+
"overhead": {"average_ms": 14.2, "min_ms": 4.5, "max_ms": 42.7},
|
| 44 |
+
"ttfb": {"average_ms": 1320.0, "min_ms": 420.0, "max_ms": 2900.0},
|
| 45 |
+
"latency": {"average_ms": 1510.0, "min_ms": 520.0, "max_ms": 3300.0},
|
| 46 |
+
"waste_signals": {"json_bloat": 95_000, "repetition": 48_000},
|
| 47 |
+
"savings_history": [
|
| 48 |
+
["2026-04-01T00:00:00Z", 12_000],
|
| 49 |
+
["2026-04-02T00:00:00Z", 38_000],
|
| 50 |
+
["2026-04-03T00:00:00Z", 57_000],
|
| 51 |
+
["2026-04-04T00:00:00Z", 102_000],
|
| 52 |
+
["2026-04-05T00:00:00Z", 143_000],
|
| 53 |
+
],
|
| 54 |
+
"persistent_savings": {
|
| 55 |
+
"display_session": {},
|
| 56 |
+
"lifetime": {"tokens_saved": 143_000, "compression_savings_usd": 12.34},
|
| 57 |
+
},
|
| 58 |
+
"pipeline_timing": {},
|
| 59 |
+
"compression_cache": {"mode": "cache"},
|
| 60 |
+
"prefix_cache": {
|
| 61 |
+
"by_provider": {
|
| 62 |
+
"anthropic": {
|
| 63 |
+
"cache_read_tokens": 9_800_000,
|
| 64 |
+
"cache_write_tokens": 420_000,
|
| 65 |
+
"cache_write_5m_tokens": 185_000,
|
| 66 |
+
"cache_write_1h_tokens": 235_000,
|
| 67 |
+
"cache_write_5m_requests": 18,
|
| 68 |
+
"cache_write_1h_requests": 24,
|
| 69 |
+
"requests": 128,
|
| 70 |
+
"hit_requests": 96,
|
| 71 |
+
"hit_rate": 75.0,
|
| 72 |
+
"bust_count": 0,
|
| 73 |
+
"bust_write_tokens": 0,
|
| 74 |
+
"read_discount": "90%",
|
| 75 |
+
"write_premium": "25%",
|
| 76 |
+
"savings_usd": 5.67,
|
| 77 |
+
"write_premium_usd": 0.42,
|
| 78 |
+
"net_savings_usd": 5.67,
|
| 79 |
+
"label": "Explicit breakpoints, 5-min TTL",
|
| 80 |
+
"observed_ttl_buckets": {
|
| 81 |
+
"5m": {"tokens": 185_000, "requests": 18},
|
| 82 |
+
"1h": {"tokens": 235_000, "requests": 24},
|
| 83 |
+
},
|
| 84 |
+
"observed_ttl_mix": {
|
| 85 |
+
"5m_pct": 44.0,
|
| 86 |
+
"1h_pct": 56.0,
|
| 87 |
+
"active_buckets": ["5m", "1h"],
|
| 88 |
+
},
|
| 89 |
+
}
|
| 90 |
+
},
|
| 91 |
+
"totals": {
|
| 92 |
+
"cache_read_tokens": 9_800_000,
|
| 93 |
+
"cache_write_tokens": 420_000,
|
| 94 |
+
"cache_write_5m_tokens": 185_000,
|
| 95 |
+
"cache_write_1h_tokens": 235_000,
|
| 96 |
+
"cache_write_5m_requests": 18,
|
| 97 |
+
"cache_write_1h_requests": 24,
|
| 98 |
+
"requests": 128,
|
| 99 |
+
"hit_requests": 96,
|
| 100 |
+
"bust_count": 0,
|
| 101 |
+
"bust_write_tokens": 0,
|
| 102 |
+
"savings_usd": 5.67,
|
| 103 |
+
"write_premium_usd": 0.42,
|
| 104 |
+
"net_savings_usd": 5.67,
|
| 105 |
+
"hit_rate": 75.0,
|
| 106 |
+
"observed_ttl_buckets": {
|
| 107 |
+
"5m": {"tokens": 185_000, "requests": 18},
|
| 108 |
+
"1h": {"tokens": 235_000, "requests": 24},
|
| 109 |
+
},
|
| 110 |
+
"observed_ttl_mix": {
|
| 111 |
+
"5m_pct": 44.0,
|
| 112 |
+
"1h_pct": 56.0,
|
| 113 |
+
"active_buckets": ["5m", "1h"],
|
| 114 |
+
},
|
| 115 |
+
},
|
| 116 |
+
"prefix_freeze": {
|
| 117 |
+
"busts_avoided": 0,
|
| 118 |
+
"tokens_preserved": 0,
|
| 119 |
+
"compression_foregone_tokens": 0,
|
| 120 |
+
"net_benefit_tokens": 0,
|
| 121 |
+
},
|
| 122 |
+
"attribution": "Observed provider TTL buckets.",
|
| 123 |
+
},
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _sample_history() -> dict:
|
| 128 |
+
return {
|
| 129 |
+
"history": [
|
| 130 |
+
{
|
| 131 |
+
"timestamp": "2026-04-01T00:00:00Z",
|
| 132 |
+
"total_tokens_saved": 12_000,
|
| 133 |
+
"compression_savings_usd": 0.6,
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"timestamp": "2026-04-05T00:00:00Z",
|
| 137 |
+
"total_tokens_saved": 143_000,
|
| 138 |
+
"compression_savings_usd": 12.34,
|
| 139 |
+
},
|
| 140 |
+
],
|
| 141 |
+
"series": {
|
| 142 |
+
"daily": [
|
| 143 |
+
{
|
| 144 |
+
"timestamp": "2026-04-05T00:00:00Z",
|
| 145 |
+
"tokens_saved": 20_000,
|
| 146 |
+
"total_tokens_saved": 143_000,
|
| 147 |
+
"compression_savings_usd_delta": 1.7,
|
| 148 |
+
}
|
| 149 |
+
],
|
| 150 |
+
"weekly": [],
|
| 151 |
+
"monthly": [],
|
| 152 |
+
},
|
| 153 |
+
"lifetime": {"tokens_saved": 143_000, "compression_savings_usd": 12.34},
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _install_dashboard_routes(page: Page) -> None:
|
| 158 |
+
stats = _sample_stats()
|
| 159 |
+
history = _sample_history()
|
| 160 |
+
health = {"status": "healthy", "version": "0.3.0"}
|
| 161 |
+
dashboard_html = get_dashboard_html()
|
| 162 |
+
|
| 163 |
+
def handler(route) -> None: # type: ignore[no-untyped-def]
|
| 164 |
+
url = route.request.url
|
| 165 |
+
if url.endswith("/dashboard") or url == "http://headroom.local/":
|
| 166 |
+
route.fulfill(status=200, content_type="text/html", body=dashboard_html)
|
| 167 |
+
return
|
| 168 |
+
if url.endswith("/stats"):
|
| 169 |
+
route.fulfill(status=200, content_type="application/json", body=json.dumps(stats))
|
| 170 |
+
return
|
| 171 |
+
if "/stats-history" in url:
|
| 172 |
+
route.fulfill(
|
| 173 |
+
status=200,
|
| 174 |
+
content_type="application/json",
|
| 175 |
+
body=json.dumps(history),
|
| 176 |
+
)
|
| 177 |
+
return
|
| 178 |
+
if url.endswith("/health"):
|
| 179 |
+
route.fulfill(status=200, content_type="application/json", body=json.dumps(health))
|
| 180 |
+
return
|
| 181 |
+
route.continue_()
|
| 182 |
+
|
| 183 |
+
page.route("**/*", handler)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def test_dashboard_renders_observed_ttl_metrics_and_can_capture_screenshot() -> None:
|
| 187 |
+
artifact_dir = os.environ.get("HEADROOM_PLAYWRIGHT_ARTIFACT_DIR")
|
| 188 |
+
|
| 189 |
+
with sync_playwright() as pw:
|
| 190 |
+
browser = pw.chromium.launch()
|
| 191 |
+
page = browser.new_page(viewport={"width": 1720, "height": 1400}, color_scheme="dark")
|
| 192 |
+
_install_dashboard_routes(page)
|
| 193 |
+
page.goto("http://headroom.local/dashboard", wait_until="load")
|
| 194 |
+
|
| 195 |
+
expect(page.get_by_text("Observed TTL Buckets")).to_be_visible()
|
| 196 |
+
expect(page.get_by_text("Provider-reported cache write mix")).to_be_visible()
|
| 197 |
+
expect(page.get_by_test_id("ttl-bucket-headline")).to_have_text("1h leaning")
|
| 198 |
+
expect(page.get_by_test_id("ttl-bucket-mix-1h-pct")).to_have_text("1h 56.0%")
|
| 199 |
+
expect(page.get_by_test_id("ttl-bucket-mix-5m-pct")).to_have_text("5m 44.0%")
|
| 200 |
+
expect(page.get_by_test_id("ttl-bucket-1h-value")).to_have_text("235.0k")
|
| 201 |
+
expect(page.get_by_test_id("ttl-bucket-5m-value")).to_have_text("185.0k")
|
| 202 |
+
expect(page.get_by_text("TTL 1h 56.0% / 5m 44.0%")).to_be_visible()
|
| 203 |
+
|
| 204 |
+
screenshot_path = (
|
| 205 |
+
Path(artifact_dir) / "dashboard-cache-ttl-main.png"
|
| 206 |
+
if artifact_dir
|
| 207 |
+
else Path.cwd() / "dashboard-cache-ttl-main.png"
|
| 208 |
+
)
|
| 209 |
+
screenshot_path.parent.mkdir(parents=True, exist_ok=True)
|
| 210 |
+
page.screenshot(path=str(screenshot_path), full_page=True)
|
| 211 |
+
browser.close()
|
tests/test_proxy_cache_ttl_metrics.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for observed Anthropic cache TTL bucket metrics."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import asyncio
|
| 6 |
+
|
| 7 |
+
import pytest
|
| 8 |
+
|
| 9 |
+
from headroom.proxy.cost import CostTracker, build_prefix_cache_stats
|
| 10 |
+
from headroom.proxy.prometheus_metrics import PrometheusMetrics
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def test_prometheus_metrics_tracks_observed_ttl_buckets() -> None:
|
| 14 |
+
metrics = PrometheusMetrics()
|
| 15 |
+
|
| 16 |
+
asyncio.run(
|
| 17 |
+
metrics.record_request(
|
| 18 |
+
provider="anthropic",
|
| 19 |
+
model="claude-opus-4-6",
|
| 20 |
+
input_tokens=100,
|
| 21 |
+
output_tokens=20,
|
| 22 |
+
tokens_saved=5,
|
| 23 |
+
latency_ms=10.0,
|
| 24 |
+
cache_read_tokens=40,
|
| 25 |
+
cache_write_tokens=60,
|
| 26 |
+
cache_write_5m_tokens=10,
|
| 27 |
+
cache_write_1h_tokens=50,
|
| 28 |
+
)
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
stats = metrics.cache_by_provider["anthropic"]
|
| 32 |
+
assert stats["cache_write_5m_tokens"] == 10
|
| 33 |
+
assert stats["cache_write_1h_tokens"] == 50
|
| 34 |
+
assert stats["cache_write_5m_requests"] == 1
|
| 35 |
+
assert stats["cache_write_1h_requests"] == 1
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_cost_tracker_exposes_observed_ttl_buckets_per_model() -> None:
|
| 39 |
+
tracker = CostTracker()
|
| 40 |
+
tracker.record_tokens(
|
| 41 |
+
"claude-opus-4-6",
|
| 42 |
+
tokens_saved=10,
|
| 43 |
+
tokens_sent=90,
|
| 44 |
+
cache_read_tokens=40,
|
| 45 |
+
cache_write_tokens=60,
|
| 46 |
+
cache_write_5m_tokens=10,
|
| 47 |
+
cache_write_1h_tokens=50,
|
| 48 |
+
uncached_tokens=20,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
stats = tracker.stats()
|
| 52 |
+
assert stats["cache_write_5m_tokens"] == 10
|
| 53 |
+
assert stats["cache_write_1h_tokens"] == 50
|
| 54 |
+
assert stats["per_model"]["claude-opus-4-6"]["cache_write_5m_tokens"] == 10
|
| 55 |
+
assert stats["per_model"]["claude-opus-4-6"]["cache_write_1h_tokens"] == 50
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def test_prefix_cache_stats_include_observed_ttl_mix() -> None:
|
| 59 |
+
metrics = PrometheusMetrics()
|
| 60 |
+
provider_stats = metrics.cache_by_provider["anthropic"]
|
| 61 |
+
provider_stats["requests"] = 2
|
| 62 |
+
provider_stats["hit_requests"] = 1
|
| 63 |
+
provider_stats["cache_read_tokens"] = 40
|
| 64 |
+
provider_stats["cache_write_tokens"] = 60
|
| 65 |
+
provider_stats["cache_write_5m_tokens"] = 15
|
| 66 |
+
provider_stats["cache_write_1h_tokens"] = 45
|
| 67 |
+
provider_stats["cache_write_5m_requests"] = 1
|
| 68 |
+
provider_stats["cache_write_1h_requests"] = 1
|
| 69 |
+
|
| 70 |
+
stats = build_prefix_cache_stats(metrics, None)
|
| 71 |
+
anthropic = stats["by_provider"]["anthropic"]
|
| 72 |
+
|
| 73 |
+
assert anthropic["observed_ttl_buckets"]["5m"]["tokens"] == 15
|
| 74 |
+
assert anthropic["observed_ttl_buckets"]["1h"]["tokens"] == 45
|
| 75 |
+
assert anthropic["observed_ttl_mix"]["5m_pct"] == 25.0
|
| 76 |
+
assert anthropic["observed_ttl_mix"]["1h_pct"] == 75.0
|
| 77 |
+
assert stats["totals"]["observed_ttl_buckets"]["5m"]["tokens"] == 15
|
| 78 |
+
assert stats["totals"]["observed_ttl_buckets"]["1h"]["tokens"] == 45
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def test_streaming_parser_extracts_anthropic_ttl_bucket_usage() -> None:
|
| 82 |
+
from headroom.proxy.server import HeadroomProxy, ProxyConfig
|
| 83 |
+
|
| 84 |
+
proxy = HeadroomProxy(
|
| 85 |
+
ProxyConfig(
|
| 86 |
+
optimize=False,
|
| 87 |
+
cache_enabled=False,
|
| 88 |
+
rate_limit_enabled=False,
|
| 89 |
+
cost_tracking_enabled=False,
|
| 90 |
+
log_requests=False,
|
| 91 |
+
ccr_inject_tool=False,
|
| 92 |
+
ccr_handle_responses=False,
|
| 93 |
+
ccr_context_tracking=False,
|
| 94 |
+
)
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
chunk = (
|
| 98 |
+
b'data: {"type":"message_start","message":{"usage":{"input_tokens":12,'
|
| 99 |
+
b'"cache_read_input_tokens":3,"cache_creation_input_tokens":9,'
|
| 100 |
+
b'"cache_creation":{"ephemeral_5m_input_tokens":4,"ephemeral_1h_input_tokens":5}}}}\n\n'
|
| 101 |
+
)
|
| 102 |
+
usage = proxy._parse_sse_usage(chunk, "anthropic")
|
| 103 |
+
|
| 104 |
+
assert usage is not None
|
| 105 |
+
assert usage["cache_creation_ephemeral_5m_input_tokens"] == 4
|
| 106 |
+
assert usage["cache_creation_ephemeral_1h_input_tokens"] == 5
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def test_stats_endpoint_reports_observed_ttl_buckets() -> None:
|
| 110 |
+
pytest.importorskip("fastapi")
|
| 111 |
+
from fastapi.testclient import TestClient
|
| 112 |
+
|
| 113 |
+
from headroom.proxy.server import ProxyConfig, create_app
|
| 114 |
+
|
| 115 |
+
app = create_app(
|
| 116 |
+
ProxyConfig(
|
| 117 |
+
optimize=False,
|
| 118 |
+
cache_enabled=False,
|
| 119 |
+
rate_limit_enabled=False,
|
| 120 |
+
cost_tracking_enabled=False,
|
| 121 |
+
log_requests=False,
|
| 122 |
+
ccr_inject_tool=False,
|
| 123 |
+
ccr_handle_responses=False,
|
| 124 |
+
ccr_context_tracking=False,
|
| 125 |
+
)
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
proxy = app.state.proxy
|
| 129 |
+
provider_stats = proxy.metrics.cache_by_provider["anthropic"]
|
| 130 |
+
provider_stats["requests"] = 1
|
| 131 |
+
provider_stats["hit_requests"] = 1
|
| 132 |
+
provider_stats["cache_read_tokens"] = 30
|
| 133 |
+
provider_stats["cache_write_tokens"] = 70
|
| 134 |
+
provider_stats["cache_write_5m_tokens"] = 20
|
| 135 |
+
provider_stats["cache_write_1h_tokens"] = 50
|
| 136 |
+
provider_stats["cache_write_5m_requests"] = 1
|
| 137 |
+
provider_stats["cache_write_1h_requests"] = 1
|
| 138 |
+
|
| 139 |
+
with TestClient(app) as client:
|
| 140 |
+
response = client.get("/stats")
|
| 141 |
+
|
| 142 |
+
assert response.status_code == 200
|
| 143 |
+
prefix_cache = response.json()["prefix_cache"]
|
| 144 |
+
anthropic = prefix_cache["by_provider"]["anthropic"]
|
| 145 |
+
assert anthropic["observed_ttl_buckets"]["5m"]["tokens"] == 20
|
| 146 |
+
assert anthropic["observed_ttl_buckets"]["1h"]["tokens"] == 50
|
| 147 |
+
assert prefix_cache["totals"]["observed_ttl_mix"]["active_buckets"] == ["5m", "1h"]
|