JerrettDavis commited on
Commit
b4d2977
·
1 Parent(s): 52bd5e8

Add observed cache TTL metrics and dashboard coverage

Browse files
docs/metrics.md CHANGED
@@ -53,6 +53,41 @@ for backward compatibility. The new `persistent_savings` block is durable local
53
  proxy compression history stored by default at `~/.headroom/proxy_savings.json`.
54
  Use `HEADROOM_SAVINGS_PATH` to override the file location.
55
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  ### Historical Savings Endpoint
57
 
58
  ```bash
 
53
  proxy compression history stored by default at `~/.headroom/proxy_savings.json`.
54
  Use `HEADROOM_SAVINGS_PATH` to override the file location.
55
 
56
+ For Anthropic-style providers that return cache-write TTL buckets, `/stats`
57
+ also surfaces observed cache TTL usage under `prefix_cache`:
58
+
59
+ ```json
60
+ {
61
+ "prefix_cache": {
62
+ "by_provider": {
63
+ "anthropic": {
64
+ "observed_ttl_buckets": {
65
+ "5m": {"tokens": 20000, "requests": 8},
66
+ "1h": {"tokens": 50000, "requests": 12}
67
+ },
68
+ "observed_ttl_mix": {
69
+ "5m_pct": 28.6,
70
+ "1h_pct": 71.4,
71
+ "active_buckets": ["5m", "1h"]
72
+ }
73
+ }
74
+ },
75
+ "totals": {
76
+ "observed_ttl_buckets": {
77
+ "5m": {"tokens": 20000, "requests": 8},
78
+ "1h": {"tokens": 50000, "requests": 12}
79
+ }
80
+ }
81
+ }
82
+ }
83
+ ```
84
+
85
+ These fields are observational only:
86
+
87
+ - they reflect provider-reported cache write buckets
88
+ - they do not configure TTL
89
+ - they do not represent remaining expiration time
90
+
91
  ### Historical Savings Endpoint
92
 
93
  ```bash
docs/proxy.md CHANGED
@@ -133,6 +133,16 @@ curl http://localhost:8787/stats
133
  small recent preview. The existing `savings_history` field is still present and
134
  remains session-scoped for backward compatibility.
135
 
 
 
 
 
 
 
 
 
 
 
136
  ### Historical Savings
137
 
138
  ```bash
 
133
  small recent preview. The existing `savings_history` field is still present and
134
  remains session-scoped for backward compatibility.
135
 
136
+ For providers that return cache-write TTL bucket usage, `/stats` also includes
137
+ observed TTL breakdowns under `prefix_cache`:
138
+
139
+ - `observed_ttl_buckets.5m.tokens`
140
+ - `observed_ttl_buckets.1h.tokens`
141
+ - `observed_ttl_mix`
142
+
143
+ These are provider-reported observations, not configured TTL and not remaining
144
+ expiration time.
145
+
146
  ### Historical Savings
147
 
148
  ```bash
headroom/dashboard/templates/dashboard.html CHANGED
@@ -208,6 +208,60 @@
208
  <span class="flex items-center gap-1"><span class="w-2 h-2 rounded-full bg-border inline-block"></span> Uncached</span>
209
  </div>
210
  </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
211
  <!-- Per-provider breakdown -->
212
  <template x-if="Object.keys(stats.prefix_cache?.by_provider || {}).length > 0">
213
  <div class="mt-4 border-t border-border pt-3">
@@ -219,11 +273,14 @@
219
  <span class="px-2 py-0.5 bg-border rounded text-xs font-mono" x-text="prov"></span>
220
  <span class="text-xs text-gray-500" x-text="pc.label"></span>
221
  </div>
222
- <div class="flex items-center gap-4 text-xs font-mono">
223
  <span class="text-emerald-400" x-text="formatNumber(pc.cache_read_tokens) + ' reads (' + pc.read_discount + ' off)'"></span>
224
  <template x-if="pc.write_premium !== 'none'">
225
  <span class="text-amber-400" x-text="formatNumber(pc.cache_write_tokens) + ' writes (+' + pc.write_premium + ')'"></span>
226
  </template>
 
 
 
227
  <span :class="pc.bust_count > 0 ? 'text-red-400' : 'text-gray-500'" x-text="pc.bust_count + ' busts'"></span>
228
  <span class="text-emerald-400" x-text="'$' + formatCurrency(pc.net_savings_usd)"></span>
229
  </div>
@@ -1054,6 +1111,26 @@
1054
  return Math.round((t.cache_write_tokens || 0) / total * 100);
1055
  },
1056
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1057
  // --- Waste Signals ---
1058
 
1059
  wasteSignalLabel(signal) {
 
208
  <span class="flex items-center gap-1"><span class="w-2 h-2 rounded-full bg-border inline-block"></span> Uncached</span>
209
  </div>
210
  </div>
211
+ <template x-if="hasObservedTtlBuckets">
212
+ <div class="mt-5 border-t border-border pt-4">
213
+ <div class="flex items-center justify-between mb-3">
214
+ <div>
215
+ <div class="text-xs text-gray-500 uppercase tracking-[0.22em]">Observed TTL Buckets</div>
216
+ <div class="text-sm text-gray-300 mt-1">Provider-reported cache write mix</div>
217
+ </div>
218
+ <div class="text-xs text-gray-500 font-mono" x-text="observedTtlWindowLabel"></div>
219
+ </div>
220
+ <div class="grid grid-cols-1 xl:grid-cols-[1.3fr_0.9fr] gap-4">
221
+ <div class="rounded-2xl border border-cyan-500/20 bg-[linear-gradient(180deg,rgba(34,211,238,0.10),rgba(12,16,22,0.88))] p-4 shadow-[inset_0_1px_0_rgba(255,255,255,0.03)]">
222
+ <div class="flex items-start justify-between gap-3">
223
+ <div>
224
+ <div class="text-[11px] uppercase tracking-[0.22em] text-cyan-300/75">Bucket Mix</div>
225
+ <div class="mt-1 text-2xl font-light text-cyan-50">
226
+ <span data-testid="ttl-bucket-mix-1h-total" x-text="formatNumber(stats.prefix_cache?.totals?.cache_write_1h_tokens || 0)"></span>
227
+ <span class="text-sm text-cyan-300/70">1h</span>
228
+ <span class="mx-2 text-cyan-500/60">/</span>
229
+ <span data-testid="ttl-bucket-mix-5m-total" x-text="formatNumber(stats.prefix_cache?.totals?.cache_write_5m_tokens || 0)"></span>
230
+ <span class="text-sm text-cyan-300/70">5m</span>
231
+ </div>
232
+ </div>
233
+ <div data-testid="ttl-bucket-headline" class="rounded-full border border-cyan-400/20 bg-black/20 px-3 py-1 text-[11px] uppercase tracking-[0.18em] text-cyan-200/80" x-text="observedTtlHeadline"></div>
234
+ </div>
235
+ <div class="mt-4">
236
+ <div class="flex justify-between text-xs text-gray-500 mb-1">
237
+ <span>Observed write-token split</span>
238
+ <span x-text="(stats.prefix_cache?.totals?.observed_ttl_mix?.active_buckets || []).join(' + ')"></span>
239
+ </div>
240
+ <div class="h-3 overflow-hidden rounded-full bg-black/30 ring-1 ring-white/5 flex">
241
+ <div class="h-full bg-cyan-400 transition-all duration-500" :style="'width:' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['1h_pct'] || 0) + '%'"></div>
242
+ <div class="h-full bg-violet-400 transition-all duration-500" :style="'width:' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['5m_pct'] || 0) + '%'"></div>
243
+ </div>
244
+ <div class="mt-2 flex flex-wrap gap-3 text-xs text-gray-400">
245
+ <span class="inline-flex items-center gap-1.5"><span class="inline-block h-2 w-2 rounded-full bg-cyan-400"></span><span data-testid="ttl-bucket-mix-1h-pct" x-text="'1h ' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['1h_pct'] || 0).toFixed(1) + '%'"></span></span>
246
+ <span class="inline-flex items-center gap-1.5"><span class="inline-block h-2 w-2 rounded-full bg-violet-400"></span><span data-testid="ttl-bucket-mix-5m-pct" x-text="'5m ' + (stats.prefix_cache?.totals?.observed_ttl_mix?.['5m_pct'] || 0).toFixed(1) + '%'"></span></span>
247
+ </div>
248
+ </div>
249
+ </div>
250
+ <div class="grid grid-cols-1 sm:grid-cols-2 xl:grid-cols-1 gap-3">
251
+ <div class="rounded-2xl border border-white/8 bg-black/20 p-4">
252
+ <div class="text-[11px] uppercase tracking-[0.18em] text-cyan-300/75">1h Cache Writes</div>
253
+ <div data-testid="ttl-bucket-1h-value" class="mt-2 text-3xl font-light text-cyan-50" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['1h']?.tokens || 0)"></div>
254
+ <div class="mt-1 text-xs text-gray-500" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['1h']?.requests || 0) + ' requests observed'"></div>
255
+ </div>
256
+ <div class="rounded-2xl border border-white/8 bg-black/20 p-4">
257
+ <div class="text-[11px] uppercase tracking-[0.18em] text-violet-300/75">5m Cache Writes</div>
258
+ <div data-testid="ttl-bucket-5m-value" class="mt-2 text-3xl font-light text-violet-50" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['5m']?.tokens || 0)"></div>
259
+ <div class="mt-1 text-xs text-gray-500" x-text="formatNumber(stats.prefix_cache?.totals?.observed_ttl_buckets?.['5m']?.requests || 0) + ' requests observed'"></div>
260
+ </div>
261
+ </div>
262
+ </div>
263
+ </div>
264
+ </template>
265
  <!-- Per-provider breakdown -->
266
  <template x-if="Object.keys(stats.prefix_cache?.by_provider || {}).length > 0">
267
  <div class="mt-4 border-t border-border pt-3">
 
273
  <span class="px-2 py-0.5 bg-border rounded text-xs font-mono" x-text="prov"></span>
274
  <span class="text-xs text-gray-500" x-text="pc.label"></span>
275
  </div>
276
+ <div class="flex flex-wrap items-center justify-end gap-4 text-xs font-mono">
277
  <span class="text-emerald-400" x-text="formatNumber(pc.cache_read_tokens) + ' reads (' + pc.read_discount + ' off)'"></span>
278
  <template x-if="pc.write_premium !== 'none'">
279
  <span class="text-amber-400" x-text="formatNumber(pc.cache_write_tokens) + ' writes (+' + pc.write_premium + ')'"></span>
280
  </template>
281
+ <template x-if="pc.observed_ttl_mix">
282
+ <span class="text-cyan-300" x-text="'TTL 1h ' + (pc.observed_ttl_mix['1h_pct'] || 0).toFixed(1) + '% / 5m ' + (pc.observed_ttl_mix['5m_pct'] || 0).toFixed(1) + '%'"></span>
283
+ </template>
284
  <span :class="pc.bust_count > 0 ? 'text-red-400' : 'text-gray-500'" x-text="pc.bust_count + ' busts'"></span>
285
  <span class="text-emerald-400" x-text="'$' + formatCurrency(pc.net_savings_usd)"></span>
286
  </div>
 
1111
  return Math.round((t.cache_write_tokens || 0) / total * 100);
1112
  },
1113
 
1114
+ get hasObservedTtlBuckets() {
1115
+ const buckets = this.stats.prefix_cache?.totals?.observed_ttl_buckets || {};
1116
+ return ((buckets['5m']?.tokens || 0) + (buckets['1h']?.tokens || 0)) > 0;
1117
+ },
1118
+
1119
+ get observedTtlHeadline() {
1120
+ const mix = this.stats.prefix_cache?.totals?.observed_ttl_mix || {};
1121
+ const oneHour = mix['1h_pct'] || 0;
1122
+ const fiveMinute = mix['5m_pct'] || 0;
1123
+ if (oneHour === fiveMinute) return 'Balanced';
1124
+ return oneHour > fiveMinute ? '1h leaning' : '5m leaning';
1125
+ },
1126
+
1127
+ get observedTtlWindowLabel() {
1128
+ const mix = this.stats.prefix_cache?.totals?.observed_ttl_mix || {};
1129
+ const active = mix.active_buckets || [];
1130
+ if (!active.length) return 'No TTL bucket data';
1131
+ return active.length === 1 ? active[0] + ' only' : active.join(' / ');
1132
+ },
1133
+
1134
  // --- Waste Signals ---
1135
 
1136
  wasteSignalLabel(signal) {
headroom/proxy/cost.py CHANGED
@@ -78,6 +78,10 @@ def build_prefix_cache_stats(
78
  totals = {
79
  "cache_read_tokens": 0,
80
  "cache_write_tokens": 0,
 
 
 
 
81
  "requests": 0,
82
  "hit_requests": 0,
83
  "bust_count": 0,
@@ -119,6 +123,10 @@ def build_prefix_cache_stats(
119
  # for observability but don't penalise our savings number.
120
  read_tokens: int = pc["cache_read_tokens"] # type: ignore[assignment]
121
  write_tokens: int = pc["cache_write_tokens"] # type: ignore[assignment]
 
 
 
 
122
  savings_usd = 0.0
123
  write_premium_usd = 0.0
124
 
@@ -134,6 +142,10 @@ def build_prefix_cache_stats(
134
  provider_stats = {
135
  "cache_read_tokens": read_tokens,
136
  "cache_write_tokens": write_tokens,
 
 
 
 
137
  "requests": pc["requests"],
138
  "hit_requests": pc["hit_requests"],
139
  "hit_rate": hit_rate,
@@ -145,12 +157,37 @@ def build_prefix_cache_stats(
145
  "write_premium_usd": round(write_premium_usd, 4),
146
  "net_savings_usd": round(savings_usd, 4),
147
  "label": str(econ["label"]),
 
 
 
 
 
 
 
 
 
 
148
  }
 
 
 
 
 
 
 
 
 
 
 
149
  by_provider[provider] = provider_stats
150
 
151
  # Accumulate totals
152
  totals["cache_read_tokens"] += read_tokens
153
  totals["cache_write_tokens"] += write_tokens
 
 
 
 
154
  totals["requests"] += pc["requests"]
155
  totals["hit_requests"] += pc["hit_requests"]
156
  totals["bust_count"] += pc["bust_count"]
@@ -164,6 +201,33 @@ def build_prefix_cache_stats(
164
  totals["hit_rate"] = (
165
  round(totals["hit_requests"] / totals["requests"] * 100, 1) if totals["requests"] > 0 else 0
166
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
 
168
  return {
169
  "by_provider": by_provider,
@@ -180,7 +244,9 @@ def build_prefix_cache_stats(
180
  "Prefix caching is performed by the LLM provider (Anthropic, OpenAI). "
181
  "Headroom reports cache stats as observed from API responses. "
182
  "CacheAligner and prefix freeze improve cache hit rates by stabilizing "
183
- "the message prefix, but baseline caching happens without Headroom."
 
 
184
  ),
185
  }
186
 
@@ -356,6 +422,8 @@ class CostTracker:
356
  # API-reported cache breakdown per model (for accurate cost calculation)
357
  self._api_cache_read_by_model: dict[str, int] = {}
358
  self._api_cache_write_by_model: dict[str, int] = {}
 
 
359
  self._api_uncached_by_model: dict[str, int] = {}
360
 
361
  # Cache resolved model names to avoid repeated litellm lookups.
@@ -480,6 +548,8 @@ class CostTracker:
480
  tokens_sent: int,
481
  cache_read_tokens: int = 0,
482
  cache_write_tokens: int = 0,
 
 
483
  uncached_tokens: int = 0,
484
  ):
485
  """Record token counts per model.
@@ -503,6 +573,12 @@ class CostTracker:
503
  self._api_cache_write_by_model[model] = (
504
  self._api_cache_write_by_model.get(model, 0) + cache_write_tokens
505
  )
 
 
 
 
 
 
506
  self._api_uncached_by_model[model] = (
507
  self._api_uncached_by_model.get(model, 0) + uncached_tokens
508
  )
@@ -574,6 +650,8 @@ class CostTracker:
574
  "requests": reqs,
575
  "tokens_saved": saved,
576
  "tokens_sent": sent,
 
 
577
  "reduction_pct": round(saved / (saved + sent) * 100, 1)
578
  if (saved + sent) > 0
579
  else 0,
@@ -625,6 +703,8 @@ class CostTracker:
625
  "total_tokens_saved": total_saved,
626
  "total_input_tokens": total_input_tokens,
627
  "total_input_cost_usd": round(cost_with_headroom, 4),
 
 
628
  "per_model": per_model,
629
  "cost_with_headroom_usd": round(cost_with_headroom, 4),
630
  "savings_usd": round(savings_usd, 4),
 
78
  totals = {
79
  "cache_read_tokens": 0,
80
  "cache_write_tokens": 0,
81
+ "cache_write_5m_tokens": 0,
82
+ "cache_write_1h_tokens": 0,
83
+ "cache_write_5m_requests": 0,
84
+ "cache_write_1h_requests": 0,
85
  "requests": 0,
86
  "hit_requests": 0,
87
  "bust_count": 0,
 
123
  # for observability but don't penalise our savings number.
124
  read_tokens: int = pc["cache_read_tokens"] # type: ignore[assignment]
125
  write_tokens: int = pc["cache_write_tokens"] # type: ignore[assignment]
126
+ write_5m_tokens: int = pc["cache_write_5m_tokens"] # type: ignore[assignment]
127
+ write_1h_tokens: int = pc["cache_write_1h_tokens"] # type: ignore[assignment]
128
+ write_5m_requests: int = pc["cache_write_5m_requests"] # type: ignore[assignment]
129
+ write_1h_requests: int = pc["cache_write_1h_requests"] # type: ignore[assignment]
130
  savings_usd = 0.0
131
  write_premium_usd = 0.0
132
 
 
142
  provider_stats = {
143
  "cache_read_tokens": read_tokens,
144
  "cache_write_tokens": write_tokens,
145
+ "cache_write_5m_tokens": write_5m_tokens,
146
+ "cache_write_1h_tokens": write_1h_tokens,
147
+ "cache_write_5m_requests": write_5m_requests,
148
+ "cache_write_1h_requests": write_1h_requests,
149
  "requests": pc["requests"],
150
  "hit_requests": pc["hit_requests"],
151
  "hit_rate": hit_rate,
 
157
  "write_premium_usd": round(write_premium_usd, 4),
158
  "net_savings_usd": round(savings_usd, 4),
159
  "label": str(econ["label"]),
160
+ "observed_ttl_buckets": {
161
+ "5m": {
162
+ "tokens": write_5m_tokens,
163
+ "requests": write_5m_requests,
164
+ },
165
+ "1h": {
166
+ "tokens": write_1h_tokens,
167
+ "requests": write_1h_requests,
168
+ },
169
+ },
170
  }
171
+ total_observed_ttl_tokens = write_5m_tokens + write_1h_tokens
172
+ if total_observed_ttl_tokens > 0:
173
+ provider_stats["observed_ttl_mix"] = {
174
+ "5m_pct": round(write_5m_tokens / total_observed_ttl_tokens * 100, 1),
175
+ "1h_pct": round(write_1h_tokens / total_observed_ttl_tokens * 100, 1),
176
+ "active_buckets": [
177
+ bucket
178
+ for bucket, tokens in (("5m", write_5m_tokens), ("1h", write_1h_tokens))
179
+ if tokens > 0
180
+ ],
181
+ }
182
  by_provider[provider] = provider_stats
183
 
184
  # Accumulate totals
185
  totals["cache_read_tokens"] += read_tokens
186
  totals["cache_write_tokens"] += write_tokens
187
+ totals["cache_write_5m_tokens"] += write_5m_tokens
188
+ totals["cache_write_1h_tokens"] += write_1h_tokens
189
+ totals["cache_write_5m_requests"] += write_5m_requests
190
+ totals["cache_write_1h_requests"] += write_1h_requests
191
  totals["requests"] += pc["requests"]
192
  totals["hit_requests"] += pc["hit_requests"]
193
  totals["bust_count"] += pc["bust_count"]
 
201
  totals["hit_rate"] = (
202
  round(totals["hit_requests"] / totals["requests"] * 100, 1) if totals["requests"] > 0 else 0
203
  )
204
+ total_observed_ttl_tokens = totals["cache_write_5m_tokens"] + totals["cache_write_1h_tokens"]
205
+ totals["observed_ttl_buckets"] = {
206
+ "5m": {
207
+ "tokens": totals["cache_write_5m_tokens"],
208
+ "requests": totals["cache_write_5m_requests"],
209
+ },
210
+ "1h": {
211
+ "tokens": totals["cache_write_1h_tokens"],
212
+ "requests": totals["cache_write_1h_requests"],
213
+ },
214
+ }
215
+ totals["observed_ttl_mix"] = {
216
+ "5m_pct": round(totals["cache_write_5m_tokens"] / total_observed_ttl_tokens * 100, 1)
217
+ if total_observed_ttl_tokens > 0
218
+ else 0.0,
219
+ "1h_pct": round(totals["cache_write_1h_tokens"] / total_observed_ttl_tokens * 100, 1)
220
+ if total_observed_ttl_tokens > 0
221
+ else 0.0,
222
+ "active_buckets": [
223
+ bucket
224
+ for bucket, tokens in (
225
+ ("5m", totals["cache_write_5m_tokens"]),
226
+ ("1h", totals["cache_write_1h_tokens"]),
227
+ )
228
+ if tokens > 0
229
+ ],
230
+ }
231
 
232
  return {
233
  "by_provider": by_provider,
 
244
  "Prefix caching is performed by the LLM provider (Anthropic, OpenAI). "
245
  "Headroom reports cache stats as observed from API responses. "
246
  "CacheAligner and prefix freeze improve cache hit rates by stabilizing "
247
+ "the message prefix, but baseline caching happens without Headroom. "
248
+ "Observed TTL bucket metrics reflect provider-reported cache write usage "
249
+ "(for example Anthropic 5m vs 1h), not configured or remaining TTL."
250
  ),
251
  }
252
 
 
422
  # API-reported cache breakdown per model (for accurate cost calculation)
423
  self._api_cache_read_by_model: dict[str, int] = {}
424
  self._api_cache_write_by_model: dict[str, int] = {}
425
+ self._api_cache_write_5m_by_model: dict[str, int] = {}
426
+ self._api_cache_write_1h_by_model: dict[str, int] = {}
427
  self._api_uncached_by_model: dict[str, int] = {}
428
 
429
  # Cache resolved model names to avoid repeated litellm lookups.
 
548
  tokens_sent: int,
549
  cache_read_tokens: int = 0,
550
  cache_write_tokens: int = 0,
551
+ cache_write_5m_tokens: int = 0,
552
+ cache_write_1h_tokens: int = 0,
553
  uncached_tokens: int = 0,
554
  ):
555
  """Record token counts per model.
 
573
  self._api_cache_write_by_model[model] = (
574
  self._api_cache_write_by_model.get(model, 0) + cache_write_tokens
575
  )
576
+ self._api_cache_write_5m_by_model[model] = (
577
+ self._api_cache_write_5m_by_model.get(model, 0) + cache_write_5m_tokens
578
+ )
579
+ self._api_cache_write_1h_by_model[model] = (
580
+ self._api_cache_write_1h_by_model.get(model, 0) + cache_write_1h_tokens
581
+ )
582
  self._api_uncached_by_model[model] = (
583
  self._api_uncached_by_model.get(model, 0) + uncached_tokens
584
  )
 
650
  "requests": reqs,
651
  "tokens_saved": saved,
652
  "tokens_sent": sent,
653
+ "cache_write_5m_tokens": self._api_cache_write_5m_by_model.get(model, 0),
654
+ "cache_write_1h_tokens": self._api_cache_write_1h_by_model.get(model, 0),
655
  "reduction_pct": round(saved / (saved + sent) * 100, 1)
656
  if (saved + sent) > 0
657
  else 0,
 
703
  "total_tokens_saved": total_saved,
704
  "total_input_tokens": total_input_tokens,
705
  "total_input_cost_usd": round(cost_with_headroom, 4),
706
+ "cache_write_5m_tokens": sum(self._api_cache_write_5m_by_model.values()),
707
+ "cache_write_1h_tokens": sum(self._api_cache_write_1h_by_model.values()),
708
  "per_model": per_model,
709
  "cost_with_headroom_usd": round(cost_with_headroom, 4),
710
  "savings_usd": round(savings_usd, 4),
headroom/proxy/handlers/anthropic.py CHANGED
@@ -40,6 +40,24 @@ class AnthropicHandlerMixin:
40
  canonical = str(tool)
41
  return (name, canonical)
42
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  @classmethod
44
  def _sort_tools_deterministically(
45
  cls, tools: list[dict[str, Any]] | None
@@ -1196,12 +1214,15 @@ class AnthropicHandlerMixin:
1196
  output_tokens = 0
1197
  cr_tokens = 0
1198
  cw_tokens = 0
 
 
1199
  uncached_input_tokens = 0
1200
  if resp_json:
1201
  usage = resp_json.get("usage", {})
1202
  output_tokens = usage.get("output_tokens", 0)
1203
  cr_tokens = usage.get("cache_read_input_tokens", 0)
1204
  cw_tokens = usage.get("cache_creation_input_tokens", 0)
 
1205
  uncached_input_tokens = usage.get("input_tokens", 0)
1206
 
1207
  # Update prefix cache tracker for next turn
@@ -1225,6 +1246,8 @@ class AnthropicHandlerMixin:
1225
  optimized_tokens,
1226
  cache_read_tokens=cr_tokens,
1227
  cache_write_tokens=cw_tokens,
 
 
1228
  uncached_tokens=uncached_input_tokens,
1229
  )
1230
 
@@ -1252,6 +1275,8 @@ class AnthropicHandlerMixin:
1252
  waste_signals=waste_signals_dict,
1253
  cache_read_tokens=cr_tokens,
1254
  cache_write_tokens=cw_tokens,
 
 
1255
  uncached_input_tokens=uncached_input_tokens,
1256
  )
1257
 
 
40
  canonical = str(tool)
41
  return (name, canonical)
42
 
43
+ @staticmethod
44
+ def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
45
+ """Extract observed Anthropic cache-write TTL bucket usage.
46
+
47
+ Returns (cache_write_5m_tokens, cache_write_1h_tokens) from provider usage.
48
+ These are observational metrics only; they do not imply configured or
49
+ remaining TTL.
50
+ """
51
+ if not isinstance(usage, dict):
52
+ return (0, 0)
53
+ cache_creation = usage.get("cache_creation")
54
+ if not isinstance(cache_creation, dict):
55
+ return (0, 0)
56
+ return (
57
+ int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
58
+ int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
59
+ )
60
+
61
  @classmethod
62
  def _sort_tools_deterministically(
63
  cls, tools: list[dict[str, Any]] | None
 
1214
  output_tokens = 0
1215
  cr_tokens = 0
1216
  cw_tokens = 0
1217
+ cw_5m_tokens = 0
1218
+ cw_1h_tokens = 0
1219
  uncached_input_tokens = 0
1220
  if resp_json:
1221
  usage = resp_json.get("usage", {})
1222
  output_tokens = usage.get("output_tokens", 0)
1223
  cr_tokens = usage.get("cache_read_input_tokens", 0)
1224
  cw_tokens = usage.get("cache_creation_input_tokens", 0)
1225
+ cw_5m_tokens, cw_1h_tokens = self._extract_anthropic_cache_ttl_metrics(usage)
1226
  uncached_input_tokens = usage.get("input_tokens", 0)
1227
 
1228
  # Update prefix cache tracker for next turn
 
1246
  optimized_tokens,
1247
  cache_read_tokens=cr_tokens,
1248
  cache_write_tokens=cw_tokens,
1249
+ cache_write_5m_tokens=cw_5m_tokens,
1250
+ cache_write_1h_tokens=cw_1h_tokens,
1251
  uncached_tokens=uncached_input_tokens,
1252
  )
1253
 
 
1275
  waste_signals=waste_signals_dict,
1276
  cache_read_tokens=cr_tokens,
1277
  cache_write_tokens=cw_tokens,
1278
+ cache_write_5m_tokens=cw_5m_tokens,
1279
+ cache_write_1h_tokens=cw_1h_tokens,
1280
  uncached_input_tokens=uncached_input_tokens,
1281
  )
1282
 
headroom/proxy/handlers/streaming.py CHANGED
@@ -24,6 +24,19 @@ logger = logging.getLogger("headroom.proxy")
24
  class StreamingMixin:
25
  """Mixin providing streaming response methods for HeadroomProxy."""
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:
28
  """Parse usage information from SSE chunk.
29
 
@@ -31,7 +44,9 @@ class StreamingMixin:
31
  For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)
32
  For Gemini: Looks for usageMetadata in each chunk
33
 
34
- Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens
 
 
35
  Returns None if no usage found in this chunk.
36
  """
37
  try:
@@ -67,6 +82,11 @@ class StreamingMixin:
67
  usage["cache_creation_input_tokens"] = msg_usage.get(
68
  "cache_creation_input_tokens", 0
69
  )
 
 
 
 
 
70
 
71
  elif event_type == "message_delta":
72
  delta_usage = data.get("usage", {})
@@ -148,6 +168,11 @@ class StreamingMixin:
148
  usage_found["cache_creation_input_tokens"] = msg_usage.get(
149
  "cache_creation_input_tokens", 0
150
  )
 
 
 
 
 
151
  # INFO logging for cache token tracking (temporary for debugging)
152
  logger.info(
153
  f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "
@@ -450,6 +475,8 @@ class StreamingMixin:
450
  "output_tokens": None,
451
  "cache_read_input_tokens": 0,
452
  "cache_creation_input_tokens": 0,
 
 
453
  "total_bytes": 0,
454
  "sse_buffer": "", # Buffer for incomplete SSE events
455
  "ttfb_ms": None, # Time to first byte from upstream
@@ -550,6 +577,14 @@ class StreamingMixin:
550
  stream_state["cache_creation_input_tokens"] = usage[
551
  "cache_creation_input_tokens"
552
  ]
 
 
 
 
 
 
 
 
553
 
554
  # Memory tool handling after stream completes
555
  # Chunks were already yielded in real-time above, so we only
@@ -637,6 +672,8 @@ class StreamingMixin:
637
  # misleading for aggregation (often just 1 with prompt caching).
638
  cache_read_tokens = stream_state["cache_read_input_tokens"]
639
  cache_write_tokens = stream_state["cache_creation_input_tokens"]
 
 
640
  uncached_input_tokens = stream_state.get("input_tokens") or 0
641
 
642
  # Structured perf log line for `headroom perf` analysis
@@ -672,6 +709,8 @@ class StreamingMixin:
672
  optimized_tokens,
673
  cache_read_tokens=cache_read_tokens,
674
  cache_write_tokens=cache_write_tokens,
 
 
675
  uncached_tokens=uncached_input_tokens,
676
  )
677
 
@@ -687,6 +726,8 @@ class StreamingMixin:
687
  pipeline_timing=pipeline_timing,
688
  cache_read_tokens=cache_read_tokens,
689
  cache_write_tokens=cache_write_tokens,
 
 
690
  uncached_input_tokens=uncached_input_tokens,
691
  )
692
 
 
24
  class StreamingMixin:
25
  """Mixin providing streaming response methods for HeadroomProxy."""
26
 
27
+ @staticmethod
28
+ def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
29
+ """Extract observed Anthropic cache-write TTL bucket usage."""
30
+ if not isinstance(usage, dict):
31
+ return (0, 0)
32
+ cache_creation = usage.get("cache_creation")
33
+ if not isinstance(cache_creation, dict):
34
+ return (0, 0)
35
+ return (
36
+ int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
37
+ int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
38
+ )
39
+
40
  def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:
41
  """Parse usage information from SSE chunk.
42
 
 
44
  For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)
45
  For Gemini: Looks for usageMetadata in each chunk
46
 
47
+ Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens,
48
+ cache_creation_input_tokens, cache_creation_ephemeral_5m_input_tokens,
49
+ cache_creation_ephemeral_1h_input_tokens
50
  Returns None if no usage found in this chunk.
51
  """
52
  try:
 
82
  usage["cache_creation_input_tokens"] = msg_usage.get(
83
  "cache_creation_input_tokens", 0
84
  )
85
+ cache_write_5m, cache_write_1h = (
86
+ self._extract_anthropic_cache_ttl_metrics(msg_usage)
87
+ )
88
+ usage["cache_creation_ephemeral_5m_input_tokens"] = cache_write_5m
89
+ usage["cache_creation_ephemeral_1h_input_tokens"] = cache_write_1h
90
 
91
  elif event_type == "message_delta":
92
  delta_usage = data.get("usage", {})
 
168
  usage_found["cache_creation_input_tokens"] = msg_usage.get(
169
  "cache_creation_input_tokens", 0
170
  )
171
+ cache_write_5m, cache_write_1h = (
172
+ self._extract_anthropic_cache_ttl_metrics(msg_usage)
173
+ )
174
+ usage_found["cache_creation_ephemeral_5m_input_tokens"] = cache_write_5m
175
+ usage_found["cache_creation_ephemeral_1h_input_tokens"] = cache_write_1h
176
  # INFO logging for cache token tracking (temporary for debugging)
177
  logger.info(
178
  f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "
 
475
  "output_tokens": None,
476
  "cache_read_input_tokens": 0,
477
  "cache_creation_input_tokens": 0,
478
+ "cache_creation_ephemeral_5m_input_tokens": 0,
479
+ "cache_creation_ephemeral_1h_input_tokens": 0,
480
  "total_bytes": 0,
481
  "sse_buffer": "", # Buffer for incomplete SSE events
482
  "ttfb_ms": None, # Time to first byte from upstream
 
577
  stream_state["cache_creation_input_tokens"] = usage[
578
  "cache_creation_input_tokens"
579
  ]
580
+ if "cache_creation_ephemeral_5m_input_tokens" in usage:
581
+ stream_state["cache_creation_ephemeral_5m_input_tokens"] = usage[
582
+ "cache_creation_ephemeral_5m_input_tokens"
583
+ ]
584
+ if "cache_creation_ephemeral_1h_input_tokens" in usage:
585
+ stream_state["cache_creation_ephemeral_1h_input_tokens"] = usage[
586
+ "cache_creation_ephemeral_1h_input_tokens"
587
+ ]
588
 
589
  # Memory tool handling after stream completes
590
  # Chunks were already yielded in real-time above, so we only
 
672
  # misleading for aggregation (often just 1 with prompt caching).
673
  cache_read_tokens = stream_state["cache_read_input_tokens"]
674
  cache_write_tokens = stream_state["cache_creation_input_tokens"]
675
+ cache_write_5m_tokens = stream_state["cache_creation_ephemeral_5m_input_tokens"]
676
+ cache_write_1h_tokens = stream_state["cache_creation_ephemeral_1h_input_tokens"]
677
  uncached_input_tokens = stream_state.get("input_tokens") or 0
678
 
679
  # Structured perf log line for `headroom perf` analysis
 
709
  optimized_tokens,
710
  cache_read_tokens=cache_read_tokens,
711
  cache_write_tokens=cache_write_tokens,
712
+ cache_write_5m_tokens=cache_write_5m_tokens,
713
+ cache_write_1h_tokens=cache_write_1h_tokens,
714
  uncached_tokens=uncached_input_tokens,
715
  )
716
 
 
726
  pipeline_timing=pipeline_timing,
727
  cache_read_tokens=cache_read_tokens,
728
  cache_write_tokens=cache_write_tokens,
729
+ cache_write_5m_tokens=cache_write_5m_tokens,
730
+ cache_write_1h_tokens=cache_write_1h_tokens,
731
  uncached_input_tokens=uncached_input_tokens,
732
  )
733
 
headroom/proxy/prometheus_metrics.py CHANGED
@@ -77,6 +77,10 @@ class PrometheusMetrics:
77
  lambda: {
78
  "cache_read_tokens": 0,
79
  "cache_write_tokens": 0,
 
 
 
 
80
  "requests": 0,
81
  "hit_requests": 0, # requests with cache_read > 0
82
  "bust_count": 0,
@@ -158,6 +162,8 @@ class PrometheusMetrics:
158
  waste_signals: dict[str, int] | None = None,
159
  cache_read_tokens: int = 0,
160
  cache_write_tokens: int = 0,
 
 
161
  uncached_input_tokens: int = 0,
162
  ):
163
  """Record metrics for a request."""
@@ -178,6 +184,12 @@ class PrometheusMetrics:
178
  pc = self.cache_by_provider[provider]
179
  pc["cache_read_tokens"] += cache_read_tokens
180
  pc["cache_write_tokens"] += cache_write_tokens
 
 
 
 
 
 
181
  pc["requests"] += 1
182
  if cache_read_tokens > 0:
183
  pc["hit_requests"] += 1
 
77
  lambda: {
78
  "cache_read_tokens": 0,
79
  "cache_write_tokens": 0,
80
+ "cache_write_5m_tokens": 0,
81
+ "cache_write_1h_tokens": 0,
82
+ "cache_write_5m_requests": 0,
83
+ "cache_write_1h_requests": 0,
84
  "requests": 0,
85
  "hit_requests": 0, # requests with cache_read > 0
86
  "bust_count": 0,
 
162
  waste_signals: dict[str, int] | None = None,
163
  cache_read_tokens: int = 0,
164
  cache_write_tokens: int = 0,
165
+ cache_write_5m_tokens: int = 0,
166
+ cache_write_1h_tokens: int = 0,
167
  uncached_input_tokens: int = 0,
168
  ):
169
  """Record metrics for a request."""
 
184
  pc = self.cache_by_provider[provider]
185
  pc["cache_read_tokens"] += cache_read_tokens
186
  pc["cache_write_tokens"] += cache_write_tokens
187
+ pc["cache_write_5m_tokens"] += cache_write_5m_tokens
188
+ pc["cache_write_1h_tokens"] += cache_write_1h_tokens
189
+ if cache_write_5m_tokens > 0:
190
+ pc["cache_write_5m_requests"] += 1
191
+ if cache_write_1h_tokens > 0:
192
+ pc["cache_write_1h_requests"] += 1
193
  pc["requests"] += 1
194
  if cache_read_tokens > 0:
195
  pc["hit_requests"] += 1
tests/test_dashboard_cache_ttl_playwright.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Behavior-driven Playwright validation for dashboard TTL cache metrics."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+
11
+ from headroom.dashboard import get_dashboard_html
12
+
13
+ playwright = pytest.importorskip("playwright.sync_api")
14
+ Page = playwright.Page
15
+ expect = playwright.expect
16
+ sync_playwright = playwright.sync_playwright
17
+
18
+
19
+ def _sample_stats() -> dict:
20
+ return {
21
+ "cost": {
22
+ "savings_usd": 12.34,
23
+ "compression_savings_usd": 12.34,
24
+ "cache_savings_usd": 5.67,
25
+ "cli_tokens_avoided": 0,
26
+ },
27
+ "requests": {
28
+ "total": 128,
29
+ "cached": 96,
30
+ "rate_limited": 0,
31
+ "failed": 0,
32
+ "by_provider": {"anthropic": 128},
33
+ "by_model": {"claude-opus-4-6": 128},
34
+ },
35
+ "tokens": {
36
+ "input": 245_000,
37
+ "output": 88_000,
38
+ "saved": 143_000,
39
+ "cli_tokens_avoided": 0,
40
+ "total_before_compression": 388_000,
41
+ "savings_percent": 36.86,
42
+ },
43
+ "overhead": {"average_ms": 14.2, "min_ms": 4.5, "max_ms": 42.7},
44
+ "ttfb": {"average_ms": 1320.0, "min_ms": 420.0, "max_ms": 2900.0},
45
+ "latency": {"average_ms": 1510.0, "min_ms": 520.0, "max_ms": 3300.0},
46
+ "waste_signals": {"json_bloat": 95_000, "repetition": 48_000},
47
+ "savings_history": [
48
+ ["2026-04-01T00:00:00Z", 12_000],
49
+ ["2026-04-02T00:00:00Z", 38_000],
50
+ ["2026-04-03T00:00:00Z", 57_000],
51
+ ["2026-04-04T00:00:00Z", 102_000],
52
+ ["2026-04-05T00:00:00Z", 143_000],
53
+ ],
54
+ "persistent_savings": {
55
+ "display_session": {},
56
+ "lifetime": {"tokens_saved": 143_000, "compression_savings_usd": 12.34},
57
+ },
58
+ "pipeline_timing": {},
59
+ "compression_cache": {"mode": "cache"},
60
+ "prefix_cache": {
61
+ "by_provider": {
62
+ "anthropic": {
63
+ "cache_read_tokens": 9_800_000,
64
+ "cache_write_tokens": 420_000,
65
+ "cache_write_5m_tokens": 185_000,
66
+ "cache_write_1h_tokens": 235_000,
67
+ "cache_write_5m_requests": 18,
68
+ "cache_write_1h_requests": 24,
69
+ "requests": 128,
70
+ "hit_requests": 96,
71
+ "hit_rate": 75.0,
72
+ "bust_count": 0,
73
+ "bust_write_tokens": 0,
74
+ "read_discount": "90%",
75
+ "write_premium": "25%",
76
+ "savings_usd": 5.67,
77
+ "write_premium_usd": 0.42,
78
+ "net_savings_usd": 5.67,
79
+ "label": "Explicit breakpoints, 5-min TTL",
80
+ "observed_ttl_buckets": {
81
+ "5m": {"tokens": 185_000, "requests": 18},
82
+ "1h": {"tokens": 235_000, "requests": 24},
83
+ },
84
+ "observed_ttl_mix": {
85
+ "5m_pct": 44.0,
86
+ "1h_pct": 56.0,
87
+ "active_buckets": ["5m", "1h"],
88
+ },
89
+ }
90
+ },
91
+ "totals": {
92
+ "cache_read_tokens": 9_800_000,
93
+ "cache_write_tokens": 420_000,
94
+ "cache_write_5m_tokens": 185_000,
95
+ "cache_write_1h_tokens": 235_000,
96
+ "cache_write_5m_requests": 18,
97
+ "cache_write_1h_requests": 24,
98
+ "requests": 128,
99
+ "hit_requests": 96,
100
+ "bust_count": 0,
101
+ "bust_write_tokens": 0,
102
+ "savings_usd": 5.67,
103
+ "write_premium_usd": 0.42,
104
+ "net_savings_usd": 5.67,
105
+ "hit_rate": 75.0,
106
+ "observed_ttl_buckets": {
107
+ "5m": {"tokens": 185_000, "requests": 18},
108
+ "1h": {"tokens": 235_000, "requests": 24},
109
+ },
110
+ "observed_ttl_mix": {
111
+ "5m_pct": 44.0,
112
+ "1h_pct": 56.0,
113
+ "active_buckets": ["5m", "1h"],
114
+ },
115
+ },
116
+ "prefix_freeze": {
117
+ "busts_avoided": 0,
118
+ "tokens_preserved": 0,
119
+ "compression_foregone_tokens": 0,
120
+ "net_benefit_tokens": 0,
121
+ },
122
+ "attribution": "Observed provider TTL buckets.",
123
+ },
124
+ }
125
+
126
+
127
+ def _sample_history() -> dict:
128
+ return {
129
+ "history": [
130
+ {
131
+ "timestamp": "2026-04-01T00:00:00Z",
132
+ "total_tokens_saved": 12_000,
133
+ "compression_savings_usd": 0.6,
134
+ },
135
+ {
136
+ "timestamp": "2026-04-05T00:00:00Z",
137
+ "total_tokens_saved": 143_000,
138
+ "compression_savings_usd": 12.34,
139
+ },
140
+ ],
141
+ "series": {
142
+ "daily": [
143
+ {
144
+ "timestamp": "2026-04-05T00:00:00Z",
145
+ "tokens_saved": 20_000,
146
+ "total_tokens_saved": 143_000,
147
+ "compression_savings_usd_delta": 1.7,
148
+ }
149
+ ],
150
+ "weekly": [],
151
+ "monthly": [],
152
+ },
153
+ "lifetime": {"tokens_saved": 143_000, "compression_savings_usd": 12.34},
154
+ }
155
+
156
+
157
+ def _install_dashboard_routes(page: Page) -> None:
158
+ stats = _sample_stats()
159
+ history = _sample_history()
160
+ health = {"status": "healthy", "version": "0.3.0"}
161
+ dashboard_html = get_dashboard_html()
162
+
163
+ def handler(route) -> None: # type: ignore[no-untyped-def]
164
+ url = route.request.url
165
+ if url.endswith("/dashboard") or url == "http://headroom.local/":
166
+ route.fulfill(status=200, content_type="text/html", body=dashboard_html)
167
+ return
168
+ if url.endswith("/stats"):
169
+ route.fulfill(status=200, content_type="application/json", body=json.dumps(stats))
170
+ return
171
+ if "/stats-history" in url:
172
+ route.fulfill(
173
+ status=200,
174
+ content_type="application/json",
175
+ body=json.dumps(history),
176
+ )
177
+ return
178
+ if url.endswith("/health"):
179
+ route.fulfill(status=200, content_type="application/json", body=json.dumps(health))
180
+ return
181
+ route.continue_()
182
+
183
+ page.route("**/*", handler)
184
+
185
+
186
+ def test_dashboard_renders_observed_ttl_metrics_and_can_capture_screenshot() -> None:
187
+ artifact_dir = os.environ.get("HEADROOM_PLAYWRIGHT_ARTIFACT_DIR")
188
+
189
+ with sync_playwright() as pw:
190
+ browser = pw.chromium.launch()
191
+ page = browser.new_page(viewport={"width": 1720, "height": 1400}, color_scheme="dark")
192
+ _install_dashboard_routes(page)
193
+ page.goto("http://headroom.local/dashboard", wait_until="load")
194
+
195
+ expect(page.get_by_text("Observed TTL Buckets")).to_be_visible()
196
+ expect(page.get_by_text("Provider-reported cache write mix")).to_be_visible()
197
+ expect(page.get_by_test_id("ttl-bucket-headline")).to_have_text("1h leaning")
198
+ expect(page.get_by_test_id("ttl-bucket-mix-1h-pct")).to_have_text("1h 56.0%")
199
+ expect(page.get_by_test_id("ttl-bucket-mix-5m-pct")).to_have_text("5m 44.0%")
200
+ expect(page.get_by_test_id("ttl-bucket-1h-value")).to_have_text("235.0k")
201
+ expect(page.get_by_test_id("ttl-bucket-5m-value")).to_have_text("185.0k")
202
+ expect(page.get_by_text("TTL 1h 56.0% / 5m 44.0%")).to_be_visible()
203
+
204
+ screenshot_path = (
205
+ Path(artifact_dir) / "dashboard-cache-ttl-main.png"
206
+ if artifact_dir
207
+ else Path.cwd() / "dashboard-cache-ttl-main.png"
208
+ )
209
+ screenshot_path.parent.mkdir(parents=True, exist_ok=True)
210
+ page.screenshot(path=str(screenshot_path), full_page=True)
211
+ browser.close()
tests/test_proxy_cache_ttl_metrics.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for observed Anthropic cache TTL bucket metrics."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+
7
+ import pytest
8
+
9
+ from headroom.proxy.cost import CostTracker, build_prefix_cache_stats
10
+ from headroom.proxy.prometheus_metrics import PrometheusMetrics
11
+
12
+
13
+ def test_prometheus_metrics_tracks_observed_ttl_buckets() -> None:
14
+ metrics = PrometheusMetrics()
15
+
16
+ asyncio.run(
17
+ metrics.record_request(
18
+ provider="anthropic",
19
+ model="claude-opus-4-6",
20
+ input_tokens=100,
21
+ output_tokens=20,
22
+ tokens_saved=5,
23
+ latency_ms=10.0,
24
+ cache_read_tokens=40,
25
+ cache_write_tokens=60,
26
+ cache_write_5m_tokens=10,
27
+ cache_write_1h_tokens=50,
28
+ )
29
+ )
30
+
31
+ stats = metrics.cache_by_provider["anthropic"]
32
+ assert stats["cache_write_5m_tokens"] == 10
33
+ assert stats["cache_write_1h_tokens"] == 50
34
+ assert stats["cache_write_5m_requests"] == 1
35
+ assert stats["cache_write_1h_requests"] == 1
36
+
37
+
38
+ def test_cost_tracker_exposes_observed_ttl_buckets_per_model() -> None:
39
+ tracker = CostTracker()
40
+ tracker.record_tokens(
41
+ "claude-opus-4-6",
42
+ tokens_saved=10,
43
+ tokens_sent=90,
44
+ cache_read_tokens=40,
45
+ cache_write_tokens=60,
46
+ cache_write_5m_tokens=10,
47
+ cache_write_1h_tokens=50,
48
+ uncached_tokens=20,
49
+ )
50
+
51
+ stats = tracker.stats()
52
+ assert stats["cache_write_5m_tokens"] == 10
53
+ assert stats["cache_write_1h_tokens"] == 50
54
+ assert stats["per_model"]["claude-opus-4-6"]["cache_write_5m_tokens"] == 10
55
+ assert stats["per_model"]["claude-opus-4-6"]["cache_write_1h_tokens"] == 50
56
+
57
+
58
+ def test_prefix_cache_stats_include_observed_ttl_mix() -> None:
59
+ metrics = PrometheusMetrics()
60
+ provider_stats = metrics.cache_by_provider["anthropic"]
61
+ provider_stats["requests"] = 2
62
+ provider_stats["hit_requests"] = 1
63
+ provider_stats["cache_read_tokens"] = 40
64
+ provider_stats["cache_write_tokens"] = 60
65
+ provider_stats["cache_write_5m_tokens"] = 15
66
+ provider_stats["cache_write_1h_tokens"] = 45
67
+ provider_stats["cache_write_5m_requests"] = 1
68
+ provider_stats["cache_write_1h_requests"] = 1
69
+
70
+ stats = build_prefix_cache_stats(metrics, None)
71
+ anthropic = stats["by_provider"]["anthropic"]
72
+
73
+ assert anthropic["observed_ttl_buckets"]["5m"]["tokens"] == 15
74
+ assert anthropic["observed_ttl_buckets"]["1h"]["tokens"] == 45
75
+ assert anthropic["observed_ttl_mix"]["5m_pct"] == 25.0
76
+ assert anthropic["observed_ttl_mix"]["1h_pct"] == 75.0
77
+ assert stats["totals"]["observed_ttl_buckets"]["5m"]["tokens"] == 15
78
+ assert stats["totals"]["observed_ttl_buckets"]["1h"]["tokens"] == 45
79
+
80
+
81
+ def test_streaming_parser_extracts_anthropic_ttl_bucket_usage() -> None:
82
+ from headroom.proxy.server import HeadroomProxy, ProxyConfig
83
+
84
+ proxy = HeadroomProxy(
85
+ ProxyConfig(
86
+ optimize=False,
87
+ cache_enabled=False,
88
+ rate_limit_enabled=False,
89
+ cost_tracking_enabled=False,
90
+ log_requests=False,
91
+ ccr_inject_tool=False,
92
+ ccr_handle_responses=False,
93
+ ccr_context_tracking=False,
94
+ )
95
+ )
96
+
97
+ chunk = (
98
+ b'data: {"type":"message_start","message":{"usage":{"input_tokens":12,'
99
+ b'"cache_read_input_tokens":3,"cache_creation_input_tokens":9,'
100
+ b'"cache_creation":{"ephemeral_5m_input_tokens":4,"ephemeral_1h_input_tokens":5}}}}\n\n'
101
+ )
102
+ usage = proxy._parse_sse_usage(chunk, "anthropic")
103
+
104
+ assert usage is not None
105
+ assert usage["cache_creation_ephemeral_5m_input_tokens"] == 4
106
+ assert usage["cache_creation_ephemeral_1h_input_tokens"] == 5
107
+
108
+
109
+ def test_stats_endpoint_reports_observed_ttl_buckets() -> None:
110
+ pytest.importorskip("fastapi")
111
+ from fastapi.testclient import TestClient
112
+
113
+ from headroom.proxy.server import ProxyConfig, create_app
114
+
115
+ app = create_app(
116
+ ProxyConfig(
117
+ optimize=False,
118
+ cache_enabled=False,
119
+ rate_limit_enabled=False,
120
+ cost_tracking_enabled=False,
121
+ log_requests=False,
122
+ ccr_inject_tool=False,
123
+ ccr_handle_responses=False,
124
+ ccr_context_tracking=False,
125
+ )
126
+ )
127
+
128
+ proxy = app.state.proxy
129
+ provider_stats = proxy.metrics.cache_by_provider["anthropic"]
130
+ provider_stats["requests"] = 1
131
+ provider_stats["hit_requests"] = 1
132
+ provider_stats["cache_read_tokens"] = 30
133
+ provider_stats["cache_write_tokens"] = 70
134
+ provider_stats["cache_write_5m_tokens"] = 20
135
+ provider_stats["cache_write_1h_tokens"] = 50
136
+ provider_stats["cache_write_5m_requests"] = 1
137
+ provider_stats["cache_write_1h_requests"] = 1
138
+
139
+ with TestClient(app) as client:
140
+ response = client.get("/stats")
141
+
142
+ assert response.status_code == 200
143
+ prefix_cache = response.json()["prefix_cache"]
144
+ anthropic = prefix_cache["by_provider"]["anthropic"]
145
+ assert anthropic["observed_ttl_buckets"]["5m"]["tokens"] == 20
146
+ assert anthropic["observed_ttl_buckets"]["1h"]["tokens"] == 50
147
+ assert prefix_cache["totals"]["observed_ttl_mix"]["active_buckets"] == ["5m", "1h"]