Spaces:
Build error
Build error
Commit ·
0a80b52
1
Parent(s): ee68d8e
Bump to v0.5.19: cache stats fixes, Codex WS fix, compression-vs-cache tracking
Browse filesv0.5.19 includes:
- Fix Codex WebSocket HTTP 500: ChatGPT auth routing to chatgpt.com,
correct beta header, HTTP POST streaming fallback
- Token-level cache hit rate (not binary per-request)
- Compression-vs-cache tracking: tokens_saved vs cache_bust_tokens per session
- Pass uncached_input_tokens for OpenAI and Gemini handlers
- Fix async test failures in test_proxy_scalability
- headroom/__init__.py +1 -1
- headroom/proxy/handlers/gemini.py +4 -0
- headroom/proxy/handlers/openai.py +5 -0
- pyproject.toml +1 -1
headroom/__init__.py
CHANGED
|
@@ -153,7 +153,7 @@ from .transforms import (
|
|
| 153 |
TransformPipeline,
|
| 154 |
)
|
| 155 |
|
| 156 |
-
__version__ = "0.5.
|
| 157 |
|
| 158 |
__all__ = [
|
| 159 |
# Main client
|
|
|
|
| 153 |
TransformPipeline,
|
| 154 |
)
|
| 155 |
|
| 156 |
+
__version__ = "0.5.19"
|
| 157 |
|
| 158 |
__all__ = [
|
| 159 |
# Main client
|
headroom/proxy/handlers/gemini.py
CHANGED
|
@@ -341,12 +341,15 @@ class GeminiHandlerMixin:
|
|
| 341 |
f"[{request_id}] Failed to extract cached tokens from Gemini response: {e}"
|
| 342 |
)
|
| 343 |
|
|
|
|
|
|
|
| 344 |
if self.cost_tracker:
|
| 345 |
self.cost_tracker.record_tokens(
|
| 346 |
model,
|
| 347 |
tokens_saved,
|
| 348 |
optimized_tokens,
|
| 349 |
cache_read_tokens=cache_read_tokens,
|
|
|
|
| 350 |
)
|
| 351 |
|
| 352 |
await self.metrics.record_request(
|
|
@@ -359,6 +362,7 @@ class GeminiHandlerMixin:
|
|
| 359 |
overhead_ms=optimization_latency,
|
| 360 |
waste_signals=waste_signals_dict,
|
| 361 |
cache_read_tokens=cache_read_tokens,
|
|
|
|
| 362 |
)
|
| 363 |
|
| 364 |
if tokens_saved > 0:
|
|
|
|
| 341 |
f"[{request_id}] Failed to extract cached tokens from Gemini response: {e}"
|
| 342 |
)
|
| 343 |
|
| 344 |
+
uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)
|
| 345 |
+
|
| 346 |
if self.cost_tracker:
|
| 347 |
self.cost_tracker.record_tokens(
|
| 348 |
model,
|
| 349 |
tokens_saved,
|
| 350 |
optimized_tokens,
|
| 351 |
cache_read_tokens=cache_read_tokens,
|
| 352 |
+
uncached_tokens=uncached_input_tokens,
|
| 353 |
)
|
| 354 |
|
| 355 |
await self.metrics.record_request(
|
|
|
|
| 362 |
overhead_ms=optimization_latency,
|
| 363 |
waste_signals=waste_signals_dict,
|
| 364 |
cache_read_tokens=cache_read_tokens,
|
| 365 |
+
uncached_input_tokens=uncached_input_tokens,
|
| 366 |
)
|
| 367 |
|
| 368 |
if tokens_saved > 0:
|
headroom/proxy/handlers/openai.py
CHANGED
|
@@ -512,12 +512,16 @@ class OpenAIHandlerMixin:
|
|
| 512 |
messages=optimized_messages,
|
| 513 |
)
|
| 514 |
|
|
|
|
|
|
|
|
|
|
| 515 |
if self.cost_tracker:
|
| 516 |
self.cost_tracker.record_tokens(
|
| 517 |
model,
|
| 518 |
tokens_saved,
|
| 519 |
optimized_tokens,
|
| 520 |
cache_read_tokens=cache_read_tokens,
|
|
|
|
| 521 |
)
|
| 522 |
|
| 523 |
# Cache
|
|
@@ -537,6 +541,7 @@ class OpenAIHandlerMixin:
|
|
| 537 |
pipeline_timing=pipeline_timing,
|
| 538 |
waste_signals=waste_signals_dict,
|
| 539 |
cache_read_tokens=cache_read_tokens,
|
|
|
|
| 540 |
)
|
| 541 |
|
| 542 |
if tokens_saved > 0:
|
|
|
|
| 512 |
messages=optimized_messages,
|
| 513 |
)
|
| 514 |
|
| 515 |
+
# OpenAI has no write penalty — uncached = total - cached
|
| 516 |
+
uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)
|
| 517 |
+
|
| 518 |
if self.cost_tracker:
|
| 519 |
self.cost_tracker.record_tokens(
|
| 520 |
model,
|
| 521 |
tokens_saved,
|
| 522 |
optimized_tokens,
|
| 523 |
cache_read_tokens=cache_read_tokens,
|
| 524 |
+
uncached_tokens=uncached_input_tokens,
|
| 525 |
)
|
| 526 |
|
| 527 |
# Cache
|
|
|
|
| 541 |
pipeline_timing=pipeline_timing,
|
| 542 |
waste_signals=waste_signals_dict,
|
| 543 |
cache_read_tokens=cache_read_tokens,
|
| 544 |
+
uncached_input_tokens=uncached_input_tokens,
|
| 545 |
)
|
| 546 |
|
| 547 |
if tokens_saved > 0:
|
pyproject.toml
CHANGED
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "headroom-ai"
|
| 7 |
-
version = "0.5.
|
| 8 |
description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
|
| 9 |
readme = "README.md"
|
| 10 |
license = "Apache-2.0"
|
|
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "headroom-ai"
|
| 7 |
+
version = "0.5.19"
|
| 8 |
description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
|
| 9 |
readme = "README.md"
|
| 10 |
license = "Apache-2.0"
|