Deploy tai-helper ea13623

.env.example

@@ -11,6 +11,11 @@ HELPER_SITE_WIDE_HOSTS=towardsai.com,www.towardsai.com
 HELPER_PRIMARY_BASE_URL=https://openrouter.ai/api/v1
 HELPER_PRIMARY_API_KEY=
 HELPER_PRIMARY_MODEL=deepseek/deepseek-v4-flash
+# OpenRouter serves one model id from many independent hosts at different
+# quantisations, and they are NOT interchangeable for extractive grounding.
+# Unpinned routing sent every request to one host that answered 0/6 on the
+# main starter prompt. Preference order, first entry wins; fallbacks stay on.
+HELPER_OPENROUTER_PROVIDER_ORDER=novita
 
 # Backup model: Gemini 3.7 Flash, on Google's API rather than OpenRouter so the
 # fallback does not share a gateway with the primary.

README.md

@@ -49,6 +49,16 @@ out of balance and took the helper down, while OpenRouter draws on a shared
 balance that is kept topped up. `HELPER_PRIMARY_MODEL` can name any OpenRouter
 model without code changes.
 
+**Pin the OpenRouter host.** OpenRouter serves one model id from many
+independent providers at different quantisations, and they are not
+interchangeable for this workload. Measured on identical grounding prompts, one
+host answered 3/3 while another answered 0/3, and the host the default routing
+kept choosing returned `not_found` on the main starter prompt every single time
+— 0/6 in production, 6/6 once pinned. `HELPER_OPENROUTER_PROVIDER_ORDER` sets
+the preference order; `require_parameters` additionally drops hosts that would
+ignore `response_format` and answer with prose. Fallbacks stay enabled, so one
+host going down degrades quality rather than taking the helper offline.
+
 Gemini 3.7 Flash is the fallback, deliberately kept on Google's own API rather
 than routed through OpenRouter, so the backup does not share a gateway with the
 primary. It only runs when the primary fails and it passes the grounding

tai_helper/llm.py

@@ -638,6 +638,22 @@ def _reasoning_control() -> dict[str, Any]:
     return {"reasoning": {"enabled": settings.primary_reasoning_enabled}}
 
 
+def _provider_routing() -> dict[str, Any]:
+    """Constrain which OpenRouter host serves the request.
+
+    ``require_parameters`` drops hosts that do not honour what is sent, so a
+    host that ignores ``response_format`` cannot answer with prose that then
+    fails to parse. The preference order puts a host measured as reliable for
+    extractive grounding first, with fallbacks left on so a single host being
+    down degrades quality instead of taking the helper offline.
+    """
+
+    routing: dict[str, Any] = {"require_parameters": True, "allow_fallbacks": True}
+    if settings.openrouter_provider_order:
+        routing["order"] = list(settings.openrouter_provider_order)
+    return {"provider": routing}
+
+
 def _generate_primary_answer(prompt: str) -> LLMResult:
     if not settings.primary_api_key:
         raise RuntimeError("HELPER_PRIMARY_API_KEY is not configured")
@@ -652,6 +668,7 @@ def _generate_primary_answer(prompt: str) -> LLMResult:
                 {"role": "user", "content": prompt},
             ],
             **_reasoning_control(),
+            **_provider_routing(),
             "response_format": {"type": "json_object"},
             "temperature": DEFAULT_TEMPERATURE,
             "max_tokens": settings.max_output_tokens,

tai_helper/settings.py

@@ -103,6 +103,16 @@ class Settings:
             or DEFAULT_PRIMARY_MODEL
         )
     )
+    # OpenRouter serves one model id from many independent hosts at different
+    # quantisations, and they are not interchangeable for extractive grounding:
+    # measured on the same prompts, one host answered 3/3 while another
+    # answered 0/3 and a third returned not_found every time. Left unpinned,
+    # routing silently picks one and answer quality swings with it.
+    openrouter_provider_order: tuple[str, ...] = field(
+        default_factory=lambda: _csv_env(
+            "HELPER_OPENROUTER_PROVIDER_ORDER", "novita"
+        )
+    )
     gemini_api_key: str = field(
         default_factory=lambda: os.getenv("GEMINI_API_KEY", "").strip()
     )

tests/test_llm.py

@@ -260,3 +260,66 @@ def test_openrouter_style_disables_reasoning_with_openrouter_parameter(
     assert captured["reasoning"] == {"enabled": False}
     assert "thinking" not in captured
     assert captured["model"] == "deepseek/deepseek-v4-flash"
+
+
+def test_openrouter_request_pins_provider_routing(monkeypatch) -> None:
+    """One OpenRouter model id is served by many independent hosts.
+
+    They are not interchangeable for extractive grounding: on identical prompts
+    one host answered 3/3 while another answered 0/3, and the host the default
+    routing kept selecting returned not_found every time. require_parameters
+    also drops hosts that would ignore response_format and reply with prose.
+    """
+
+    captured: dict[str, object] = {}
+
+    def fake_post(_url, **kwargs):
+        captured.update(kwargs["json"])
+        return FakeOpenRouterResponse(
+            200, {"choices": [{"message": {"content": "{}"}}]}, ""
+        )
+
+    monkeypatch.setattr(
+        llm,
+        "settings",
+        Settings(
+            primary_api_key="openrouter-key",
+            openrouter_provider_order=("novita", "alibaba"),
+        ),
+    )
+    monkeypatch.setattr(llm.requests, "post", fake_post)
+
+    llm.generate_answer("Visitor prompt")
+
+    assert captured["provider"] == {
+        "require_parameters": True,
+        "allow_fallbacks": True,
+        "order": ["novita", "alibaba"],
+    }
+
+
+def test_openrouter_routing_omits_order_when_unset(monkeypatch) -> None:
+    """An empty order must not pin routing to nothing and 404 the request."""
+
+    captured: dict[str, object] = {}
+
+    def fake_post(_url, **kwargs):
+        captured.update(kwargs["json"])
+        return FakeOpenRouterResponse(
+            200, {"choices": [{"message": {"content": "{}"}}]}, ""
+        )
+
+    monkeypatch.setattr(
+        llm,
+        "settings",
+        Settings(primary_api_key="openrouter-key", openrouter_provider_order=()),
+    )
+    monkeypatch.setattr(llm.requests, "post", fake_post)
+
+    llm.generate_answer("Visitor prompt")
+
+    assert captured["provider"] == {
+        "require_parameters": True,
+        "allow_fallbacks": True,
+    }
+    assert "order" not in captured["provider"]