fix: eagerly load LLM at startup so /api/health surfaces the real error

Root cause of persistent generic 'model not loaded' on the Space:
agents.get_llm() was lazy. The first /api/health call (before any
chat/insight/mentor hit) returned llm_status='uninitialized' and an
empty llm_error. The frontend fallback '(h.llm_error || model not loaded)'
showed the generic message. No real error reason ever reached the UI.

Fixes:
- app.py: eagerly call agents.get_llm() at startup after the download.
The Space's first health check now reflects the TRUE status
('loaded' or 'error' with the exception type+message). The model
load (~10-60s) happens during container startup, not on first
user request. If the download failed, get_llm() raises
FileNotFoundError immediately and the real reason is captured.
- download_model.py: retry each download mode (with-token, anonymous)
up to 3 times with exponential backoff to survive transient
network failures on cold starts.
- tests/e2e_test.py: inject a fake download_model so the test
process never tries to download 2.84 GB from the Hub.
- tests/verify_llm_status.py: Playwright-driven end-to-end check
that simulates 'model missing' (fake download returning a
nonexistent path) and confirms the real FileNotFoundError
reaches both /api/health and the frontend status line. Also
confirms chat replies via deterministic fallback with no
'error: format only' leak.

Verified via Playwright: status line now reads
'LLM offline (FileNotFoundError: Model file not found at ...).
Chat/mentor use deterministic fallbacks. Game still works.'
instead of the generic 'model not loaded'.

110 tests pass: 12 unit + 42 FastAPI E2E + 56 Playwright.

app.py

@@ -20,12 +20,19 @@ app = FastAPI(title="Retro Alpha")
 ROOT = Path(__file__).resolve().parent
 STATIC_DIR = ROOT / "static"
 
-# Load the local LLM once at startup. All endpoints share this single LLM.
+# Ensure the GGUF is on disk, then eagerly load it into RAM so
+# /api/health reflects the REAL status (not "uninitialized") as soon
+# as the container is up. Lazy-loading would race the first health
+# check and surface a generic "model not loaded" with no error reason.
 try:
     agents.MODEL_PATH = download_model.download()
-    print(f"Model path set to: {agents.MODEL_PATH}")
+    print(f"Model path: {agents.MODEL_PATH}")
+    print("Eagerly loading LLM into memory (this may take ~10-60s)...")
+    _ = agents.get_llm()  # triggers Llama(...) load; sets status + error
+    err = agents.llm_error()
+    print(f"LLM status: {agents.llm_status()} ({err or 'ok'})")
 except Exception as e:
-    print(f"Model download wrapper failed: {e}. Endpoints will use deterministic fallbacks.")
+    print(f"Startup LLM init failed: {e}")
 
 
 app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")

download_model.py

@@ -24,7 +24,11 @@ def _local_path() -> Path:
 def download() -> str:
     """Ensure the GGUF model is available locally. Returns the local path
     as a string. Never raises — returns the expected path even on
-    failure so callers can surface a precise error to the user."""
+    failure so callers can surface a precise error to the user.
+
+    Retries each download mode up to 3 times with exponential backoff
+    to survive transient network failures on cold starts."""
+    import time
     MODEL_DIR.mkdir(parents=True, exist_ok=True)
     local = _local_path()
 
@@ -34,29 +38,32 @@ def download() -> str:
         return str(local)
 
     token = os.getenv("HF_TOKEN")
-    # Try with token first (for private repos), then anonymously (public).
     for attempt_token, label in [(token, "with token"), (None, "anonymously")]:
         if attempt_token == "":
             attempt_token = None
-        try:
-            print(f"Downloading {MODEL_FILE} from {MODEL_REPO} ({label})...")
-            from huggingface_hub import hf_hub_download
-            path = hf_hub_download(
-                repo_id=MODEL_REPO,
-                filename=MODEL_FILE,
-                local_dir=str(MODEL_DIR),
-                local_dir_use_symlinks=False,
-                token=attempt_token,
-            )
-            print(f"Download complete: {path}")
-            return str(path)
-        except Exception as e:
-            print(f"Download attempt failed ({label}): {type(e).__name__}: {e}")
-            if attempt_token is None:
-                break  # don't retry anonymous again
+        for attempt in range(1, 4):
+            try:
+                print(f"Downloading {MODEL_FILE} from {MODEL_REPO} ({label}, attempt {attempt}/3)...")
+                from huggingface_hub import hf_hub_download
+                path = hf_hub_download(
+                    repo_id=MODEL_REPO,
+                    filename=MODEL_FILE,
+                    local_dir=str(MODEL_DIR),
+                    local_dir_use_symlinks=False,
+                    token=attempt_token,
+                )
+                print(f"Download complete: {path}")
+                return str(path)
+            except Exception as e:
+                print(f"Download attempt {attempt} failed ({label}): {type(e).__name__}: {e}")
+                if attempt < 3:
+                    time.sleep(2 ** attempt)  # 2s, 4s
+                else:
+                    break
+        # If anonymous mode also failed, no point retrying it
+        if attempt_token is None:
+            break
 
-    # Download failed; return the expected path so the app can report
-    # a clear "model not found" error rather than crashing.
     print(f"Model download failed. Expected at: {local}")
     return str(local)
 

tests/e2e_test.py

@@ -2,6 +2,7 @@
 
 import os
 import sys
+import types
 
 # Force UTF-8 stdout for ₹ symbol on Windows
 try:
@@ -14,6 +15,21 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 # Force mock LLM for tests
 os.environ["MOCK_LLM"] = "1"
 
+# Inject a fake download_model so app.py startup does NOT try to
+# download the 2.84 GB model from the Hub. The MOCK_LLM flag means
+# the model is never loaded into RAM, so the path is irrelevant.
+_MODEL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "models")
+_FAKE_PATH = os.path.join(os.path.abspath(_MODEL_DIR), "TEST_FAKE_IGNORED.gguf")
+for mod in list(sys.modules):
+    if mod in ("download_model",):
+        del sys.modules[mod]
+fake_dm = types.ModuleType("download_model")
+fake_dm.download = lambda: _FAKE_PATH
+fake_dm.MODEL_REPO = "fake/test"
+fake_dm.MODEL_FILE = "fake.gguf"
+fake_dm.MODEL_DIR = os.path.abspath(_MODEL_DIR)
+sys.modules["download_model"] = fake_dm
+
 import agents
 agents._llm = "mock"
 agents._llm_status = "mock"

tests/verify_llm_status.py

@@ -0,0 +1,168 @@
+"""Verify the LLM error reason reaches the UI when the model can't load.
+
+This is the exact scenario the user is hitting on the Space: the model
+file is missing (download failed / not present), so get_llm() raises.
+We simulate it WITHOUT triggering a real 2.84 GB download by injecting
+a fake download_model that returns a nonexistent path.
+
+Confirms:
+  - /api/health returns llm='error' (not 'uninitialized') and a non-empty
+    llm_error describing the real failure
+  - the frontend status line shows the real error (not the generic
+    'model not loaded' fallback)
+  - chat still works via deterministic fallback (no 'error: format only'
+    sentinel leak)
+"""
+
+import os
+import sys
+import time
+import socket
+import types
+
+try:
+    sys.stdout.reconfigure(encoding="utf-8")
+except Exception:
+    pass
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+PORT = 7861
+BASE_URL = f"http://localhost:{PORT}"
+SCREENSHOT_DIR = os.path.join(os.path.dirname(__file__), "screenshots")
+os.makedirs(SCREENSHOT_DIR, exist_ok=True)
+MODEL_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "models")
+FAKE_MISSING = os.path.join(MODEL_DIR, "FAKE_MISSING_MODEL.gguf")
+
+PASSED = 0
+FAILED = 0
+
+
+def check(name, condition, detail=""):
+    global PASSED, FAILED
+    if condition:
+        PASSED += 1
+        print(f"  PASS: {name}")
+    else:
+        FAILED += 1
+        print(f"  FAIL: {name} {detail}")
+
+
+def wait_for_http(url, timeout=30):
+    """Wait until the URL returns any HTTP response (not just port open)."""
+    import urllib.request
+    deadline = time.time() + timeout
+    last = None
+    while time.time() < deadline:
+        try:
+            with urllib.request.urlopen(url, timeout=2) as r:
+                r.read(1)
+                return True
+        except Exception as e:
+            last = e
+            time.sleep(0.3)
+    print(f"  (wait_for_http timed out: {last})")
+    return False
+
+
+# Clear cached modules
+for mod in list(sys.modules):
+    if mod in ("app", "agents", "download_model", "mentor", "engine", "events"):
+        del sys.modules[mod]
+
+# Inject fake download_model -> returns nonexistent path
+fake_dm = types.ModuleType("download_model")
+fake_dm.download = lambda: FAKE_MISSING
+fake_dm.MODEL_REPO = "fake/repo"
+fake_dm.MODEL_FILE = "fake.gguf"
+fake_dm.MODEL_DIR = MODEL_DIR
+sys.modules["download_model"] = fake_dm
+
+# Ensure MOCK_LLM is off so the real llama_cpp loader runs (and fails)
+for k in ("MOCK_LLM", "HF_TOKEN", "MODEL_PATH"):
+    os.environ.pop(k, None)
+
+import uvicorn
+import app as app_module  # noqa: E402
+
+print("Starting server (MOCK_LLM=0, fake missing model)...")
+config = uvicorn.Config(app_module.app, host="127.0.0.1", port=PORT, log_level="warning")
+server = uvicorn.Server(config)
+import threading
+thread = threading.Thread(target=server.run, daemon=True)
+thread.start()
+
+if not wait_for_http(BASE_URL + "/", timeout=20):
+    print("Server failed to start. Stopping.")
+    server.should_exit = True
+    sys.exit(1)
+time.sleep(1.0)  # let eager get_llm() complete (FileNotFoundError is instant)
+print("Server ready.\n")
+
+try:
+    import urllib.request, json
+    with urllib.request.urlopen(BASE_URL + "/api/health", timeout=5) as r:
+        health = json.loads(r.read().decode())
+    print(f"  /api/health response: {json.dumps(health, indent=2)}\n")
+    check("health.llm is 'error' (real load was attempted)",
+          health.get("llm") == "error", f"got '{health.get('llm')}'")
+    check("health.llm_error is NON-EMPTY (real reason)",
+          bool(health.get("llm_error")), f"got '{health.get('llm_error')}'")
+    check("health.llm_error mentions the file or the path",
+          "FAKE_MISSING" in str(health.get("llm_error", "")) or "not found" in str(health.get("llm_error", "")).lower(),
+          f"err='{health.get('llm_error')}'")
+    check("health.model_exists is False",
+          health.get("model_exists") is False)
+
+    from playwright.sync_api import sync_playwright
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_context(viewport={"width": 1500, "height": 950}).new_page()
+        page.on("dialog", lambda d: d.accept())
+        page.goto(BASE_URL, wait_until="networkidle")
+        time.sleep(0.5)
+
+        status_text = page.locator("#status-line").inner_text()
+        print(f"  status-line: '{status_text}'")
+        check("status-line shows 'LLM offline'", "offline" in status_text.lower())
+        check("status-line does NOT show generic 'model not loaded' fallback",
+              "model not loaded" not in status_text.lower(),
+              f"status='{status_text}'")
+        check("status-line mentions the fake model path (real error)",
+              "FAKE_MISSING" in status_text,
+              f"status='{status_text}'")
+
+        llm_tag = page.locator("#llm-status").inner_text()
+        check("topbar shows OFFLINE", "OFFLINE" in llm_tag)
+        tip = page.locator("#llm-status").get_attribute("title") or ""
+        check("topbar tooltip has real error", len(tip) > 10 and "FAKE" in tip,
+              f"tooltip='{tip}'")
+
+        # Chat via deterministic fallback (no 'error: format only' leak)
+        page.fill("#chat-input", "Should I buy Nifty?")
+        page.click("#chat-form button")
+        page.wait_for_function(
+            "() => document.getElementById('chat-log').children.length >= 1",
+            timeout=8000,
+        )
+        chat_text = page.locator("#chat-log").inner_text()
+        print(f"  chat reply: '{chat_text[:120]}'")
+        check("chat returned a reply", len(chat_text.strip()) > 0)
+        check("chat did NOT leak 'error: format only'",
+              "error: format only" not in chat_text,
+              f"chat='{chat_text[:120]}'")
+        check("chat reply is a real, useful sentence",
+              len(chat_text.split()) >= 4, f"chat='{chat_text[:120]}'")
+
+        page.screenshot(path=os.path.join(SCREENSHOT_DIR, "llm_missing.png"), full_page=True)
+        browser.close()
+finally:
+    server.should_exit = True
+    time.sleep(0.5)
+
+print(f"\n{'='*60}")
+print(f"VERIFY LLM STATUS — PASSED: {PASSED}  FAILED: {FAILED}")
+print(f"{'='*60}")
+if FAILED:
+    sys.exit(1)
+print("ALL CHECKS PASSED — the real LLM error reason reaches the UI.")