Upload main.py with huggingface_hub

main.py

@@ -2,7 +2,12 @@
 Blood-Brain Omics Benchmark API
 
 FastAPI + DuckDB backend serving benchmark results.
-Loads Parquet files at startup and queries them via DuckDB in-memory.
+Queries Parquet files remotely via DuckDB httpfs — no data downloaded to disk.
+DuckDB reads only the needed columns/row groups per query via HTTP range requests.
+
+Data source modes (in priority order):
+  1. Remote: HF Dataset repo via httpfs (default, zero RAM footprint)
+  2. Local: files in BENCHMARK_DATA_DIR (for local dev)
 
 Endpoints:
     GET /api/v1/registry              - Full registry metadata
@@ -22,6 +27,7 @@ Endpoints:
 
 import json
 import os
+import urllib.request
 from typing import Optional
 
 import duckdb
@@ -32,6 +38,13 @@ from fastapi.middleware.cors import CORSMiddleware
 # Configuration
 # =============================================================================
 
+# HF Dataset repo for remote Parquet queries
+HF_DATASET = os.environ.get(
+    "HF_DATASET", "stasaking/blood-brain-benchmark"
+)
+HF_BASE_URL = f"https://huggingface.co/datasets/{HF_DATASET}/resolve/main"
+
+# Local fallback for dev
 DATA_DIR = os.environ.get(
     "BENCHMARK_DATA_DIR",
     os.path.join(os.path.dirname(__file__), "..", "data")
@@ -51,67 +64,67 @@ app.add_middleware(
 )
 
 # =============================================================================
-# Startup: load data into DuckDB
+# Startup: configure DuckDB with httpfs views or local tables
 # =============================================================================
 
 db = None
 registry = None
 
 
+def _resolve_parquet(filename: str) -> str:
+    """Return remote URL or local path for a Parquet file."""
+    local_path = os.path.join(DATA_DIR, filename)
+    if os.path.exists(local_path):
+        return local_path
+    return f"{HF_BASE_URL}/{filename}"
+
+
 @app.on_event("startup")
 def startup():
     global db, registry
 
-    # Load registry
-    registry_path = os.path.join(DATA_DIR, "benchmark_registry.json")
-    if os.path.exists(registry_path):
-        with open(registry_path) as f:
-            registry = json.load(f)
-    else:
-        registry = {"project": {}, "blood_platforms": {}, "brain_targets": {},
-                     "phases": {}, "models": {}}
-
-    # Initialize DuckDB
+    # Initialize DuckDB with httpfs
     db = duckdb.connect(":memory:")
+    db.execute("INSTALL httpfs; LOAD httpfs;")
+
+    # Resolve data sources (remote or local)
+    results_src = _resolve_parquet("benchmark_results.parquet")
+    features_src = _resolve_parquet("feature_importance.parquet")
+
+    is_remote = results_src.startswith("http")
+    mode = "remote httpfs" if is_remote else "local files"
+    print(f"Data mode: {mode}")
+
+    # Create views that query Parquet directly (no data copied into RAM)
+    db.execute(f"""
+        CREATE VIEW results AS
+        SELECT * FROM read_parquet('{results_src}')
+    """)
+    db.execute(f"""
+        CREATE VIEW features AS
+        SELECT * FROM read_parquet('{features_src}')
+    """)
 
-    results_path = os.path.join(DATA_DIR, "benchmark_results.parquet")
-    features_path = os.path.join(DATA_DIR, "feature_importance.parquet")
+    n_results = db.execute("SELECT COUNT(*) FROM results").fetchone()[0]
+    n_features = db.execute("SELECT COUNT(*) FROM features").fetchone()[0]
+    print(f"Views created: {n_results} results, {n_features} features")
 
-    if os.path.exists(results_path):
-        db.execute(f"""
-            CREATE TABLE results AS
-            SELECT * FROM read_parquet('{results_path}')
-        """)
-        n = db.execute("SELECT COUNT(*) FROM results").fetchone()[0]
-        print(f"Loaded results: {n} rows")
-    else:
-        print(f"WARNING: {results_path} not found")
-        db.execute("""
-            CREATE TABLE results (
-                phase VARCHAR, blood_platform VARCHAR, brain_target VARCHAR,
-                target VARCHAR, model VARCHAR, h2h_pair VARCHAR,
-                temporal_bin VARCHAR, n_samples INT, n_features INT,
-                include_covariates BOOLEAN, r2 DOUBLE, pearson DOUBLE,
-                mse DOUBLE, mae DOUBLE
-            )
-        """)
-
-    if os.path.exists(features_path):
-        db.execute(f"""
-            CREATE TABLE features AS
-            SELECT * FROM read_parquet('{features_path}')
-        """)
-        n = db.execute("SELECT COUNT(*) FROM features").fetchone()[0]
-        print(f"Loaded features: {n} rows")
+    # Load registry JSON (small file, always downloaded)
+    registry_local = os.path.join(DATA_DIR, "benchmark_registry.json")
+    if os.path.exists(registry_local):
+        with open(registry_local) as f:
+            registry = json.load(f)
     else:
-        print(f"WARNING: {features_path} not found")
-        db.execute("""
-            CREATE TABLE features (
-                phase VARCHAR, blood_platform VARCHAR, brain_target VARCHAR,
-                target VARCHAR, model VARCHAR, feature_name VARCHAR,
-                importance DOUBLE, rank SMALLINT
-            )
-        """)
+        # Download from HF
+        registry_url = f"{HF_BASE_URL}/benchmark_registry.json"
+        try:
+            with urllib.request.urlopen(registry_url) as resp:
+                registry = json.loads(resp.read().decode())
+            print(f"Downloaded registry from {registry_url}")
+        except Exception as e:
+            print(f"WARNING: Could not load registry: {e}")
+            registry = {"project": {}, "blood_platforms": {},
+                         "brain_targets": {}, "phases": {}, "models": {}}
 
 
 # =============================================================================